apple_epf 1.0.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 Artem Kramrenko
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# AppleEpf
+
+## Installation
+
+    gem 'apple_epf'
+
+## Download incremental
+
+    AppleEpf.get_incremental( 'current',
+                                      lambda { |header| puts header },
+                                      lambda { |row| puts row },
+                                      lambda { |success| puts 'Yeah!' if success } )
+
+## Setup
+
+  Put this in your initializer.rb if you are using Rails.
+
+    AppleEpf.configure do |config|
+      config.apple_id = 'username'
+      config.apple_password = 'password'
+      config.download_retry_count = 3 #
+      config.keep_tbz_after_extract = false
+      config.extract_dir = '' # where to extract to
+      config.log_file = 'path to log file' #absolute path to log file
+      config.files_matrix = {} # {popularity: ['application_popularity_per_genre']}
+    end
+
+  All of this can be redefined for every downloader.
+
+## Manual manipulations
+
+    # Manually download one file
+    downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
+    downloader.download #=> will return local filepath to downloaded file or fire exception
+
+
+    # Manually extract one archive
+    extractor = AppleEpf::Extractor.new(filename, files_to_extract)
+    # filename - full path to local file
+    # files_to_extract - Files to be extracted from Archive (application, application_detail)
+    file_entry = extractor.perform #=> will return instance of FileEntry
+    file_entry.tbz_file #=> original file that was parsed. It is removed after untaring
+    file_entry.extracted_files #=> newly created(unpacked) files
+
+    #Manually parse file
+    parser = AppleEpf::Parser.new(filename, header_block, row_block)
+    # filename - full local path to file
+    # header_block - will return header info from file
+    # row_block - will return each row from file
+
+## Download and Extract
+  If you want to combine downloading and extracting your can use one of following
+  methonds. My personal feeling is to parsing should we something live alone and should not be combined in one stack with download and extract. And of cource it is better to download and extract files one by one.
+
+    manager = AppleEpf::Incremental.new('10-10-2012', {popularity: ['application_popularity_per_genre']})
+
+    manager = AppleEpf::Full.new('10-10-2012', {popularity: ['application_popularity_per_genre']})
+
+    manager.download_all_files #will download all files for this date for all keys "popularity", 'pricing', 'itunes' etc
+
+    manager.download_and_extract_all_files #will first download and than extract all files
+
+    manager.download_and_extract('itunes', ['application', 'application_detail']) #will donwload only 'itunes' and extract only ['application', 'application_detail']. This actually ignored matrix passed to initializer
+
+    manager.download('itunes') #will only download file
+
+  You can omit where to store files by setting it directly to downloader instance
+
+    manager.store_dir = '/whatever_dir_you_like'
+    manager.download('itunes')
+
+  OR
+
+    downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
+    downloader.dirpath = '/whatever_dir_you_like'
+    downloader.download
+
+  You can also omit if you want to store initial tbz files after they will be unpacked
+
+    extractor.keep_tbz_after_extract = true
+
+  OR
+
+    manager.keep_tbz_after_extract = true
+
+
+## Get list of current files avaliable for download
+
+  AppleEpf::Incremental.get_current_list #=> current incremental files
+  AppleEpf::Full.get_current_list #=> current full files

data/Rakefile

@@ -0,0 +1,35 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'AppleEpf'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task :default => :test

data/lib/apple_epf.rb

@@ -0,0 +1,43 @@
+require 'tmpdir'
+require 'fileutils'
+require 'chronic'
+require 'core_ext/array'
+require 'core_ext/module'
+require 'apple_epf/errors'
+require 'apple_epf/logging'
+require 'apple_epf/main'
+require 'apple_epf/downloader'
+require 'apple_epf/extractor'
+require 'apple_epf/parser'
+
+module AppleEpf
+  FILE_TYPES = %w( full incremental )
+
+  mattr_accessor :apple_id
+  @@apple_id = "test"
+
+  mattr_accessor :apple_password
+  @@apple_password = "test"
+
+  mattr_accessor :download_retry_count
+  @@download_retry_count = 3
+
+  mattr_accessor :keep_tbz_after_extract
+  @@keep_tbz_after_extract = false
+
+  mattr_accessor :extract_dir
+  @@extract_dir = [Dir.tmpdir, 'epm_files'].join('/')
+
+  mattr_accessor :log_to_console
+  @@keep_tbz_after_extract = log_to_console
+
+  mattr_accessor :log_file
+  @@log_file = '/dev/null'
+
+  mattr_accessor :files_matrix
+  @@files_matrix = {popularity: ['application_popularity_per_genre']}.freeze
+
+  def self.configure
+    yield self
+  end
+end

data/lib/apple_epf/downloader.rb

@@ -0,0 +1,164 @@
+require 'net/http'
+require 'date'
+require 'curb'
+require 'digest/md5'
+
+module AppleEpf
+  class Downloader
+    include AppleEpf::Logging
+    ITUNES_FLAT_FEED_URL = 'http://feeds.itunes.apple.com/feeds/epf/v3/full'.freeze
+
+    attr_accessor :type, :filename, :filedate
+
+    attr_reader :download_to, :apple_filename_full
+    attr_writer :dirpath
+    def initialize(type, filename, filedate)
+      @type = type
+      @filename = filename #itunes, popularity, match, pricing
+      @filedate = filedate
+    end
+
+    def download
+      _prepare_folders
+      get_filename_by_date_and_type
+
+      @apple_filename_full = apple_filename_full_url(@apple_filename_full_path)
+      @download_to = File.join(dirpath, File.basename(@apple_filename_full))
+
+      logger_info "Download file: #{@apple_filename_full}"
+      logger_info "Download to: #{@download_to}"
+
+      @download_retry = 0
+      start_download
+      download_and_compare_md5_checksum
+      @download_to
+    end
+
+    def dirpath
+      File.join((@dirpath || AppleEpf.extract_dir), @type)
+    end
+
+    #TODO combine with start_download
+    def download_and_compare_md5_checksum
+      begin
+        curl = Curl::Easy.new("#{@apple_filename_full}.md5")
+        curl.http_auth_types = :basic
+        curl.username = AppleEpf.apple_id
+        curl.password = AppleEpf.apple_password
+        curl.perform
+        @md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
+      rescue NoMethodError
+        raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
+      end
+
+      if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
+        raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
+      end
+
+      @md5_checksum
+    end
+
+    def get_filename_by_date_and_type
+      #today = DateTime.now
+      path = ""
+      case @type
+        when "full"
+          path = "#{main_dir_date}/#{@filename}#{main_dir_date}.tbz"
+
+        when "incremental"
+          date_of_file = date_to_epf_format(@filedate)
+          path = "#{main_dir_date}/incremental/#{date_of_file}/#{@filename}#{date_of_file}.tbz"
+
+        when "file"
+          #TODO: FIX THIS
+          # date = date_to_epf_format( @filedate, check_if_in_previous_week, check_if_in_thursday )
+          # path = "#{file}#{date}.tbz"
+      end
+
+      # Return false if no url was suggested or file does not exist
+      raise AppleEpf::DownloaderError.new("Unable to find out what file do you want to download") if path.empty?
+      raise AppleEpf::FileNotExist.new("File does not exist #{path}") unless file_exists?(path)
+
+      @apple_filename_full_path = path
+      @apple_filename_full_path
+    end
+
+    def downloaded_file_base_name
+      File.basename(@download_to, '.tbz') #popularity20130109
+    end
+
+    private
+
+    def apple_filename_full_url(path)
+      File.join(ITUNES_FLAT_FEED_URL, path)
+    end
+
+    def _prepare_folders
+      logger_info "Create folders for path: #{dirpath}"
+      FileUtils.mkpath(dirpath)
+    end
+
+    def main_dir_date
+      if @type == "incremental"
+        # from Mon to Thurday dumps are in prev week folder
+        this_or_last = @filedate.wday <= 4 ? 'last' : 'this'
+      elsif @type == "full"
+        # full downloads usually are done only once. user can determine when it should be done
+        this_or_last = 'this'
+      end
+
+      main_folder_date = Chronic.parse("#{this_or_last} week wednesday", :now => @filedate.to_time).to_date
+      date_to_epf_format(main_folder_date)
+    end
+
+    def date_to_epf_format(date)
+      date.strftime("%Y%m%d")
+    end
+
+    def file_exists?(path_to_check)
+      full_url = apple_filename_full_url(path_to_check)
+      logger_info "Checking file at URL: #{full_url}"
+
+      uri = URI.parse(full_url)
+
+      request = Net::HTTP::Head.new(full_url)
+      request.basic_auth(AppleEpf.apple_id, AppleEpf.apple_password)
+
+      response = Net::HTTP.new(uri.host, uri.port).start { |http| http.request(request) }
+
+      raise AppleEpf::BadCredentialsError.new('Bad credentials') if response.code == "401"
+
+      response.code == "200"
+    end
+
+    def start_download
+      begin
+        curl = Curl::Easy.new(@apple_filename_full)
+
+        # Authentication
+        curl.http_auth_types = :basic
+        curl.username = AppleEpf.apple_id
+        curl.password = AppleEpf.apple_password
+
+        File.open(@download_to, 'wb') do |f|
+          curl.on_body { |data|
+            f << data;
+            data.size
+          }
+          curl.perform
+        end
+      rescue Curl::Err::PartialFileError => ex
+        if @download_retry < AppleEpf.download_retry_count
+          @download_retry += 1
+
+          logger_info "Curl::Err::PartialFileError happened..."
+          logger_info "Restarting download"
+          start_download
+        else
+          raise AppleEpf::CurlError.new("Unable to download file.")
+        end
+      end
+    end
+
+  end
+end

data/lib/apple_epf/errors.rb

@@ -0,0 +1,8 @@
+  module AppleEpf
+    class DownloaderError < StandardError; end
+    class FileNotExist < DownloaderError; end
+    class CurlError < DownloaderError; end
+    class Md5CompareError < DownloaderError; end
+
+    class BadCredentialsError < StandardError; end
+  end

data/lib/apple_epf/extractor.rb

@@ -0,0 +1,43 @@
+module AppleEpf
+  class Extractor
+    class FileEntry <  Struct.new(:tbz_file, :extracted_files); end
+
+    attr_reader :file_entry, :filename, :dirname, :basename
+    attr_accessor :keep_tbz_after_extract
+
+    def initialize(filename, files_to_extract)
+      @filename = filename
+      @files_to_extract = files_to_extract
+
+      @dirname = File.dirname(@filename)
+      @basename = File.basename(@filename)
+    end
+
+    #TODO use multithread uncompressing tool
+    def perform
+      @extracted_files = Array.new
+      @files_to_extract.each do |f|
+        @extracted_files.push File.basename(@filename, '.tbz') + '/' + f
+      end
+
+      result = system "cd #{@dirname} && tar -xjf #{@basename} #{@extracted_files.join(' ')}"
+
+      if result
+        _extracted_files = @extracted_files.map{|f| File.join(@dirname, f)}
+        @file_entry = FileEntry.new(@filename, Hash[@files_to_extract.zip(_extracted_files)])
+        FileUtils.remove_file(@filename, true) unless keep_tbz_after_extract?
+      else
+        raise "Unable to extract files '#{@files_to_extract.join(' ')}' from #{@filename}"
+      end
+
+      @file_entry
+    end
+
+    private
+    def keep_tbz_after_extract?
+      !!keep_tbz_after_extract || AppleEpf.keep_tbz_after_extract
+    end
+  end
+
+
+end

data/lib/apple_epf/logging.rb

@@ -0,0 +1,32 @@
+require 'logger'
+
+module AppleEpf
+  module Logging
+    def self.logger
+      @logger ||= initialize_logger
+    end
+
+    def logger
+      AppleEpf::Logging.logger
+    end
+
+    def logger_info(data)
+      logger.info(data)
+      p data if AppleEpf.log_to_console
+    end
+
+    private
+
+    def self.initialize_logger
+      begin
+        logfile = File.open(AppleEpf::log_file, File::WRONLY | File::APPEND | File::CREAT)
+        logger = Logger.new(logfile, 'weekly')
+        logger.level = Logger::DEBUG
+        logger
+      rescue
+        p "Unable to create logger"
+        raise $!
+      end
+    end
+  end
+end

data/lib/apple_epf/main.rb

@@ -0,0 +1,113 @@
+require 'nokogiri'
+
+module AppleEpf
+  class Main
+    attr_reader :downloader, :filedate, :files_matrix
+    attr_accessor :store_dir, :keep_tbz_after_extract
+
+    def initialize(filedate, files_matrix = nil, store_dir = nil, keep_tbz_after_extract = nil)
+      @filedate = filedate
+      @files_matrix = files_matrix || AppleEpf.files_matrix
+      @store_dir = store_dir
+      @keep_tbz_after_extract = !!keep_tbz_after_extract || AppleEpf.keep_tbz_after_extract
+    end
+
+    def self.get_current_list
+        curl = Curl::Easy.new(self.current_url)
+        curl.http_auth_types = :basic
+        curl.username = AppleEpf.apple_id
+        curl.password = AppleEpf.apple_password
+        curl.perform
+        body = curl.body_str
+        files =  Nokogiri::HTML(body).xpath("//td[2]/a").map(&:text).select{|s| s=~/.*tbz$/}.map{|f| f.chomp('.tbz')}
+        files.inject({}) do |all, e|
+          e.match(/([a-z]*)(\d*)/)
+          all[$1] = $2
+          all
+        end
+    end
+
+    module BaseActions
+      def download_all_files(&block)
+        downloaded_files = []
+
+        @files_matrix.each_pair do |filename, extractables|
+          begin
+            downloaded = self.download(filename)
+            downloaded_files << downloaded
+            block.call(downloaded) if block_given?
+          rescue AppleEpf::DownloaderError
+            AppleEpf::Logging.logger.fatal "Failed to download file #{filename}"
+            AppleEpf::Logging.logger.fatal $!
+            next
+          end
+        end
+
+        downloaded_files
+      end
+
+      def download_and_extract_all_files(&block)
+        extracted_files = []
+
+        @files_matrix.each_pair do |filename, extractables|
+          begin
+            extracted_file = download_and_extract(filename.to_s, extractables)
+            extracted_files << extracted_file
+            block.call(extracted_file) if block_given?
+          rescue
+            AppleEpf::Logging.logger.fatal "Failed to download and parse file #{filename}"
+            next
+          end
+        end
+
+        extracted_files
+      end
+
+      # will return array of filepath of extracted files
+      def download_and_extract(filename, extractables)
+        downloader = self.download(filename.to_s)
+        downloaded_file = downloader.download_to
+        self.extract(downloaded_file, extractables)
+      end
+
+      def download(filename)
+        downloader = AppleEpf::Downloader.new(type, filename.to_s, @filedate)
+        downloader.dirpath = @store_dir if @store_dir
+        downloader.download
+        downloader
+      end
+
+      def extract(downloaded_file, extractables)
+        extractor = AppleEpf::Extractor.new(downloaded_file, extractables)
+        extractor.keep_tbz_after_extract = @keep_tbz_after_extract if @keep_tbz_after_extract
+        extractor.perform
+        extractor.file_entry
+      end
+    end
+  end
+
+  class Incremental <  Main
+    include BaseActions
+
+    def type
+      'incremental'
+    end
+
+    def self.current_url
+      'http://feeds.itunes.apple.com/feeds/epf/v3/full/current/incremental/current'
+    end
+  end
+
+  class Full < Main
+    include BaseActions
+
+    def type
+      'full'
+    end
+
+    def self.current_url
+      'http://feeds.itunes.apple.com/feeds/epf/v3/full/current'
+    end
+  end
+
+end