apple_epf 1.2.1 → 1.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a87068a1d217c86b226a7710fadef9ec1330bd7
+  data.tar.gz: 1dc1aa8b23d85f3e4e2f66a37e75ed16b7fb93dc
+SHA512:
+  metadata.gz: faa56def47d9748e3910d7d6a4066ad527640ac45980fc201b95e2f7f324e3ddffdfe92e056292ed166c449a96a6728bc059eaa80b5cf193291596651eec3492
+  data.tar.gz: 4f65225dd120a03d2b1f8e2e8fd3cf5bebeafe2238e753e8ee7ee8b551276db2ca2a307882336e477ac9b0a80cfe5c07b6b2ed9e953b1f3b52bd49d3e3684990

data/README.md

@@ -1,74 +1,95 @@
 # AppleEpf
 
 ## Installation
-
-    gem 'apple_epf'
+```ruby
+  gem 'apple_epf'
+```
 
 ## Setup
 
   Put this in your initializer.rb if you are using Rails.
-
-    AppleEpf.configure do |config|
-      config.apple_id = 'username'
-      config.apple_password = 'password'
-      config.download_retry_count = 3 #
-      config.keep_tbz_after_extract = false
-      config.extract_dir = '' # where to extract to
-      config.log_file = 'path to log file' #absolute path to log file
-      config.files_matrix = {} # {popularity: ['application_popularity_per_genre']}
-    end
+```ruby
+AppleEpf.configure do |config|
+  config.apple_id = 'username'
+  config.apple_password = 'password'
+  config.download_retry_count = 3 #
+  config.keep_tbz_after_extract = false
+  config.extract_dir = '' # where to extract to
+  config.files_matrix = {} # {popularity: ['application_popularity_per_genre']}
+  #config.files_matrix = {itunes: [], pricing: [], popularity: []}
+  config.download_processor = AppleEpf::AriaDownloadProcessor
+  config.concurrent_downloads = 16
+  config.log_file = "#{Rails.root}/log/apple_epf_#{Rails.env}.log"
+end
+```
 
   All of this can be redefined for every downloader.
 
 ## Manual manipulations
 
-    # Manually download one file
-    downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
-    downloader.download #=> will return local filepath to downloaded file or fire exception
+```ruby
+  # Manually download one file
+  downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
+  downloader.download #=> will return local filepath to downloaded file or fire exception
 
+  # Manually extract one archive
+  extractor = AppleEpf::Extractor.new(filename, files_to_extract)
+  # filename - full path to local file
+  # files_to_extract - Files to be extracted from Archive (application, application_detail)
+  file_entry = extractor.perform #=> will return instance of FileEntry
+  file_entry.tbz_file #=> original file that was parsed. It is removed after untaring
+  file_entry.extracted_files #=> newly created(unpacked) files
 
-    # Manually extract one archive
-    extractor = AppleEpf::Extractor.new(filename, files_to_extract)
-    # filename - full path to local file
-    # files_to_extract - Files to be extracted from Archive (application, application_detail)
-    file_entry = extractor.perform #=> will return instance of FileEntry
-    file_entry.tbz_file #=> original file that was parsed. It is removed after untaring
-    file_entry.extracted_files #=> newly created(unpacked) files
+  #Manually parse file
+  parser = AppleEpf::Parser.new(filename)
+  # filename - full local path to file
 
-    #Manually parse file
-    parser = AppleEpf::Parser.new(filename)
-    # filename - full local path to file
-    
-    parser.process_rows do |r|
-      puts "row is #{r}"
-    end
+  parser.process_rows do |r|
+    puts "row is #{r}"
+  end
+```
 
 ## Download and Extract
   If you want to combine downloading and extracting your can use one of following
   methonds. My personal feeling is to parsing should we something live alone and should not be combined in one stack with download and extract. And of cource it is better to download and extract files one by one.
 
-    manager = AppleEpf::Incremental.new('10-10-2012', {popularity: ['application_popularity_per_genre']})
+```ruby
+  manager = AppleEpf::Incremental.new('10-10-2012', 
+    { popularity: ['application_popularity_per_genre'] })
+
+
+  manager = AppleEpf::Full.new('10-10-2012', 
+    { popularity: ['application_popularity_per_genre'] })
 
-    manager = AppleEpf::Full.new('10-10-2012', {popularity: ['application_popularity_per_genre']})
 
-    manager.download_all_files #will download all files for this date for all keys "popularity", 'pricing', 'itunes' etc
+  manager.download_all_files 
+  # will download all files for this date 
+  # for all keys "popularity", 'pricing', 'itunes' etc
 
-    manager.download_and_extract_all_files #will first download and than extract all files
+  manager.download_and_extract_all_files 
+  #will first download and than extract all files
 
-    manager.download_and_extract('itunes', ['application', 'application_detail']) #will donwload only 'itunes' and extract only ['application', 'application_detail']. This actually ignored matrix passed to initializer
+  manager.download_and_extract('itunes', ['application', 'application_detail']) 
+  # will download only 'itunes' and extract only ['application', 'application_detail']. 
+  # This actually ignores matrix passed to initializer
 
-    manager.download('itunes') #will only download file
+  manager.download('itunes') #will only download file
+```
 
   You can omit where to store files by setting it directly to downloader instance
 
-    manager.store_dir = '/whatever_dir_you_like'
-    manager.download('itunes')
+```ruby
+  manager.store_dir = '/whatever_dir_you_like'
+  manager.download('itunes')
+```
 
   OR
 
-    downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
-    downloader.dirpath = '/whatever_dir_you_like'
-    downloader.download
+```ruby
+  downloader = AppleEpf::Downloader.new('incremental', 'popularity', Date.parse('17-01-2013'))
+  downloader.dirpath = '/whatever_dir_you_like'
+  downloader.download
+```
 
   You can also omit if you want to store initial tbz files after they will be unpacked
 
@@ -80,6 +101,22 @@
 
 
 ## Get list of current files avaliable for download
+```ruby
+AppleEpf::Incremental.get_current_list #=> current incremental files
+AppleEpf::Full.get_current_list #=> current full files
+```
+
+## Make sure to try AriaDownloadProcessor
+  There are 2 downloaders avaliable for use:
+  1. `CurbDownloadProcessor` - default one
+  2. `AriaDownloadProcessor` - we use in production
+
+  I suggest using last one, as in can do download in parallel. I sugest set 
+  `config.concurrent_downloads = 16 or 8`. If you chose to use aria, make sure
+  you have `aria2c` in your `PATH`.
+
+  And of cource you write your own processor.
 
-  AppleEpf::Incremental.get_current_list #=> current incremental files
-  AppleEpf::Full.get_current_list #=> current full files
+## Make sure to try lbzip2
+  lbzip2 uncompress files in parallel. Thanks [@funkyboy](https://github.com/funkyboy) for his help.
+  Just set `config.use_lbzip2 = true` and make sure you have it in `PATH`. On mac you can use `brew install lbzip2` for it.

data/lib/apple_epf.rb

@@ -46,6 +46,9 @@ module AppleEpf
   mattr_accessor :files_matrix
   @@files_matrix = {popularity: ['application_popularity_per_genre']}.freeze
 
+  mattr_accessor :use_lbzip2
+  @@use_lbzip2 = false
+
   def self.configure
     yield self
   end

data/lib/apple_epf/extractor.rb

@@ -20,7 +20,11 @@ module AppleEpf
         @extracted_files.push File.basename(@filename, '.tbz') + '/' + f
       end
 
-      result = system "cd #{@dirname} && tar -xjf #{@basename} #{@extracted_files.join(' ')}"
+      result = if AppleEpf.use_lbzip2
+        system "cd #{@dirname} && tar -xj #{@basename} #{@extracted_files.join(' ')} --use-compress-program lbzip2"
+      else
+        system "cd #{@dirname} && tar -xjf #{@basename} #{@extracted_files.join(' ')} "
+      end
 
       if result
         _extracted_files = @extracted_files.map{|f| File.join(@dirname, f)}

data/lib/apple_epf/version.rb

@@ -1,3 +1,3 @@
 module AppleEpf
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

metadata

@@ -1,190 +1,167 @@
 --- !ruby/object:Gem::Specification
 name: apple_epf
 version: !ruby/object:Gem::Version
-  version: 1.2.1
-  prerelease: 
+  version: 1.2.2
 platform: ruby
 authors:
 - Artem Kramarenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2015-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.12'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.3
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rb-fsevent
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: thin
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: curb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: chronic
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.6
 description: Downloader, Extractor and Parser for Apple Epf Affiliate files
@@ -194,28 +171,24 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/apple_epf.rb
+- lib/apple_epf/download_processor.rb
 - lib/apple_epf/download_processor/aria_download_processor.rb
-- lib/apple_epf/download_processor/aria_download_processor.rb~
 - lib/apple_epf/download_processor/curb_download_processor.rb
-- lib/apple_epf/download_processor/curb_download_processor.rb~
-- lib/apple_epf/download_processor.rb
-- lib/apple_epf/download_processor.rb~
 - lib/apple_epf/downloader.rb
 - lib/apple_epf/errors.rb
 - lib/apple_epf/extractor.rb
 - lib/apple_epf/finder.rb
-- lib/apple_epf/finder.rb~
 - lib/apple_epf/logging.rb
 - lib/apple_epf/main.rb
 - lib/apple_epf/parser.rb
 - lib/apple_epf/version.rb
-- lib/apple_epf.rb
 - lib/core_ext/array.rb
 - lib/core_ext/module.rb
 - lib/tasks/apple_epf_tasks.rake
-- MIT-LICENSE
-- Rakefile
-- README.md
 - spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
 - spec/lib/apple_epf/downloader_spec.rb
 - spec/lib/apple_epf/exctractor_spec.rb
@@ -236,33 +209,26 @@ files:
 - spec/support/fixtures/itunes/epf/index_of_folders.html
 homepage: 
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2191365109496733098
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2191365109496733098
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Downloader, Extractor and Parser for Apple Epf Affiliate files
 test_files:
 - spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb