apple_epf 1.1.4 → 1.2.0

data/lib/apple_epf.rb

@@ -6,6 +6,7 @@ require 'core_ext/module'
 require 'apple_epf/errors'
 require 'apple_epf/logging'
 require 'apple_epf/main'
+require 'apple_epf/download_processor'
 require 'apple_epf/downloader'
 require 'apple_epf/extractor'
 require 'apple_epf/parser'
@@ -22,6 +23,12 @@ module AppleEpf
   mattr_accessor :download_retry_count
   @@download_retry_count = 3
 
+  mattr_accessor :concurrent_downloads
+  @@concurrent_downloads = 2
+
+  mattr_accessor :download_processor
+  @@download_processor = AppleEpf::CurbDownloadProcessor
+
   mattr_accessor :keep_tbz_after_extract
   @@keep_tbz_after_extract = false
 

data/lib/apple_epf/download_processor.rb

@@ -0,0 +1,29 @@
+module AppleEpf
+  class DownloadProcessor
+
+    def initialize(apple_filename_full, download_to)
+      @apple_filename_full = apple_filename_full
+      @download_to = download_to
+    end
+
+    def download_and_check
+      raise 'should be implemented in subclass'
+    end
+
+    def get_file_md5
+      begin
+        curl = Curl::Easy.new("#{@apple_filename_full}.md5")
+        curl.http_auth_types = :basic
+        curl.username = AppleEpf.apple_id
+        curl.password = AppleEpf.apple_password
+        curl.perform
+        @md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
+      rescue NoMethodError
+        raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
+      end
+    end
+  end
+end
+
+require "apple_epf/download_processor/curb_download_processor"
+require "apple_epf/download_processor/aria_download_processor"

data/lib/apple_epf/download_processor/aria_download_processor.rb

@@ -0,0 +1,16 @@
+class AppleEpf::AriaDownloadProcessor < AppleEpf::DownloadProcessor
+  def download_and_check
+    get_file_md5
+    download
+  end
+
+  def download
+    command = "cd #{File.dirname(@download_to)} && aria2c --continue --check-integrity=true --checksum=md5=#{@md5_checksum} -x#{AppleEpf.concurrent_downloads} -j#{AppleEpf.concurrent_downloads} -s#{AppleEpf.concurrent_downloads} --http-user=#{AppleEpf.apple_id} --http-passwd=#{AppleEpf.apple_password} -o #{File.basename(@download_to)} #{@apple_filename_full}"
+    result = system(command)
+
+    unless result
+      raise AppleEpf::DownloaderError.new("Unable to download file. #{$?}")
+    end
+  end
+
+end

data/lib/apple_epf/download_processor/curb_download_processor.rb

@@ -0,0 +1,46 @@
+class AppleEpf::CurbDownloadProcessor < AppleEpf::DownloadProcessor
+  def download_and_check
+    @download_retry = 0
+    get_file_md5
+    download
+    compare_md5_checksum
+    @download_to
+  end
+
+  def compare_md5_checksum
+    if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
+      raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
+    end
+  end
+
+  private
+  def download
+    begin
+      curl = Curl::Easy.new(@apple_filename_full)
+
+      # Authentication
+      curl.http_auth_types = :basic
+      curl.username = AppleEpf.apple_id
+      curl.password = AppleEpf.apple_password
+
+      File.open(@download_to, 'wb') do |f|
+        curl.on_body { |data|
+          f << data;
+          data.size
+        }
+        curl.perform
+      end
+    rescue Curl::Err::PartialFileError => ex
+      if @download_retry < AppleEpf.download_retry_count
+        @download_retry += 1
+
+        logger_info "Curl::Err::PartialFileError happened..."
+        logger_info "Restarting download"
+        download
+      else
+        raise AppleEpf::CurlError.new("Unable to download file.")
+      end
+    end
+  end
+
+end

data/lib/apple_epf/downloader.rb

@@ -19,7 +19,7 @@ module AppleEpf
       @force_url = force_url
     end
 
-    def download
+    def prepare
       _prepare_folders
       if @force_url
         @apple_filename_full = @force_url
@@ -27,15 +27,13 @@ module AppleEpf
         get_filename_by_date_and_type
         @apple_filename_full = apple_filename_full_url(@apple_filename_full_path)
       end
-
       @download_to = File.join(dirpath, File.basename(@apple_filename_full))
+    end
 
-      logger_info "Download file: #{@apple_filename_full}"
-      logger_info "Download to: #{@download_to}"
-
-      @download_retry = 0
-      start_download
-      download_and_compare_md5_checksum
+    def download
+      prepare
+      @download_processor = AppleEpf.download_processor.new(@apple_filename_full, @download_to)
+      @download_processor.download_and_check
       @download_to
     end
 
@@ -43,25 +41,6 @@ module AppleEpf
       File.join((@dirpath || AppleEpf.extract_dir), @type)
     end
 
-    #TODO combine with start_download
-    def download_and_compare_md5_checksum
-      begin
-        curl = Curl::Easy.new("#{@apple_filename_full}.md5")
-        curl.http_auth_types = :basic
-        curl.username = AppleEpf.apple_id
-        curl.password = AppleEpf.apple_password
-        curl.perform
-        @md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
-      rescue NoMethodError
-        raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
-      end
-
-      if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
-        raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
-      end
-
-      @md5_checksum
-    end
 
     def get_filename_by_date_and_type
       #today = DateTime.now
@@ -146,35 +125,5 @@ module AppleEpf
 
       response.code == "200"
     end
-
-    def start_download
-      begin
-        curl = Curl::Easy.new(@apple_filename_full)
-
-        # Authentication
-        curl.http_auth_types = :basic
-        curl.username = AppleEpf.apple_id
-        curl.password = AppleEpf.apple_password
-
-        File.open(@download_to, 'wb') do |f|
-          curl.on_body { |data|
-            f << data;
-            data.size
-          }
-          curl.perform
-        end
-      rescue Curl::Err::PartialFileError => ex
-        if @download_retry < AppleEpf.download_retry_count
-          @download_retry += 1
-
-          logger_info "Curl::Err::PartialFileError happened..."
-          logger_info "Restarting download"
-          start_download
-        else
-          raise AppleEpf::CurlError.new("Unable to download file.")
-        end
-      end
-    end
-
   end
 end

data/lib/apple_epf/errors.rb

@@ -5,4 +5,4 @@
     class Md5CompareError < DownloaderError; end
 
     class BadCredentialsError < StandardError; end
-  end
+  end

data/lib/apple_epf/version.rb

@@ -1,3 +1,3 @@
 module AppleEpf
-  VERSION = "1.1.4"
+  VERSION = "1.2.0"
 end

data/spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+require File.expand_path('../../../../spec_helper', __FILE__)
+require 'rack'
+require 'thin'
+
+describe AppleEpf::AriaDownloadProcessor do
+
+  describe "download" do
+    before(:all) do
+      @mockServer = Rack::File.new(apple_epf_dir)
+      @server_thread = Thread.new do
+        Rack::Handler::Thin.run @mockServer, :Port => 4400
+      end
+      sleep(2)
+    end
+
+    after(:all) do
+      @server_thread.kill
+    end
+
+    before do
+      @tmp_dir = [Dir.tmpdir, 'epm_files'].join('/')
+
+      FileUtils.mkpath @tmp_dir
+
+      AppleEpf.configure do |config|
+        config.apple_id = 'test'
+        config.apple_password = 'test'
+        config.extract_dir = @tmp_dir
+      end
+
+    end
+
+    after do
+      FileUtils.remove_dir(@tmp_dir)
+    end
+
+    it "should process if md5 is fine" do
+      downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
+
+      #correct md5
+      downloader.instance_variable_set(:@md5_checksum, '6fad1fb7823075d92296260fae3e317e')
+      downloader.download
+
+      File.read("#{@tmp_dir}/popularity20130111.tbz").should ==
+        File.read(apple_epf_inc_filename('popularity20130111.tbz'))
+    end
+
+    it "should return error if md5 is not correct" do
+      downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
+
+      downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
+      expect {
+        downloader.download
+      }.to raise_error(AppleEpf::DownloaderError)
+    end
+
+    it 'should return error if file is not found' do
+      downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130112.tbz", "#{@tmp_dir}/popularity20130112.tbz")
+
+      downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
+      expect {
+        downloader.download
+      }.to raise_error(AppleEpf::DownloaderError)
+    end
+
+    describe "download_and_check" do
+      it "should download md5 and file and compare" do
+        downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
+
+        stub_request(:get, "http://test:test@localhost:4400/popularity20130111.tbz.md5").
+          to_return(:status => 200, :body => "MD5 (popularity20130111.tbz) = 6fad1fb7823075d92296260fae3e317e\n", :headers => {})
+
+        expect {
+          downloader.download_and_check
+        }.to_not raise_error
+      end
+
+    end
+  end
+end

data/spec/lib/apple_epf/downloader_spec.rb

@@ -146,14 +146,14 @@ describe AppleEpf::Downloader do
     it "should properly set url for download" do
       downloader.stub(:file_exists?){ file_exists }
       downloader.stub(:start_download)
-      downloader.download
+      downloader.prepare
       downloader.apple_filename_full.should eq("https://feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz")
     end
 
     it "should properly set local file to store file in" do
       downloader.stub(:file_exists?){ file_exists }
       downloader.stub(:start_download)
-      downloader.download
+      downloader.prepare
       downloader.download_to.should eq("#{@tmp_dir}/incremental/popularity20130121.tbz")
     end
 
@@ -161,6 +161,9 @@ describe AppleEpf::Downloader do
       stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz").
         to_return(:status => 200, :body => "Test\nWow", :headers => {})
 
+      stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz.md5").
+        to_return(:status => 200, :body => "MD5 (popularity20130116.tbz) = 0371a79664856494e840af9e1e6c0152\n", :headers => {})
+
       downloader = AppleEpf::Downloader.new('full', file, filedate)
       downloader.stub(:download_and_compare_md5_checksum)
       downloader.stub(:file_exists?){ file_exists }
@@ -175,7 +178,8 @@ describe AppleEpf::Downloader do
     describe "dirpath" do
       before do
         downloader.stub(:file_exists?){ file_exists }
-        downloader.stub(:start_download)
+        #downloader.stub(:start_download)
+        AppleEpf.download_processor.any_instance.stub(:download_and_check)
       end
 
       it "should be able to change dir where to save files" do
@@ -222,4 +226,4 @@ describe AppleEpf::Downloader do
     end
   end
 
-end
+end

data/spec/support/fixture_helper.rb

@@ -1,4 +1,8 @@
 module FixtureHelper
+  def apple_epf_dir
+    File.expand_path("../../support/fixtures/itunes/epf/incremental", __FILE__)
+  end
+
   def apple_epf_inc_filename(filename)
     File.expand_path("../../support/fixtures/itunes/epf/incremental/#{filename}", __FILE__)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apple_epf
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-13 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -123,6 +123,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: curb
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +194,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/apple_epf/download_processor/aria_download_processor.rb
+- lib/apple_epf/download_processor/aria_download_processor.rb~
+- lib/apple_epf/download_processor/curb_download_processor.rb
+- lib/apple_epf/download_processor/curb_download_processor.rb~
+- lib/apple_epf/download_processor.rb
+- lib/apple_epf/download_processor.rb~
 - lib/apple_epf/downloader.rb
 - lib/apple_epf/errors.rb
 - lib/apple_epf/extractor.rb
@@ -192,6 +214,7 @@ files:
 - MIT-LICENSE
 - Rakefile
 - README.md
+- spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
 - spec/lib/apple_epf/downloader_spec.rb
 - spec/lib/apple_epf/exctractor_spec.rb
 - spec/lib/apple_epf/main_spec.rb
@@ -221,7 +244,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2434904396329215938
+      hash: -6031403435138943
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -230,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2434904396329215938
+      hash: -6031403435138943
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -238,6 +261,7 @@ signing_key:
 specification_version: 3
 summary: Downloader, Extractor and Parser for Apple Epf Affiliate files
 test_files:
+- spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
 - spec/lib/apple_epf/downloader_spec.rb
 - spec/lib/apple_epf/exctractor_spec.rb
 - spec/lib/apple_epf/main_spec.rb