apple_epf 1.1.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ require 'core_ext/module'
6
6
  require 'apple_epf/errors'
7
7
  require 'apple_epf/logging'
8
8
  require 'apple_epf/main'
9
+ require 'apple_epf/download_processor'
9
10
  require 'apple_epf/downloader'
10
11
  require 'apple_epf/extractor'
11
12
  require 'apple_epf/parser'
@@ -22,6 +23,12 @@ module AppleEpf
22
23
  mattr_accessor :download_retry_count
23
24
  @@download_retry_count = 3
24
25
 
26
+ mattr_accessor :concurrent_downloads
27
+ @@concurrent_downloads = 2
28
+
29
+ mattr_accessor :download_processor
30
+ @@download_processor = AppleEpf::CurbDownloadProcessor
31
+
25
32
  mattr_accessor :keep_tbz_after_extract
26
33
  @@keep_tbz_after_extract = false
27
34
 
@@ -0,0 +1,29 @@
1
+ module AppleEpf
2
+ class DownloadProcessor
3
+
4
+ def initialize(apple_filename_full, download_to)
5
+ @apple_filename_full = apple_filename_full
6
+ @download_to = download_to
7
+ end
8
+
9
+ def download_and_check
10
+ raise 'should be implemented in subclass'
11
+ end
12
+
13
+ def get_file_md5
14
+ begin
15
+ curl = Curl::Easy.new("#{@apple_filename_full}.md5")
16
+ curl.http_auth_types = :basic
17
+ curl.username = AppleEpf.apple_id
18
+ curl.password = AppleEpf.apple_password
19
+ curl.perform
20
+ @md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
21
+ rescue NoMethodError
22
+ raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ require "apple_epf/download_processor/curb_download_processor"
29
+ require "apple_epf/download_processor/aria_download_processor"
File without changes
@@ -0,0 +1,16 @@
1
+ class AppleEpf::AriaDownloadProcessor < AppleEpf::DownloadProcessor
2
+ def download_and_check
3
+ get_file_md5
4
+ download
5
+ end
6
+
7
+ def download
8
+ command = "cd #{File.dirname(@download_to)} && aria2c --continue --check-integrity=true --checksum=md5=#{@md5_checksum} -x#{AppleEpf.concurrent_downloads} -j#{AppleEpf.concurrent_downloads} -s#{AppleEpf.concurrent_downloads} --http-user=#{AppleEpf.apple_id} --http-passwd=#{AppleEpf.apple_password} -o #{File.basename(@download_to)} #{@apple_filename_full}"
9
+ result = system(command)
10
+
11
+ unless result
12
+ raise AppleEpf::DownloaderError.new("Unable to download file. #{$?}")
13
+ end
14
+ end
15
+
16
+ end
@@ -0,0 +1,46 @@
1
+ class AppleEpf::CurbDownloadProcessor < AppleEpf::DownloadProcessor
2
+ def download_and_check
3
+ @download_retry = 0
4
+ get_file_md5
5
+ download
6
+ compare_md5_checksum
7
+ @download_to
8
+ end
9
+
10
+ def compare_md5_checksum
11
+ if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
12
+ raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
13
+ end
14
+ end
15
+
16
+ private
17
+ def download
18
+ begin
19
+ curl = Curl::Easy.new(@apple_filename_full)
20
+
21
+ # Authentication
22
+ curl.http_auth_types = :basic
23
+ curl.username = AppleEpf.apple_id
24
+ curl.password = AppleEpf.apple_password
25
+
26
+ File.open(@download_to, 'wb') do |f|
27
+ curl.on_body { |data|
28
+ f << data;
29
+ data.size
30
+ }
31
+ curl.perform
32
+ end
33
+ rescue Curl::Err::PartialFileError => ex
34
+ if @download_retry < AppleEpf.download_retry_count
35
+ @download_retry += 1
36
+
37
+ logger_info "Curl::Err::PartialFileError happened..."
38
+ logger_info "Restarting download"
39
+ download
40
+ else
41
+ raise AppleEpf::CurlError.new("Unable to download file.")
42
+ end
43
+ end
44
+ end
45
+
46
+ end
@@ -19,7 +19,7 @@ module AppleEpf
19
19
  @force_url = force_url
20
20
  end
21
21
 
22
- def download
22
+ def prepare
23
23
  _prepare_folders
24
24
  if @force_url
25
25
  @apple_filename_full = @force_url
@@ -27,15 +27,13 @@ module AppleEpf
27
27
  get_filename_by_date_and_type
28
28
  @apple_filename_full = apple_filename_full_url(@apple_filename_full_path)
29
29
  end
30
-
31
30
  @download_to = File.join(dirpath, File.basename(@apple_filename_full))
31
+ end
32
32
 
33
- logger_info "Download file: #{@apple_filename_full}"
34
- logger_info "Download to: #{@download_to}"
35
-
36
- @download_retry = 0
37
- start_download
38
- download_and_compare_md5_checksum
33
+ def download
34
+ prepare
35
+ @download_processor = AppleEpf.download_processor.new(@apple_filename_full, @download_to)
36
+ @download_processor.download_and_check
39
37
  @download_to
40
38
  end
41
39
 
@@ -43,25 +41,6 @@ module AppleEpf
43
41
  File.join((@dirpath || AppleEpf.extract_dir), @type)
44
42
  end
45
43
 
46
- #TODO combine with start_download
47
- def download_and_compare_md5_checksum
48
- begin
49
- curl = Curl::Easy.new("#{@apple_filename_full}.md5")
50
- curl.http_auth_types = :basic
51
- curl.username = AppleEpf.apple_id
52
- curl.password = AppleEpf.apple_password
53
- curl.perform
54
- @md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
55
- rescue NoMethodError
56
- raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
57
- end
58
-
59
- if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
60
- raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
61
- end
62
-
63
- @md5_checksum
64
- end
65
44
 
66
45
  def get_filename_by_date_and_type
67
46
  #today = DateTime.now
@@ -146,35 +125,5 @@ module AppleEpf
146
125
 
147
126
  response.code == "200"
148
127
  end
149
-
150
- def start_download
151
- begin
152
- curl = Curl::Easy.new(@apple_filename_full)
153
-
154
- # Authentication
155
- curl.http_auth_types = :basic
156
- curl.username = AppleEpf.apple_id
157
- curl.password = AppleEpf.apple_password
158
-
159
- File.open(@download_to, 'wb') do |f|
160
- curl.on_body { |data|
161
- f << data;
162
- data.size
163
- }
164
- curl.perform
165
- end
166
- rescue Curl::Err::PartialFileError => ex
167
- if @download_retry < AppleEpf.download_retry_count
168
- @download_retry += 1
169
-
170
- logger_info "Curl::Err::PartialFileError happened..."
171
- logger_info "Restarting download"
172
- start_download
173
- else
174
- raise AppleEpf::CurlError.new("Unable to download file.")
175
- end
176
- end
177
- end
178
-
179
128
  end
180
129
  end
@@ -5,4 +5,4 @@
5
5
  class Md5CompareError < DownloaderError; end
6
6
 
7
7
  class BadCredentialsError < StandardError; end
8
- end
8
+ end
@@ -1,3 +1,3 @@
1
1
  module AppleEpf
2
- VERSION = "1.1.4"
2
+ VERSION = "1.2.0"
3
3
  end
@@ -0,0 +1,81 @@
1
+ # encoding: UTF-8
2
+ require File.expand_path('../../../../spec_helper', __FILE__)
3
+ require 'rack'
4
+ require 'thin'
5
+
6
+ describe AppleEpf::AriaDownloadProcessor do
7
+
8
+ describe "download" do
9
+ before(:all) do
10
+ @mockServer = Rack::File.new(apple_epf_dir)
11
+ @server_thread = Thread.new do
12
+ Rack::Handler::Thin.run @mockServer, :Port => 4400
13
+ end
14
+ sleep(2)
15
+ end
16
+
17
+ after(:all) do
18
+ @server_thread.kill
19
+ end
20
+
21
+ before do
22
+ @tmp_dir = [Dir.tmpdir, 'epm_files'].join('/')
23
+
24
+ FileUtils.mkpath @tmp_dir
25
+
26
+ AppleEpf.configure do |config|
27
+ config.apple_id = 'test'
28
+ config.apple_password = 'test'
29
+ config.extract_dir = @tmp_dir
30
+ end
31
+
32
+ end
33
+
34
+ after do
35
+ FileUtils.remove_dir(@tmp_dir)
36
+ end
37
+
38
+ it "should process if md5 is fine" do
39
+ downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
40
+
41
+ #correct md5
42
+ downloader.instance_variable_set(:@md5_checksum, '6fad1fb7823075d92296260fae3e317e')
43
+ downloader.download
44
+
45
+ File.read("#{@tmp_dir}/popularity20130111.tbz").should ==
46
+ File.read(apple_epf_inc_filename('popularity20130111.tbz'))
47
+ end
48
+
49
+ it "should return error if md5 is not correct" do
50
+ downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
51
+
52
+ downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
53
+ expect {
54
+ downloader.download
55
+ }.to raise_error(AppleEpf::DownloaderError)
56
+ end
57
+
58
+ it 'should return error if file is not found' do
59
+ downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130112.tbz", "#{@tmp_dir}/popularity20130112.tbz")
60
+
61
+ downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
62
+ expect {
63
+ downloader.download
64
+ }.to raise_error(AppleEpf::DownloaderError)
65
+ end
66
+
67
+ describe "download_and_check" do
68
+ it "should download md5 and file and compare" do
69
+ downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
70
+
71
+ stub_request(:get, "http://test:test@localhost:4400/popularity20130111.tbz.md5").
72
+ to_return(:status => 200, :body => "MD5 (popularity20130111.tbz) = 6fad1fb7823075d92296260fae3e317e\n", :headers => {})
73
+
74
+ expect {
75
+ downloader.download_and_check
76
+ }.to_not raise_error
77
+ end
78
+
79
+ end
80
+ end
81
+ end
@@ -146,14 +146,14 @@ describe AppleEpf::Downloader do
146
146
  it "should properly set url for download" do
147
147
  downloader.stub(:file_exists?){ file_exists }
148
148
  downloader.stub(:start_download)
149
- downloader.download
149
+ downloader.prepare
150
150
  downloader.apple_filename_full.should eq("https://feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz")
151
151
  end
152
152
 
153
153
  it "should properly set local file to store file in" do
154
154
  downloader.stub(:file_exists?){ file_exists }
155
155
  downloader.stub(:start_download)
156
- downloader.download
156
+ downloader.prepare
157
157
  downloader.download_to.should eq("#{@tmp_dir}/incremental/popularity20130121.tbz")
158
158
  end
159
159
 
@@ -161,6 +161,9 @@ describe AppleEpf::Downloader do
161
161
  stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz").
162
162
  to_return(:status => 200, :body => "Test\nWow", :headers => {})
163
163
 
164
+ stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz.md5").
165
+ to_return(:status => 200, :body => "MD5 (popularity20130116.tbz) = 0371a79664856494e840af9e1e6c0152\n", :headers => {})
166
+
164
167
  downloader = AppleEpf::Downloader.new('full', file, filedate)
165
168
  downloader.stub(:download_and_compare_md5_checksum)
166
169
  downloader.stub(:file_exists?){ file_exists }
@@ -175,7 +178,8 @@ describe AppleEpf::Downloader do
175
178
  describe "dirpath" do
176
179
  before do
177
180
  downloader.stub(:file_exists?){ file_exists }
178
- downloader.stub(:start_download)
181
+ #downloader.stub(:start_download)
182
+ AppleEpf.download_processor.any_instance.stub(:download_and_check)
179
183
  end
180
184
 
181
185
  it "should be able to change dir where to save files" do
@@ -222,4 +226,4 @@ describe AppleEpf::Downloader do
222
226
  end
223
227
  end
224
228
 
225
- end
229
+ end
@@ -1,4 +1,8 @@
1
1
  module FixtureHelper
2
+ def apple_epf_dir
3
+ File.expand_path("../../support/fixtures/itunes/epf/incremental", __FILE__)
4
+ end
5
+
2
6
  def apple_epf_inc_filename(filename)
3
7
  File.expand_path("../../support/fixtures/itunes/epf/incremental/#{filename}", __FILE__)
4
8
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apple_epf
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.4
4
+ version: 1.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-13 00:00:00.000000000 Z
12
+ date: 2014-03-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rspec
@@ -123,6 +123,22 @@ dependencies:
123
123
  - - ! '>='
124
124
  - !ruby/object:Gem::Version
125
125
  version: '0'
126
+ - !ruby/object:Gem::Dependency
127
+ name: thin
128
+ requirement: !ruby/object:Gem::Requirement
129
+ none: false
130
+ requirements:
131
+ - - ! '>='
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ type: :development
135
+ prerelease: false
136
+ version_requirements: !ruby/object:Gem::Requirement
137
+ none: false
138
+ requirements:
139
+ - - ! '>='
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
126
142
  - !ruby/object:Gem::Dependency
127
143
  name: curb
128
144
  requirement: !ruby/object:Gem::Requirement
@@ -178,6 +194,12 @@ executables: []
178
194
  extensions: []
179
195
  extra_rdoc_files: []
180
196
  files:
197
+ - lib/apple_epf/download_processor/aria_download_processor.rb
198
+ - lib/apple_epf/download_processor/aria_download_processor.rb~
199
+ - lib/apple_epf/download_processor/curb_download_processor.rb
200
+ - lib/apple_epf/download_processor/curb_download_processor.rb~
201
+ - lib/apple_epf/download_processor.rb
202
+ - lib/apple_epf/download_processor.rb~
181
203
  - lib/apple_epf/downloader.rb
182
204
  - lib/apple_epf/errors.rb
183
205
  - lib/apple_epf/extractor.rb
@@ -192,6 +214,7 @@ files:
192
214
  - MIT-LICENSE
193
215
  - Rakefile
194
216
  - README.md
217
+ - spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
195
218
  - spec/lib/apple_epf/downloader_spec.rb
196
219
  - spec/lib/apple_epf/exctractor_spec.rb
197
220
  - spec/lib/apple_epf/main_spec.rb
@@ -221,7 +244,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
221
244
  version: '0'
222
245
  segments:
223
246
  - 0
224
- hash: 2434904396329215938
247
+ hash: -6031403435138943
225
248
  required_rubygems_version: !ruby/object:Gem::Requirement
226
249
  none: false
227
250
  requirements:
@@ -230,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
230
253
  version: '0'
231
254
  segments:
232
255
  - 0
233
- hash: 2434904396329215938
256
+ hash: -6031403435138943
234
257
  requirements: []
235
258
  rubyforge_project:
236
259
  rubygems_version: 1.8.23
@@ -238,6 +261,7 @@ signing_key:
238
261
  specification_version: 3
239
262
  summary: Downloader, Extractor and Parser for Apple Epf Affiliate files
240
263
  test_files:
264
+ - spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
241
265
  - spec/lib/apple_epf/downloader_spec.rb
242
266
  - spec/lib/apple_epf/exctractor_spec.rb
243
267
  - spec/lib/apple_epf/main_spec.rb