apple_epf 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/apple_epf.rb +7 -0
- data/lib/apple_epf/download_processor.rb +29 -0
- data/lib/apple_epf/download_processor.rb~ +0 -0
- data/lib/apple_epf/download_processor/aria_download_processor.rb +16 -0
- data/lib/apple_epf/download_processor/aria_download_processor.rb~ +0 -0
- data/lib/apple_epf/download_processor/curb_download_processor.rb +46 -0
- data/lib/apple_epf/download_processor/curb_download_processor.rb~ +0 -0
- data/lib/apple_epf/downloader.rb +6 -57
- data/lib/apple_epf/errors.rb +1 -1
- data/lib/apple_epf/version.rb +1 -1
- data/spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb +81 -0
- data/spec/lib/apple_epf/downloader_spec.rb +8 -4
- data/spec/support/fixture_helper.rb +4 -0
- metadata +28 -4
data/lib/apple_epf.rb
CHANGED
@@ -6,6 +6,7 @@ require 'core_ext/module'
|
|
6
6
|
require 'apple_epf/errors'
|
7
7
|
require 'apple_epf/logging'
|
8
8
|
require 'apple_epf/main'
|
9
|
+
require 'apple_epf/download_processor'
|
9
10
|
require 'apple_epf/downloader'
|
10
11
|
require 'apple_epf/extractor'
|
11
12
|
require 'apple_epf/parser'
|
@@ -22,6 +23,12 @@ module AppleEpf
|
|
22
23
|
mattr_accessor :download_retry_count
|
23
24
|
@@download_retry_count = 3
|
24
25
|
|
26
|
+
mattr_accessor :concurrent_downloads
|
27
|
+
@@concurrent_downloads = 2
|
28
|
+
|
29
|
+
mattr_accessor :download_processor
|
30
|
+
@@download_processor = AppleEpf::CurbDownloadProcessor
|
31
|
+
|
25
32
|
mattr_accessor :keep_tbz_after_extract
|
26
33
|
@@keep_tbz_after_extract = false
|
27
34
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module AppleEpf
|
2
|
+
class DownloadProcessor
|
3
|
+
|
4
|
+
def initialize(apple_filename_full, download_to)
|
5
|
+
@apple_filename_full = apple_filename_full
|
6
|
+
@download_to = download_to
|
7
|
+
end
|
8
|
+
|
9
|
+
def download_and_check
|
10
|
+
raise 'should be implemented in subclass'
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_file_md5
|
14
|
+
begin
|
15
|
+
curl = Curl::Easy.new("#{@apple_filename_full}.md5")
|
16
|
+
curl.http_auth_types = :basic
|
17
|
+
curl.username = AppleEpf.apple_id
|
18
|
+
curl.password = AppleEpf.apple_password
|
19
|
+
curl.perform
|
20
|
+
@md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
|
21
|
+
rescue NoMethodError
|
22
|
+
raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
require "apple_epf/download_processor/curb_download_processor"
|
29
|
+
require "apple_epf/download_processor/aria_download_processor"
|
File without changes
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class AppleEpf::AriaDownloadProcessor < AppleEpf::DownloadProcessor
|
2
|
+
def download_and_check
|
3
|
+
get_file_md5
|
4
|
+
download
|
5
|
+
end
|
6
|
+
|
7
|
+
def download
|
8
|
+
command = "cd #{File.dirname(@download_to)} && aria2c --continue --check-integrity=true --checksum=md5=#{@md5_checksum} -x#{AppleEpf.concurrent_downloads} -j#{AppleEpf.concurrent_downloads} -s#{AppleEpf.concurrent_downloads} --http-user=#{AppleEpf.apple_id} --http-passwd=#{AppleEpf.apple_password} -o #{File.basename(@download_to)} #{@apple_filename_full}"
|
9
|
+
result = system(command)
|
10
|
+
|
11
|
+
unless result
|
12
|
+
raise AppleEpf::DownloaderError.new("Unable to download file. #{$?}")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
File without changes
|
@@ -0,0 +1,46 @@
|
|
1
|
+
class AppleEpf::CurbDownloadProcessor < AppleEpf::DownloadProcessor
|
2
|
+
def download_and_check
|
3
|
+
@download_retry = 0
|
4
|
+
get_file_md5
|
5
|
+
download
|
6
|
+
compare_md5_checksum
|
7
|
+
@download_to
|
8
|
+
end
|
9
|
+
|
10
|
+
def compare_md5_checksum
|
11
|
+
if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
|
12
|
+
raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
def download
|
18
|
+
begin
|
19
|
+
curl = Curl::Easy.new(@apple_filename_full)
|
20
|
+
|
21
|
+
# Authentication
|
22
|
+
curl.http_auth_types = :basic
|
23
|
+
curl.username = AppleEpf.apple_id
|
24
|
+
curl.password = AppleEpf.apple_password
|
25
|
+
|
26
|
+
File.open(@download_to, 'wb') do |f|
|
27
|
+
curl.on_body { |data|
|
28
|
+
f << data;
|
29
|
+
data.size
|
30
|
+
}
|
31
|
+
curl.perform
|
32
|
+
end
|
33
|
+
rescue Curl::Err::PartialFileError => ex
|
34
|
+
if @download_retry < AppleEpf.download_retry_count
|
35
|
+
@download_retry += 1
|
36
|
+
|
37
|
+
logger_info "Curl::Err::PartialFileError happened..."
|
38
|
+
logger_info "Restarting download"
|
39
|
+
download
|
40
|
+
else
|
41
|
+
raise AppleEpf::CurlError.new("Unable to download file.")
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
File without changes
|
data/lib/apple_epf/downloader.rb
CHANGED
@@ -19,7 +19,7 @@ module AppleEpf
|
|
19
19
|
@force_url = force_url
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def prepare
|
23
23
|
_prepare_folders
|
24
24
|
if @force_url
|
25
25
|
@apple_filename_full = @force_url
|
@@ -27,15 +27,13 @@ module AppleEpf
|
|
27
27
|
get_filename_by_date_and_type
|
28
28
|
@apple_filename_full = apple_filename_full_url(@apple_filename_full_path)
|
29
29
|
end
|
30
|
-
|
31
30
|
@download_to = File.join(dirpath, File.basename(@apple_filename_full))
|
31
|
+
end
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
@
|
37
|
-
start_download
|
38
|
-
download_and_compare_md5_checksum
|
33
|
+
def download
|
34
|
+
prepare
|
35
|
+
@download_processor = AppleEpf.download_processor.new(@apple_filename_full, @download_to)
|
36
|
+
@download_processor.download_and_check
|
39
37
|
@download_to
|
40
38
|
end
|
41
39
|
|
@@ -43,25 +41,6 @@ module AppleEpf
|
|
43
41
|
File.join((@dirpath || AppleEpf.extract_dir), @type)
|
44
42
|
end
|
45
43
|
|
46
|
-
#TODO combine with start_download
|
47
|
-
def download_and_compare_md5_checksum
|
48
|
-
begin
|
49
|
-
curl = Curl::Easy.new("#{@apple_filename_full}.md5")
|
50
|
-
curl.http_auth_types = :basic
|
51
|
-
curl.username = AppleEpf.apple_id
|
52
|
-
curl.password = AppleEpf.apple_password
|
53
|
-
curl.perform
|
54
|
-
@md5_checksum = curl.body_str.match(/.*=(.*)/)[1].strip
|
55
|
-
rescue NoMethodError
|
56
|
-
raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
|
57
|
-
end
|
58
|
-
|
59
|
-
if Digest::MD5.file(@download_to).hexdigest != @md5_checksum
|
60
|
-
raise AppleEpf::Md5CompareError.new('Md5 of downloaded file is not the same as apple provide')
|
61
|
-
end
|
62
|
-
|
63
|
-
@md5_checksum
|
64
|
-
end
|
65
44
|
|
66
45
|
def get_filename_by_date_and_type
|
67
46
|
#today = DateTime.now
|
@@ -146,35 +125,5 @@ module AppleEpf
|
|
146
125
|
|
147
126
|
response.code == "200"
|
148
127
|
end
|
149
|
-
|
150
|
-
def start_download
|
151
|
-
begin
|
152
|
-
curl = Curl::Easy.new(@apple_filename_full)
|
153
|
-
|
154
|
-
# Authentication
|
155
|
-
curl.http_auth_types = :basic
|
156
|
-
curl.username = AppleEpf.apple_id
|
157
|
-
curl.password = AppleEpf.apple_password
|
158
|
-
|
159
|
-
File.open(@download_to, 'wb') do |f|
|
160
|
-
curl.on_body { |data|
|
161
|
-
f << data;
|
162
|
-
data.size
|
163
|
-
}
|
164
|
-
curl.perform
|
165
|
-
end
|
166
|
-
rescue Curl::Err::PartialFileError => ex
|
167
|
-
if @download_retry < AppleEpf.download_retry_count
|
168
|
-
@download_retry += 1
|
169
|
-
|
170
|
-
logger_info "Curl::Err::PartialFileError happened..."
|
171
|
-
logger_info "Restarting download"
|
172
|
-
start_download
|
173
|
-
else
|
174
|
-
raise AppleEpf::CurlError.new("Unable to download file.")
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
128
|
end
|
180
129
|
end
|
data/lib/apple_epf/errors.rb
CHANGED
data/lib/apple_epf/version.rb
CHANGED
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require File.expand_path('../../../../spec_helper', __FILE__)
|
3
|
+
require 'rack'
|
4
|
+
require 'thin'
|
5
|
+
|
6
|
+
describe AppleEpf::AriaDownloadProcessor do
|
7
|
+
|
8
|
+
describe "download" do
|
9
|
+
before(:all) do
|
10
|
+
@mockServer = Rack::File.new(apple_epf_dir)
|
11
|
+
@server_thread = Thread.new do
|
12
|
+
Rack::Handler::Thin.run @mockServer, :Port => 4400
|
13
|
+
end
|
14
|
+
sleep(2)
|
15
|
+
end
|
16
|
+
|
17
|
+
after(:all) do
|
18
|
+
@server_thread.kill
|
19
|
+
end
|
20
|
+
|
21
|
+
before do
|
22
|
+
@tmp_dir = [Dir.tmpdir, 'epm_files'].join('/')
|
23
|
+
|
24
|
+
FileUtils.mkpath @tmp_dir
|
25
|
+
|
26
|
+
AppleEpf.configure do |config|
|
27
|
+
config.apple_id = 'test'
|
28
|
+
config.apple_password = 'test'
|
29
|
+
config.extract_dir = @tmp_dir
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
after do
|
35
|
+
FileUtils.remove_dir(@tmp_dir)
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should process if md5 is fine" do
|
39
|
+
downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
|
40
|
+
|
41
|
+
#correct md5
|
42
|
+
downloader.instance_variable_set(:@md5_checksum, '6fad1fb7823075d92296260fae3e317e')
|
43
|
+
downloader.download
|
44
|
+
|
45
|
+
File.read("#{@tmp_dir}/popularity20130111.tbz").should ==
|
46
|
+
File.read(apple_epf_inc_filename('popularity20130111.tbz'))
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should return error if md5 is not correct" do
|
50
|
+
downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
|
51
|
+
|
52
|
+
downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
|
53
|
+
expect {
|
54
|
+
downloader.download
|
55
|
+
}.to raise_error(AppleEpf::DownloaderError)
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'should return error if file is not found' do
|
59
|
+
downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130112.tbz", "#{@tmp_dir}/popularity20130112.tbz")
|
60
|
+
|
61
|
+
downloader.instance_variable_set(:@md5_checksum, '0371a79664856494e840af9e1e6c0152')
|
62
|
+
expect {
|
63
|
+
downloader.download
|
64
|
+
}.to raise_error(AppleEpf::DownloaderError)
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "download_and_check" do
|
68
|
+
it "should download md5 and file and compare" do
|
69
|
+
downloader = AppleEpf::AriaDownloadProcessor.new("http://localhost:4400/popularity20130111.tbz", "#{@tmp_dir}/popularity20130111.tbz")
|
70
|
+
|
71
|
+
stub_request(:get, "http://test:test@localhost:4400/popularity20130111.tbz.md5").
|
72
|
+
to_return(:status => 200, :body => "MD5 (popularity20130111.tbz) = 6fad1fb7823075d92296260fae3e317e\n", :headers => {})
|
73
|
+
|
74
|
+
expect {
|
75
|
+
downloader.download_and_check
|
76
|
+
}.to_not raise_error
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -146,14 +146,14 @@ describe AppleEpf::Downloader do
|
|
146
146
|
it "should properly set url for download" do
|
147
147
|
downloader.stub(:file_exists?){ file_exists }
|
148
148
|
downloader.stub(:start_download)
|
149
|
-
downloader.
|
149
|
+
downloader.prepare
|
150
150
|
downloader.apple_filename_full.should eq("https://feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz")
|
151
151
|
end
|
152
152
|
|
153
153
|
it "should properly set local file to store file in" do
|
154
154
|
downloader.stub(:file_exists?){ file_exists }
|
155
155
|
downloader.stub(:start_download)
|
156
|
-
downloader.
|
156
|
+
downloader.prepare
|
157
157
|
downloader.download_to.should eq("#{@tmp_dir}/incremental/popularity20130121.tbz")
|
158
158
|
end
|
159
159
|
|
@@ -161,6 +161,9 @@ describe AppleEpf::Downloader do
|
|
161
161
|
stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz").
|
162
162
|
to_return(:status => 200, :body => "Test\nWow", :headers => {})
|
163
163
|
|
164
|
+
stub_request(:get, "https://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz.md5").
|
165
|
+
to_return(:status => 200, :body => "MD5 (popularity20130116.tbz) = 0371a79664856494e840af9e1e6c0152\n", :headers => {})
|
166
|
+
|
164
167
|
downloader = AppleEpf::Downloader.new('full', file, filedate)
|
165
168
|
downloader.stub(:download_and_compare_md5_checksum)
|
166
169
|
downloader.stub(:file_exists?){ file_exists }
|
@@ -175,7 +178,8 @@ describe AppleEpf::Downloader do
|
|
175
178
|
describe "dirpath" do
|
176
179
|
before do
|
177
180
|
downloader.stub(:file_exists?){ file_exists }
|
178
|
-
downloader.stub(:start_download)
|
181
|
+
#downloader.stub(:start_download)
|
182
|
+
AppleEpf.download_processor.any_instance.stub(:download_and_check)
|
179
183
|
end
|
180
184
|
|
181
185
|
it "should be able to change dir where to save files" do
|
@@ -222,4 +226,4 @@ describe AppleEpf::Downloader do
|
|
222
226
|
end
|
223
227
|
end
|
224
228
|
|
225
|
-
end
|
229
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apple_epf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-03-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
@@ -123,6 +123,22 @@ dependencies:
|
|
123
123
|
- - ! '>='
|
124
124
|
- !ruby/object:Gem::Version
|
125
125
|
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: thin
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
126
142
|
- !ruby/object:Gem::Dependency
|
127
143
|
name: curb
|
128
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,6 +194,12 @@ executables: []
|
|
178
194
|
extensions: []
|
179
195
|
extra_rdoc_files: []
|
180
196
|
files:
|
197
|
+
- lib/apple_epf/download_processor/aria_download_processor.rb
|
198
|
+
- lib/apple_epf/download_processor/aria_download_processor.rb~
|
199
|
+
- lib/apple_epf/download_processor/curb_download_processor.rb
|
200
|
+
- lib/apple_epf/download_processor/curb_download_processor.rb~
|
201
|
+
- lib/apple_epf/download_processor.rb
|
202
|
+
- lib/apple_epf/download_processor.rb~
|
181
203
|
- lib/apple_epf/downloader.rb
|
182
204
|
- lib/apple_epf/errors.rb
|
183
205
|
- lib/apple_epf/extractor.rb
|
@@ -192,6 +214,7 @@ files:
|
|
192
214
|
- MIT-LICENSE
|
193
215
|
- Rakefile
|
194
216
|
- README.md
|
217
|
+
- spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
|
195
218
|
- spec/lib/apple_epf/downloader_spec.rb
|
196
219
|
- spec/lib/apple_epf/exctractor_spec.rb
|
197
220
|
- spec/lib/apple_epf/main_spec.rb
|
@@ -221,7 +244,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
221
244
|
version: '0'
|
222
245
|
segments:
|
223
246
|
- 0
|
224
|
-
hash:
|
247
|
+
hash: -6031403435138943
|
225
248
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
226
249
|
none: false
|
227
250
|
requirements:
|
@@ -230,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
230
253
|
version: '0'
|
231
254
|
segments:
|
232
255
|
- 0
|
233
|
-
hash:
|
256
|
+
hash: -6031403435138943
|
234
257
|
requirements: []
|
235
258
|
rubyforge_project:
|
236
259
|
rubygems_version: 1.8.23
|
@@ -238,6 +261,7 @@ signing_key:
|
|
238
261
|
specification_version: 3
|
239
262
|
summary: Downloader, Extractor and Parser for Apple Epf Affiliate files
|
240
263
|
test_files:
|
264
|
+
- spec/lib/apple_epf/download_processor/aria_download_processor_spec.rb
|
241
265
|
- spec/lib/apple_epf/downloader_spec.rb
|
242
266
|
- spec/lib/apple_epf/exctractor_spec.rb
|
243
267
|
- spec/lib/apple_epf/main_spec.rb
|