downspout 0.2.10 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +23 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +43 -0
- data/Rakefile +3 -27
- data/downspout.gemspec +29 -0
- data/lib/downspout/base.rb +6 -2
- data/lib/downspout/downloader.rb +55 -41
- data/lib/downspout/version.rb +3 -0
- data/lib/downspout.rb +1 -4
- data/test/downspout_test.rb +6 -5
- data/test/servlet.rb +3 -4
- data/test/unit/base_test.rb +8 -8
- metadata +56 -36
- data/VERSION +0 -1
data/.gitignore
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# OS generated files #
|
2
|
+
######################
|
3
|
+
.DS_Store?
|
4
|
+
ehthumbs.db
|
5
|
+
Icon?
|
6
|
+
Thumbs.db
|
7
|
+
|
8
|
+
# Logs and databases #
|
9
|
+
######################
|
10
|
+
*.log
|
11
|
+
*.sql
|
12
|
+
*.sqlite
|
13
|
+
|
14
|
+
# Output Folders #
|
15
|
+
##################
|
16
|
+
tmp/*
|
17
|
+
tmp/**/*
|
18
|
+
pkg/*
|
19
|
+
rdoc/*
|
20
|
+
rdoc/**/*
|
21
|
+
bin
|
22
|
+
coverage
|
23
|
+
vendor
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
downspout (0.3.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
activesupport (3.2.12)
|
10
|
+
i18n (~> 0.6)
|
11
|
+
multi_json (~> 1.0)
|
12
|
+
bourne (1.1.2)
|
13
|
+
mocha (= 0.10.5)
|
14
|
+
curb (0.8.3)
|
15
|
+
i18n (0.6.4)
|
16
|
+
json (1.7.7)
|
17
|
+
metaclass (0.0.1)
|
18
|
+
mocha (0.10.5)
|
19
|
+
metaclass (~> 0.0.1)
|
20
|
+
multi_json (1.6.1)
|
21
|
+
rake (10.0.3)
|
22
|
+
rcov (1.0.0)
|
23
|
+
rdoc (4.0.0)
|
24
|
+
json (~> 1.4)
|
25
|
+
shoulda (3.3.2)
|
26
|
+
shoulda-context (~> 1.0.1)
|
27
|
+
shoulda-matchers (~> 1.4.1)
|
28
|
+
shoulda-context (1.0.2)
|
29
|
+
shoulda-matchers (1.4.2)
|
30
|
+
activesupport (>= 3.0.0)
|
31
|
+
bourne (~> 1.1.2)
|
32
|
+
|
33
|
+
PLATFORMS
|
34
|
+
ruby
|
35
|
+
|
36
|
+
DEPENDENCIES
|
37
|
+
curb (>= 0.7.15)
|
38
|
+
downspout!
|
39
|
+
mocha (>= 0.9.12)
|
40
|
+
rake (>= 0.9.6)
|
41
|
+
rcov (>= 0.9.9)
|
42
|
+
rdoc (>= 2.4.2)
|
43
|
+
shoulda (>= 2.11.3)
|
data/Rakefile
CHANGED
@@ -1,32 +1,8 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
|
4
|
-
$LOAD_PATH.unshift('lib')
|
5
|
-
|
6
|
-
require 'rake'
|
7
|
-
require 'jeweler'
|
8
|
-
|
9
|
-
Jeweler::Tasks.new do |gem|
|
10
|
-
# gem is a Gem::Specification...
|
11
|
-
# see http://docs.rubygems.org/read/chapter/20 for more options
|
12
|
-
gem.name = "downspout"
|
13
|
-
gem.homepage = "http://github.com/sci-phi/downspout"
|
14
|
-
gem.license = "MIT"
|
15
|
-
gem.summary = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs.}
|
16
|
-
gem.description = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)}
|
17
|
-
gem.email = "phi.sanders@sciphi.me"
|
18
|
-
gem.authors = ["Phi.Sanders"]
|
19
|
-
gem.files = FileList['lib/**/*.rb', '[A-Z]*', 'test/**/*'].to_a
|
20
|
-
gem.add_development_dependency "jeweler", "~> 1.5.2"
|
21
|
-
gem.add_development_dependency "rcov", ">= 0.9.9"
|
22
|
-
gem.add_development_dependency "shoulda", ">= 2.11.3"
|
23
|
-
gem.add_development_dependency "mocha", ">= 0.9.12"
|
24
|
-
gem.add_development_dependency "curb", ">= 0.7.15"
|
25
|
-
end
|
26
|
-
Jeweler::RubygemsDotOrgTasks.new
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
27
3
|
|
28
4
|
require 'rdoc'
|
29
|
-
require '
|
5
|
+
require 'rdoc/task'
|
30
6
|
Rake::RDocTask.new do |rdoc|
|
31
7
|
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
32
8
|
|
data/downspout.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/downspout/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "downspout"
|
6
|
+
gem.license = "MIT"
|
7
|
+
gem.authors = ["Phi.Sanders"]
|
8
|
+
gem.email = ["phi.sanders@sciphi.me"]
|
9
|
+
gem.homepage = "http://github.com/sci-phi/downspout"
|
10
|
+
gem.summary = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs.}
|
11
|
+
gem.description = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)}
|
12
|
+
|
13
|
+
#gem.files = FileList['lib/**/*.rb', '[A-Z]*', 'test/**/*'].to_a
|
14
|
+
gem.files = `git ls-files`.split($\)
|
15
|
+
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
19
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
20
|
+
|
21
|
+
gem.version = Downspout::VERSION
|
22
|
+
|
23
|
+
gem.add_development_dependency "rake", ">= 0.9.6"
|
24
|
+
gem.add_development_dependency "rdoc", ">= 2.4.2"
|
25
|
+
gem.add_development_dependency "rcov", ">= 0.9.9"
|
26
|
+
gem.add_development_dependency "shoulda", ">= 2.11.3"
|
27
|
+
gem.add_development_dependency "mocha", ">= 0.9.12"
|
28
|
+
gem.add_development_dependency "curb", ">= 0.7.15"
|
29
|
+
end
|
data/lib/downspout/base.rb
CHANGED
@@ -53,7 +53,7 @@ module Downspout
|
|
53
53
|
begin
|
54
54
|
# remove user/password prefix if provided
|
55
55
|
clean_url = self.extract_credentials_from_url!( url_string )
|
56
|
-
|
56
|
+
|
57
57
|
uri = URI.parse( clean_url )
|
58
58
|
rescue URI::InvalidURIError
|
59
59
|
$logger.warn("downspout | viable_url? | The format of the url is not valid : #{url_string}")
|
@@ -88,6 +88,10 @@ module Downspout
|
|
88
88
|
return true
|
89
89
|
when "URI::FTP"
|
90
90
|
return true
|
91
|
+
when "FTPS"
|
92
|
+
return true
|
93
|
+
when "URI::FTPS"
|
94
|
+
return true
|
91
95
|
else
|
92
96
|
$logger.warn("downspout | supported_protocol? | #{protocol_string} is not supported by Downspout.")
|
93
97
|
end
|
@@ -100,7 +104,7 @@ module Downspout
|
|
100
104
|
def self.supported_protocols
|
101
105
|
return ["HTTP", "HTTPS", "FTP"]
|
102
106
|
end
|
103
|
-
|
107
|
+
|
104
108
|
def self.extract_credentials_from_url!( some_url )
|
105
109
|
|
106
110
|
begin
|
data/lib/downspout/downloader.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Downspout
|
2
|
-
# The object returned by a call to fetch_url()
|
2
|
+
# The object returned by a call to fetch_url()
|
3
3
|
class Downloader < Base
|
4
4
|
|
5
5
|
# returns the path to the downloaded file
|
@@ -24,7 +24,7 @@ module Downspout
|
|
24
24
|
@started_at = nil
|
25
25
|
@finished_at = nil
|
26
26
|
@redirected_url = nil
|
27
|
-
|
27
|
+
|
28
28
|
if options.respond_to?(:keys) then
|
29
29
|
options.each do |key, value|
|
30
30
|
self.send("#{key}=", value)
|
@@ -48,25 +48,25 @@ module Downspout
|
|
48
48
|
def duration
|
49
49
|
return nil unless @started_at
|
50
50
|
return nil unless @finished_at
|
51
|
-
|
51
|
+
|
52
52
|
return @finished_at - @started_at
|
53
53
|
end
|
54
54
|
|
55
55
|
# Extracts the file name from the URL or uses a default name based on the content-type header
|
56
56
|
def basename
|
57
57
|
return @basename unless @basename.nil?
|
58
|
-
|
58
|
+
|
59
59
|
if !(@path.nil?) then
|
60
60
|
@basename = File.basename( @path )
|
61
61
|
else
|
62
62
|
if !(@uri.path.nil? || @uri.path.empty? || @uri.path == '/')
|
63
|
-
@basename = File.basename( @uri.path )
|
63
|
+
@basename = File.basename( @uri.path )
|
64
64
|
else
|
65
|
-
$logger.debug("downspout | downloader | basename | Bad URI path")
|
65
|
+
$logger.debug("downspout | downloader | basename | Bad URI path")
|
66
66
|
@basename = 'file.downspout'
|
67
67
|
end
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
$logger.debug("downspout | downloader | basename | #{@basename} ")
|
71
71
|
|
72
72
|
return @basename
|
@@ -140,7 +140,7 @@ module Downspout
|
|
140
140
|
$logger.debug("downspout | downloader | download! | Renaming #{@basename} to #{new_name} ...")
|
141
141
|
new_path = File.join( File.dirname( tf.path ), new_name)
|
142
142
|
begin
|
143
|
-
FileUtils.mv( tf.path, new_path )
|
143
|
+
FileUtils.mv( tf.path, new_path )
|
144
144
|
rescue Exception => e
|
145
145
|
$logger.error("downspout | downloader | download! | Exception while renaming #{@basename} : #{e}")
|
146
146
|
raise e
|
@@ -149,7 +149,7 @@ module Downspout
|
|
149
149
|
end
|
150
150
|
|
151
151
|
$logger.debug("downspout | downloader | download! | Started: #{@started_at.utc}, Finished: #{@finished_at.utc}, Duration: #{duration}")
|
152
|
-
|
152
|
+
|
153
153
|
return downloaded
|
154
154
|
end
|
155
155
|
|
@@ -162,33 +162,47 @@ module Downspout
|
|
162
162
|
end
|
163
163
|
end
|
164
164
|
|
165
|
-
def
|
166
|
-
|
165
|
+
def get_ftp_credential
|
166
|
+
# look up the credentials for this FTP host, preferring the FTPS scheme
|
167
|
+
cred = Downspout::Config.credentials.select{|c| c.scheme == "ftps" }.select{ |c| c.host == @uri.host }.first
|
167
168
|
|
168
|
-
|
169
|
-
|
169
|
+
unless cred
|
170
|
+
cred = Downspout::Config.credentials.select{|c| c.scheme =~ /ftp/ }.select{ |c| c.host == @uri.host }.first
|
171
|
+
end
|
170
172
|
|
171
|
-
if cred
|
172
|
-
$logger.
|
173
|
+
if cred then
|
174
|
+
$logger.debug("downspout | downloader | get_ftp_credential | Loaded credentials for #{cred.host} ...")
|
175
|
+
else
|
176
|
+
$logger.warn("downspout | downloader | get_ftp_credential | No established credentials found for '#{@uri.host}'.")
|
173
177
|
|
174
178
|
# attempt to extract credential from the URL
|
175
179
|
cred = Downspout::Credential.create_from_url( @url )
|
176
|
-
|
177
|
-
|
180
|
+
|
181
|
+
unless cred
|
182
|
+
$logger.warn("downspout | downloader | get_ftp_credential | No embedded credentials found in URL.")
|
183
|
+
return nil
|
184
|
+
end
|
178
185
|
end
|
179
|
-
|
186
|
+
|
187
|
+
$logger.debug("downspout | downloader | get_ftp_credential | Using embedded credentials found in URL with user: #{cred.user_name}.")
|
188
|
+
return cred
|
189
|
+
end
|
190
|
+
|
191
|
+
def net_ftp_download
|
192
|
+
$logger.debug("downspout | downloader | net_ftp_download | Downloading #{@url} ...")
|
193
|
+
|
194
|
+
cred = get_ftp_credential
|
195
|
+
|
180
196
|
if cred.nil? then
|
181
|
-
$logger.warn("downspout | downloader | net_ftp_download | No embedded credentials found in URL.")
|
182
197
|
# proceed anyway - slight possibility it's an un-authorized FTP account...
|
183
|
-
|
184
|
-
$logger.warn("downspout | downloader | net_ftp_download | Using embedded credentials found in URL with user: #{cred.user_name}.")
|
198
|
+
$logger.warn("downspout | downloader | net_ftp_download | Proceeding without credentials, assuming unauthorized service ...")
|
185
199
|
end
|
186
|
-
|
200
|
+
|
187
201
|
begin
|
188
202
|
ftp = Net::FTP.open( @uri.host ) do |ftp|
|
189
203
|
ftp.login( cred.user_name, cred.pass_word ) unless cred.nil?
|
190
204
|
ftp.passive
|
191
|
-
ftp.chdir( File.dirname( @uri.path ) )
|
205
|
+
ftp.chdir( File.dirname( @uri.path ) )
|
192
206
|
ftp.getbinaryfile( File.basename(@uri.path), @path )
|
193
207
|
end
|
194
208
|
rescue Exception => e
|
@@ -217,7 +231,7 @@ module Downspout
|
|
217
231
|
remove_file_at_target_path
|
218
232
|
raise dns_err
|
219
233
|
end
|
220
|
-
|
234
|
+
|
221
235
|
$logger.debug("downspout | downloader | net_http_download | Response Code : #{response.code}")
|
222
236
|
|
223
237
|
# populate the response headers from net/http headers...
|
@@ -226,18 +240,18 @@ module Downspout
|
|
226
240
|
new_header_str += "#{k}: #{v}\r\n"
|
227
241
|
end
|
228
242
|
parse_headers_from_string!( new_header_str )
|
229
|
-
|
243
|
+
|
230
244
|
if ((response.code.to_i != 200) and (response.code.to_i != 202)) then
|
231
245
|
# missing file, failed download - delete the response body [if downloaded]
|
232
246
|
remove_file_at_target_path
|
233
247
|
return false
|
234
248
|
end
|
235
|
-
|
249
|
+
|
236
250
|
if !( File.exist?( @path ) ) then
|
237
251
|
$logger.error("downspout | downloader | net_http_download | Missing File at download path : #{@path}")
|
238
252
|
return false
|
239
253
|
end
|
240
|
-
|
254
|
+
|
241
255
|
return true
|
242
256
|
end
|
243
257
|
|
@@ -256,9 +270,9 @@ module Downspout
|
|
256
270
|
# convert to Invalid URI as that's the more pertinent issue
|
257
271
|
raise URI::InvalidURIError, e.to_s
|
258
272
|
end
|
259
|
-
|
273
|
+
|
260
274
|
http = Net::HTTP.new( u.host, u.port )
|
261
|
-
|
275
|
+
|
262
276
|
if (u.scheme == "https") then
|
263
277
|
http.use_ssl = true
|
264
278
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE if !(Downspout::Config.ssl_verification?)
|
@@ -283,7 +297,7 @@ module Downspout
|
|
283
297
|
@response.error!
|
284
298
|
end
|
285
299
|
end
|
286
|
-
|
300
|
+
|
287
301
|
def curb_http_download
|
288
302
|
$logger.debug("downspout | downloader | curb_http_download | Downloading #{@url} ...")
|
289
303
|
|
@@ -293,7 +307,7 @@ module Downspout
|
|
293
307
|
$logger.error("downspout | downloader | curb_http_download | Curb/Curl DNS Error | #{@uri.host}")
|
294
308
|
raise dns_err
|
295
309
|
end
|
296
|
-
|
310
|
+
|
297
311
|
$logger.debug("downspout | downloader | curb_http_download | Response Code : #{curb.response_code}")
|
298
312
|
|
299
313
|
if ((curb.response_code != 200) and (curb.response_code != 202)) then
|
@@ -313,16 +327,16 @@ module Downspout
|
|
313
327
|
# populate a 'proxy' HTTPResponse object with the Curb data...
|
314
328
|
hr_klass = Net::HTTPResponse.send('response_class', curb.response_code.to_s)
|
315
329
|
$logger.debug("downspout | downloader | curb_http_download | Response Type : #{hr_klass.name}")
|
316
|
-
|
330
|
+
|
317
331
|
@response = hr_klass.new( @response_headers["HTTP"][:version],
|
318
332
|
curb.response_code,
|
319
333
|
@response_headers["HTTP"][:message] )
|
320
|
-
|
334
|
+
|
321
335
|
if !( File.exist?( @path ) ) then
|
322
336
|
$logger.error("downspout | downloader | curb_http_download | Missing File at download path : #{@path}")
|
323
337
|
return false
|
324
338
|
end
|
325
|
-
|
339
|
+
|
326
340
|
return true
|
327
341
|
end
|
328
342
|
|
@@ -349,10 +363,10 @@ module Downspout
|
|
349
363
|
http_hash[:version] = http_info.split(" ")[0].match("HTTP/([0-9\.]+)")[1]
|
350
364
|
http_hash[:code] = (http_info.split("\r\n")[0].split(" ")[1]).to_i
|
351
365
|
http_hash[:message] = http_info.split("\r\n")[0].split(" ")[2]
|
352
|
-
|
366
|
+
|
353
367
|
header_hash["HTTP"] = http_hash
|
354
368
|
|
355
|
-
headers[1..-1].each do |line|
|
369
|
+
headers[1..-1].each do |line|
|
356
370
|
next if line.nil? || line.empty?
|
357
371
|
begin
|
358
372
|
matches = line.match(/([\w\-\s]+)\:\s?(.*)/)
|
@@ -386,7 +400,7 @@ module Downspout
|
|
386
400
|
|
387
401
|
$logger.debug("downspout | downloader | file_name_from_content_disposition | cd key : #{cd_key}")
|
388
402
|
return nil if cd_key.nil?
|
389
|
-
|
403
|
+
|
390
404
|
if cd_key then
|
391
405
|
disposition = @response_headers[cd_key]
|
392
406
|
if disposition then
|
@@ -396,7 +410,7 @@ module Downspout
|
|
396
410
|
end
|
397
411
|
|
398
412
|
$logger.debug("downspout | downloader | file_name_from_content_disposition | #{file_name}")
|
399
|
-
return file_name
|
413
|
+
return file_name
|
400
414
|
end
|
401
415
|
|
402
416
|
def file_name_from_content_type
|
@@ -421,11 +435,11 @@ module Downspout
|
|
421
435
|
if !(my_uri.path.nil? || my_uri.path.empty? || my_uri.path == '/')
|
422
436
|
return File.basename( my_uri.path )
|
423
437
|
else
|
424
|
-
$logger.debug("downspout | downloader | basename | Bad URI path")
|
438
|
+
$logger.debug("downspout | downloader | basename | Bad URI path")
|
425
439
|
return nil
|
426
440
|
end
|
427
441
|
end
|
428
|
-
|
442
|
+
|
429
443
|
end
|
430
|
-
|
444
|
+
|
431
445
|
end
|
data/lib/downspout.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'rubygems'
|
1
|
+
#require 'rubygems'
|
2
2
|
|
3
3
|
# common dependencies
|
4
4
|
require 'fileutils'
|
@@ -7,9 +7,6 @@ require 'net/http'
|
|
7
7
|
require 'net/https'
|
8
8
|
require 'net/ftp'
|
9
9
|
|
10
|
-
# add this directory to the path...
|
11
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
12
|
-
|
13
10
|
# customized logger
|
14
11
|
require 'downspout/logger'
|
15
12
|
|
data/test/downspout_test.rb
CHANGED
@@ -10,8 +10,9 @@ class DownspoutTest < Test::Unit::TestCase
|
|
10
10
|
|
11
11
|
$test_ws ||= WEBrick::HTTPServer.new(:Port => 8899,
|
12
12
|
:DocumentRoot => @test_ws_root,
|
13
|
-
#
|
14
|
-
:Logger =>
|
13
|
+
# TODO : Use Log/BasicLog from WEBrick to reduce spam in tests
|
14
|
+
# :Logger => Log.new(nil, BasicLog::WARN),
|
15
|
+
:Logger => $logger,
|
15
16
|
:AccessLog => [])
|
16
17
|
|
17
18
|
|
@@ -23,7 +24,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
23
24
|
num = req.query['n'].to_i
|
24
25
|
|
25
26
|
if num > 1 then
|
26
|
-
resp.set_redirect( HTTPStatus::MovedPermanently, "/deep/?n=#{num - 1}")
|
27
|
+
resp.set_redirect( HTTPStatus::MovedPermanently, "/deep/?n=#{num - 1}")
|
27
28
|
else
|
28
29
|
resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
|
29
30
|
end
|
@@ -101,7 +102,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
101
102
|
assert_equal 2, Downspout::Config.max_redirects
|
102
103
|
|
103
104
|
too_deep_url = "http://127.0.0.1:8899/deep/?n=3&test=no-curb"
|
104
|
-
|
105
|
+
|
105
106
|
assert_raise Downspout::ExcessiveRedirects do
|
106
107
|
dl = Downspout.fetch_url( too_deep_url )
|
107
108
|
end
|
@@ -140,7 +141,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
140
141
|
should "download to custom path" do
|
141
142
|
gfx_path = File.join( Test::App.root, "tmp", "download-test", "image.png" )
|
142
143
|
FileUtils.mkdir_p( File.dirname( gfx_path ) )
|
143
|
-
|
144
|
+
|
144
145
|
dl = Downspout.download_url_to_path( $test_image_url, gfx_path )
|
145
146
|
end
|
146
147
|
|
data/test/servlet.rb
CHANGED
@@ -4,8 +4,7 @@ ws_root = File.expand_path( File.dirname( File.dirname( __FILE__ ) ) )
|
|
4
4
|
|
5
5
|
ws_app = WEBrick::HTTPServer.new(:Port => 8899,
|
6
6
|
:DocumentRoot => @test_ws_root,
|
7
|
-
:Logger => Log.new(nil, BasicLog::WARN),
|
8
|
-
# :Logger => $logger,
|
7
|
+
:Logger => Log.new(nil, BasicLog::WARN),
|
9
8
|
:AccessLog => [])
|
10
9
|
|
11
10
|
ws_app.mount(TestServlet.path, TestServlet)
|
@@ -24,8 +23,8 @@ redir_proc = Proc.new { |req, resp|
|
|
24
23
|
|
25
24
|
ws_app.mount('/one/deep/', HTTPServlet::ProcHandler.new(redir_proc) )
|
26
25
|
|
27
|
-
ws_thread = Thread.new { ws_app.start }
|
28
|
-
|
26
|
+
ws_thread = Thread.new { ws_app.start }
|
27
|
+
|
29
28
|
read_me_url = "#{TestServlet.url}/READ_ME.doc"
|
30
29
|
|
31
30
|
puts "Request #{read_me_url}"
|
data/test/unit/base_test.rb
CHANGED
@@ -21,12 +21,12 @@ class DownspoutTest < Test::Unit::TestCase
|
|
21
21
|
end
|
22
22
|
|
23
23
|
should "respond to fetch_url" do
|
24
|
-
assert Downspout.respond_to?(:
|
24
|
+
assert Downspout.respond_to?(:fetch_url)
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
context "protocols" do
|
29
|
-
|
29
|
+
|
30
30
|
should "not support SFTP" do
|
31
31
|
assert !( Downspout.supported_protocol?( "sftp" ) )
|
32
32
|
end
|
@@ -38,7 +38,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
38
38
|
should "not support SCP" do
|
39
39
|
assert !( Downspout.supported_protocol?( "scp" ) )
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
should "support all secret herbs and spices" do
|
43
43
|
assert_not_nil Downspout.supported_protocols
|
44
44
|
assert_equal Array, Downspout.supported_protocols.class
|
@@ -86,8 +86,8 @@ class DownspoutTest < Test::Unit::TestCase
|
|
86
86
|
assert !( Downspout.viable_url?( "sftp://host.domain.tld/resource/file.format" ) )
|
87
87
|
end
|
88
88
|
|
89
|
-
should "be
|
90
|
-
assert
|
89
|
+
should "be accepted with FTPS protocol" do
|
90
|
+
assert Downspout.viable_url?( "ftps://host.domain.tld/resource/file.format" )
|
91
91
|
end
|
92
92
|
|
93
93
|
end
|
@@ -115,7 +115,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
115
115
|
|
116
116
|
should "be accepted with FTP protocol containing user & password" do
|
117
117
|
num = Downspout::Config.credentials.size
|
118
|
-
|
118
|
+
|
119
119
|
assert Downspout.viable_url?( "ftp://uzer:passw0rd@host.domain.tld/resource/file.format" )
|
120
120
|
|
121
121
|
assert_equal num + 1, Downspout::Config.credentials.size
|
@@ -135,6 +135,6 @@ class DownspoutTest < Test::Unit::TestCase
|
|
135
135
|
assert_equal URI::parse( test_url ).host, URI::parse( new_url ).host
|
136
136
|
assert_equal URI::parse( test_url ).path, URI::parse( new_url ).path
|
137
137
|
end
|
138
|
-
|
138
|
+
|
139
139
|
end
|
140
140
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: downspout
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 19
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 3
|
9
|
+
- 0
|
10
|
+
version: 0.3.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Phi.Sanders
|
@@ -15,29 +15,42 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date:
|
19
|
-
default_executable:
|
18
|
+
date: 2013-03-14 00:00:00 Z
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
22
|
-
name: jeweler
|
23
|
-
prerelease: false
|
24
21
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
22
|
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
hash:
|
26
|
+
hash: 55
|
30
27
|
segments:
|
31
|
-
-
|
32
|
-
-
|
33
|
-
-
|
34
|
-
version:
|
28
|
+
- 0
|
29
|
+
- 9
|
30
|
+
- 6
|
31
|
+
version: 0.9.6
|
32
|
+
prerelease: false
|
35
33
|
type: :development
|
34
|
+
name: rake
|
36
35
|
version_requirements: *id001
|
37
36
|
- !ruby/object:Gem::Dependency
|
38
|
-
name: rcov
|
39
|
-
prerelease: false
|
40
37
|
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 27
|
43
|
+
segments:
|
44
|
+
- 2
|
45
|
+
- 4
|
46
|
+
- 2
|
47
|
+
version: 2.4.2
|
48
|
+
prerelease: false
|
49
|
+
type: :development
|
50
|
+
name: rdoc
|
51
|
+
version_requirements: *id002
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
54
|
none: false
|
42
55
|
requirements:
|
43
56
|
- - ">="
|
@@ -48,12 +61,12 @@ dependencies:
|
|
48
61
|
- 9
|
49
62
|
- 9
|
50
63
|
version: 0.9.9
|
64
|
+
prerelease: false
|
51
65
|
type: :development
|
52
|
-
|
66
|
+
name: rcov
|
67
|
+
version_requirements: *id003
|
53
68
|
- !ruby/object:Gem::Dependency
|
54
|
-
|
55
|
-
prerelease: false
|
56
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
69
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
57
70
|
none: false
|
58
71
|
requirements:
|
59
72
|
- - ">="
|
@@ -64,12 +77,12 @@ dependencies:
|
|
64
77
|
- 11
|
65
78
|
- 3
|
66
79
|
version: 2.11.3
|
80
|
+
prerelease: false
|
67
81
|
type: :development
|
68
|
-
|
82
|
+
name: shoulda
|
83
|
+
version_requirements: *id004
|
69
84
|
- !ruby/object:Gem::Dependency
|
70
|
-
|
71
|
-
prerelease: false
|
72
|
-
requirement: &id004 !ruby/object:Gem::Requirement
|
85
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
86
|
none: false
|
74
87
|
requirements:
|
75
88
|
- - ">="
|
@@ -80,12 +93,12 @@ dependencies:
|
|
80
93
|
- 9
|
81
94
|
- 12
|
82
95
|
version: 0.9.12
|
96
|
+
prerelease: false
|
83
97
|
type: :development
|
84
|
-
|
98
|
+
name: mocha
|
99
|
+
version_requirements: *id005
|
85
100
|
- !ruby/object:Gem::Dependency
|
86
|
-
|
87
|
-
prerelease: false
|
88
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
101
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
89
102
|
none: false
|
90
103
|
requirements:
|
91
104
|
- - ">="
|
@@ -96,22 +109,27 @@ dependencies:
|
|
96
109
|
- 7
|
97
110
|
- 15
|
98
111
|
version: 0.7.15
|
112
|
+
prerelease: false
|
99
113
|
type: :development
|
100
|
-
|
114
|
+
name: curb
|
115
|
+
version_requirements: *id006
|
101
116
|
description: Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)
|
102
|
-
email:
|
117
|
+
email:
|
118
|
+
- phi.sanders@sciphi.me
|
103
119
|
executables: []
|
104
120
|
|
105
121
|
extensions: []
|
106
122
|
|
107
|
-
extra_rdoc_files:
|
108
|
-
|
109
|
-
- README.rdoc
|
123
|
+
extra_rdoc_files: []
|
124
|
+
|
110
125
|
files:
|
126
|
+
- .gitignore
|
127
|
+
- Gemfile
|
128
|
+
- Gemfile.lock
|
111
129
|
- LICENSE.txt
|
112
130
|
- README.rdoc
|
113
131
|
- Rakefile
|
114
|
-
-
|
132
|
+
- downspout.gemspec
|
115
133
|
- lib/downspout.rb
|
116
134
|
- lib/downspout/base.rb
|
117
135
|
- lib/downspout/config.rb
|
@@ -119,6 +137,7 @@ files:
|
|
119
137
|
- lib/downspout/downloader.rb
|
120
138
|
- lib/downspout/logger.rb
|
121
139
|
- lib/downspout/tmp_file.rb
|
140
|
+
- lib/downspout/version.rb
|
122
141
|
- test/downspout_test.rb
|
123
142
|
- test/fixtures/faux_headers.txt
|
124
143
|
- test/fixtures/ruby.png
|
@@ -133,7 +152,6 @@ files:
|
|
133
152
|
- test/unit/downloader_test.rb
|
134
153
|
- test/unit/tmp_file_test.rb
|
135
154
|
- test/watchr.rb
|
136
|
-
has_rdoc: true
|
137
155
|
homepage: http://github.com/sci-phi/downspout
|
138
156
|
licenses:
|
139
157
|
- MIT
|
@@ -163,12 +181,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
163
181
|
requirements: []
|
164
182
|
|
165
183
|
rubyforge_project:
|
166
|
-
rubygems_version: 1.
|
184
|
+
rubygems_version: 1.8.24
|
167
185
|
signing_key:
|
168
186
|
specification_version: 3
|
169
187
|
summary: Downspout is an easy-to-use ruby library for downloading files from URLs.
|
170
188
|
test_files:
|
171
189
|
- test/downspout_test.rb
|
190
|
+
- test/fixtures/faux_headers.txt
|
191
|
+
- test/fixtures/ruby.png
|
172
192
|
- test/focused_test.rb
|
173
193
|
- test/servlet.rb
|
174
194
|
- test/test_helper.rb
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.2.10
|