downspout 0.2.10 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +23 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +43 -0
- data/Rakefile +3 -27
- data/downspout.gemspec +29 -0
- data/lib/downspout/base.rb +6 -2
- data/lib/downspout/downloader.rb +55 -41
- data/lib/downspout/version.rb +3 -0
- data/lib/downspout.rb +1 -4
- data/test/downspout_test.rb +6 -5
- data/test/servlet.rb +3 -4
- data/test/unit/base_test.rb +8 -8
- metadata +56 -36
- data/VERSION +0 -1
data/.gitignore
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# OS generated files #
|
2
|
+
######################
|
3
|
+
.DS_Store?
|
4
|
+
ehthumbs.db
|
5
|
+
Icon?
|
6
|
+
Thumbs.db
|
7
|
+
|
8
|
+
# Logs and databases #
|
9
|
+
######################
|
10
|
+
*.log
|
11
|
+
*.sql
|
12
|
+
*.sqlite
|
13
|
+
|
14
|
+
# Output Folders #
|
15
|
+
##################
|
16
|
+
tmp/*
|
17
|
+
tmp/**/*
|
18
|
+
pkg/*
|
19
|
+
rdoc/*
|
20
|
+
rdoc/**/*
|
21
|
+
bin
|
22
|
+
coverage
|
23
|
+
vendor
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
downspout (0.3.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
activesupport (3.2.12)
|
10
|
+
i18n (~> 0.6)
|
11
|
+
multi_json (~> 1.0)
|
12
|
+
bourne (1.1.2)
|
13
|
+
mocha (= 0.10.5)
|
14
|
+
curb (0.8.3)
|
15
|
+
i18n (0.6.4)
|
16
|
+
json (1.7.7)
|
17
|
+
metaclass (0.0.1)
|
18
|
+
mocha (0.10.5)
|
19
|
+
metaclass (~> 0.0.1)
|
20
|
+
multi_json (1.6.1)
|
21
|
+
rake (10.0.3)
|
22
|
+
rcov (1.0.0)
|
23
|
+
rdoc (4.0.0)
|
24
|
+
json (~> 1.4)
|
25
|
+
shoulda (3.3.2)
|
26
|
+
shoulda-context (~> 1.0.1)
|
27
|
+
shoulda-matchers (~> 1.4.1)
|
28
|
+
shoulda-context (1.0.2)
|
29
|
+
shoulda-matchers (1.4.2)
|
30
|
+
activesupport (>= 3.0.0)
|
31
|
+
bourne (~> 1.1.2)
|
32
|
+
|
33
|
+
PLATFORMS
|
34
|
+
ruby
|
35
|
+
|
36
|
+
DEPENDENCIES
|
37
|
+
curb (>= 0.7.15)
|
38
|
+
downspout!
|
39
|
+
mocha (>= 0.9.12)
|
40
|
+
rake (>= 0.9.6)
|
41
|
+
rcov (>= 0.9.9)
|
42
|
+
rdoc (>= 2.4.2)
|
43
|
+
shoulda (>= 2.11.3)
|
data/Rakefile
CHANGED
@@ -1,32 +1,8 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
|
4
|
-
$LOAD_PATH.unshift('lib')
|
5
|
-
|
6
|
-
require 'rake'
|
7
|
-
require 'jeweler'
|
8
|
-
|
9
|
-
Jeweler::Tasks.new do |gem|
|
10
|
-
# gem is a Gem::Specification...
|
11
|
-
# see http://docs.rubygems.org/read/chapter/20 for more options
|
12
|
-
gem.name = "downspout"
|
13
|
-
gem.homepage = "http://github.com/sci-phi/downspout"
|
14
|
-
gem.license = "MIT"
|
15
|
-
gem.summary = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs.}
|
16
|
-
gem.description = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)}
|
17
|
-
gem.email = "phi.sanders@sciphi.me"
|
18
|
-
gem.authors = ["Phi.Sanders"]
|
19
|
-
gem.files = FileList['lib/**/*.rb', '[A-Z]*', 'test/**/*'].to_a
|
20
|
-
gem.add_development_dependency "jeweler", "~> 1.5.2"
|
21
|
-
gem.add_development_dependency "rcov", ">= 0.9.9"
|
22
|
-
gem.add_development_dependency "shoulda", ">= 2.11.3"
|
23
|
-
gem.add_development_dependency "mocha", ">= 0.9.12"
|
24
|
-
gem.add_development_dependency "curb", ">= 0.7.15"
|
25
|
-
end
|
26
|
-
Jeweler::RubygemsDotOrgTasks.new
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
27
3
|
|
28
4
|
require 'rdoc'
|
29
|
-
require '
|
5
|
+
require 'rdoc/task'
|
30
6
|
Rake::RDocTask.new do |rdoc|
|
31
7
|
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
32
8
|
|
data/downspout.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/downspout/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "downspout"
|
6
|
+
gem.license = "MIT"
|
7
|
+
gem.authors = ["Phi.Sanders"]
|
8
|
+
gem.email = ["phi.sanders@sciphi.me"]
|
9
|
+
gem.homepage = "http://github.com/sci-phi/downspout"
|
10
|
+
gem.summary = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs.}
|
11
|
+
gem.description = %Q{Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)}
|
12
|
+
|
13
|
+
#gem.files = FileList['lib/**/*.rb', '[A-Z]*', 'test/**/*'].to_a
|
14
|
+
gem.files = `git ls-files`.split($\)
|
15
|
+
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
19
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
20
|
+
|
21
|
+
gem.version = Downspout::VERSION
|
22
|
+
|
23
|
+
gem.add_development_dependency "rake", ">= 0.9.6"
|
24
|
+
gem.add_development_dependency "rdoc", ">= 2.4.2"
|
25
|
+
gem.add_development_dependency "rcov", ">= 0.9.9"
|
26
|
+
gem.add_development_dependency "shoulda", ">= 2.11.3"
|
27
|
+
gem.add_development_dependency "mocha", ">= 0.9.12"
|
28
|
+
gem.add_development_dependency "curb", ">= 0.7.15"
|
29
|
+
end
|
data/lib/downspout/base.rb
CHANGED
@@ -53,7 +53,7 @@ module Downspout
|
|
53
53
|
begin
|
54
54
|
# remove user/password prefix if provided
|
55
55
|
clean_url = self.extract_credentials_from_url!( url_string )
|
56
|
-
|
56
|
+
|
57
57
|
uri = URI.parse( clean_url )
|
58
58
|
rescue URI::InvalidURIError
|
59
59
|
$logger.warn("downspout | viable_url? | The format of the url is not valid : #{url_string}")
|
@@ -88,6 +88,10 @@ module Downspout
|
|
88
88
|
return true
|
89
89
|
when "URI::FTP"
|
90
90
|
return true
|
91
|
+
when "FTPS"
|
92
|
+
return true
|
93
|
+
when "URI::FTPS"
|
94
|
+
return true
|
91
95
|
else
|
92
96
|
$logger.warn("downspout | supported_protocol? | #{protocol_string} is not supported by Downspout.")
|
93
97
|
end
|
@@ -100,7 +104,7 @@ module Downspout
|
|
100
104
|
def self.supported_protocols
|
101
105
|
return ["HTTP", "HTTPS", "FTP"]
|
102
106
|
end
|
103
|
-
|
107
|
+
|
104
108
|
def self.extract_credentials_from_url!( some_url )
|
105
109
|
|
106
110
|
begin
|
data/lib/downspout/downloader.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Downspout
|
2
|
-
# The object returned by a call to fetch_url()
|
2
|
+
# The object returned by a call to fetch_url()
|
3
3
|
class Downloader < Base
|
4
4
|
|
5
5
|
# returns the path to the downloaded file
|
@@ -24,7 +24,7 @@ module Downspout
|
|
24
24
|
@started_at = nil
|
25
25
|
@finished_at = nil
|
26
26
|
@redirected_url = nil
|
27
|
-
|
27
|
+
|
28
28
|
if options.respond_to?(:keys) then
|
29
29
|
options.each do |key, value|
|
30
30
|
self.send("#{key}=", value)
|
@@ -48,25 +48,25 @@ module Downspout
|
|
48
48
|
def duration
|
49
49
|
return nil unless @started_at
|
50
50
|
return nil unless @finished_at
|
51
|
-
|
51
|
+
|
52
52
|
return @finished_at - @started_at
|
53
53
|
end
|
54
54
|
|
55
55
|
# Extracts the file name from the URL or uses a default name based on the content-type header
|
56
56
|
def basename
|
57
57
|
return @basename unless @basename.nil?
|
58
|
-
|
58
|
+
|
59
59
|
if !(@path.nil?) then
|
60
60
|
@basename = File.basename( @path )
|
61
61
|
else
|
62
62
|
if !(@uri.path.nil? || @uri.path.empty? || @uri.path == '/')
|
63
|
-
@basename = File.basename( @uri.path )
|
63
|
+
@basename = File.basename( @uri.path )
|
64
64
|
else
|
65
|
-
$logger.debug("downspout | downloader | basename | Bad URI path")
|
65
|
+
$logger.debug("downspout | downloader | basename | Bad URI path")
|
66
66
|
@basename = 'file.downspout'
|
67
67
|
end
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
$logger.debug("downspout | downloader | basename | #{@basename} ")
|
71
71
|
|
72
72
|
return @basename
|
@@ -140,7 +140,7 @@ module Downspout
|
|
140
140
|
$logger.debug("downspout | downloader | download! | Renaming #{@basename} to #{new_name} ...")
|
141
141
|
new_path = File.join( File.dirname( tf.path ), new_name)
|
142
142
|
begin
|
143
|
-
FileUtils.mv( tf.path, new_path )
|
143
|
+
FileUtils.mv( tf.path, new_path )
|
144
144
|
rescue Exception => e
|
145
145
|
$logger.error("downspout | downloader | download! | Exception while renaming #{@basename} : #{e}")
|
146
146
|
raise e
|
@@ -149,7 +149,7 @@ module Downspout
|
|
149
149
|
end
|
150
150
|
|
151
151
|
$logger.debug("downspout | downloader | download! | Started: #{@started_at.utc}, Finished: #{@finished_at.utc}, Duration: #{duration}")
|
152
|
-
|
152
|
+
|
153
153
|
return downloaded
|
154
154
|
end
|
155
155
|
|
@@ -162,33 +162,47 @@ module Downspout
|
|
162
162
|
end
|
163
163
|
end
|
164
164
|
|
165
|
-
def
|
166
|
-
|
165
|
+
def get_ftp_credential
|
166
|
+
# look up the credentials for this FTP host, preferring the FTPS scheme
|
167
|
+
cred = Downspout::Config.credentials.select{|c| c.scheme == "ftps" }.select{ |c| c.host == @uri.host }.first
|
167
168
|
|
168
|
-
|
169
|
-
|
169
|
+
unless cred
|
170
|
+
cred = Downspout::Config.credentials.select{|c| c.scheme =~ /ftp/ }.select{ |c| c.host == @uri.host }.first
|
171
|
+
end
|
170
172
|
|
171
|
-
if cred
|
172
|
-
$logger.
|
173
|
+
if cred then
|
174
|
+
$logger.debug("downspout | downloader | get_ftp_credential | Loaded credentials for #{cred.host} ...")
|
175
|
+
else
|
176
|
+
$logger.warn("downspout | downloader | get_ftp_credential | No established credentials found for '#{@uri.host}'.")
|
173
177
|
|
174
178
|
# attempt to extract credential from the URL
|
175
179
|
cred = Downspout::Credential.create_from_url( @url )
|
176
|
-
|
177
|
-
|
180
|
+
|
181
|
+
unless cred
|
182
|
+
$logger.warn("downspout | downloader | get_ftp_credential | No embedded credentials found in URL.")
|
183
|
+
return nil
|
184
|
+
end
|
178
185
|
end
|
179
|
-
|
186
|
+
|
187
|
+
$logger.debug("downspout | downloader | get_ftp_credential | Using embedded credentials found in URL with user: #{cred.user_name}.")
|
188
|
+
return cred
|
189
|
+
end
|
190
|
+
|
191
|
+
def net_ftp_download
|
192
|
+
$logger.debug("downspout | downloader | net_ftp_download | Downloading #{@url} ...")
|
193
|
+
|
194
|
+
cred = get_ftp_credential
|
195
|
+
|
180
196
|
if cred.nil? then
|
181
|
-
$logger.warn("downspout | downloader | net_ftp_download | No embedded credentials found in URL.")
|
182
197
|
# proceed anyway - slight possibility it's an un-authorized FTP account...
|
183
|
-
|
184
|
-
$logger.warn("downspout | downloader | net_ftp_download | Using embedded credentials found in URL with user: #{cred.user_name}.")
|
198
|
+
$logger.warn("downspout | downloader | net_ftp_download | Proceeding without credentials, assuming unauthorized service ...")
|
185
199
|
end
|
186
|
-
|
200
|
+
|
187
201
|
begin
|
188
202
|
ftp = Net::FTP.open( @uri.host ) do |ftp|
|
189
203
|
ftp.login( cred.user_name, cred.pass_word ) unless cred.nil?
|
190
204
|
ftp.passive
|
191
|
-
ftp.chdir( File.dirname( @uri.path ) )
|
205
|
+
ftp.chdir( File.dirname( @uri.path ) )
|
192
206
|
ftp.getbinaryfile( File.basename(@uri.path), @path )
|
193
207
|
end
|
194
208
|
rescue Exception => e
|
@@ -217,7 +231,7 @@ module Downspout
|
|
217
231
|
remove_file_at_target_path
|
218
232
|
raise dns_err
|
219
233
|
end
|
220
|
-
|
234
|
+
|
221
235
|
$logger.debug("downspout | downloader | net_http_download | Response Code : #{response.code}")
|
222
236
|
|
223
237
|
# populate the response headers from net/http headers...
|
@@ -226,18 +240,18 @@ module Downspout
|
|
226
240
|
new_header_str += "#{k}: #{v}\r\n"
|
227
241
|
end
|
228
242
|
parse_headers_from_string!( new_header_str )
|
229
|
-
|
243
|
+
|
230
244
|
if ((response.code.to_i != 200) and (response.code.to_i != 202)) then
|
231
245
|
# missing file, failed download - delete the response body [if downloaded]
|
232
246
|
remove_file_at_target_path
|
233
247
|
return false
|
234
248
|
end
|
235
|
-
|
249
|
+
|
236
250
|
if !( File.exist?( @path ) ) then
|
237
251
|
$logger.error("downspout | downloader | net_http_download | Missing File at download path : #{@path}")
|
238
252
|
return false
|
239
253
|
end
|
240
|
-
|
254
|
+
|
241
255
|
return true
|
242
256
|
end
|
243
257
|
|
@@ -256,9 +270,9 @@ module Downspout
|
|
256
270
|
# convert to Invalid URI as that's the more pertinent issue
|
257
271
|
raise URI::InvalidURIError, e.to_s
|
258
272
|
end
|
259
|
-
|
273
|
+
|
260
274
|
http = Net::HTTP.new( u.host, u.port )
|
261
|
-
|
275
|
+
|
262
276
|
if (u.scheme == "https") then
|
263
277
|
http.use_ssl = true
|
264
278
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE if !(Downspout::Config.ssl_verification?)
|
@@ -283,7 +297,7 @@ module Downspout
|
|
283
297
|
@response.error!
|
284
298
|
end
|
285
299
|
end
|
286
|
-
|
300
|
+
|
287
301
|
def curb_http_download
|
288
302
|
$logger.debug("downspout | downloader | curb_http_download | Downloading #{@url} ...")
|
289
303
|
|
@@ -293,7 +307,7 @@ module Downspout
|
|
293
307
|
$logger.error("downspout | downloader | curb_http_download | Curb/Curl DNS Error | #{@uri.host}")
|
294
308
|
raise dns_err
|
295
309
|
end
|
296
|
-
|
310
|
+
|
297
311
|
$logger.debug("downspout | downloader | curb_http_download | Response Code : #{curb.response_code}")
|
298
312
|
|
299
313
|
if ((curb.response_code != 200) and (curb.response_code != 202)) then
|
@@ -313,16 +327,16 @@ module Downspout
|
|
313
327
|
# populate a 'proxy' HTTPResponse object with the Curb data...
|
314
328
|
hr_klass = Net::HTTPResponse.send('response_class', curb.response_code.to_s)
|
315
329
|
$logger.debug("downspout | downloader | curb_http_download | Response Type : #{hr_klass.name}")
|
316
|
-
|
330
|
+
|
317
331
|
@response = hr_klass.new( @response_headers["HTTP"][:version],
|
318
332
|
curb.response_code,
|
319
333
|
@response_headers["HTTP"][:message] )
|
320
|
-
|
334
|
+
|
321
335
|
if !( File.exist?( @path ) ) then
|
322
336
|
$logger.error("downspout | downloader | curb_http_download | Missing File at download path : #{@path}")
|
323
337
|
return false
|
324
338
|
end
|
325
|
-
|
339
|
+
|
326
340
|
return true
|
327
341
|
end
|
328
342
|
|
@@ -349,10 +363,10 @@ module Downspout
|
|
349
363
|
http_hash[:version] = http_info.split(" ")[0].match("HTTP/([0-9\.]+)")[1]
|
350
364
|
http_hash[:code] = (http_info.split("\r\n")[0].split(" ")[1]).to_i
|
351
365
|
http_hash[:message] = http_info.split("\r\n")[0].split(" ")[2]
|
352
|
-
|
366
|
+
|
353
367
|
header_hash["HTTP"] = http_hash
|
354
368
|
|
355
|
-
headers[1..-1].each do |line|
|
369
|
+
headers[1..-1].each do |line|
|
356
370
|
next if line.nil? || line.empty?
|
357
371
|
begin
|
358
372
|
matches = line.match(/([\w\-\s]+)\:\s?(.*)/)
|
@@ -386,7 +400,7 @@ module Downspout
|
|
386
400
|
|
387
401
|
$logger.debug("downspout | downloader | file_name_from_content_disposition | cd key : #{cd_key}")
|
388
402
|
return nil if cd_key.nil?
|
389
|
-
|
403
|
+
|
390
404
|
if cd_key then
|
391
405
|
disposition = @response_headers[cd_key]
|
392
406
|
if disposition then
|
@@ -396,7 +410,7 @@ module Downspout
|
|
396
410
|
end
|
397
411
|
|
398
412
|
$logger.debug("downspout | downloader | file_name_from_content_disposition | #{file_name}")
|
399
|
-
return file_name
|
413
|
+
return file_name
|
400
414
|
end
|
401
415
|
|
402
416
|
def file_name_from_content_type
|
@@ -421,11 +435,11 @@ module Downspout
|
|
421
435
|
if !(my_uri.path.nil? || my_uri.path.empty? || my_uri.path == '/')
|
422
436
|
return File.basename( my_uri.path )
|
423
437
|
else
|
424
|
-
$logger.debug("downspout | downloader | basename | Bad URI path")
|
438
|
+
$logger.debug("downspout | downloader | basename | Bad URI path")
|
425
439
|
return nil
|
426
440
|
end
|
427
441
|
end
|
428
|
-
|
442
|
+
|
429
443
|
end
|
430
|
-
|
444
|
+
|
431
445
|
end
|
data/lib/downspout.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'rubygems'
|
1
|
+
#require 'rubygems'
|
2
2
|
|
3
3
|
# common dependencies
|
4
4
|
require 'fileutils'
|
@@ -7,9 +7,6 @@ require 'net/http'
|
|
7
7
|
require 'net/https'
|
8
8
|
require 'net/ftp'
|
9
9
|
|
10
|
-
# add this directory to the path...
|
11
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
12
|
-
|
13
10
|
# customized logger
|
14
11
|
require 'downspout/logger'
|
15
12
|
|
data/test/downspout_test.rb
CHANGED
@@ -10,8 +10,9 @@ class DownspoutTest < Test::Unit::TestCase
|
|
10
10
|
|
11
11
|
$test_ws ||= WEBrick::HTTPServer.new(:Port => 8899,
|
12
12
|
:DocumentRoot => @test_ws_root,
|
13
|
-
#
|
14
|
-
:Logger =>
|
13
|
+
# TODO : Use Log/BasicLog from WEBrick to reduce spam in tests
|
14
|
+
# :Logger => Log.new(nil, BasicLog::WARN),
|
15
|
+
:Logger => $logger,
|
15
16
|
:AccessLog => [])
|
16
17
|
|
17
18
|
|
@@ -23,7 +24,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
23
24
|
num = req.query['n'].to_i
|
24
25
|
|
25
26
|
if num > 1 then
|
26
|
-
resp.set_redirect( HTTPStatus::MovedPermanently, "/deep/?n=#{num - 1}")
|
27
|
+
resp.set_redirect( HTTPStatus::MovedPermanently, "/deep/?n=#{num - 1}")
|
27
28
|
else
|
28
29
|
resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
|
29
30
|
end
|
@@ -101,7 +102,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
101
102
|
assert_equal 2, Downspout::Config.max_redirects
|
102
103
|
|
103
104
|
too_deep_url = "http://127.0.0.1:8899/deep/?n=3&test=no-curb"
|
104
|
-
|
105
|
+
|
105
106
|
assert_raise Downspout::ExcessiveRedirects do
|
106
107
|
dl = Downspout.fetch_url( too_deep_url )
|
107
108
|
end
|
@@ -140,7 +141,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
140
141
|
should "download to custom path" do
|
141
142
|
gfx_path = File.join( Test::App.root, "tmp", "download-test", "image.png" )
|
142
143
|
FileUtils.mkdir_p( File.dirname( gfx_path ) )
|
143
|
-
|
144
|
+
|
144
145
|
dl = Downspout.download_url_to_path( $test_image_url, gfx_path )
|
145
146
|
end
|
146
147
|
|
data/test/servlet.rb
CHANGED
@@ -4,8 +4,7 @@ ws_root = File.expand_path( File.dirname( File.dirname( __FILE__ ) ) )
|
|
4
4
|
|
5
5
|
ws_app = WEBrick::HTTPServer.new(:Port => 8899,
|
6
6
|
:DocumentRoot => @test_ws_root,
|
7
|
-
:Logger => Log.new(nil, BasicLog::WARN),
|
8
|
-
# :Logger => $logger,
|
7
|
+
:Logger => Log.new(nil, BasicLog::WARN),
|
9
8
|
:AccessLog => [])
|
10
9
|
|
11
10
|
ws_app.mount(TestServlet.path, TestServlet)
|
@@ -24,8 +23,8 @@ redir_proc = Proc.new { |req, resp|
|
|
24
23
|
|
25
24
|
ws_app.mount('/one/deep/', HTTPServlet::ProcHandler.new(redir_proc) )
|
26
25
|
|
27
|
-
ws_thread = Thread.new { ws_app.start }
|
28
|
-
|
26
|
+
ws_thread = Thread.new { ws_app.start }
|
27
|
+
|
29
28
|
read_me_url = "#{TestServlet.url}/READ_ME.doc"
|
30
29
|
|
31
30
|
puts "Request #{read_me_url}"
|
data/test/unit/base_test.rb
CHANGED
@@ -21,12 +21,12 @@ class DownspoutTest < Test::Unit::TestCase
|
|
21
21
|
end
|
22
22
|
|
23
23
|
should "respond to fetch_url" do
|
24
|
-
assert Downspout.respond_to?(:
|
24
|
+
assert Downspout.respond_to?(:fetch_url)
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
context "protocols" do
|
29
|
-
|
29
|
+
|
30
30
|
should "not support SFTP" do
|
31
31
|
assert !( Downspout.supported_protocol?( "sftp" ) )
|
32
32
|
end
|
@@ -38,7 +38,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
38
38
|
should "not support SCP" do
|
39
39
|
assert !( Downspout.supported_protocol?( "scp" ) )
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
should "support all secret herbs and spices" do
|
43
43
|
assert_not_nil Downspout.supported_protocols
|
44
44
|
assert_equal Array, Downspout.supported_protocols.class
|
@@ -86,8 +86,8 @@ class DownspoutTest < Test::Unit::TestCase
|
|
86
86
|
assert !( Downspout.viable_url?( "sftp://host.domain.tld/resource/file.format" ) )
|
87
87
|
end
|
88
88
|
|
89
|
-
should "be
|
90
|
-
assert
|
89
|
+
should "be accepted with FTPS protocol" do
|
90
|
+
assert Downspout.viable_url?( "ftps://host.domain.tld/resource/file.format" )
|
91
91
|
end
|
92
92
|
|
93
93
|
end
|
@@ -115,7 +115,7 @@ class DownspoutTest < Test::Unit::TestCase
|
|
115
115
|
|
116
116
|
should "be accepted with FTP protocol containing user & password" do
|
117
117
|
num = Downspout::Config.credentials.size
|
118
|
-
|
118
|
+
|
119
119
|
assert Downspout.viable_url?( "ftp://uzer:passw0rd@host.domain.tld/resource/file.format" )
|
120
120
|
|
121
121
|
assert_equal num + 1, Downspout::Config.credentials.size
|
@@ -135,6 +135,6 @@ class DownspoutTest < Test::Unit::TestCase
|
|
135
135
|
assert_equal URI::parse( test_url ).host, URI::parse( new_url ).host
|
136
136
|
assert_equal URI::parse( test_url ).path, URI::parse( new_url ).path
|
137
137
|
end
|
138
|
-
|
138
|
+
|
139
139
|
end
|
140
140
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: downspout
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 19
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 3
|
9
|
+
- 0
|
10
|
+
version: 0.3.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Phi.Sanders
|
@@ -15,29 +15,42 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date:
|
19
|
-
default_executable:
|
18
|
+
date: 2013-03-14 00:00:00 Z
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
22
|
-
name: jeweler
|
23
|
-
prerelease: false
|
24
21
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
22
|
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
hash:
|
26
|
+
hash: 55
|
30
27
|
segments:
|
31
|
-
-
|
32
|
-
-
|
33
|
-
-
|
34
|
-
version:
|
28
|
+
- 0
|
29
|
+
- 9
|
30
|
+
- 6
|
31
|
+
version: 0.9.6
|
32
|
+
prerelease: false
|
35
33
|
type: :development
|
34
|
+
name: rake
|
36
35
|
version_requirements: *id001
|
37
36
|
- !ruby/object:Gem::Dependency
|
38
|
-
name: rcov
|
39
|
-
prerelease: false
|
40
37
|
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 27
|
43
|
+
segments:
|
44
|
+
- 2
|
45
|
+
- 4
|
46
|
+
- 2
|
47
|
+
version: 2.4.2
|
48
|
+
prerelease: false
|
49
|
+
type: :development
|
50
|
+
name: rdoc
|
51
|
+
version_requirements: *id002
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
54
|
none: false
|
42
55
|
requirements:
|
43
56
|
- - ">="
|
@@ -48,12 +61,12 @@ dependencies:
|
|
48
61
|
- 9
|
49
62
|
- 9
|
50
63
|
version: 0.9.9
|
64
|
+
prerelease: false
|
51
65
|
type: :development
|
52
|
-
|
66
|
+
name: rcov
|
67
|
+
version_requirements: *id003
|
53
68
|
- !ruby/object:Gem::Dependency
|
54
|
-
|
55
|
-
prerelease: false
|
56
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
69
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
57
70
|
none: false
|
58
71
|
requirements:
|
59
72
|
- - ">="
|
@@ -64,12 +77,12 @@ dependencies:
|
|
64
77
|
- 11
|
65
78
|
- 3
|
66
79
|
version: 2.11.3
|
80
|
+
prerelease: false
|
67
81
|
type: :development
|
68
|
-
|
82
|
+
name: shoulda
|
83
|
+
version_requirements: *id004
|
69
84
|
- !ruby/object:Gem::Dependency
|
70
|
-
|
71
|
-
prerelease: false
|
72
|
-
requirement: &id004 !ruby/object:Gem::Requirement
|
85
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
86
|
none: false
|
74
87
|
requirements:
|
75
88
|
- - ">="
|
@@ -80,12 +93,12 @@ dependencies:
|
|
80
93
|
- 9
|
81
94
|
- 12
|
82
95
|
version: 0.9.12
|
96
|
+
prerelease: false
|
83
97
|
type: :development
|
84
|
-
|
98
|
+
name: mocha
|
99
|
+
version_requirements: *id005
|
85
100
|
- !ruby/object:Gem::Dependency
|
86
|
-
|
87
|
-
prerelease: false
|
88
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
101
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
89
102
|
none: false
|
90
103
|
requirements:
|
91
104
|
- - ">="
|
@@ -96,22 +109,27 @@ dependencies:
|
|
96
109
|
- 7
|
97
110
|
- 15
|
98
111
|
version: 0.7.15
|
112
|
+
prerelease: false
|
99
113
|
type: :development
|
100
|
-
|
114
|
+
name: curb
|
115
|
+
version_requirements: *id006
|
101
116
|
description: Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)
|
102
|
-
email:
|
117
|
+
email:
|
118
|
+
- phi.sanders@sciphi.me
|
103
119
|
executables: []
|
104
120
|
|
105
121
|
extensions: []
|
106
122
|
|
107
|
-
extra_rdoc_files:
|
108
|
-
|
109
|
-
- README.rdoc
|
123
|
+
extra_rdoc_files: []
|
124
|
+
|
110
125
|
files:
|
126
|
+
- .gitignore
|
127
|
+
- Gemfile
|
128
|
+
- Gemfile.lock
|
111
129
|
- LICENSE.txt
|
112
130
|
- README.rdoc
|
113
131
|
- Rakefile
|
114
|
-
-
|
132
|
+
- downspout.gemspec
|
115
133
|
- lib/downspout.rb
|
116
134
|
- lib/downspout/base.rb
|
117
135
|
- lib/downspout/config.rb
|
@@ -119,6 +137,7 @@ files:
|
|
119
137
|
- lib/downspout/downloader.rb
|
120
138
|
- lib/downspout/logger.rb
|
121
139
|
- lib/downspout/tmp_file.rb
|
140
|
+
- lib/downspout/version.rb
|
122
141
|
- test/downspout_test.rb
|
123
142
|
- test/fixtures/faux_headers.txt
|
124
143
|
- test/fixtures/ruby.png
|
@@ -133,7 +152,6 @@ files:
|
|
133
152
|
- test/unit/downloader_test.rb
|
134
153
|
- test/unit/tmp_file_test.rb
|
135
154
|
- test/watchr.rb
|
136
|
-
has_rdoc: true
|
137
155
|
homepage: http://github.com/sci-phi/downspout
|
138
156
|
licenses:
|
139
157
|
- MIT
|
@@ -163,12 +181,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
163
181
|
requirements: []
|
164
182
|
|
165
183
|
rubyforge_project:
|
166
|
-
rubygems_version: 1.
|
184
|
+
rubygems_version: 1.8.24
|
167
185
|
signing_key:
|
168
186
|
specification_version: 3
|
169
187
|
summary: Downspout is an easy-to-use ruby library for downloading files from URLs.
|
170
188
|
test_files:
|
171
189
|
- test/downspout_test.rb
|
190
|
+
- test/fixtures/faux_headers.txt
|
191
|
+
- test/fixtures/ruby.png
|
172
192
|
- test/focused_test.rb
|
173
193
|
- test/servlet.rb
|
174
194
|
- test/test_helper.rb
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.2.10
|