downer 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +21 -0
- data/LICENSE +20 -0
- data/README.rdoc +14 -0
- data/Rakefile +45 -0
- data/bin/downer +6 -0
- data/downer.gemspec +76 -0
- data/features/step_definitions/downer_steps.rb +38 -0
- data/features/support/env.rb +2 -0
- data/features/user_starts_downer_with_valid_args.feature +10 -0
- data/features/user_starts_downer_without_args.feature +10 -0
- data/lib/downer/application.rb +41 -0
- data/lib/downer/download_item.rb +58 -0
- data/lib/downer/download_manager.rb +39 -0
- data/lib/downer/download_worker.rb +32 -0
- data/lib/downer/options.rb +21 -0
- data/lib/downer.rb +8 -0
- data/spec/downer/application_spec.rb +22 -0
- data/spec/downer/download_item_spec.rb +43 -0
- data/spec/downer/download_manager_spec.rb +41 -0
- data/spec/downer/download_worker_spec.rb +35 -0
- data/spec/downer/generator_spec.rb +1 -0
- data/spec/fixtures/some_images.txt +4 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +18 -0
- data/version.yml +5 -0
- metadata +113 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Nate Miller
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
= Downer
|
2
|
+
|
3
|
+
Downer is a simple personal tool used to download a sequential list of URLs contained in a text file.
|
4
|
+
|
5
|
+
== Usage:
|
6
|
+
Create a single text file, each line containing a url you wish to download.
|
7
|
+
|
8
|
+
=== From bash:
|
9
|
+
$ downer myfile.txt ~/Desktop/myFolder # will donwload each file into the myFolder directory
|
10
|
+
|
11
|
+
|
12
|
+
== Copyright
|
13
|
+
|
14
|
+
Copyright (c) 2010 Nate Miller. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "downer"
|
8
|
+
gem.summary = "Bulk downloading utitlity"
|
9
|
+
gem.description = "Downer is a tool used to download a list of urls from a website thorugh HTTP."
|
10
|
+
gem.email = "nate@natemiller.org"
|
11
|
+
gem.homepage = "http://github.com/nate63179/downer"
|
12
|
+
gem.authors = ["Nate Miller"]
|
13
|
+
gem.add_development_dependency "rspec", ">= 1.2.9"
|
14
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
15
|
+
end
|
16
|
+
Jeweler::GemcutterTasks.new
|
17
|
+
rescue LoadError
|
18
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'spec/rake/spectask'
|
22
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
23
|
+
spec.libs << 'lib' << 'spec'
|
24
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
25
|
+
end
|
26
|
+
|
27
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
28
|
+
spec.libs << 'lib' << 'spec'
|
29
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
30
|
+
spec.rcov = true
|
31
|
+
end
|
32
|
+
|
33
|
+
task :spec => :check_dependencies
|
34
|
+
|
35
|
+
task :default => :spec
|
36
|
+
|
37
|
+
require 'rake/rdoctask'
|
38
|
+
Rake::RDocTask.new do |rdoc|
|
39
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
40
|
+
|
41
|
+
rdoc.rdoc_dir = 'rdoc'
|
42
|
+
rdoc.title = "downer #{version}"
|
43
|
+
rdoc.rdoc_files.include('README*')
|
44
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
45
|
+
end
|
data/bin/downer
ADDED
data/downer.gemspec
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{downer}
|
8
|
+
s.version = "0.1.1"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Nate Miller"]
|
12
|
+
s.date = %q{2010-07-16}
|
13
|
+
s.default_executable = %q{downer}
|
14
|
+
s.description = %q{Downer is a tool used to download a list of urls from a website thorugh HTTP.}
|
15
|
+
s.email = %q{nate@natemiller.org}
|
16
|
+
s.executables = ["downer"]
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE",
|
19
|
+
"README.rdoc"
|
20
|
+
]
|
21
|
+
s.files = [
|
22
|
+
".document",
|
23
|
+
".gitignore",
|
24
|
+
"LICENSE",
|
25
|
+
"README.rdoc",
|
26
|
+
"Rakefile",
|
27
|
+
"bin/downer",
|
28
|
+
"downer.gemspec",
|
29
|
+
"features/step_definitions/downer_steps.rb",
|
30
|
+
"features/support/env.rb",
|
31
|
+
"features/user_starts_downer_with_valid_args.feature",
|
32
|
+
"features/user_starts_downer_without_args.feature",
|
33
|
+
"lib/downer.rb",
|
34
|
+
"lib/downer/application.rb",
|
35
|
+
"lib/downer/download_item.rb",
|
36
|
+
"lib/downer/download_manager.rb",
|
37
|
+
"lib/downer/download_worker.rb",
|
38
|
+
"lib/downer/options.rb",
|
39
|
+
"spec/downer/application_spec.rb",
|
40
|
+
"spec/downer/download_item_spec.rb",
|
41
|
+
"spec/downer/download_manager_spec.rb",
|
42
|
+
"spec/downer/download_worker_spec.rb",
|
43
|
+
"spec/downer/generator_spec.rb",
|
44
|
+
"spec/fixtures/some_images.txt",
|
45
|
+
"spec/spec.opts",
|
46
|
+
"spec/spec_helper.rb",
|
47
|
+
"version.yml"
|
48
|
+
]
|
49
|
+
s.homepage = %q{http://github.com/nate63179/downer}
|
50
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
51
|
+
s.require_paths = ["lib"]
|
52
|
+
s.rubygems_version = %q{1.3.7}
|
53
|
+
s.summary = %q{Bulk downloading utitlity}
|
54
|
+
s.test_files = [
|
55
|
+
"spec/downer/application_spec.rb",
|
56
|
+
"spec/downer/download_item_spec.rb",
|
57
|
+
"spec/downer/download_manager_spec.rb",
|
58
|
+
"spec/downer/download_worker_spec.rb",
|
59
|
+
"spec/downer/generator_spec.rb",
|
60
|
+
"spec/spec_helper.rb"
|
61
|
+
]
|
62
|
+
|
63
|
+
if s.respond_to? :specification_version then
|
64
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
65
|
+
s.specification_version = 3
|
66
|
+
|
67
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
68
|
+
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
|
69
|
+
else
|
70
|
+
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
71
|
+
end
|
72
|
+
else
|
73
|
+
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Mock objec for simulating outpu
|
2
|
+
class Output
|
3
|
+
def messages
|
4
|
+
@messages ||= []
|
5
|
+
end
|
6
|
+
|
7
|
+
def puts(message)
|
8
|
+
messages << message
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def output
|
13
|
+
@output ||= Output.new
|
14
|
+
end
|
15
|
+
|
16
|
+
# Create temporary file with urls
|
17
|
+
def fixture_directory
|
18
|
+
File.expand_path('../../../spec/fixtures', __FILE__)
|
19
|
+
end
|
20
|
+
|
21
|
+
Given /^I have not started application$/ do
|
22
|
+
end
|
23
|
+
|
24
|
+
When /^I start a new application without arguments$/ do
|
25
|
+
app = Downer::Application.new(output)
|
26
|
+
app.run!
|
27
|
+
end
|
28
|
+
|
29
|
+
Then /^I should see "([^"]*)"$/ do |message|
|
30
|
+
output.messages.should include(message)
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
|
35
|
+
When /^I start a new application with valid arguments$/ do
|
36
|
+
app = Downer::Application.new(output)
|
37
|
+
app.run!(fixture_directory + '/some_images.txt', '/tmp')
|
38
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
Feature: user starts downer
|
2
|
+
|
3
|
+
As a user
|
4
|
+
I want to start downer
|
5
|
+
So that I can download a list of files
|
6
|
+
|
7
|
+
Scenario: start downer
|
8
|
+
Given I have not started application
|
9
|
+
When I start a new application with valid arguments
|
10
|
+
Then I should see "Starting download on 4 files"
|
@@ -0,0 +1,10 @@
|
|
1
|
+
Feature: user starts downer
|
2
|
+
|
3
|
+
As a user
|
4
|
+
I want to start downer
|
5
|
+
So that I can download a list of files
|
6
|
+
|
7
|
+
Scenario: start downer
|
8
|
+
Given I have not started application
|
9
|
+
When I start a new application without arguments
|
10
|
+
Then I should see "Usage: downer URL_SOURCE DESTINATION_DIR"
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Downer
|
2
|
+
class Application
|
3
|
+
|
4
|
+
attr_accessor :output
|
5
|
+
|
6
|
+
def initialize(output = nil)
|
7
|
+
@output = (output) ? output : $stdout
|
8
|
+
@options = nil
|
9
|
+
end
|
10
|
+
|
11
|
+
def run!(*arguments)
|
12
|
+
@options = Downer::Options.new(arguments)
|
13
|
+
|
14
|
+
if @options[:invalid_argument]
|
15
|
+
@output.puts @options[:invalid_argument]
|
16
|
+
@options[:show_help] = true
|
17
|
+
end
|
18
|
+
|
19
|
+
return exit_with_help_banner if @options[:file_manifest].nil?
|
20
|
+
return exit_with_help_banner if @options[:target_directory].nil?
|
21
|
+
return exit_with_help_banner if @options[:show_help]
|
22
|
+
|
23
|
+
begin
|
24
|
+
manager = Downer::DownloadManager.new(@options[:file_manifest], @options[:target_directory], @output)
|
25
|
+
manager.start
|
26
|
+
return 0
|
27
|
+
rescue Downer::WriteFailed
|
28
|
+
@output.puts %Q{Insufficient permissions to write to directory}
|
29
|
+
return 1
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def exit_with_help_banner
|
36
|
+
@output.puts @options.opts.banner
|
37
|
+
return 1
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Downer
|
2
|
+
|
3
|
+
class FailedDownload < StandardError
|
4
|
+
attr_accessor :http_code, :url
|
5
|
+
end
|
6
|
+
|
7
|
+
class DownloadItem
|
8
|
+
attr_reader :url
|
9
|
+
attr_reader :content
|
10
|
+
|
11
|
+
def initialize(url, destination)
|
12
|
+
@url, @destination = sanitize_url(url), destination
|
13
|
+
@uri = URI.parse(url)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Returns the name which this file will be saved as
|
17
|
+
def get_save_filename
|
18
|
+
file_name = @uri.to_s
|
19
|
+
file_name = file_name.gsub(/https?:\/\//,'').split('/').last
|
20
|
+
file_name = file_name.gsub('%5B', '[')
|
21
|
+
file_name = file_name.gsub('%5D', ']')
|
22
|
+
file_name
|
23
|
+
# TODO : refine to auto append file extentions based on content type
|
24
|
+
end
|
25
|
+
|
26
|
+
def download
|
27
|
+
@http = Net::HTTP.new(@uri.host, @uri.port)
|
28
|
+
req = Net::HTTP::Get.new(@uri.request_uri)
|
29
|
+
response = @http.request(req)
|
30
|
+
|
31
|
+
if response.code != '200'
|
32
|
+
fd = FailedDownload.new
|
33
|
+
fd.http_code = response.code
|
34
|
+
fd.url = @url
|
35
|
+
raise fd
|
36
|
+
else
|
37
|
+
@content = response.body
|
38
|
+
write_to_file
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def write_to_file
|
45
|
+
file_out_path = @destination + "/#{get_save_filename}"
|
46
|
+
fout = File.new(file_out_path, 'w')
|
47
|
+
fout.puts @content
|
48
|
+
end
|
49
|
+
|
50
|
+
def sanitize_url(unsafe_url)
|
51
|
+
url = unsafe_url
|
52
|
+
url.gsub!('[', '%5B')
|
53
|
+
url.gsub!(']', '%5D')
|
54
|
+
url.gsub!(',', '%2C')
|
55
|
+
url
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Downer
|
2
|
+
class WriteFailed < StandardError; end
|
3
|
+
class BadUrl < StandardError; end
|
4
|
+
class MalformedManifest < StandardError; end
|
5
|
+
class NoManifestFileGiven < StandardError; end
|
6
|
+
class NoTargetDirectoryGiven < StandardError; end
|
7
|
+
class URLSourceDoesNotExist < StandardError; end
|
8
|
+
|
9
|
+
class DownloadManager
|
10
|
+
|
11
|
+
attr_accessor :file_manifest, :target_directory, :output
|
12
|
+
attr_reader :urls
|
13
|
+
|
14
|
+
def initialize(url_source, target_directory, output)
|
15
|
+
@file_manifest, @target_directory, @output = url_source, target_directory, output
|
16
|
+
dir_last_char_is_slash = (@target_directory[-1,1] == '/')
|
17
|
+
@target_directory = @target_directory + '/' unless dir_last_char_is_slash
|
18
|
+
@urls = []
|
19
|
+
raise URLSourceDoesNotExist unless File.exists?(@file_manifest)
|
20
|
+
get_urls
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
raise WriteFailed unless File.writable?(@target_directory)
|
25
|
+
|
26
|
+
@output.puts "Starting download on #{@urls.size} files"
|
27
|
+
worker = DownloadWorker.new(@urls, @target_directory, @output)
|
28
|
+
worker.start
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def get_urls
|
34
|
+
f = File.open(@file_manifest, 'r')
|
35
|
+
f.each_line { |line| @urls << line.chomp }
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Downer
|
2
|
+
class DownloadWorker
|
3
|
+
|
4
|
+
attr_reader :items
|
5
|
+
|
6
|
+
def initialize(urls, target_directory, output)
|
7
|
+
@urls, @target_directory, @output = urls, target_directory, output
|
8
|
+
@items = []
|
9
|
+
@urls.each { |url| @items << DownloadItem.new(url, target_directory) }
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
if @urls.empty?
|
14
|
+
@output.puts "No URLs specified, exiting."
|
15
|
+
return
|
16
|
+
end
|
17
|
+
|
18
|
+
@items.each { |item| try_download_item(item) }
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def try_download_item(item)
|
24
|
+
begin
|
25
|
+
item.download
|
26
|
+
@output.puts "Downloaded #{item.url}"
|
27
|
+
rescue Downer::FailedDownload => e
|
28
|
+
@output.puts "Could not download #{e.url}, received http code #{e.http_code}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Downer
|
2
|
+
class Options < Hash
|
3
|
+
attr_reader :opts, :orig_args
|
4
|
+
|
5
|
+
def initialize(args)
|
6
|
+
|
7
|
+
@orig_args = args.clone
|
8
|
+
@opts = OptionParser.new do |o|
|
9
|
+
o.banner = "Usage: downer URL_SOURCE DESTINATION_DIR"
|
10
|
+
end
|
11
|
+
|
12
|
+
begin
|
13
|
+
@opts.parse!(args)
|
14
|
+
self[:file_manifest] = args.shift
|
15
|
+
self[:target_directory] = args.shift
|
16
|
+
rescue OptionParser::InvalidOption => e
|
17
|
+
self[:invalid_argument] = e.message
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/downer.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
|
4
|
+
module Downer
|
5
|
+
describe Downer do
|
6
|
+
describe "#run!" do
|
7
|
+
it "when run without arguments displays a usage hint" do
|
8
|
+
output = double('output')
|
9
|
+
app = Application.new(output)
|
10
|
+
output.should_receive(:puts).with('Usage: downer URL_SOURCE DESTINATION_DIR')
|
11
|
+
app.run!
|
12
|
+
end
|
13
|
+
|
14
|
+
it "when run with valid arguments displays the number of files to download" do
|
15
|
+
output = double('output').as_null_object
|
16
|
+
app = Application.new(output)
|
17
|
+
output.should_receive(:puts).with("Starting download on 4 files")
|
18
|
+
app.run!(fixture_directory + "/some_images.txt", '/tmp')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
|
2
|
+
|
3
|
+
module Downer
|
4
|
+
describe DownloadItem do
|
5
|
+
|
6
|
+
describe "#initialize" do
|
7
|
+
it "should escape bad url tokens" do
|
8
|
+
item = DownloadItem.new('http://www.urbaninfluence.com/my[place].html', '/tmp')
|
9
|
+
item.url.should == 'http://www.urbaninfluence.com/my%5Bplace%5D.html'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe "#get_save_filename" do
|
14
|
+
it "should generate a filename from its download url" do
|
15
|
+
item = DownloadItem.new('http://www.urbaninfluence.com/sites/default/files/user_uploads/images/mapsAndAtlases2.png', '/tmp')
|
16
|
+
item.get_save_filename.should == 'mapsAndAtlases2.png'
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
it "should restore bad url tokens into valid ascii characters" do
|
21
|
+
item = DownloadItem.new('http://www.urbaninfluence.com/my[place].html', '/tmp')
|
22
|
+
item.get_save_filename.should == 'my[place].html'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "#download" do
|
27
|
+
before(:each) do
|
28
|
+
FileUtils.rm_r(Dir.glob(tmp_directory + '/*'))
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should write the url to the target directory" do
|
32
|
+
item = DownloadItem.new('http://www.urbaninfluence.com/sites/default/files/user_uploads/images/mapsAndAtlases2.png', tmp_directory)
|
33
|
+
item.download
|
34
|
+
Dir.entries(tmp_directory).size.should > 2
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should raise an error when a url is unable to be downloaded" do
|
38
|
+
item = DownloadItem.new('http://www.urbaninfluence.com/will_never_complete', tmp_directory)
|
39
|
+
lambda { item.download }.should raise_error(Downer::FailedDownload)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
|
2
|
+
|
3
|
+
module Downer
|
4
|
+
describe DownloadManager do
|
5
|
+
let(:output) { double('output').as_null_object }
|
6
|
+
let(:fixture_file_path) { fixture_directory + '/some_images.txt' }
|
7
|
+
let(:manager) { DownloadManager.new(fixture_file_path, 'myoutputdir', output) }
|
8
|
+
|
9
|
+
describe "#new" do
|
10
|
+
it "should add a slash as the last character of the target directory if one is not present" do
|
11
|
+
manager.target_directory.should == 'myoutputdir/'
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should retrieve all urls from the url source" do
|
15
|
+
manager.urls.should_not be_empty
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe "#start" do
|
20
|
+
|
21
|
+
it "should raise an error when the file does not exist" do
|
22
|
+
File.should_receive(:exists?).with(fixture_file_path)
|
23
|
+
lambda { manager.start }.should raise_error(Downer::URLSourceDoesNotExist)
|
24
|
+
end
|
25
|
+
it "should raise a WriteFailed exception if the target directory is not writable" do
|
26
|
+
File.should_receive(:writable?).with('myoutputdir/').and_return(false)
|
27
|
+
lambda { manager.start }.should raise_error(Downer::WriteFailed)
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
it "should create a download worker to begin the downloading" do
|
32
|
+
File.should_receive(:writable?).and_return(true)
|
33
|
+
worker = double('worker')
|
34
|
+
worker.should_receive(:start)
|
35
|
+
DownloadWorker.should_receive(:new).and_return(worker)
|
36
|
+
manager.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
|
2
|
+
|
3
|
+
module Downer
|
4
|
+
describe DownloadWorker do
|
5
|
+
let (:output) { double('output') }
|
6
|
+
let (:urls) { DownloadManager.new(fixture_directory + '/some_images.txt', '/tmp', output).urls }
|
7
|
+
|
8
|
+
describe '#start' do
|
9
|
+
it "should write a message to output when no urls exist to be downloaded" do
|
10
|
+
output.should_receive(:puts).with("No URLs specified, exiting.")
|
11
|
+
worker = DownloadWorker.new([], '/tmp', output)
|
12
|
+
worker.start
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should create a download object for each url to be downloaded" do
|
16
|
+
worker = DownloadWorker.new(urls, '/tmp', output)
|
17
|
+
worker.items.size.should == urls.size
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should write a message to output feed when a url cannot be downloaded" do
|
21
|
+
bad_url = "http://www.urbaninfluence.com/will_never_succeed"
|
22
|
+
worker = DownloadWorker.new([bad_url], '/tmp', output)
|
23
|
+
output.should_receive(:puts).with("Could not download #{bad_url}, received http code 404")
|
24
|
+
worker.start
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should write a message to output feed when a url is successfully downloaded" do
|
28
|
+
good_url = "http://www.urbaninfluence.com/sites/default/files/user_uploads/images/4th.png"
|
29
|
+
worker = DownloadWorker.new([good_url], '/tmp', output)
|
30
|
+
output.should_receive(:puts).with("Downloaded #{good_url}")
|
31
|
+
worker.start
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
@@ -0,0 +1,4 @@
|
|
1
|
+
http://www.urbaninfluence.com/sites/default/files/user_uploads/images/4th.png
|
2
|
+
http://www.urbaninfluence.com/sites/default/files/user_uploads/images/mapsAndAtlases2.png
|
3
|
+
http://www.urbaninfluence.com/sites/default/files/user_uploads/images/homerstranglesbart1.png
|
4
|
+
http://www.urbaninfluence.com/sites/default/files/user_uploads/images/propane-smoker-large.png
|
data/spec/spec.opts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
3
|
+
require 'downer'
|
4
|
+
require 'spec'
|
5
|
+
require 'spec/autorun'
|
6
|
+
|
7
|
+
Spec::Runner.configure do |config|
|
8
|
+
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def fixture_directory
|
13
|
+
File.expand_path('../fixtures', __FILE__)
|
14
|
+
end
|
15
|
+
|
16
|
+
def tmp_directory
|
17
|
+
File.expand_path('../../tmp', __FILE__)
|
18
|
+
end
|
data/version.yml
ADDED
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: downer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 25
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 1
|
10
|
+
version: 0.1.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Nate Miller
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-07-16 00:00:00 -07:00
|
19
|
+
default_executable: downer
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: rspec
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 13
|
30
|
+
segments:
|
31
|
+
- 1
|
32
|
+
- 2
|
33
|
+
- 9
|
34
|
+
version: 1.2.9
|
35
|
+
type: :development
|
36
|
+
version_requirements: *id001
|
37
|
+
description: Downer is a tool used to download a list of urls from a website thorugh HTTP.
|
38
|
+
email: nate@natemiller.org
|
39
|
+
executables:
|
40
|
+
- downer
|
41
|
+
extensions: []
|
42
|
+
|
43
|
+
extra_rdoc_files:
|
44
|
+
- LICENSE
|
45
|
+
- README.rdoc
|
46
|
+
files:
|
47
|
+
- .document
|
48
|
+
- .gitignore
|
49
|
+
- LICENSE
|
50
|
+
- README.rdoc
|
51
|
+
- Rakefile
|
52
|
+
- bin/downer
|
53
|
+
- downer.gemspec
|
54
|
+
- features/step_definitions/downer_steps.rb
|
55
|
+
- features/support/env.rb
|
56
|
+
- features/user_starts_downer_with_valid_args.feature
|
57
|
+
- features/user_starts_downer_without_args.feature
|
58
|
+
- lib/downer.rb
|
59
|
+
- lib/downer/application.rb
|
60
|
+
- lib/downer/download_item.rb
|
61
|
+
- lib/downer/download_manager.rb
|
62
|
+
- lib/downer/download_worker.rb
|
63
|
+
- lib/downer/options.rb
|
64
|
+
- spec/downer/application_spec.rb
|
65
|
+
- spec/downer/download_item_spec.rb
|
66
|
+
- spec/downer/download_manager_spec.rb
|
67
|
+
- spec/downer/download_worker_spec.rb
|
68
|
+
- spec/downer/generator_spec.rb
|
69
|
+
- spec/fixtures/some_images.txt
|
70
|
+
- spec/spec.opts
|
71
|
+
- spec/spec_helper.rb
|
72
|
+
- version.yml
|
73
|
+
has_rdoc: true
|
74
|
+
homepage: http://github.com/nate63179/downer
|
75
|
+
licenses: []
|
76
|
+
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options:
|
79
|
+
- --charset=UTF-8
|
80
|
+
require_paths:
|
81
|
+
- lib
|
82
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ">="
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
hash: 3
|
88
|
+
segments:
|
89
|
+
- 0
|
90
|
+
version: "0"
|
91
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
|
+
none: false
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
hash: 3
|
97
|
+
segments:
|
98
|
+
- 0
|
99
|
+
version: "0"
|
100
|
+
requirements: []
|
101
|
+
|
102
|
+
rubyforge_project:
|
103
|
+
rubygems_version: 1.3.7
|
104
|
+
signing_key:
|
105
|
+
specification_version: 3
|
106
|
+
summary: Bulk downloading utitlity
|
107
|
+
test_files:
|
108
|
+
- spec/downer/application_spec.rb
|
109
|
+
- spec/downer/download_item_spec.rb
|
110
|
+
- spec/downer/download_manager_spec.rb
|
111
|
+
- spec/downer/download_worker_spec.rb
|
112
|
+
- spec/downer/generator_spec.rb
|
113
|
+
- spec/spec_helper.rb
|