downer 0.2.2 → 0.3.0

data/.gitignore

@@ -19,3 +19,4 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+*._*

data/Rakefile

@@ -11,6 +11,7 @@ begin
     gem.homepage = "http://github.com/nate63179/downer"
     gem.authors = ["Nate Miller"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency 'nokogiri'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/downer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{downer}
-  s.version = "0.2.2"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Nate Miller"]
-  s.date = %q{2010-07-16}
+  s.date = %q{2010-08-08}
   s.default_executable = %q{downer}
   s.description = %q{Downer is a tool used to download a list of urls from a website thorugh HTTP.}
   s.email = %q{nate@natemiller.org}
@@ -34,13 +34,22 @@ Gem::Specification.new do |s|
      "lib/downer/application.rb",
      "lib/downer/download_item.rb",
      "lib/downer/download_manager.rb",
+     "lib/downer/download_strategy.rb",
      "lib/downer/download_worker.rb",
+     "lib/downer/generic_strategy.rb",
      "lib/downer/options.rb",
+     "lib/downer/strategies/flat_file_strategy.rb",
+     "lib/downer/strategies/website_strategy.rb",
      "spec/downer/application_spec.rb",
      "spec/downer/download_item_spec.rb",
      "spec/downer/download_manager_spec.rb",
+     "spec/downer/download_strategy_spec.rb",
      "spec/downer/download_worker_spec.rb",
      "spec/downer/generator_spec.rb",
+     "spec/downer/options_spec.rb",
+     "spec/downer/strategies/flat_file_stragtegy_spec.rb",
+     "spec/downer/strategies/website_strategy_spec.rb",
+     "spec/fixtures/basic_page.html",
      "spec/fixtures/some_images.txt",
      "spec/spec.opts",
      "spec/spec_helper.rb",
@@ -55,8 +64,12 @@ Gem::Specification.new do |s|
     "spec/downer/application_spec.rb",
      "spec/downer/download_item_spec.rb",
      "spec/downer/download_manager_spec.rb",
+     "spec/downer/download_strategy_spec.rb",
      "spec/downer/download_worker_spec.rb",
      "spec/downer/generator_spec.rb",
+     "spec/downer/options_spec.rb",
+     "spec/downer/strategies/flat_file_stragtegy_spec.rb",
+     "spec/downer/strategies/website_strategy_spec.rb",
      "spec/spec_helper.rb"
   ]
 
@@ -66,11 +79,14 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
   end
 end
 

data/lib/downer.rb

@@ -1,8 +1,7 @@
+require 'rubygems'
 require 'net/http'
+require 'nokogiri'
 require 'optparse'
+require 'open-uri'
 
-require 'downer/application'
-require 'downer/download_manager'
-require 'downer/download_worker'
-require 'downer/download_item'
-require 'downer/options'
+Dir.glob(File.dirname(__FILE__) + '/**/*.rb').each { |file| require file }

data/lib/downer/application.rb

@@ -2,6 +2,7 @@ module Downer
   class Application
     
     attr_accessor :output
+    attr_reader :options
     
     def initialize(output = nil)
       @output = (output) ? output : $stdout
@@ -10,18 +11,29 @@ module Downer
     
     def run!(*arguments)
       @options = Downer::Options.new(arguments)
+      download_options = {}
       
+
+      
+      # begin analysis of arguments
       if @options[:invalid_argument]
         @output.puts @options[:invalid_argument]
         @options[:show_help] = true
       end
       
+      if @options[:images_only]
+        print_image_only_message
+        download_options[:images_only] = true
+      end
+      
+      # Immediately exit if this will never complete
       return exit_with_help_banner if @options[:file_manifest].nil?
       return exit_with_help_banner if @options[:target_directory].nil?
       return exit_with_help_banner if @options[:show_help]
       
+
       begin
-        manager = Downer::DownloadManager.new(@options[:file_manifest], @options[:target_directory], @output)
+        manager = Downer::DownloadManager.new(@options[:file_manifest], @options[:target_directory], @output, download_options)
         manager.start
         return 0
       rescue Downer::WriteFailed
@@ -31,6 +43,10 @@ module Downer
     end
     
     private
+
+    def print_image_only_message
+      @output.puts "Images only filter selected...downloading PNG,JPG,GIF, and TIFF files"
+    end
     
     def exit_with_help_banner
       @output.puts @options.opts.banner

data/lib/downer/download_item.rb

@@ -25,18 +25,23 @@ module Downer
     
     def download
       @http = Net::HTTP.new(@uri.host, @uri.port)
-      req = Net::HTTP::Get.new(@uri.request_uri)
-      response = @http.request(req)
-      
-      if response.code != '200'
-        fd = FailedDownload.new
-        fd.http_code = response.code
-        fd.url = @url
-        raise fd
+      if @uri.respond_to? :request_uri
+        req = Net::HTTP::Get.new(@uri.request_uri)
+        response = @http.request(req)
+
+        if response.code != '200'
+          fd = FailedDownload.new
+          fd.http_code = response.code
+          fd.url = @url
+          raise fd
+        else
+          @content = response.body
+          write_to_file
+        end
       else
-        @content = response.body
-        write_to_file
+        puts "WARNING: Ignored download for #{@uri.inspect}"
       end
+
     end
     
     private

data/lib/downer/download_manager.rb

@@ -5,34 +5,69 @@ module Downer
   class NoManifestFileGiven < StandardError; end
   class NoTargetDirectoryGiven < StandardError; end
   class URLSourceDoesNotExist < StandardError; end
+
   
   class DownloadManager
     
-    attr_accessor :file_manifest, :target_directory, :output
-    attr_reader :urls
+    attr_accessor :target_directory, :output, :source_type
+    attr_reader :urls, :downloaded_files, :strategy
     
-    def initialize(url_source, target_directory, output)
-      @file_manifest, @target_directory, @output = url_source, target_directory, output
-      dir_last_char_is_slash = (@target_directory[-1,1] == '/')
-      @target_directory = @target_directory + '/' unless dir_last_char_is_slash
-      @urls = []
-      raise URLSourceDoesNotExist unless File.exists?(@file_manifest) 
-      get_urls      
+    def initialize(url_source, target_directory, output, options ={})
+      @url_source = url_source
+      @output = output
+      @target_directory = append_slash_to_path(target_directory) 
+      @strategy = StrategyFinder::find_strategy(@url_source, options)
     end
     
     def start
-      raise WriteFailed unless File.writable?(@target_directory)
-
-      @output.puts "Starting download on #{@urls.size} files"
-      worker = DownloadWorker.new(@urls, @target_directory, @output)
-      worker.start
+      check_directory
+      
+      if @strategy && @strategy.source_valid?
+        urls = @strategy.get_urls
+        @output.puts "Starting download on #{urls.size} files"
+        worker = DownloadWorker.new(urls, @target_directory, @output)
+        @downloaded_files = worker.start
+        print_session_summary(worker)
+      else
+        @output.puts "Could not open url source #{@url_source}"
+      end
     end
     
-    protected
+    # Tells us what worked and what failed
+    def print_session_summary(worker)
+      @output.puts "Session complete."
+      @output.puts "\n\n"
+      @output.puts "----------------------------------"
+      @output.puts "SUMMARY:"
+      @output.puts "Successful downloads: (#{worker.successful_downloads.size})"
+      @output.puts "Failed downloads: (#{worker.failed_downloads.size}): "
+      worker.failed_downloads.each do |fail|
+        @output.puts "  * #{fail}"
+      end
+    end
     
-      def get_urls
-        f = File.open(@file_manifest, 'r')
-        f.each_line { |line| @urls << line.chomp }
+    def source_type
+      @strategy.source_type
+    end
+    
+    # Determine whether the direcotry specified exists. If it does not, see if its parent is writable. If not, give up
+    # and throw error
+    def check_directory
+      if File.writable?(@target_directory)
+        return true
+      else
+        #FileUtils.mkdir(@target_directory)
+        raise WriteFailed
+      end
+    end
+    
+    protected
+      
+      # Put a slash on the directory name if one was ommited
+      def append_slash_to_path(dir_name)
+        dir_last_char_is_slash = (dir_name[-1,1] == '/')
+        dir_name = dir_name + '/' unless dir_last_char_is_slash
+        dir_name
       end
     
   end

data/lib/downer/download_strategy.rb

@@ -0,0 +1,34 @@
+module Downer
+  
+  class StrategyFinder
+    class << self
+      
+      # Determines a strategy for extracting urls from a media type
+      def find_strategy(url_source, options ={})
+        strategy = nil
+                
+        if is_local_file?(url_source)
+          strategy = DownloadStrategy::FlatFileStrategy.new(url_source, options)
+        elsif is_remote_source?(url_source)
+          strategy = DownloadStrategy::WebsiteStrategy.new(url_source, options)
+        else
+          raise "Could not find strategy"
+        end
+        strategy
+      end
+      
+      # Determine whether the source is located on a local file system
+      def is_local_file?(url_source)
+        File.exist?(url_source)
+      end
+      
+      # Determine if this is something that lives online
+      def is_remote_source?(url_source)
+        #if url_source =~ /(ftp|https?).*$/
+        url_source.match(/(ftp|https?).*$/) ? true : false
+      end
+      
+    end
+  end
+  
+end

data/lib/downer/download_worker.rb

@@ -1,11 +1,15 @@
 module Downer
   class DownloadWorker
     
-    attr_reader :items
+    attr_reader :items, :successful_downloads, :failed_downloads
     
     def initialize(urls, target_directory, output)
       @urls, @target_directory, @output = urls, target_directory, output
+      @urls.delete_if { |url| url == nil }
       @items = []
+      @successful_downloads = []
+      @failed_downloads = []
+      
       @urls.each { |url| @items << DownloadItem.new(url, target_directory) }
     end
     
@@ -16,6 +20,7 @@ module Downer
       end
       
       @items.each { |item| try_download_item(item) }
+      successful_downloads
     end
     
     private
@@ -23,9 +28,14 @@ module Downer
       def try_download_item(item)
         begin
           item.download
+          @successful_downloads << item.url
           @output.puts "Downloaded #{item.url}"
         rescue Downer::FailedDownload => e
           @output.puts "Could not download #{e.url}, received http code #{e.http_code}"
+          @failed_downloads << item.url
+        rescue SocketError => e
+          @output.puts "SocketError encountered on url #{item.url}"
+          @failed_downloads << item.url
         end
       end
   end

data/lib/downer/generic_strategy.rb

@@ -0,0 +1,31 @@
+module Downer
+ 
+  class SubclassMethodUndefined < StandardError; end
+  
+  class GenericStrategy
+        
+    def initialize(source, search_options = {})
+      @url_source = source
+      @search_options = search_options
+    end
+    
+    def options
+      @search_options
+    end
+    
+    def get_urls
+      raise SubclassMethodUndefined
+    end
+    
+    def source_valid?
+      raise SubclassMethodUndefined
+    end
+    
+    def source_type
+      name = self.class.name.gsub(/Downer::DownloadStrategy::/,'')
+      name.gsub(/Strategy/,'').downcase
+    end
+  end
+
+end
+  

data/lib/downer/options.rb

@@ -3,10 +3,20 @@ module Downer
     attr_reader :opts, :orig_args
     
     def initialize(args)
-      
-      @orig_args = args.clone
+      @opts = args.clone
+      self[:is_website] = false
+      self[:images_only] = false
+
       @opts = OptionParser.new do |o|
-        o.banner = "Usage: downer URL_SOURCE DESTINATION_DIR"
+        o.banner = "Usage: downer -flags URL_SOURCE DESTINATION_DIR"
+        
+        o.on('-w', '--web', 'Declare source as a url') do |url|
+          self[:is_website] = true
+        end
+        
+        o.on('-i', '--image', 'When combined with w will download JPG,GIF,PNG formats') do
+          self[:images_only] = true
+        end
       end
       
       begin

data/lib/downer/strategies/flat_file_strategy.rb

@@ -0,0 +1,19 @@
+module Downer
+  module DownloadStrategy
+    
+    class FlatFileStrategy < GenericStrategy
+            
+      def get_urls
+        urls = []
+        f = File.open(@url_source, 'r')
+        f.each_line { |line| urls << line.chomp}
+        urls
+      end
+      
+      def source_valid?
+        File.exist?(@url_source)
+      end
+      
+    end
+  end
+end

data/lib/downer/strategies/website_strategy.rb

@@ -0,0 +1,75 @@
+module Downer
+  module DownloadStrategy
+    
+    class WebsiteStrategy < GenericStrategy      
+      
+      # Create the downloading strategy, set any behavior flags in the options hash
+      def initialize(url_source, search_options = {})
+        super(url_source, search_options)
+        uri = URI.parse(url_source)
+        @host_prefix = uri.scheme + "://" + uri.host
+      end
+      
+      # Retrieve urls from an HTML page. Behavior is dependent upon options passed
+      # to constructor
+      def get_urls
+        @noko = Nokogiri::HTML(download_page)
+        urls = []
+        
+        if @search_options[:images_only]
+          urls = image_urls
+        else
+          urls = urls.concat document_links
+          urls = urls.concat image_urls
+        end
+        urls.uniq
+      end
+      
+      # read an html page into memory
+      def download_page
+        @downloaded_page ||= open(@url_source)
+      end
+      
+      # Return all image urls from document
+      def image_urls
+        urls = []      
+        @noko.css('img').each do |img| 
+          urls << absolutify_link(img['src'])
+        end
+        urls
+      end
+      
+      # Return all links stored within the document
+      def document_links
+        urls = []
+        @noko.css('a').each do |alink|
+          link = alink['href']
+          urls << absolutify_link(link)
+        end
+        urls
+      end
+      
+      # Converts non absolute urls to absolute ones
+      def absolutify_link(link)
+        
+        # Auto prepend any links which refer use releative reference like '../'
+        if link[0,1] == '.'
+          link = '/' + link
+        end
+        
+        if link =~ /(https?|ftp).*/
+          url = link
+        elsif link[0,1] != '/'
+          link = "/" + link
+        else
+          url = @host_prefix + link
+        end
+      end
+      
+      def source_valid?
+        URI.parse(@url_source)
+      end
+      
+    end
+  end
+end

data/spec/downer/application_spec.rb

@@ -4,19 +4,28 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 module Downer
   describe Downer do
     describe "#run!" do
+      before(:each) do
+        @output = double('output').as_null_object
+        @app = Application.new(@output)
+      end
+      
       it "when run without arguments displays a usage hint" do
-        output = double('output')
-        app = Application.new(output)
-        output.should_receive(:puts).with('Usage: downer URL_SOURCE DESTINATION_DIR')
-        app.run!
+        @output.should_receive(:puts).with('Usage: downer -flags URL_SOURCE DESTINATION_DIR')
+        @app.run!
       end
       
-      it "when run with valid arguments displays the number of files to download" do
-        output = double('output').as_null_object
-        app = Application.new(output)
-        output.should_receive(:puts).with("Starting download on 4 files")
-        app.run!(fixture_directory + "/some_images.txt", '/tmp')
+      it "when run with -i argument it will download only images" do
+        @output.should_receive(:puts).with("Images only filter selected...downloading PNG,JPG,GIF, and TIFF files")
+        @app.run!("-i")
+        @app.options[:images_only].should == true
       end
+            
+      # it "when run with a -w DATA_SOURCE argument it should start a web download" do
+      #   host = "http://www.urbaninfluence.com"
+      #   @output.should_receive(:puts).with("Requesting from host #{host}")
+      #   arg_cmd = %w{-wi http://www.urbaninfluence.com /tmp}
+      #   @app.run!(arg_cmd)
+      # end
     end
   end
 end

data/spec/downer/download_manager_spec.rb

@@ -1,6 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
 
 module Downer
+
   describe DownloadManager do
     let(:output) { double('output').as_null_object }
     let(:fixture_file_path) { fixture_directory + '/some_images.txt' }
@@ -10,31 +11,48 @@ module Downer
       it "should add a slash as the last character of the target directory if one is not present" do
         manager.target_directory.should == 'myoutputdir/'
       end
-            
-      it "should retrieve all urls from the url source" do
-        manager.urls.should_not be_empty
+      
+      it "when passed an option hash containing :images_only, only images will be downloaded for this session" do
+        mgr = DownloadManager.new("http://localhost/basic_page.html", '/tmp', output, {:images_only => true})
+        mgr.strategy.options[:images_only].should == true
       end
     end
     
     describe "#start" do
       
-      it "should raise an error when the file does not exist" do
-        File.should_receive(:exists?).with(fixture_file_path)
-        lambda { manager.start }.should raise_error(Downer::URLSourceDoesNotExist)
-      end
       it "should raise a WriteFailed exception if the target directory is not writable" do
         File.should_receive(:writable?).with('myoutputdir/').and_return(false)
         lambda { manager.start }.should raise_error(Downer::WriteFailed)
       end
-
+      
+      it "should create the specified directory if it does not exist and its parent directory is writable" 
+      ## do
+      #         if File.exist?('/tmp/new-directory')
+      #           FileUtils.rm_rf('/tmp/new-directory')
+      #         end
+      #         create_mgr = DownloadManager.new(fixture_file_path, '/tmp/new-directory', output)
+      #         create_mgr.start
+      #         File.exist?('/tmp/new-directory').should == true
+      #       end
       
       it "should create a download worker to begin the downloading" do
         File.should_receive(:writable?).and_return(true)
-        worker = double('worker')
+        worker = double('worker').as_null_object
         worker.should_receive(:start)
         DownloadWorker.should_receive(:new).and_return(worker)
         manager.start
       end
+      
+      it "should resolve to use a flat file strategy when it receives a file data source" do
+        dm = DownloadManager.new(fixture_file_path, 'myoutputdir', output)
+        dm.source_type.should == "flatfile"
+      end
+      
+      it "should resolve to use a web strategy when it receives a data source that looks like a url" do
+        dm = DownloadManager.new('http://www.msn.com', 'myoutputdir', output)
+        dm.source_type.should == "website"
+      end
+      
     end
       
   end

data/spec/downer/download_strategy_spec.rb

@@ -0,0 +1,19 @@
+require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
+
+module Downer
+  describe StrategyFinder do
+    describe "Class method find_strategy" do
+      it "should return flat file strategy when url source is a local file" do
+        strategy = StrategyFinder.find_strategy(fixture_directory + '/some_images.txt')
+        strategy.should respond_to :get_urls
+        strategy.source_type.should == 'flatfile'
+      end
+      
+      it "should return a website strategy when url source is a web resource" do
+        strategy = StrategyFinder.find_strategy('http://www.example.com')
+        strategy.should respond_to :get_urls
+        strategy.source_type.should == 'website'
+      end
+    end
+  end
+end

data/spec/downer/download_worker_spec.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
 module Downer
   describe DownloadWorker do
     let (:output) { double('output') }
-    let (:urls) { DownloadManager.new(fixture_directory + '/some_images.txt', '/tmp', output).urls }
+    let (:urls) { DownloadStrategy::FlatFileStrategy.new(fixture_directory + "/some_images.txt").get_urls }
     
     describe '#start' do 
       it "should write a message to output when no urls exist to be downloaded" do
@@ -12,6 +12,12 @@ module Downer
         worker.start
       end
       
+      it "should ignore arrays with nils" do
+        output.should_receive(:puts).with("No URLs specified, exiting.")
+        worker = DownloadWorker.new([nil,nil], '/tmp', output)
+        worker.start
+      end
+      
       it "should create a download object for each url to be downloaded" do
         worker = DownloadWorker.new(urls, '/tmp', output)
         worker.items.size.should == urls.size
@@ -21,14 +27,16 @@ module Downer
         bad_url = "http://www.urbaninfluence.com/will_never_succeed"
         worker = DownloadWorker.new([bad_url], '/tmp', output)
         output.should_receive(:puts).with("Could not download #{bad_url}, received http code 404")
-        worker.start
+        results = worker.start
+        results.size.should == 0
       end
       
       it "should write a message to output feed when a url is successfully downloaded" do
         good_url = "http://www.urbaninfluence.com/sites/default/files/user_uploads/images/4th.png"
         worker = DownloadWorker.new([good_url], '/tmp', output)
         output.should_receive(:puts).with("Downloaded #{good_url}")
-        worker.start
+        results = worker.start
+        results.size.should == 1
       end
     end
   end

data/spec/downer/options_spec.rb

@@ -0,0 +1,10 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+module Downer
+  describe Options do
+    it "when passed '-i' the option image_only will be set to true" do
+      op_struct = Options.new(['-i'])
+      op_struct[:images_only].should == true
+    end
+  end
+end

data/spec/downer/strategies/flat_file_stragtegy_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+module Downer
+  module DownloadStrategy
+    describe FlatFileStrategy do
+      
+      describe "#get_urls" do
+        before(:each) do
+          @flat_file_strategy = FlatFileStrategy.new(fixture_directory + '/some_images.txt')
+        end
+        
+        it "should retrieve all urls from a text file" do
+          @flat_file_strategy.get_urls.size.should == 4
+        end
+        
+      end
+      
+      describe "#source_valid?" do
+        it "should return false when the local file does not exist" do
+          flat_file_strategy = FlatFileStrategy.new('foobar')
+          flat_file_strategy.source_valid?.should == false
+        end
+      end
+    end
+  end
+end

data/spec/downer/strategies/website_strategy_spec.rb

@@ -0,0 +1,58 @@
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+module Downer
+  module DownloadStrategy
+    describe WebsiteStrategy do
+      
+      it "should automatically prepend the host for relative urls" do
+        @web_strategy = WebsiteStrategy.new('http://www.example.com')
+        downloaded_page_mock = IO.read(fixture_directory + '/basic_page.html')
+        @web_strategy.should_receive(:download_page).and_return(downloaded_page_mock)
+        @web_strategy.get_urls.should include('http://www.example.com/clickhere.html')
+      end
+      
+      describe "default get_urls behavior" do
+        before(:each) do
+          @web_strategy = WebsiteStrategy.new('http://www.example.com')
+          downloaded_page_mock = IO.read(fixture_directory + '/basic_page.html')
+          @web_strategy.should_receive(:download_page).and_return(downloaded_page_mock)
+        end
+        
+        it "should retrieve 5 urls fom basic_page.html fixture" do
+          @web_strategy.get_urls.size.should == 5
+        end
+      end
+      
+      describe "get images only behavior" do
+        before(:each) do
+          @web_strategy = WebsiteStrategy.new('http://www.example.com', {:images_only => true} )
+          downloaded_page_mock = IO.read(fixture_directory + '/basic_page.html')
+          @web_strategy.should_receive(:download_page).and_return(downloaded_page_mock)
+        end
+        
+        it "should retrieve 3 urls from basic_page.html fixture with images only mode enabled" do
+          @web_strategy.get_urls.size.should == 3
+        end
+      end
+      
+      describe "image searching" do
+        it "should skip duplicate urls" do
+          @web_strategy = WebsiteStrategy.new('http://www.example.com', :images_only => true)
+          downloaded_page_mock = <<-PAGE
+            <img src="/sites/image.png" />
+            <img src="/sites/image.png" />
+          PAGE
+          @web_strategy.should_receive(:download_page).and_return(downloaded_page_mock)
+          @web_strategy.get_urls.size.should == 1
+        end
+        
+        it "should retrieve 3 images from basic_page.html fixture" do
+          @web_strategy = WebsiteStrategy.new('http://www.example.com', :images_only => true)
+          downloaded_page_mock = IO.read(fixture_directory + '/basic_page.html')
+          @web_strategy.should_receive(:download_page).and_return(downloaded_page_mock)
+          @web_strategy.get_urls.size.should == 3
+        end
+      end
+    end
+  end
+end

data/spec/fixtures/basic_page.html

@@ -0,0 +1,18 @@
+<html>
+<head>
+<title>Hello, Page</title>
+</head>
+
+<body>
+  <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+  
+  <img width="186" height="65" title="Urban Influence Brand Studio" alt="Urban Influence Brand Studio" src="/sites/all/themes/urbaninfluence/images/ui-logo-orange.png">
+  
+  <img style="width: 600px; height: 1083px;" src="/sites/default/files/user_uploads/images/Essentials01(1).jpg" alt="Essentials">
+  
+  <IMG style="width: 600px; height: 133px; border-width: 0px; border-style: solid;" src="/sites/default/files/user_uploads/images/RatRace.tiff" alt="Workin For Our Money!">
+  
+  <a href="http://www.boobs.com">Free boobs!</a>
+  <a href="/clickhere.html">Click me!!!</a>
+</body>
+</html>

data/version.yml

@@ -1,5 +1,5 @@
 --- 
-:minor: 2
-:build: 
-:patch: 2
+:patch: 0
 :major: 0
+:build: 
+:minor: 3

metadata

@@ -5,9 +5,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 2
-  version: 0.2.2
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Nate Miller
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-16 00:00:00 -07:00
+date: 2010-08-08 00:00:00 -07:00
 default_executable: downer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -34,6 +34,20 @@ dependencies:
         version: 1.2.9
   type: :development
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
 description: Downer is a tool used to download a list of urls from a website thorugh HTTP.
 email: nate@natemiller.org
 executables: 
@@ -59,13 +73,22 @@ files:
 - lib/downer/application.rb
 - lib/downer/download_item.rb
 - lib/downer/download_manager.rb
+- lib/downer/download_strategy.rb
 - lib/downer/download_worker.rb
+- lib/downer/generic_strategy.rb
 - lib/downer/options.rb
+- lib/downer/strategies/flat_file_strategy.rb
+- lib/downer/strategies/website_strategy.rb
 - spec/downer/application_spec.rb
 - spec/downer/download_item_spec.rb
 - spec/downer/download_manager_spec.rb
+- spec/downer/download_strategy_spec.rb
 - spec/downer/download_worker_spec.rb
 - spec/downer/generator_spec.rb
+- spec/downer/options_spec.rb
+- spec/downer/strategies/flat_file_stragtegy_spec.rb
+- spec/downer/strategies/website_strategy_spec.rb
+- spec/fixtures/basic_page.html
 - spec/fixtures/some_images.txt
 - spec/spec.opts
 - spec/spec_helper.rb
@@ -108,6 +131,10 @@ test_files:
 - spec/downer/application_spec.rb
 - spec/downer/download_item_spec.rb
 - spec/downer/download_manager_spec.rb
+- spec/downer/download_strategy_spec.rb
 - spec/downer/download_worker_spec.rb
 - spec/downer/generator_spec.rb
+- spec/downer/options_spec.rb
+- spec/downer/strategies/flat_file_stragtegy_spec.rb
+- spec/downer/strategies/website_strategy_spec.rb
 - spec/spec_helper.rb