fizx-rwget 0.5.1 → 0.5.2

data/Rakefile

@@ -10,7 +10,7 @@ begin
     gem.homepage = "http://github.com/fizx/rwget"
     gem.authors = ["Kyle Maxwell"]
     gem.add_dependency("curb", ["> 0.0.0"])
-    gem.add_dependency("hpricot", ["> 0.0.0", "< 0.7"])
+    gem.add_dependency("hpricot", ["> 0.0.0"])
     gem.add_dependency("fizx-robots", [">= 0.3.1"])
     gem.add_dependency("bloomfilter", ["> 0.0.0"])
     gem.add_dependency("libxml-ruby", ["> 0.9"])

data/VERSION

@@ -1 +1 @@
-0.5.1
+0.5.2

data/lib/rwget/controller.rb

@@ -78,6 +78,7 @@ class RWGet::Controller
         puts "storing at #{key}"
         @store.put(key, tmpfile)
         sleep options[:wait]
+        tmpfile.close rescue nil
       else
         puts "unable to download"
       end  
@@ -102,12 +103,13 @@ class RWGet::Controller
   
   def key_for(uri)
     arr = []
-    arr << options[:prefix]     if options[:prefix]
-    arr << @start_time        if options[:timestampize]
-    arr << uri.scheme           if options[:protocol_directories]
-    arr << uri.host             unless options[:no_host_directories]
+    arr << options[:prefix]           if options[:prefix]
+    arr << @start_time                if options[:timestampize]
+    arr << uri.scheme                 if options[:protocol_directories]
+    arr << uri.host                   unless options[:no_host_directories]
     paths = uri.path.split("/")
-    paths.shift                 if paths.first.to_s.empty?
+    paths << paths.pop + "?" + uri.query     if uri.query
+    paths.shift                       if paths.first.to_s.empty?
     File.join(arr + paths)
   end
   

data/lib/rwget/rwget_option_parser.rb

@@ -9,7 +9,8 @@ class RWGetOptionParser < OptionParser
   
   def parse!
     super
-    options[:seeds] = ARGV
+    options[:seeds] ||= []
+    options[:seeds] += ARGV
   end
   
   def initialize
@@ -49,10 +50,6 @@ class RWGetOptionParser < OptionParser
         options[:reject_patterns] ||= []
         options[:reject_patterns] << Regexp.new(r)
       end
-
-      opts.on("--require=RUBY_SCRIPT", "Will execute 'require RUBY_SCRIPT'") do |s|
-        require s
-      end
   
       opts.on("--limit-rate=RATE", "limit download rate to RATE.") do |r|
         rate = r.to_i
@@ -74,23 +71,27 @@ class RWGetOptionParser < OptionParser
         options[:proxy_password] = p
       end
   
-      opts.on("--fetch-class=RUBY_CLASS", "Must implement fetch(uri, user_agent_string) #=> [final_redirected_url, file_object]") do |c|
+      opts.on("--require=RUBY_SCRIPT", "Will execute 'require RUBY_SCRIPT'") do |s|
+        require s
+      end
+
+      opts.on("--fetch-class=RUBY_CLASS", "Must implement fetch(uri, user_agent_string) #=> [final_redirected_url, file_object] (Load the class with --require)") do |c|
         options[:fetch_class] = c
       end
   
-      opts.on("--store-class=RUBY_CLASS", "Must implement put(key_string, temp_file)") do |c|
+      opts.on("--store-class=RUBY_CLASS", "Must implement put(key_string, temp_file) (Load the class with --require)") do |c|
         options[:store_class] = c
       end
   
-      opts.on("--dupes-class=RUBY_CLASS", "Must implement dupe?(uri)") do |c|
+      opts.on("--dupes-class=RUBY_CLASS", "Must implement dupe?(uri) (Load the class with --require)") do |c|
         options[:dupes_class] = c
       end
   
-      opts.on("--queue-class=RUBY_CLASS", "Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int]") do |c|
+      opts.on("--queue-class=RUBY_CLASS", "Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int] (Load the class with --require)") do |c|
         options[:queue_class] = c
       end
   
-      opts.on("--queue-class=RUBY_CLASS", "Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int]") do |c|
+      opts.on("--queue-class=RUBY_CLASS", "Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int] (Load the class with --require)") do |c|
         options[:queue_class] = c
       end
   
@@ -99,6 +100,7 @@ class RWGetOptionParser < OptionParser
       end
       
       opts.on("-S", "--sitemap=URL", "URL of a sitemap to crawl (will ignore inter-page links)") do |url|
+        options[:seeds] ||= []
         options[:seeds] << url
         options[:links_class] = "RWGet::SitemapLinks"
       end

data/lib/rwget/store.rb

@@ -11,7 +11,15 @@ class RWGet::Store
   def put(key, tmpfile)
     path = File.join(@root, key)
     path = File.join(path, "index.html")     unless path.split("/").last =~ /\.|\?/
-    mkdir_p(File.dirname(path))
+    dir = File.dirname(path)
+    if(File.file?(dir))
+      tmp = "#{dir}.index.html.#{Time.now.to_f}"
+      mv dir, tmp
+      mkdir_p(dir)
+      mv tmp, File.join(dir, "index.html")
+    else
+      mkdir_p(dir)
+    end
     mv tmpfile.path, path
   end
 end

data/rwget.gemspec

@@ -1,12 +1,15 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{rwget}
-  s.version = "0.5.1"
+  s.version = "0.5.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2009-06-19}
+  s.date = %q{2009-09-10}
   s.default_executable = %q{rwget}
   s.email = %q{kyle@kylemaxwell.com}
   s.executables = ["rwget"]
@@ -42,11 +45,10 @@ Gem::Specification.new do |s|
      "test/sitemap_links_test.rb",
      "test/store_test.rb"
   ]
-  s.has_rdoc = true
   s.homepage = %q{http://github.com/fizx/rwget}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.2}
+  s.rubygems_version = %q{1.3.5}
   s.summary = %q{Ruby port of wget, emphasis on recursive/crawler}
   s.test_files = [
     "test/controller_test.rb",
@@ -65,20 +67,20 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<curb>, ["> 0.0.0"])
-      s.add_runtime_dependency(%q<hpricot>, ["> 0.0.0", "< 0.7"])
+      s.add_runtime_dependency(%q<hpricot>, ["> 0.0.0"])
       s.add_runtime_dependency(%q<fizx-robots>, [">= 0.3.1"])
       s.add_runtime_dependency(%q<bloomfilter>, ["> 0.0.0"])
       s.add_runtime_dependency(%q<libxml-ruby>, ["> 0.9"])
     else
       s.add_dependency(%q<curb>, ["> 0.0.0"])
-      s.add_dependency(%q<hpricot>, ["> 0.0.0", "< 0.7"])
+      s.add_dependency(%q<hpricot>, ["> 0.0.0"])
       s.add_dependency(%q<fizx-robots>, [">= 0.3.1"])
       s.add_dependency(%q<bloomfilter>, ["> 0.0.0"])
       s.add_dependency(%q<libxml-ruby>, ["> 0.9"])
     end
   else
     s.add_dependency(%q<curb>, ["> 0.0.0"])
-    s.add_dependency(%q<hpricot>, ["> 0.0.0", "< 0.7"])
+    s.add_dependency(%q<hpricot>, ["> 0.0.0"])
     s.add_dependency(%q<fizx-robots>, [">= 0.3.1"])
     s.add_dependency(%q<bloomfilter>, ["> 0.0.0"])
     s.add_dependency(%q<libxml-ruby>, ["> 0.9"])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: fizx-rwget
 version: !ruby/object:Gem::Version 
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-19 00:00:00 -07:00
+date: 2009-09-10 00:00:00 -07:00
 default_executable: rwget
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,9 +31,6 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version 
         version: 0.0.0
-    - - <
-      - !ruby/object:Gem::Version 
-        version: "0.7"
     version: 
 - !ruby/object:Gem::Dependency 
   name: fizx-robots
@@ -101,8 +98,9 @@ files:
 - test/server.rb
 - test/sitemap_links_test.rb
 - test/store_test.rb
-has_rdoc: true
+has_rdoc: false
 homepage: http://github.com/fizx/rwget
+licenses: 
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -123,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Ruby port of wget, emphasis on recursive/crawler