websitary 0.2.0 → 0.2.1

data/History.txt

@@ -1,3 +1,9 @@
+= 0.2.1
+
+* Use URI.merge for constructing robots.txt uri.
+* Fixed minor show-stopper.
+
+
 = 0.2.0
 
 * Renamed the project from websitiary to websitary (without the 
@@ -51,6 +57,7 @@
       :months => N (calculated on basis of  the calendar month, not the 
       number of days)
 
+
 == 0.1.0 / 2007-07-16
 
 * Initial release

data/lib/websitary.rb

@@ -1,5 +1,5 @@
 # websitary.rb
-# @Last Change: 2007-09-11.
+# @Last Change: 2007-09-16.
 # Author::      Thomas Link (micathom AT gmail com)
 # License::     GPL (see http://www.gnu.org/licenses/gpl.txt)
 # Created::     2007-09-08.
@@ -37,8 +37,8 @@ end
 
 module Websitary
     APPNAME     = 'websitary'
-    VERSION     = '0.2.0'
-    REVISION    = '2403'
+    VERSION     = '0.2.1'
+    REVISION    = '2405'
 end
 
 require 'websitary/applog'

data/lib/websitary/configuration.rb

@@ -55,6 +55,7 @@ class Websitary::Configuration
         @default_profiles  = [@quicklist_profile]
         @done              = []
         @mtimes            = Websitary::FileMTimes.new(self)
+        @options           = {}
         @outfile           = {}
         @profiles          = []
         @robots            = {}
@@ -162,6 +163,11 @@ class Websitary::Configuration
                 @logger.set_level(:verbose)
             end
 
+            opts.on('--version', 'Run verbosely') do |v|
+                puts Websitary::VERSION
+                exit 1
+            end
+
             opts.on_tail('-h', '--help', 'Show this message') do
                 puts opts
                 exit 1
@@ -786,7 +792,7 @@ HTML
                 pn   = guess_dir(uri.path)
                 href = rewrite_href(href, url, uri0, pn0, true)
                 curl = canonic_url(href)
-                next if href.nil? or @done.include?(curl) or @todo.include?(curl)
+                next if !href or href.nil? or @done.include?(curl) or @todo.include?(curl)
                 next unless robots_allowed?(curl, uri)
                 # pn   = Pathname.new(guess_dir(File.expand_path(uri.path)))
                 uri  = URI.parse(href)
@@ -1278,7 +1284,8 @@ OUT
             if doc
                 return if robots?(doc, 'noindex')
                 push_hrefs(url, doc) do |uri0, pn0, uri, pn|
-                    eligible_path?(url, uri0.path, uri.path) &&
+                    uri.host && uri0.host &&
+                        eligible_path?(url, uri0.path, uri.path) &&
                         uri.host == uri0.host &&
                         (pn.to_s == '.' || pn.relative_path_from(pn0).to_s == '.')
                 end
@@ -1331,6 +1338,11 @@ OUT
 
 
     def open_url(url)
+        if url.nil? or url.empty?
+            $logger.fatal "Internal error: url is nil"
+            puts caller.join("\n")
+            exit 5
+        end
         $logger.debug "Open URL: #{url}"
         uri = URI.parse(url)
         if uri.instance_of?(URI::Generic) or uri.scheme == 'file'
@@ -1400,7 +1412,7 @@ EOT
 
             unless (rules = @robots[host])
                 rurl = robots_uri(uri).to_s
-                return true unless rurl
+                return true if rurl.nil? or rurl.empty?
                 begin
                     robots_txt = open_url(rurl).read
                     rules      = RobotRules.new(@user_agent)
@@ -1409,8 +1421,7 @@ EOT
                     $logger.info "Loaded #{rurl} for #{@user_agent}"
                     $logger.debug robots_txt
                 rescue Exception => e
-                    puts e
-                    puts robots_txt
+                    $logger.info "#{rurl}: #{e}"
                 end
             end
 
@@ -1436,9 +1447,11 @@ EOT
     # Get the robots.txt uri for uri.
     def robots_uri(uri)
         unless uri.relative?
-            ruri = uri.dup
-            ruri.path = '/robots.txt'
-            ruri
+            # ruri = uri.dup
+            # ruri.path = '/robots.txt'
+            # ruri.query = nil
+            # ruri
+            uri.merge '/robots.txt'
         end
     end
 

data/lib/websitary/filemtimes.rb

@@ -20,6 +20,12 @@ class Websitary::FileMTimes
     def swap_in
         if File.exist?(@store)
             @data = YAML.load_file(@store)
+            case @data
+            when Hash
+            else
+                $logger.error 'mtime.yml stored malformed data'
+                @data = {}
+            end
             File.delete(@store)
         end
     end
@@ -40,6 +46,8 @@ class Websitary::FileMTimes
             @data[filenamec] = mtime
             $logger.debug "Set mtime: #{filename} -> #{mtime.to_s}"
             mtime
+        else
+            nil
         end
     end
 end

metadata

@@ -3,7 +3,7 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: websitary
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 date: 2007-09-16 00:00:00 +02:00
 summary: A unified website news, rss feed, podcast monitor
 require_paths: