websitiary 0.1.0

data/Rakefile

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+load './bin/websitiary'
+
+Hoe.new('websitiary', Websitiary::VERSION) do |p|
+  p.rubyforge_name = 'websitiary'
+  p.author = 'Thomas Link'
+  p.email = 'sanobast-ruby@yahoo.de'
+  p.summary = 'A simple website monitor'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps << 'hpricot'
+  # p.need_tgz = false
+  p.need_zip = true
+end
+
+# vim: syntax=Ruby

data/bin/websitiary

@@ -0,0 +1,1351 @@
+#! /usr/bin/ruby.exe
+# websitiary.rb -- Website Monitor
+# @Last Change: 2007-07-16.
+# Author::      Thomas Link (samul AT web de)
+# License::     GPL (see http://www.gnu.org/licenses/gpl.txt)
+# Created::     2007-06-09.
+#
+# = TODO
+# * Find a ruby-based replacement for webdiff (or port webdiff to ruby)
+# * Built-in support for robots.txt
+# * Option to append to output files
+# * Option to trim output files (when appending items)
+
+
+require 'cgi'
+require 'digest/md5'
+require 'logger'
+require 'optparse'
+require 'pathname'
+require 'rbconfig'
+require 'uri'
+require 'open-uri'
+
+
+['hpricot', 'robot_rules'].each do |f|
+  begin
+    require f
+  rescue Exception => e
+    $stderr.puts <<EOT
+#{e.message}
+Library could not be loaded: #{f}
+Please see the requirements section at: http://websitiary.rubyforge.org
+EOT
+  end
+end
+
+
+
+# Basic usage:
+#   Websitiary.new(ARGV).process
+class Websitiary
+  APPNAME     = 'websitiary'
+  VERSION     = '0.1.0'
+  REVISION    = '1447'
+  MINUTE_SECS = 60
+  HOUR_SECS   = MINUTE_SECS * 60
+  DAY_SECS    = HOUR_SECS * 24
+
+  # A simple wrapper around Logger.
+  class AppLog
+    def initialize(output=nil)
+      @output = output || $stdout
+      $logger = Logger.new(@output, 'daily')
+      $logger.progname = APPNAME
+      $logger.datetime_format = "%H:%M:%S"
+      set_level
+    end
+
+
+    def set_level(level=:default)
+      case level
+      when :debug
+        $logger.level = Logger::DEBUG
+      when :verbose
+        $logger.level = Logger::INFO
+      when :quiet
+        $logger.level = Logger::ERROR
+      else
+        $logger.level = Logger::WARN
+      end
+      $logger.debug "Set logger level: #{level}"
+    end
+  end
+
+
+  # This class defines the scope in which profiles are evaluated. Most 
+  # of its methods are suitable for use in profiles.
+  class Configuration
+    # Hash (key = URL, value = Hash of options)
+    attr_accessor :urls
+    # Array of urls to be downloaded.
+    attr_accessor :todo
+    # Array of downloaded urls.
+    attr_accessor :done
+    # The user configuration directory
+    attr_accessor :cfgdir
+    # attr_accessor :default_profiles
+    # attr_accessor :options
+    # attr_accessor :cmd_edit
+
+
+    def initialize(app, args=[])
+      @logger = AppLog.new
+      $logger.debug "Configuration#initialize"
+      @app    = app
+      @urls   = {}
+      @todo   = []
+      @done   = []
+      @robots = {}
+      @allow  = {}
+
+      @suffix = {
+        'text' => 'txt'
+        # 'rss'  => 'xml'
+      }
+
+      @cfgdir = ENV['HOME'] ? File.join(ENV['HOME'], '.websitiary') : '.'
+      [
+        ENV['USERPROFILE'] && File.join(ENV['USERPROFILE'], 'websitiary'),
+        File.join(Config::CONFIG['sysconfdir'], 'websitiary')
+      ].each do |dir|
+        if File.exists?(dir)
+          @cfgdir = dir
+          break
+        end
+      end
+
+      @user_agent = "websitiary/#{Websitiary::VERSION}"
+
+      @cmd_edit = 'vi "%s"'
+
+      @options = {:global => {}}
+
+      @options[:diff] = {
+        :default    => :diff,
+        :diff       => 'diff -d -w -u2 "%s" "%s"',
+        :webdiff    => lambda do |old, new|
+        $logger.debug "webdiff: #{File.basename(new)}"
+        $logger.debug %{webdiff --hicolor=yellow -archive "#{old}" -current "#{new}" -out -}
+        difftext = `webdiff --hicolor=yellow -archive "#{old}" -current "#{new}" -out -`
+        $?.exitstatus == 1 ? difftext : ''
+        end,
+      }
+
+      @options[:format] = {
+        :default => :diff,
+        :diff    => %{<pre class="diff">\n%s\n</pre>},
+        :webdiff => "%s\n",
+      }
+
+      @options[:diffprocess] = {
+        :default => :diff,
+        :diff    => lambda {|text| text.split("\n")[2..-1].delete_if {|l| l =~ /^[^+]/}.map {|l| l[1..-1]}.join("\n")},
+        :webdiff => false,
+      }
+
+      @options[:download] = {
+        :default    => :w3m,
+      }
+
+      @options[:downloadformat] = {
+        :w3m => :text,
+        :webdiff => :html,
+      }
+
+      @options[:downloadprocess] = {
+      }
+
+      @options[:rss] = {
+        :version => '2.0',
+      }
+
+      @options[:strip_tags] = {
+          :default   => ['script', 'object', 'form', 'input', 'select', 'iframe', 'head', 'meta', 'link'],
+      }
+
+      shortcut :w3m, :delegate => :diff,
+        :download => 'w3m -no-cookie -S -F -dump "%s"'
+
+      shortcut :lynx, :delegate => :diff,
+        :download => 'lynx -dump "%s"'
+
+      shortcut :links, :delegate => :diff,
+        :download => 'links -dump "%s"'
+
+      shortcut :curl, :delegate => :webdiff,
+        :download => 'curl --silent "%s"'
+
+      shortcut :wget, :delegate => :webdiff,
+        :download => 'wget -q -O - "%s"'
+
+      shortcut :body_html, :delegate => :webdiff,
+        :strip_tags => :default,
+        :download => lambda {|url|
+        begin
+          doc = Hpricot(open(url))
+          doc = doc.at('body')
+          if doc
+            doc  = rewrite_urls(url, doc)
+            doc  = doc.inner_html
+            if (tags = get(url, :strip_tags))
+              doc = strip_tags(doc, :format => :hpricot, :tags => tags)
+            end
+          else
+            $logger.warn 'inner html: No body'
+          end
+          doc.to_s
+        rescue Exception => e
+          # $logger.error e  #DBG#
+          $logger.error e.message
+          $logger.debug e.backtrace
+          break %{<pre class="error">\n#{e.message}\n</pre>}
+        end
+      }
+
+      shortcut :openuri, :delegate => :webdiff,
+        :download => lambda {|url|
+        begin
+          open(url).read
+        rescue Exception => e
+          # $logger.error e  #DBG#
+          $logger.error e.message
+          $logger.debug e.backtrace
+          %{<pre class="error">\n#{e.to_s}\n</pre>}
+        end
+      }
+
+      shortcut :website, :delegate => :webdiff,
+        :download => lambda {|url|
+        html = @options[:download][:body_html].call(url)
+        break unless html
+        doc = Hpricot(html)
+        push_hrefs(url, doc) do |uri0, pn0, uri, pn|
+          eligible_path?(url, uri0.path, uri.path) &&
+            uri.host == uri0.host
+        end
+        html
+      }
+
+      shortcut :website_below, :delegate => :webdiff,
+        :download => lambda {|url|
+        html = @options[:download][:body_html].call(url)
+        break unless html
+        doc = Hpricot(html)
+        if doc
+          push_hrefs(url, doc) do |uri0, pn0, uri, pn|
+            eligible_path?(url, uri0.path, uri.path) &&
+              uri.host == uri0.host &&
+              pn.relative_path_from(pn0).to_s == '.'
+          end
+        end
+        html
+      }
+
+      shortcut :website_txt, :delegate => :default,
+        :download => lambda {|url|
+        success, cmd = get_option(:download, :default)
+        if success
+          html = @options[:download][:website].call(url)
+          html_to_text(html)
+        end
+      }
+
+      shortcut :website_txt_below, :delegate => :default,
+        :download => lambda {|url|
+        success, cmd = get_option(:download, :default)
+        if success
+          html = @options[:download][:website_below].call(url)
+          html_to_text(html)
+        end
+      }
+
+      @options[:page] = {:format => lambda do |ti, li, bd|
+        template = <<OUT
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<title>%s</title>
+<link rel="stylesheet" href="websitiary.css" type="text/css">
+<link rel="alternate" href="websitiary.rss" type="application/rss+xml" title="%s">
+</head>
+<body>
+<ol class="toc">
+%s
+</ol>
+<div class="contents">
+%s
+</div>
+</body>
+</html>
+OUT
+        template % [ti, ti, li, bd]
+      end
+      }
+
+      # @view = nil
+      @view = 'w3m "%s"'
+      @default_options  = {}
+      @default_profiles = []
+      @profiles         = []
+      @outfile          = {}
+      profile 'config.rb'
+      parse_command_line_args(args)
+      @output_format  ||= ['html']
+      @output_title     = %{#{APPNAME}: #{@profiles.join(", ")}}
+    end
+
+
+    def parse_command_line_args(args)
+      $logger.debug "parse_command_line_args: #{args}"
+      opts    = OptionParser.new do |opts|
+        opts.banner =  "Usage: #{APPNAME} [OPTIONS] [PROFILES] > [OUT]"
+        opts.separator ''
+        opts.separator "#{APPNAME} is a free software with ABSOLUTELY NO WARRANTY under"
+        opts.separator 'the terms of the GNU General Public License version 2 or newer.'
+        opts.separator ''
+
+        opts.separator 'General Options:'
+
+        opts.on('-c', '--cfg=DIR', String, 'Configuration directory') do |value|
+          @cfgdir = value
+        end
+
+        opts.on('-e', '--edit=PROFILE', String, 'Edit a profile') do |value|
+          edit_profile value
+          exit 0
+        end
+
+        opts.on('-f', '--output-format=FORMAT', 'Output format (html, text, rss)') do |value|
+          output_format(*value.split(/,/))
+        end
+
+        opts.on('--[no-]ignore-age', 'Ignore age limits') do |bool|
+            set :ignore_age => bool
+        end
+
+        opts.on('--log=DESTINATION', String, 'Log destination') do |value|
+          @logger = AppLog.new(value != '-' && value)
+        end
+        opts.on('-o', '--output=FILENAME', String, 'Output') do |value|
+          output_file(value)
+        end
+
+        opts.on('-s', '--set=NAME=VAR', String, 'Set a default option') do |value|
+          key, val = value.split(/=/, 2)
+          set key.intern => eval(val)
+        end
+
+        opts.on('--review', 'View last diff') do |value|
+          view_output
+          exit 0
+        end
+
+        opts.separator ''
+        opts.separator 'Available profiles:'
+        opts.separator Dir[File.join(@cfgdir, '*.rb')].map {|f| File.basename(f, '.*')}.join(', ')
+
+        opts.separator ''
+        opts.separator 'Other Options:'
+
+        opts.on('--debug', 'Show debug messages') do |v|
+          $VERBOSE = $DEBUG = true
+          @logger.set_level(:debug)
+        end
+
+        opts.on('-q', '--quiet', 'Be mostly quiet') do |v|
+          @logger.set_level(:quiet)
+        end
+
+        opts.on('-v', '--verbose', 'Run verbosely') do |v|
+          $VERBOSE = true
+          @logger.set_level(:verbose)
+        end
+
+        opts.on_tail('-h', '--help', 'Show this message') do
+          puts opts
+          exit 1
+        end
+      end
+
+      @profiles = opts.parse!(args)
+      @profiles = @default_profiles if @profiles.empty?
+      for pn in @profiles
+        profile pn
+      end
+
+      self
+    end
+
+
+    # Retrieve an option for an url
+    # url:: String
+    # opt:: Symbol
+    def get(url, opt, default=nil)
+      opts = @urls[url]
+      $logger.debug "get: opts=#{opts.inspect}"
+      case opt
+      when :diffprocess, :format
+        opt_ = opts.has_key?(opt) ? opt : :diff
+      else
+        opt_ = opt
+      end
+
+      $logger.debug "get: opt=#{opt} opt_=#{opt_} #{opts[opt_]} #{opts[:use]}"
+      if opts.has_key?(opt_)
+        val = opts[opt_]
+      elsif opts.has_key?(:use)
+        val = opts[:use]
+      else
+        val = nil
+      end
+
+      case val
+      when nil
+      when Symbol
+        $logger.debug "get: val=#{val}"
+        success, rv = get_option(opt, val)
+        $logger.debug "get: #{success}, #{rv}"
+        if success
+          return rv
+        end
+      else
+        $logger.debug "get: return val=#{val}"
+        return val
+      end
+      unless default
+        success, default1 = get_option(opt, :default)
+        default = default1 if success
+      end
+
+      $logger.debug "get: return default=#{default}"
+      return default
+    end
+
+
+    # Configuration command:
+    # Set the default profiles
+    def default(*profile_names)
+      @default_profiles = profile_names
+    end
+
+
+    # Configuration command:
+    # Load a profile
+    def profile(profile_name)
+      case profile_name
+      when '-'
+        readlines.map! {|l| l.chomp}.each {|url| source url}
+      else
+        fn = profile_filename(profile_name)
+        if fn
+          $logger.debug "Profile: #{fn}"
+          contents = File.read(fn)
+          @current_profile = fn
+          begin
+            self.instance_eval(contents)
+          ensure
+            @current_profile = nil
+          end
+          true
+        else
+          false
+        end
+      end
+    end
+
+
+    # Define a options shortcut.
+    def shortcut(symbol, args)
+      ak = args.keys
+      ok = @options.keys
+      dk = ok - ak
+
+      # :downloadprocess
+      if !ak.include?(:delegate) and
+        dk.any? {|e| [:download, :downloadformat, :diff, :format, :diffprocess].include?(e)}
+        $logger.warn "Shortcut #{symbol}: Undefined fields: #{dk.inspect}"
+      end
+
+      if ak.include?(:delegate)
+        dk.each do |field|
+          @options[field][symbol] = args[:delegate]
+        end
+      end
+
+      args.each do |field, val|
+        @options[field][symbol] = val unless field == :delegate
+      end
+    end
+
+
+    # Set the output format.
+    def output_format(*format)
+      unless format.all? {|e| ['text', 'html', 'rss'].include?(e)}
+        $logger.fatal "Unknown output format: #{format}"
+        exit 5
+      end
+      @output_format = format
+    end
+
+
+    # Set the output file.
+    def output_file(filename, outformat=nil)
+      @outfile[outformat] = filename
+    end
+
+
+    # Configuration command:
+    # Set global options.
+    # type:: Symbol
+    # options:: Hash
+    def option(type, options)
+      $logger.info "option #{type}: #{options.inspect}"
+      o = @options[type]
+      if o
+        o.merge!(options)
+      else
+        $logger.error "Unknown option type: #{type} (#{options.inspect})"
+      end
+    end
+
+
+    # Set a global option.
+    def global(options)
+      options.each do |type, value|
+        @options[type] = value
+      end
+    end
+
+
+    # Configuration command:
+    # Set the default value for source-options.
+    def set(options)
+      $logger.debug "set: #{options.inspect}"
+      @default_options.merge!(options)
+    end
+
+
+    # Configuration command:
+    # Unset a default source-option.
+    def unset(*options)
+      for option in options
+        @default_options.delete(option)
+      end
+    end
+
+
+    # Configuration command:
+    # Define a source.
+    # urls:: String
+    def source(urls, opts={})
+      urls.split("\n").flatten.compact.each do |url|
+        @urls[url] = @default_options.dup.update(opts)
+        @todo << url
+      end
+    end
+
+
+    # Configuration command:
+    # Set the default download processor. The block takes the 
+    # downloaded text (STRING) as argument.
+    def downloadprocess(&block)
+      @options[:downloadprocess][:default] = block
+    end
+
+
+    # Configuration command:
+    # Set the default diff processor. The block takes the 
+    # diff text (STRING) as argument.
+    def diffprocess(&block)
+      @options[:diff][:default] = block
+    end
+
+
+    # Configuration command:
+    # Set the editor.
+    def edit(cmd)
+      @cmd_edit = cmd
+    end
+
+
+    # Configuration command:
+    # Set the viewer.
+    def view(view)
+      @view = view
+    end
+
+
+    # Configuration command:
+    # Set the default diff program.
+    def diff(diff)
+      @options[:diff][:default] = diff
+    end
+
+
+    # Configuration command:
+    # Set the default dowloader.
+    def download(download)
+      @options[:download][:default] = download
+    end
+
+
+    # Format a diff according to URL's source options.
+    def format(url, difftext)
+      fmt = get(url, :format)
+      eval_arg(fmt, [difftext])
+    end
+
+
+    # Apply some arguments to a format.
+    # format:: String or Proc
+    # args:: Array of Arguments
+    def eval_arg(format, args, default=nil, &process_string)
+      case format
+      when nil
+        return default
+      when Proc
+        $logger.debug "eval proc: #{format} #{args.inspect}"
+        return format.call(*args)
+      else
+        ca = format % args
+        $logger.debug "eval string: #{ca}"
+        if process_string
+          return process_string.call(ca)
+        else
+          return ca
+        end
+      end
+    end
+
+
+    # Apply the argument to cmd (a format String or a Proc). If a 
+    # String, execute the command.
+    def call_cmd(cmd, args, default=nil)
+      eval_arg(cmd, args, default) {|cmd| `#{cmd}`}
+    end
+
+
+    # Generate & view the final output.
+    # difftext:: Hash
+    def show_output(difftext)
+      if difftext.empty?
+        $logger.warn 'No news is good news'
+        return
+      end
+
+      @output_format.each do |outformat|
+        meth = "get_output_#{outformat}"
+
+        unless respond_to?(meth)
+          $logger.fatal "Unknown output format: #{outformat}"
+          exit 5
+        end
+
+        out = send(meth, difftext)
+        if out
+          outfile = get_outfile(outformat)
+          case outfile
+          when '-'
+            puts out
+          else
+            File.open(outfile, 'w') {|io| io.puts out}
+            meth = "view_output_#{outformat}"
+            self.send(meth, outfile)
+          end
+        end
+      end
+    end
+
+
+    def get_output_text(difftext)
+      difftext.map do |url, difftext|
+        if difftext
+          difftext = html_to_text(difftext) if is_html?(difftext)
+          !difftext.empty? && [url, difftext_annotation(url), nil, difftext].join("\n")
+        end
+      end.compact.join("\n\n#{('-' * 68)}\n\n")
+    end
+
+
+    def get_output_rss(difftext)
+      success, rss_url = get_option(:rss, :url)
+      if success
+        success, rss_version = get_option(:rss, :version)
+        require "rss/#{rss_version}"
+
+        rss         = RSS::Rss.new(rss_version)
+        chan        = RSS::Rss::Channel.new
+        chan.title  = @output_title
+        [:description, :copyright, :category, :language, :image, :webMaster, :pubDate].each do |field|
+          ok, val = get_option(:rss, field)
+          item.send(format_symbol(field, '%s='), val) if ok
+        end
+        chan.link   = rss_url
+        rss.channel = chan
+
+        cnt = difftext.map do |url, text|
+          rss_format = get(url, :rss_format, :plain_text)
+          text = strip_tags(text, :format => rss_format)
+          next if text.empty?
+
+          item = RSS::Rss::Channel::Item.new
+          item.title = get(url, :title, File.basename(url))
+          item.link  = eval_arg(get(url, :rewrite_link, '%s'), [url])
+          [:author, :date, :enclosure, :category, :pubDate].each do |field|
+            val = get(url, format_symbol(field, 'rss_%s'))
+            item.send(format_symbol(field, '%s='), val) if val
+          end
+
+          annotation = difftext_annotation(url)
+          case rss_format
+          when :plain_text
+            annotation = "<pre>#{annotation}</pre>" if annotation
+            item.description = %{#{annotation}<pre>#{text}</pre>}
+          else
+            item.description = %{<pre>#{annotation}</pre>\n#{text}}
+          end
+          chan.items << item
+        end
+
+        return rss.to_s
+
+      else
+
+        $logger.fatal "Global option :rss[:url] not defined."
+        exit 5
+
+      end
+    end
+
+
+    def get_output_html(difftext)
+      difftext = difftext.map do |url, text|
+        tags = get(url, :strip_tags)
+        text = strip_tags(text, :tags => tags) if tags
+        text.empty? ? nil : [url, text]
+      end
+      difftext.compact!
+
+      toc = difftext.map do |url, text|
+        lab = Digest::MD5.hexdigest(url)
+        ti  = get(url, :title, File.basename(url))
+        # %{<li class="toc"><a class="toc" href="\##{lab}">#{ti}</a> <a  class="external" href="#{url}">[W]</a></li>}
+        %{<li class="toc"><a class="toc" href="\##{lab}">#{ti}</a></li>}
+      end.join("\n")
+
+      cnt = difftext.map do |url, text|
+        lab  = Digest::MD5.hexdigest(url)
+        ti   = get(url, :title, File.basename(url))
+        if (rewrite = get(url, :rewrite_link))
+          url = eval_arg(rewrite, [url])
+          ext = ''
+        else
+          old = %{<a class="old" href="#{file_url(backupname(url))}">old</a>}
+          lst = %{<a class="latest" href="#{file_url(latestname(url))}">latest</a>}
+          ext = %{ (#{old}, #{lst})}
+        end
+        <<HTML
+<div class="webpage">
+<h1 class="diff" id="#{lab}"><a class="external" href="#{url}">#{ti}</a>#{ext}</h1>
+<div class="annotation">
+#{CGI::escapeHTML(difftext_annotation(url))}
+</div>
+<div class="diff">
+#{format(url, text)}
+</div>
+</div>
+HTML
+      end.join(('<hr class="separator"/>') + "\n")
+
+      success, template = get_option(:page, :format)
+      unless success
+        success, template = get_option(:page, :simple)
+      end
+      return eval_arg(template, [@output_title, toc, cnt])
+    end
+
+
+    # Get the backup filename.
+    def backupname(url)
+      File.join(@cfgdir, 'old', encode(url))
+    end
+
+
+    # Get the filename for the freshly downloaded copy.
+    def latestname(url)
+      File.join(@cfgdir, 'latest', encode(url))
+    end
+
+
+    # Guess path's dirname.
+    #   foo/bar     -> foo
+    #   foo/bar.txt -> foo
+    #   foo/bar/    -> foo/bar
+    def guess_dir(path)
+      path[-1..-1] == '/' ? path[0..-2] : File.dirname(path)
+    end
+
+
+    # Strip the url's last part (after #).
+    def canonic_url(url)
+      url.sub(/#.*$/, '')
+    end
+
+
+    def strip_tags_default
+      success, tags = get_option(:strip_tags, :default)
+      tags.dup if success
+    end
+
+
+    def strip_tags(doc, args={})
+      tags = args[:tags] || strip_tags_default
+      case doc
+      when String
+        doc = Hpricot(doc)
+      end
+      tags.each do |tag|
+        doc.search(tag).remove
+      end
+      case args[:format]
+      when :hpricot
+        doc
+      else
+        doc.send("to_#{args[:format] || :html}")
+      end
+    end
+
+
+    # Check whether path is eligible on the basis of url or path0.
+    # This checks either for a :match option for url or the extensions 
+    # of path0 and path.
+    def eligible_path?(url, path0, path)
+      rx = get(url, :match)
+      if rx
+        return path =~ rx
+      else
+        return File.extname(path0) == File.extname(path)
+      end
+    end
+
+
+    # Scan hpricot document for hrefs and push the onto @todo if not 
+    # already included.
+    def push_hrefs(url, hpricot, &condition)
+      begin
+        depth = get(url, :depth)
+        return if depth and depth <= 0
+        uri0  = URI.parse(url)
+        pn0   = Pathname.new(guess_dir(File.expand_path(uri0.path)))
+        (hpricot / 'a').each do |a|
+          href = a['href']
+          next if href.nil?
+          curl = canonic_url(href)
+          next if @done.include?(curl) or @todo.include?(curl)
+          uri  = URI.parse(href)
+          next unless robots_allowed?(curl, uri)
+          pn   = Pathname.new(guess_dir(File.expand_path(uri.path)))
+          if condition.call(uri0, pn0, uri, pn)
+            opts = @urls[url].dup
+            opts[:title] = File.basename(curl)
+            opts[:depth] = depth - 1 if depth and depth >= 0
+            @urls[curl] = opts
+            @todo << curl
+          end
+        end
+      rescue Exception => e
+        # $logger.error e  #DBG#
+        $logger.error e.message
+        $logger.debug e.backtrace
+      end
+    end
+
+
+    # Rewrite urls in doc
+    # url:: String
+    # doc:: Hpricot document
+    def rewrite_urls(url, doc)
+      uri = URI.parse(url)
+      urd = guess_dir(uri.path)
+      (doc / 'a').each do |a|
+        href = rewrite_href(a['href'], url, uri, urd)
+        a['href'] = href
+      end
+      (doc / 'img').each do |a|
+        href = rewrite_href(a['src'], url, uri, urd)
+        a['src'] = href if href
+      end
+      doc
+    end
+
+
+    # Try to make href an absolute url.
+    def rewrite_href(href, url, uri, urd)
+      begin
+        return if !href
+        rv   = nil
+        href = href.strip
+
+        if href =~ /\w+:/
+        elsif uri.relative? and URI.parse(href).relative?
+          if uri.instance_of?(URI::Generic)
+            rv = File.join(urd, href)
+          end
+        elsif href[0..0] == '#'
+          rv = url + href
+        else
+          rv = uri.merge(href).to_s
+        end
+
+        case rv
+        when String
+          return rv
+        when nil
+        else
+          $logger.error "Internal error: href=#{href}"
+          $logger.debug caller.join("\n")
+        end
+        return
+      rescue Exception => e
+        # $logger.error e  #DBG#
+        $logger.error e.message
+        $logger.debug e.backtrace
+      end
+    end
+
+
+    # Return a Proc that takes an text as argument and highlight occurences of rx.
+    # rx:: Regular expression
+    # group:: A number (default: 0)
+    # tag:: The HTML tag to use (default: "span")
+    def highlighter(rx, group=nil, tag='span')
+      lambda {|text| text.gsub(rx, %{<#{tag} class="highlight">\\#{group || 0}</#{tag}>})}
+    end
+
+
+    private
+
+    def difftext_annotation(url)
+      bak = backupname(url)
+      lst = latestname(url)
+      if File.exist?(bak) and File.exist?(lst)
+        eval_arg(get(url, :format_annotation, '%s >>> %s'), [File.mtime(bak), File.mtime(lst)])
+      end
+    end
+
+
+    def format_symbol(name, format_string)
+      (format_string % name.to_s).intern
+    end
+
+
+    def is_html?(text)
+      text =~ /<(div|a|span|body|html|script|p|table|td|tr|th|li|dt|br|hr|em|b)\b/
+    end
+
+
+    def html_to_text(text)
+      Hpricot(text).to_plain_text
+    end
+
+
+    def robots_allowed?(url, uri)
+      if @allow.has_key?(url)
+        return @allow[url]
+      end
+
+      if defined?(RobotRules)
+        host = uri.host
+
+        unless (rules = @robots[host])
+          rurl = robots_uri(uri).to_s
+          return true unless rurl
+          begin
+            robots_txt = open(rurl).read
+            rules      = RobotRules.new(@user_agent)
+            rules.parse(rurl, robots_txt)
+            @robots[host] = rules
+            $logger.info "Loaded #{rurl} for #{@user_agent}"
+            $logger.debug robots_txt
+          rescue Exception => e
+            # puts e
+            # puts robots_txt
+          end
+        end
+
+        rv = if rules and !rules.allowed?(url)
+               $logger.info "Excluded url: #{url}"
+               false
+             else
+               true
+             end
+        @allow[url] = rv
+        return rv
+      end
+
+      unless @robots[:warning]
+        $logger.warn 'robots.txt is ignored: Please install robot_rules.rb from http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/177589 in $RUBYLIB'
+        @robots[:warning] = true
+      end
+      @allow[url] = true
+      return true
+    end
+
+
+    def robots_uri(uri)
+      uri.merge('/robots.txt') unless uri.relative?
+    end
+
+
+    def file_url(filename)
+      filename = File.join(File.basename(File.dirname(filename)), File.basename(filename))
+      # "file://#{encode(filename, ':/')}"
+      encode(filename, ':/')
+    end
+
+
+    def get_optionvalue(opt, val, default=nil)
+      ok, val = get_option(opt, val)
+      if ok
+        val
+      else
+        default
+      end
+    end
+
+
+    def get_option(opt, val)
+      vals = @options[opt]
+      $logger.debug "val=#{val} vals=#{vals.inspect}"
+      if vals and vals.has_key?(val)
+        rv = vals[val]
+        $logger.debug "get_option ok: #{opt} => #{rv.inspect}"
+        case rv
+        when Symbol
+          $logger.debug "get_option re: #{rv}"
+          return get_option(opt, rv)
+        else
+          $logger.debug "get_option true, #{rv}"
+          return [true, rv]
+        end
+      else
+        $logger.debug "get_option no: #{opt} => #{val.inspect}"
+        return [false, val]
+      end
+    end
+
+
+    def encode(text, chars='')
+      text.gsub(/[^a-zA-Z0-9,._#{chars}-]/) {|t| '%%%02x' % t[0]}
+    end
+
+
+    def output_suffix(outformat)
+      @suffix[outformat] || outformat
+    end
+
+
+    def get_outfile(outformat=nil)
+      @outfile[outformat] || File.join(@cfgdir, "websitiary.#{output_suffix(outformat || @output_format[0])}")
+    end
+
+
+    def view_output(outfile=nil)
+      send("view_output_#{@output_format[0]}", outfile || get_outfile)
+    end
+
+
+    def view_output_general(outfile)
+      if @view
+        system((@view % outfile))
+      end
+    end
+    alias :view_output_html :view_output_general
+    alias :view_output_text :view_output_general
+    alias :view_output_rss :view_output_general
+
+
+    def edit_profile(profile)
+      fn = profile_filename(profile)
+      $logger.debug "edit: #{fn}"
+            `#{@cmd_edit % fn}`
+    end
+
+
+    def profile_filename(profile_name)
+      if File.extname(profile_name) != '.rb'
+        profile_name = "#{profile_name}.rb"
+      end
+      for d in ['.', @cfgdir]
+        filename = File.join(d, profile_name)
+        if File.exists?(filename)
+          return filename
+        end
+      end
+      return nil
+    end
+
+  end
+
+
+
+  # Hash: The output of the diff commands for each url.
+  attr_reader :difftext
+
+  # The configurator
+  attr_reader :configuration
+
+
+  # args:: Array of command-line (like) arguments.
+  def initialize(args=[])
+    @configuration = Configuration.new(self, args)
+    @difftext = {}
+
+    ensure_dir(@configuration.cfgdir)
+    css = File.join(@configuration.cfgdir, 'websitiary.css')
+    unless File.exists?(css)
+      $logger.info "Copying default css file: #{css}"
+      File.open(css, 'w') do |io|
+        io.puts <<CSS
+body {
+  color: black;
+  background-color: #f0f0f0;
+}
+a.external {
+}
+a.old {
+}
+a.latest {
+}
+a.toc {
+}
+ol.toc {
+  float: left;
+  width: 200px;
+  position: fixed;
+  padding: 0;
+  margin: 0;
+}
+li.toc {
+  list-style: none;
+  border: 1px solid silver;
+  background-color: #fafafa;
+    padding: 0.5em;
+  font-size: 80%;
+  font-family: Verdana, Myriad Web, Syntax, sans-serif;
+}
+li.toc:hover {
+  background-color: #ffff8d;
+}
+div.contents {
+  margin-left: 210px;
+  min-width: 16em;
+}
+div.webpage {
+  margin: 5px 0 5px 0;
+  padding: 5px;
+  border: 1px solid silver;
+  background-color: white;
+}
+h1.diff {
+  font-family: Verdana, Myriad Web, Syntax, sans-serif;
+}
+div.diff {
+  padding-left: 2em;
+}
+pre.diff {
+  padding-left: 2em;
+}
+hr.separator {
+  width: 100%;
+  visibility: hidden;
+}
+.error {
+  color: yellow;
+  background-color: red;
+}
+.highlight {
+  background-color: #ffc730;
+}
+CSS
+      end
+    end
+  end
+
+
+  # Process the sources in @configuration.url as defined by profiles 
+  # and command-line options. The differences are stored in @difftext (a Hash).
+  # show_output:: If true, show the output with the defined viewer.
+  def process(show_output=true)
+    @configuration.todo.each do |url|
+      opts = @configuration.urls[url]
+      $logger.debug "Source: #{@configuration.get(url, :title, url)}"
+      older = @configuration.backupname(url)
+      ensure_dir(File.dirname(older))
+      $logger.debug "older: #{older}"
+      latest = @configuration.latestname(url)
+      ensure_dir(File.dirname(latest))
+      $logger.debug "latest: #{latest}"
+
+      if File.exists?(latest) and !opts[:ignore_age]
+        if (hdiff = opts[:hours])
+          tdiff = hdiff * HOUR_SECS
+          $logger.debug "hours: #{hdiff} (#{tdiff}s)"
+        elsif (ddiff = opts[:days])
+          tdiff = ddiff * DAY_SECS
+          $logger.debug "days: #{ddiff} (#{tdiff}s)"
+        else
+          tdiff = nil
+        end
+
+        if tdiff
+          tn = Time.now
+          tl = File.mtime(latest)
+          td = tn - tl
+          if td < tdiff
+            $logger.info "Skip #{@configuration.get(url, :title, url).inspect}: Only #{(td / DAY_SECS).to_i}d old (#{(tdiff / DAY_SECS).to_i}d)"
+            next
+          end
+        end
+      end
+
+      move(latest, older)
+      if download(url, latest, opts)
+        difftext = diff(url, older, latest, opts)
+        if difftext
+          $logger.debug "difftext: #{difftext}"
+          accumulate(url, difftext, opts)
+        end
+      end
+    end
+    show if show_output
+  end
+
+
+
+  private
+
+  def download(url, latest, opts)
+    if @configuration.done.include?(url)
+      $logger.info "Already downloaded: #{@configuration.get(url, :title, url).inspect}"
+      return false
+    end
+
+    $logger.info "Download: #{@configuration.get(url, :title, url).inspect}"
+    @configuration.done << url
+    text = @configuration.call_cmd(@configuration.get(url, :download), [url])
+    # $logger.debug text
+    unless text
+      $logger.warn "no contents: #{@configuration.get(url, :title, url)}"
+      return false
+    end
+
+    if opts
+      text = text.split("\n")
+      if (range = opts[:lines])
+        $logger.debug "download: lines=#{range}"
+        text = text[range] || []
+      end
+      if (range = opts[:cols])
+        $logger.debug "download: cols=#{range}"
+        text.map! {|l| l[range]}
+        text.compact!
+      end
+      if (o = opts[:sort])
+        $logger.debug "download: sort=#{o}"
+        case o
+        when true
+          text.sort!
+        when Proc
+          text.sort!(&o)
+        end
+      end
+      if (o = opts[:strip])
+        $logger.debug "download: strip!"
+        text.delete_if {|l| l !~ /\S/}
+      end
+      text = text.join("\n")
+    end
+
+    pprc = @configuration.get(url, :downloadprocess)
+    if pprc
+      $logger.debug "download process: #{pprc}"
+      text = @configuration.call_cmd(pprc, [text])
+      $logger.debug text
+    end
+
+    File.open(latest, 'w') {|io| io.puts(text)}
+    return true
+  end
+
+
+  def diff(url, old, new, opts)
+    if File.exists?(old)
+      $logger.debug "diff: #{old} <-> #{new}"
+      difftext = @configuration.call_cmd(@configuration.get(url, :diff), [old, new])
+      $logger.debug "diff: #{difftext}"
+
+      if difftext =~ /\S/
+        if (pprc = @configuration.get(url, :diffprocess))
+          $logger.debug "diff process: #{pprc}"
+          difftext = @configuration.call_cmd(pprc, [difftext])
+        end
+        $logger.debug "difftext: #{difftext}"
+        if difftext =~ /\S/
+          $logger.warn "Changed: #{@configuration.get(url, :title, url).inspect}"
+          return difftext
+        end
+      end
+
+      $logger.debug "Unchanged: #{@configuration.get(url, :title, url).inspect}"
+    else
+      $logger.info "Initial copy: #{old.inspect}"
+    end
+    return nil
+  end
+
+
+  def accumulate(url, difftext, opts)
+    @difftext[url] = difftext
+  end
+
+
+  def show
+    @configuration.show_output(@difftext)
+  end
+
+
+  def move(from, to)
+    if File.exists?(from)
+      $logger.debug "Overwriting: #{from} -> #{to}" if File.exists?(to)
+      File.rename(from, to)
+    end
+  end
+
+
+  def ensure_dir(dir, &fill_dir)
+    if File.exist?(dir)
+      unless File.directory?(dir)
+        $logger.fatal "Not a directory: #{dir}"
+        exit 5
+      end
+    else
+      Dir.mkdir(dir)
+      fill_dir.call(dir) if fill_dir
+    end
+  end
+
+end
+
+
+
+if __FILE__ == $0
+  Websitiary.new(ARGV).process
+  # sleep 5
+end
+
+
+# Local Variables:
+# revisionRx: REVISION\s\+=\s\+\'
+# End: