websitary 0.2.0

data/lib/websitary/applog.rb

@@ -0,0 +1,39 @@
+# applog.rb
+# @Last Change: 2007-09-11.
+# Author::      Thomas Link (micathom AT gmail com)
+# License::     GPL (see http://www.gnu.org/licenses/gpl.txt)
+# Created::     2007-09-08.
+
+require 'logger'
+
+
+# A simple wrapper around Logger.
+class Websitary::AppLog
+    def initialize(output=nil)
+        @output = output || $stdout
+        $logger = Logger.new(@output, 'daily')
+        $logger.progname = Websitary::APPNAME
+        $logger.datetime_format = "%H:%M:%S"
+        set_level
+    end
+
+
+    def set_level(level=:default)
+        case level
+        when :debug
+            $logger.level = Logger::DEBUG
+        when :verbose
+            $logger.level = Logger::INFO
+        when :quiet
+            $logger.level = Logger::ERROR
+        else
+            $logger.level = Logger::WARN
+        end
+        $logger.debug "Set logger level: #{level}"
+    end
+end
+
+
+# Local Variables:
+# revisionRx: REVISION\s\+=\s\+\'
+# End:

data/lib/websitary/configuration.rb

@@ -0,0 +1,1505 @@
+# configuration.rb
+# @Last Change: 2007-09-16.
+# Author::      Thomas Link (micathom AT gmail com)
+# License::     GPL (see http://www.gnu.org/licenses/gpl.txt)
+# Created::     2007-09-08.
+
+
+
+# This class defines the scope in which profiles are evaluated. Most 
+# of its methods are suitable for use in profiles.
+class Websitary::Configuration
+    # Hash (key = URL, value = Hash of options)
+    attr_accessor :urls
+    # Array of urls to be downloaded.
+    attr_accessor :todo
+    # Array of downloaded urls.
+    attr_accessor :done
+    # The user configuration directory
+    attr_accessor :cfgdir
+    # What to do
+    attr_accessor :execute
+    # Global Options
+    attr_accessor :options
+    # Cached mtimes
+    attr_accessor :mtimes
+    # The name of the quicklist profile
+    attr_accessor :quicklist_profile
+    # attr_accessor :default_profiles
+    # attr_accessor :cmd_edit
+
+
+    def initialize(app, args=[])
+        @logger = Websitary::AppLog.new
+        $logger.debug "Configuration#initialize"
+        @app    = app
+        @cfgdir = ENV['HOME'] ? File.join(ENV['HOME'], '.websitary') : '.'
+        [
+            ENV['USERPROFILE'] && File.join(ENV['USERPROFILE'], 'websitary'),
+            File.join(Config::CONFIG['sysconfdir'], 'websitary')
+        ].each do |dir|
+            if File.exists?(dir)
+                @cfgdir = dir
+                break
+            end
+        end
+
+        @cmd_edit          = 'vi "%s"'
+        @execute           = 'downdiff'
+        @quicklist_profile = 'quicklist'
+        @user_agent        = "websitary/#{Websitary::VERSION}"
+        @view              = 'w3m "%s"'
+
+        @allow             = {}
+        @default_options   = {}
+        @default_profiles  = [@quicklist_profile]
+        @done              = []
+        @mtimes            = Websitary::FileMTimes.new(self)
+        @outfile           = {}
+        @profiles          = []
+        @robots            = {}
+        @todo              = []
+        @urlencmap         = {}
+        @urls              = {}
+
+        @suffix = {
+            'text' => 'txt'
+            # 'rss'  => 'xml'
+        }
+
+        migrate
+        initialize_options
+        profile 'config.rb'
+        parse_command_line_args(args)
+
+        @output_format   ||= ['html']
+        @output_title      = %{#{Websitary::APPNAME}: #{@profiles.join(", ")}}
+    end
+
+
+    def parse_command_line_args(args)
+        $logger.debug "parse_command_line_args: #{args}"
+        opts    = OptionParser.new do |opts|
+            opts.banner =  "Usage: #{Websitary::APPNAME} [OPTIONS] [PROFILES] > [OUT]"
+            opts.separator ''
+            opts.separator "#{Websitary::APPNAME} is a free software with ABSOLUTELY NO WARRANTY under"
+            opts.separator 'the terms of the GNU General Public License version 2 or newer.'
+            opts.separator ''
+
+            opts.separator 'General Options:'
+
+            opts.on('-c', '--cfg=DIR', String, 'Configuration directory') do |value|
+                @cfgdir = value
+            end
+
+            opts.on('-e', '--execute=COMMAND', String, 'Define what to do (default: downdiff)') do |value|
+                @execute = value
+            end
+
+            # opts.on('-E', '--edit=PROFILE', String, 'Edit a profile') do |value|
+            #   edit_profile value
+            #   exit 0
+            # end
+
+            opts.on('-f', '--output-format=FORMAT', 'Output format (html, text, rss)') do |value|
+                output_format(*value.split(/,/))
+            end
+
+            opts.on('--[no-]ignore-age', 'Ignore age limits') do |bool|
+                set :ignore_age => bool
+            end
+
+            opts.on('--log=DESTINATION', String, 'Log destination') do |value|
+                @logger = Websitary::AppLog.new(value != '-' && value)
+            end
+
+            opts.on('-o', '--output=FILENAME', String, 'Output') do |value|
+                output_file(value)
+            end
+
+            opts.on('-s', '--set=NAME=VAR', String, 'Set a default option') do |value|
+                key, val = value.split(/=/, 2)
+                set key.intern => eval(val)
+            end
+
+            opts.on('-t', '--timer=N', Numeric, 'Repeat every N seconds (never exit)') do |value|
+                global(:timer => value)
+            end
+
+            # opts.on('--review', 'View last diff') do |value|
+            #   view_output
+            #   exit 0
+            # end
+
+            opts.separator ''
+            opts.separator "Available commands (default: #@execute):"
+            commands = @app.methods.map do |m|
+                mt = m.match(/^execute_(.*)$/)
+                mt && mt[1]
+            end
+            commands.compact!
+            commands.sort!
+            opts.separator commands.join(', ')
+
+            opts.separator ''
+            opts.separator 'Available profiles:'
+            opts.separator Dir[File.join(@cfgdir, '*.rb')].map {|f| File.basename(f, '.*')}.join(', ')
+
+            opts.separator ''
+            opts.separator 'Other Options:'
+
+            opts.on('--debug', 'Show debug messages') do |v|
+                $VERBOSE = $DEBUG = true
+                @logger.set_level(:debug)
+            end
+
+            opts.on('-q', '--quiet', 'Be mostly quiet') do |v|
+                @logger.set_level(:quiet)
+            end
+
+            opts.on('-v', '--verbose', 'Run verbosely') do |v|
+                $VERBOSE = true
+                @logger.set_level(:verbose)
+            end
+
+            opts.on_tail('-h', '--help', 'Show this message') do
+                puts opts
+                exit 1
+            end
+        end
+
+        @profiles = opts.parse!(args)
+        @profiles = @default_profiles if @profiles.empty?
+        cla_handler = "cmdline_arg_#{@execute}"
+        cla_handler = nil unless @app.respond_to?(cla_handler)
+        for pn in @profiles
+            if cla_handler
+                @app.send(cla_handler, self, pn)
+            else
+                profile pn
+            end
+        end
+
+        self
+    end
+
+
+    # Retrieve an option for an url
+    # url:: String
+    # opt:: Symbol
+    def get(url, opt, default=nil)
+        opts = @urls[url]
+        unless opts
+            $logger.debug "Non-registered URL: #{url}"
+            return default
+        end
+        $logger.debug "get: opts=#{opts.inspect}"
+        case opt
+        when :diffprocess, :format
+            opt_ = opts.has_key?(opt) ? opt : :diff
+        else
+            opt_ = opt
+        end
+
+        $logger.debug "get: opt=#{opt} opt_=#{opt_}"
+        $logger.debug "get: #{opts[opt_]} #{opts[:use]}" if opts
+        if opts.has_key?(opt_)
+            val = opts[opt_]
+        elsif opts.has_key?(:use)
+            val = opts[:use]
+        else
+            val = nil
+        end
+
+        case val
+        when nil
+        when Symbol
+            $logger.debug "get: val=#{val}"
+            success, rv = get_option(opt, val)
+            $logger.debug "get: #{success}, #{rv}"
+            if success
+                return rv
+            end
+        else
+            $logger.debug "get: return val=#{val}"
+            return val
+        end
+        unless default
+            success, default1 = get_option(opt, :default)
+            default = default1 if success
+        end
+
+        $logger.debug "get: return default=#{default}"
+        return default
+    end
+
+
+    def get_optionvalue(opt, val, default=nil)
+        case val
+        when Symbol
+            ok, val = get_option(opt, val)
+            if ok
+                val
+            else
+                default
+            end
+        else
+            val
+        end
+    end
+
+
+    def get_option(opt, val)
+        vals = @options[opt]
+        $logger.debug "val=#{val} vals=#{vals.inspect}"
+        if vals and vals.has_key?(val)
+            rv = vals[val]
+            $logger.debug "get_option ok: #{opt} => #{rv.inspect}"
+            case rv
+            when Symbol
+                $logger.debug "get_option re: #{rv}"
+                return get_option(opt, rv)
+            else
+                $logger.debug "get_option true, #{rv}"
+                return [true, rv]
+            end
+        else
+            $logger.debug "get_option no: #{opt} => #{val.inspect}"
+            return [false, val]
+        end
+    end
+
+
+    # Configuration command:
+    # Set the default profiles
+    def default(*profile_names)
+        @default_profiles = profile_names
+    end
+
+
+    def quicklist(profile_name)
+        @quicklist_profile = profile_name
+    end
+
+
+    # Configuration command:
+    # Load a profile
+    def profile(profile_name)
+        case profile_name
+        when '-'
+            readlines.map! {|l| l.chomp}.each {|url| source url}
+        when '__END__'
+            $logger.debug "Profile: __END__"
+            contents = DATA.read
+            return eval_profile(contents)
+        else
+            fn = profile_filename(profile_name)
+            if fn
+                $logger.debug "Profile: #{fn}"
+                contents = File.read(fn)
+                return eval_profile(contents, fn)
+            end
+        end
+        return false
+    end
+
+
+    # Define a options shortcut.
+    def shortcut(symbol, args)
+        ak = args.keys
+        ok = @options.keys
+        dk = ok - ak
+
+        # :downloadprocess
+        if !ak.include?(:delegate) and
+            dk.any? {|e| [:download, :downloadformat, :diff, :format, :diffprocess].include?(e)}
+            $logger.warn "Shortcut #{symbol}: Undefined fields: #{dk.inspect}"
+        end
+
+        if ak.include?(:delegate)
+            dk.each do |field|
+                @options[field][symbol] = args[:delegate]
+            end
+        end
+
+        args.each do |field, val|
+            @options[field][symbol] = val unless field == :delegate
+        end
+    end
+
+
+    # Set the output format.
+    def output_format(*format)
+        unless format.all? {|e| ['text', 'html', 'rss'].include?(e)}
+            $logger.fatal "Unknown output format: #{format}"
+            exit 5
+        end
+        @output_format = format
+    end
+
+
+    # Set the output file.
+    def output_file(filename, outformat=nil)
+        @outfile[outformat] = filename
+    end
+
+
+    # Configuration command:
+    # Set global options.
+    # type:: Symbol
+    # options:: Hash
+    def option(type, options)
+        $logger.info "option #{type}: #{options.inspect}"
+        o = @options[type]
+        if o
+            o.merge!(options)
+        else
+            $logger.error "Unknown option type: #{type} (#{options.inspect})"
+        end
+    end
+
+
+    # Set a global option.
+    def global(options)
+        options.each do |type, value|
+            @options[:global][type] = value
+        end
+    end
+
+
+    # Configuration command:
+    # Set the default value for source-options.
+    def set(options)
+        $logger.debug "set: #{options.inspect}"
+        @default_options.merge!(options)
+    end
+
+
+    # Configuration command:
+    # Unset a default source-option.
+    def unset(*options)
+        for option in options
+            @default_options.delete(option)
+        end
+    end
+
+
+    # Configuration command:
+    # Define a source.
+    # urls:: String
+    def source(urls, opts={})
+        urls.split("\n").flatten.compact.each do |url|
+            @urls[url] = @default_options.dup.update(opts)
+            @todo << url
+        end
+    end
+
+
+    # Configuration command:
+    # Set the default download processor. The block takes the 
+    # downloaded text (STRING) as argument.
+    def downloadprocess(&block)
+        @options[:downloadprocess][:default] = block
+    end
+
+
+    # Configuration command:
+    # Set the default diff processor. The block takes the 
+    # diff text (STRING) as argument.
+    def diffprocess(&block)
+        @options[:diff][:default] = block
+    end
+
+
+    # Configuration command:
+    # Set the editor.
+    def edit(cmd)
+        @cmd_edit = cmd
+    end
+
+
+    # Configuration command:
+    # Set the viewer.
+    def view(view)
+        @view = view
+    end
+
+
+    # Configuration command:
+    # Set the default diff program.
+    def diff(diff)
+        @options[:diff][:default] = diff
+    end
+
+
+    # Configuration command:
+    # Set the default dowloader.
+    def download(download)
+        @options[:download][:default] = download
+    end
+
+
+    # Format a diff according to URL's source options.
+    def format(url, difftext)
+        fmt = get(url, :format)
+        eval_arg(fmt, [difftext], difftext)
+    end
+
+
+    # Apply some arguments to a format.
+    # format:: String or Proc
+    # args:: Array of Arguments
+    def eval_arg(format, args, default=nil, &process_string)
+        case format
+        when nil
+            return default
+        when Proc
+            # $logger.debug "eval proc: #{format} #{args.inspect}" #DBG#
+            $logger.debug "eval proc: #{format}/#{args.size}"
+            return format.call(*args)
+        else
+            ca = format % args
+            # $logger.debug "eval string: #{ca}" #DBG#
+            if process_string
+                return process_string.call(ca)
+            else
+                return ca
+            end
+        end
+    end
+
+
+    # Apply the argument to cmd (a format String or a Proc). If a 
+    # String, execute the command.
+    def call_cmd(cmd, args, default=nil)
+        eval_arg(cmd, args, default) {|cmd| `#{cmd}`}
+    end
+
+
+    # Generate & view the final output.
+    # difftext:: Hash
+    def show_output(difftext)
+        if difftext.empty?
+            msg = ['No news is good news']
+            msg << "try again in #{@app.format_tdiff(@app.tdiff_min)}" if @app.tdiff_min
+            $logger.warn msg.join('; ')
+            return 0
+        end
+
+        @output_format.each do |outformat|
+            meth = "get_output_#{outformat}"
+
+            unless respond_to?(meth)
+                $logger.fatal "Unknown output format: #{outformat}"
+                exit 5
+            end
+
+            out = send(meth, difftext)
+            if out
+                outfile = get_outfile(outformat)
+                case outfile
+                when '-'
+                    puts out
+                else
+                    write_file(outfile) {|io| io.puts out}
+                    meth = "view_output_#{outformat}"
+                    self.send(meth, outfile)
+                end
+            end
+        end
+        return 1
+    end
+
+
+    def get_output_text(difftext)
+        difftext.map do |url, difftext|
+            if difftext
+                difftext = html_to_text(difftext) if is_html?(difftext)
+                !difftext.empty? && [
+                    eval_arg(get(url, :rewrite_link, '%s'), [url]), 
+                    difftext_annotation(url), 
+                    nil, 
+                    difftext
+                ].join("\n")
+            end
+        end.compact.join("\n\n#{('-' * 68)}\n\n")
+    end
+
+
+    def get_output_rss(difftext)
+        success, rss_url = get_option(:rss, :url)
+        if success
+            success, rss_version = get_option(:rss, :version)
+            # require "rss/#{rss_version}"
+
+            rss         = RSS::Rss.new(rss_version)
+            chan        = RSS::Rss::Channel.new
+            chan.title  = @output_title
+            [:description, :copyright, :category, :language, :image, :webMaster, :pubDate].each do |field|
+                ok, val = get_option(:rss, field)
+                item.send(format_symbol(field, '%s='), val) if ok
+            end
+            chan.link   = rss_url
+            rss.channel = chan
+
+            cnt = difftext.map do |url, text|
+                rss_format = get(url, :rss_format, 'plain_text')
+                text = strip_tags(text, :format => rss_format)
+                next if text.empty?
+
+                item = RSS::Rss::Channel::Item.new
+                item.date  = Time.now
+                item.title = get(url, :title, File.basename(url))
+                item.link  = eval_arg(get(url, :rewrite_link, '%s'), [url])
+                [:author, :date, :enclosure, :category, :pubDate].each do |field|
+                    val = get(url, format_symbol(field, 'rss_%s'))
+                    item.send(format_symbol(field, '%s='), val) if val
+                end
+
+                annotation = difftext_annotation(url)
+                annotation = "<pre>#{annotation}</pre>" if annotation
+                case rss_format
+                when 'plain_text'
+                    item.description = %{#{annotation}<pre>#{text}</pre>}
+                else
+                    item.description = %{#{annotation}\n#{text}}
+                end
+                chan.items << item
+            end
+
+            return rss.to_s
+
+        else
+
+            $logger.fatal "Global option :rss[:url] not defined."
+            exit 5
+
+        end
+    end
+
+
+    def get_output_html(difftext)
+        difftext = difftext.map do |url, text|
+            tags = get(url, :strip_tags)
+            text = strip_tags(text, :tags => tags) if tags
+            text.empty? ? nil : [url, text]
+        end
+        difftext.compact!
+        sort_difftext!(difftext)
+
+        toc = difftext.map do |url, text|
+            ti  = get(url, :title, File.basename(url))
+            tid = html_toc_id(url)
+            bid = html_body_id(url)
+            %{<li id="#{tid}" class="toc"><a class="toc" href="\##{bid}">#{ti}</a></li>}
+        end.join("\n")
+
+        idx = 0
+        cnt = difftext.map do |url, text|
+            idx += 1
+            ti   = get(url, :title, File.basename(url))
+            bid  = html_body_id(url)
+            if (rewrite = get(url, :rewrite_link))
+                urlr = eval_arg(rewrite, [url])
+                ext  = ''
+            else
+                old  = %{<a class="old" href="#{file_url(oldname(url))}">old</a>}
+                lst  = %{<a class="latest" href="#{file_url(latestname(url))}">latest</a>}
+                ext  = %{ (#{old}, #{lst})}
+                urlr = url
+            end
+            note = difftext_annotation(url)
+            <<HTML
+<div id="#{bid}" class="webpage">
+<div class="count">
+#{idx}
+</div>
+<h1 class="diff">
+<a class="external" href="#{urlr}">#{ti}</a>#{ext}
+</h1>
+<div class="annotation">
+#{note && CGI::escapeHTML(note)}
+</div>
+<div class="diff,difftext">
+#{format(url, text)}
+</div>
+</div>
+HTML
+        end.join(('<hr class="separator"/>') + "\n")
+
+        success, template = get_option(:page, :format)
+        unless success
+            success, template = get_option(:page, :simple)
+        end
+        return eval_arg(template, [@output_title, toc, cnt])
+    end
+
+
+    # Get the diff filename.
+    def diffname(url, ensure_dir=false)
+        encoded_filename('diff', url, ensure_dir, 'md5')
+    end
+
+
+    # Get the backup filename.
+    def oldname(url, ensure_dir=false, type=nil)
+        encoded_filename('old', url, ensure_dir, type)
+    end
+
+
+    # Get the filename for the freshly downloaded copy.
+    def latestname(url, ensure_dir=false, type=nil)
+        encoded_filename('latest', url, ensure_dir, type)
+    end
+
+
+    def url_from_filename(filename)
+        rv = @urlencmap[filename]
+        if rv
+            $logger.debug "Map filename: #{filename} -> #{rv}"
+        else
+            $logger.warn "Unmapped filename: #{filename}"
+        end
+        rv
+    end
+
+
+    def encoded_filename(dir, url, ensure_dir=false, type=nil)
+        type ||= get(url, :cachetype, 'tree')
+        $logger.debug "encoded_filename: type=#{type} url=#{url}"
+        rv = File.join(@cfgdir, dir, encoded_basename(url, type))
+        rd = File.dirname(rv)
+        $logger.debug "encoded_filename: rv0=#{rv}"
+        fm = get_optionvalue(:global, :filename_size, 255)
+        rdok = !ensure_dir || @app.ensure_dir(rd, false)
+        if !rdok or rv.size > fm or File.directory?(rv)
+            # $logger.debug "Filename too long (:global=>:filename_size = #{fm}), try md5 encoded filename instead: #{url}"
+            $logger.info "Can't use filename, try 'md5' instead: #{url}"
+            rv = File.join(@cfgdir, dir, encoded_basename(url, :md5))
+            rd = File.dirname(rv)
+        end
+        @urlencmap[rv] = url
+        return rv
+    end
+
+
+    def encoded_basename(url, type='tree')
+        m = "encoded_basename_#{type}"
+        if respond_to?(m)
+            return send(m, url)
+        else
+            $logger.fatal "Unknown cache type: #{type}"
+            exit 5
+        end
+    end
+
+
+    def encoded_basename_tree(url)
+        ensure_filename(encode(url, '/'))
+    end
+
+
+    def encoded_basename_flat(url)
+        encode(url)
+    end
+
+
+    def encoded_basename_md5(url)
+        Digest::MD5.hexdigest(url)
+    end
+
+
+    def urlextname(url)
+        begin
+            return File.extname(URI.parse(url).path)
+        rescue Exception => e
+        end
+    end
+
+
+    # Guess path's dirname.
+    #   foo/bar     -> foo
+    #   foo/bar.txt -> foo
+    #   foo/bar/    -> foo/bar
+    def guess_dir(path)
+        path[-1..-1] == '/' ? path[0..-2] : File.dirname(path)
+    end
+
+
+    # Strip the url's last part (after #).
+    def canonic_url(url)
+        url.sub(/#.*$/, '')
+    end
+
+
+    def strip_tags_default
+        success, tags = get_option(:strip_tags, :default)
+        tags.dup if success
+    end
+
+
+    def strip_tags(doc, args={})
+        tags = args[:tags] || strip_tags_default
+        case doc
+        when String
+            doc = Hpricot(doc)
+        end
+        tags.each do |tag|
+            doc.search(tag).remove
+        end
+        case args[:format]
+        when :hpricot
+            doc
+        else
+            doc.send("to_#{args[:format] || :html}")
+        end
+    end
+
+
+    # Check whether path is eligible on the basis of url or path0.
+    # This checks either for a :match option for url or the extensions 
+    # of path0 and path.
+    def eligible_path?(url, path0, path)
+        rx = get(url, :match)
+        if rx
+            return path =~ rx
+        else
+            return File.extname(path0) == File.extname(path)
+        end
+    end
+
+
+    # Scan hpricot document for hrefs and push the onto @todo if not 
+    # already included.
+    def push_hrefs(url, hpricot, &condition)
+        begin
+            return if robots?(hpricot, 'nofollow')
+            depth = get(url, :depth)
+            return if depth and depth <= 0
+            uri0  = URI.parse(url)
+            # pn0   = Pathname.new(guess_dir(File.expand_path(uri0.path)))
+            pn0   = Pathname.new(guess_dir(uri0.path))
+            (hpricot / 'a').each do |a|
+                href = a['href']
+                next if href.nil? or href == url or href =~ /^\s*javascript:/
+                    uri  = URI.parse(href)
+                pn   = guess_dir(uri.path)
+                href = rewrite_href(href, url, uri0, pn0, true)
+                curl = canonic_url(href)
+                next if href.nil? or @done.include?(curl) or @todo.include?(curl)
+                next unless robots_allowed?(curl, uri)
+                # pn   = Pathname.new(guess_dir(File.expand_path(uri.path)))
+                uri  = URI.parse(href)
+                pn   = Pathname.new(guess_dir(uri.path))
+                if condition.call(uri0, pn0, uri, pn)
+                    opts = @urls[url].dup
+                    # opts[:title] = File.basename(curl)
+                    opts[:title] = [opts[:title], File.basename(curl)].join(' - ')
+                    opts[:depth] = depth - 1 if depth and depth >= 0
+                    @urls[curl] = opts
+                    @todo << curl
+                end
+            end
+        rescue Exception => e
+            # $logger.error e  #DBG#
+            $logger.error e.message
+            $logger.debug e.backtrace
+        end
+    end
+
+
+    # Rewrite urls in doc
+    # url:: String
+    # doc:: Hpricot document
+    def rewrite_urls(url, doc)
+        uri = URI.parse(url)
+        urd = guess_dir(uri.path)
+        (doc / 'a').each do |a|
+            href = rewrite_href(a['href'], url, uri, urd, true)
+            a['href'] = href if href
+        end
+        (doc / 'img').each do |a|
+            href = rewrite_href(a['src'], url, uri, urd, false)
+            a['src'] = href if href
+        end
+        doc
+    end
+
+
+    # Try to make href an absolute url.
+    def rewrite_href(href, url, uri=nil, urd=nil, local=false)
+        begin
+            return if !href or href =~ /^\s*javascript:/
+            urh   = URI.parse(href)
+            uri ||= URI.parse(url)
+            urd ||= guess_dir(uri.path)
+            rv    = nil
+            href  = href.strip
+
+            # $logger.debug "DBG", uri, urh, #DBG#
+            if href =~ /\w+:/
+                # $logger.debug "DBG href=#$0" #DBG#
+                rv = href
+            elsif urh.relative?
+                # $logger.debug "DBG urh relative" #DBG#
+                if uri.relative?
+                    # $logger.debug "DBG both relative" #DBG#
+                    if uri.instance_of?(URI::Generic)
+                        rv = File.join(urd, href)
+                        # $logger.debug "DBG rv=#{rv}" #DBG#
+                    end
+                else
+                    rv = uri.merge(href).to_s
+                    # $logger.debug "DBG relativ rv=#{rv}" #DBG#
+                    if local
+                        hf = latestname(rv)
+                        if @todo.include?(rv) or @done.include?(rv) or File.exist?(hf)
+                            rv = hf
+                            # $logger.debug "DBG relativ, local rv=#{rv}" #DBG#
+                        end
+                    end
+                end
+            elsif href[0..0] == '#'
+                # $logger.debug "DBG anchor" #DBG#
+                rv = url + href
+            elsif uri.host == urh.host
+                # $logger.debug "DBG merge" #DBG#
+                rv = uri.merge(href).to_s
+            else
+                # $logger.debug "as is" #DBG#
+                rv = href
+            end
+
+            case rv
+            when String
+                return rv
+            when nil
+            else
+                $logger.error "Internal error: href=#{href}"
+                $logger.debug caller.join("\n")
+            end
+            return
+        rescue Exception => e
+            # $logger.error e  #DBG#
+            $logger.error e.message
+            $logger.debug e.backtrace
+        end
+        return nil
+    end
+
+
+    # Return a Proc that takes an text as argument and highlight occurences of rx.
+    # rx:: Regular expression
+    # color:: A string, sets the class to highlight-color (default: "yellow")
+    # group:: A number (default: 0)
+    # tag:: The HTML tag to use (default: "span")
+    def highlighter(rx, color=nil, group=nil, tag='span')
+        lambda {|text| text.gsub(rx, %{<#{tag} class="highlight-#{color || 'yellow'}">\\#{group || 0}</#{tag}>})}
+    end
+
+
+    def view_output(outfile=nil)
+        send("view_output_#{@output_format[0]}", outfile || get_outfile)
+    end
+
+
+    def edit_profile(profile=nil)
+        profile ||= @profiles
+        case profile
+        when Array
+            profile.each {|p| edit_profile p}
+        else
+            fn = profile_filename(profile)
+            $logger.debug "edit: #{fn}"
+            `#{@cmd_edit % fn}`
+        end
+    end
+
+
+    def profile_filename(profile_name, check_file_exists=true)
+        if File.extname(profile_name) != '.rb'
+            profile_name = "#{profile_name}.rb"
+        end
+        filename = nil
+        ['.', @cfgdir].each do |d|
+            filename = File.join(d, profile_name)
+            if File.exists?(filename)
+                return filename
+            end
+        end
+        return check_file_exists ? nil : filename
+    end
+
+
+    def write_file(filename, mode='w', &block)
+        File.open(filename, mode) {|io| block.call(io)}
+        @mtimes.set(filename)
+    end
+
+
+    def canonic_filename(filename)
+        call_cmd(get_optionvalue(:global, :canonic_filename), [filename], filename)        
+    end
+
+
+    private
+    def initialize_options
+        @options = {
+            :global => {
+                :downloadhtml => :openuri,
+            },
+        }
+
+        @options[:diff] = {
+            :default => :diff,
+
+            :diff   => lambda {|old, new, *args|
+                opts, _  = args
+                opts   ||= '-d -w'
+                difftext = call_cmd('diff %s -u2 "%s" "%s"', [opts, old, new])
+                difftext = difftext.split("\n")[2..-1]
+                difftext ? difftext.delete_if {|l| l =~ /^[^+]/}.map {|l| l[1..-1]}.join("\n") : ''
+            },
+
+            :binary => lambda {|old, new|
+                call_cmd(get_optionvalue(:diff, :diff), [old, new, '--binary -d -w'])
+            },
+
+            :new => lambda {|old, new|
+                difftext = call_cmd(get_optionvalue(:diff, :binary), [old, new])
+                difftext.empty? ? '' : new
+            },
+
+            :raw => :new,
+
+            :webdiff => lambda {|old, new|
+                $logger.debug "webdiff: #{File.basename(new)}"
+                $logger.debug %{webdiff --hicolor=yellow -archive "#{old}" -current "#{new}" -out -}
+                difftext = `webdiff --hicolor=yellow -archive "#{old}" -current "#{new}" -out -`
+                $?.exitstatus == 1 ? difftext : ''
+            },
+        }
+
+        @options[:format] = {
+            :default => :diff,
+            :diff    => %{<pre class="diff">\n%s\n</pre>},
+            :webdiff => "%s\n",
+            :raw     => lambda {|new| File.read(new)},
+        }
+
+        @options[:diffprocess] = {
+            :default => :diff,
+            :diff    => false,
+            :webdiff => false,
+            :raw     => false,
+        }
+
+        @options[:download] = {
+            :default    => :w3m,
+            :raw        => :openuri,
+        }
+
+        @options[:downloadformat] = {
+            :w3m => 'text',
+            :webdiff => 'html',
+            :raw => '',
+        }
+
+        @options[:downloadprocess] = {
+        }
+
+        @options[:rss] = {
+            :version => '2.0',
+        }
+
+        @options[:strip_tags] = {
+            :default => ['script', 'object', 'form', 'input', 'select', 'iframe', 'head', 'meta', 'link'],
+        }
+
+        shortcut :w3m, :delegate => :diff,
+            :download => 'w3m -S -F -dump "%s"'
+            # :download => 'w3m -no-cookie -S -F -dump "%s"'
+
+        shortcut :lynx, :delegate => :diff,
+        :download => 'lynx -dump "%s"'
+
+        shortcut :links, :delegate => :diff,
+        :download => 'links -dump "%s"'
+
+        shortcut :curl, :delegate => :webdiff,
+        :download => 'curl --silent "%s"'
+
+        shortcut :wget, :delegate => :webdiff,
+        :download => 'wget -q -O - "%s"'
+
+        shortcut :text, :delegate => :diff,
+        :download => lambda {|url| html_to_text(open_url(url).read)}
+
+        shortcut :body_html, :delegate => :webdiff,
+            :strip_tags => :default,
+            :download => lambda {|url|
+                begin
+                    doc = Hpricot(open_url(url).read)
+                    doc = doc.at('body')
+                    if doc
+                        doc  = rewrite_urls(url, doc)
+                        doc  = doc.inner_html
+                        if (tags = get(url, :strip_tags))
+                            doc = strip_tags(doc, :format => :hpricot, :tags => tags)
+                        end
+                    else
+                        $logger.warn 'inner html: No body'
+                    end
+                    doc.to_s
+                rescue Exception => e
+                    # $logger.error e  #DBG#
+                    $logger.error e.message
+                    $logger.debug e.backtrace
+                    break %{<pre class="error">\n#{e.message}\n</pre>}
+                end
+            }
+
+        shortcut :openuri, :delegate => :webdiff,
+            :download => lambda {|url|
+                begin
+                    open_url(url).read
+                rescue Exception => e
+                    # $logger.error e  #DBG#
+                    $logger.error e.message
+                    $logger.debug e.backtrace
+                    %{<pre class="error">\n#{e.to_s}\n</pre>}
+                end
+            }
+
+        shortcut :rss,
+            :delegate => :openuri,
+            :diff => lambda {|old, new|
+                success, rss_version = get_option(:rss, :version)
+                ro = RSS::Parser.parse(File.read(old), false)
+                if ro
+                    rh = {}
+                    ro.items.each do |item|
+                        rh[Digest::MD5.hexdigest(item.to_s)] = item
+                        rh[item.link] = item
+                    end
+                    rnew = []
+                    rn = RSS::Parser.parse(File.read(new), false)
+                    if rn
+                        rn.items.each do |item|
+                            rid = Digest::MD5.hexdigest(item.to_s)
+                            if !rh[rid]
+                                if (olditem = rh[item.link])
+                                    rss_diff = Websitary::Htmldiff.new(:oldtext => olditem.description, :newtext => item.description).process
+                                    rnew << format_rss_item(item, rss_diff)
+                                else
+                                    if item.enclosure and (curl = item.enclosure.url)
+                                        url   = url_from_filename(new)
+                                        dir   = get(url, :rss_enclosure)
+                                        curl  = rewrite_href(curl, url, nil, nil, true)
+                                        next unless curl
+                                        if dir
+                                            if dir == true
+                                                dir = File.join(@cfgdir, 'attachments', encode(rn.channel.title))
+                                            end
+                                            @app.ensure_dir(dir)
+                                            $logger.debug "Enclosure URL: #{curl}"
+                                            fname = File.join(dir, encode(File.basename(curl) || item.title || item.pubDate.to_s || Time.now.to_s))
+                                            $logger.debug "Enclosure save to: #{fname}"
+                                            enc   = open_url(curl).read
+                                            write_file(fname, 'wb') {|io| io.puts enc}
+                                            furl = file_url(fname)
+                                            enclosure = %{<p class="enclosure"><a href="%s" class="enclosure" />Enclosure (local copy)</a></p>} % furl
+                                            if get(url, :rss_rewrite_enclosed_urls)
+                                                item.description.gsub!(Regexp.new(Regexp.escape(curl))) {|t| furl}
+                                            end
+                                        else
+                                            enclosure = %{<p class="enclosure"><a href="%s" class="enclosure" />Original Enclosure</a></p>} % curl
+                                        end
+                                    else
+                                        enclosure = ''
+                                    end
+                                    rnew << format_rss_item(item, item.description, enclosure)
+                                end
+                            end
+                        end
+                        rnew.join("\n")
+                    end
+                end
+            }
+
+        shortcut :opml, :delegate => :rss,
+            :download => lambda {|url|
+                opml = open(url) {|io| io.read}
+                if oplm
+                    xml = Hpricot(opml)
+                    # <+TBD+>Well, maybe would should search for outline[@type=rss]?
+                    xml.search('//outline[@xmlurl]').each {|elt|
+                        if elt['type'] =~ /rss/
+                            curl = elt['xmlurl']
+                            opts = @urls[url].dup
+                            opts[:download] = :rss
+                            opts[:title] = elt['title'] || elt['text'] || elt['htmlurl'] || curl
+                            @urls[curl] = opts
+                            @todo << curl
+                        else
+                            $logger.warn "Unsupported type in OPML: #{elt.to_s}"
+                        end
+                    }
+                end
+                nil
+            }
+
+        shortcut :website, :delegate => :webdiff,
+            :download => lambda {|url| get_website(:body_html, url)}
+
+        shortcut :website_below, :delegate => :webdiff,
+            :download => lambda {|url| get_website_below(:body_html, url)}
+
+        shortcut :website_txt, :delegate => :default,
+            :download => lambda {|url| html_to_text(get_website(get(url, :downloadhtml, :openuri), url))}
+
+        shortcut :website_txt_below, :delegate => :default,
+            :download => lambda {|url| html_to_text(get_website_below(get(url, :downloadhtml, :openuri), url))}
+
+        shortcut :ftp, :delegate => :default,
+            :download => lambda {|url| get_ftp(url).join("\n")}
+
+        shortcut :ftp_recursive, :delegate => :default,
+            :download => lambda {|url|
+                list = get_ftp(url)
+                depth = get(url, :depth)
+                if !depth or depth >= 0
+                    dirs = list.find_all {|e| e =~ /^d/}
+                    dirs.each do |l|
+                        sl = l.scan(/^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+ +\S+ +\S+)\s+(.+)$/)
+                        perms, type, owner, group, size, date, dirname = sl[0]
+                        curl = File.join(url, dirname, '')
+                        opts = @urls[url].dup
+                        opts[:title] = [opts[:title], File.basename(curl)].join(' - ')
+                        opts[:depth] = depth - 1 if depth and depth >= 0
+                        @urls[curl] = opts
+                        @todo << curl
+                    end
+                end
+                list.join("\n")
+            }
+
+        shortcut :img, :delegate => :raw,
+            :format => lambda {|new|
+                file = file_url(new)
+                %{<img src="#{file}" />}
+            }
+
+        @options[:page] = {
+            :format => lambda do |ti, li, bd|
+                template = <<OUT
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<title>%s</title>
+<link rel="stylesheet" href="websitary.css" type="text/css">
+<link rel="alternate" href="websitary.rss" type="application/rss+xml" title="%s">
+</head>
+<body>
+<ol class="toc">
+%s
+</ol>
+<div class="contents">
+%s
+</div>
+</body>
+</html>
+OUT
+                template % [ti, ti, li, bd]
+            end
+        }
+    end
+
+
+    def migrate
+        store = File.join(@cfgdir, 'version.yml')
+        if File.exist?(store)
+            version = YAML.load_file(store)
+            return if version == Websitary::VERSION
+        else
+            version = '0.1.0'
+        end
+        va = version.split(/\./).map {|i| i.to_i}
+        migrate_0_1_0 if (va <=> [0, 1, 0]) != 1
+        write_file(store) {|f| YAML.dump(Websitary::VERSION, f)}
+    end
+
+
+    def migrate_0_1_0
+        $logger.warn "Migrate data from version 0.1.0"
+        ['latest', 'old'].each do |dir|
+            files = Dir[File.join(@cfgdir, dir, '*')]
+            files.each do |f|
+                url = decode(File.basename(f))
+                nfn = encoded_filename(dir, url, true)
+                @app.move(f, nfn)
+            end
+        end
+    end
+
+
+    def eval_profile(contents, profile_file=nil)
+        @current_profile = profile_file
+        begin
+            self.instance_eval(contents)
+            return true
+        ensure
+            @current_profile = nil
+        end
+    end
+
+
+    def get_website(download, url)
+        html = call_cmd(get_optionvalue(:download, download), [url])
+        if html
+            doc = Hpricot(html)
+            if doc
+                return if robots?(doc, 'noindex')
+                push_hrefs(url, doc) do |uri0, pn0, uri, pn|
+                    eligible_path?(url, uri0.path, uri.path) &&
+                        uri.host == uri0.host
+                end
+            end
+        end
+        html
+    end
+
+
+    def get_website_below(download, url)
+        dwnl = get_optionvalue(:download, download)
+        html = call_cmd(dwnl, [url])
+        if html
+            doc = Hpricot(html)
+            if doc
+                return if robots?(doc, 'noindex')
+                push_hrefs(url, doc) do |uri0, pn0, uri, pn|
+                    eligible_path?(url, uri0.path, uri.path) &&
+                        uri.host == uri0.host &&
+                        (pn.to_s == '.' || pn.relative_path_from(pn0).to_s == '.')
+                end
+            end
+        end
+        html
+    end
+
+
+    def get_ftp(url)
+        uri = URI.parse(url)
+        ftp = Net::FTP.new(uri.host)
+        ftp.passive = true
+        begin
+            ftp.login
+            ftp.chdir(uri.path)
+            return ftp.list('*')
+        rescue Exception => e
+            $logger.error e
+        ensure
+            ftp.close
+        end
+    end
+
+
+    def html_toc_id(url)
+        't%s' % Digest::MD5.hexdigest(url)
+    end
+
+
+    def html_body_id(url)
+        'b%s' % Digest::MD5.hexdigest(url)
+    end
+
+
+    def ensure_filename(filename)
+        filename = filename.gsub(/[\/]{2,}/, File::SEPARATOR)
+        # File.join(*File.split(filename))
+        if filename =~ /#{Regexp.escape(File::SEPARATOR)}$/
+            File.join(filename, '__WEBSITARY__')
+        else
+            parts = filename.split(/#{Regexp.escape(File::SEPARATOR)}/)
+            if parts.size == 2 and parts[0] =~ /^\w+%3a$/
+                File.join(filename, '__WEBSITARY__')
+            else
+                filename
+            end
+        end
+    end
+
+
+    def open_url(url)
+        $logger.debug "Open URL: #{url}"
+        uri = URI.parse(url)
+        if uri.instance_of?(URI::Generic) or uri.scheme == 'file'
+            open(url)
+        else
+            header = {"User-Agent" => @user_agent}
+            header.merge!(get(url, :header, {}))
+            open(url, header)
+        end
+    end
+
+
+    def difftext_annotation(url)
+        bak = oldname(url)
+        lst = latestname(url)
+        if File.exist?(bak) and File.exist?(lst)
+            eval_arg(get(url, :format_annotation, '%s >>> %s'), [@mtimes.mtime(bak), @mtimes.mtime(lst)])
+        end
+    end
+
+
+    def format_symbol(name, format_string)
+        (format_string % name.to_s).intern
+    end
+
+
+    def format_rss_item(item, body, enclosure='')
+        hd = [item.title]
+        hd << " (#{item.author})" if item.author
+        return <<EOT
+<h2 class="rss"><a class="rss" href="#{item.link}">#{hd.join} -- #{item.pubDate}</a></h2>
+<div class="rss">
+#{body}
+#{enclosure}
+</div>
+EOT
+    end
+
+    # Guess whether text is plain text or html.
+    def is_html?(text)
+        text =~ /<(div|a|span|body|html|script|p|table|td|tr|th|li|dt|br|hr|em|b)\b/
+    end
+
+
+    # Convert html to plain text using hpricot.
+    def html_to_text(text)
+        text && Hpricot(text).to_plain_text
+    end
+
+
+    # Retrieve any robots meta directives from the hpricot document.
+    def robots?(hpricot, *what)
+        (hpricot / '//meta[@name="robots"]').any? do |e|
+            what.any? {|w| e['content'].split(/,\s*/).include?(w)}
+        end
+    end
+
+
+    # Check whether robots are allowed to retrieve an url.
+    def robots_allowed?(url, uri)
+        if @allow.has_key?(url)
+            return @allow[url]
+        end
+
+        if defined?(RobotRules)
+            host = uri.host
+
+            unless (rules = @robots[host])
+                rurl = robots_uri(uri).to_s
+                return true unless rurl
+                begin
+                    robots_txt = open_url(rurl).read
+                    rules      = RobotRules.new(@user_agent)
+                    rules.parse(rurl, robots_txt)
+                    @robots[host] = rules
+                    $logger.info "Loaded #{rurl} for #{@user_agent}"
+                    $logger.debug robots_txt
+                rescue Exception => e
+                    puts e
+                    puts robots_txt
+                end
+            end
+
+            rv = if rules and !rules.allowed?(url)
+                     $logger.info "Excluded url: #{url}"
+                     false
+                 else
+                     true
+                 end
+            @allow[url] = rv
+            return rv
+        end
+
+        unless @robots[:warning]
+            $logger.warn 'robots.txt is ignored: Please install robot_rules.rb from http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/177589 in $RUBYLIB'
+            @robots[:warning] = true
+        end
+        @allow[url] = true
+        return true
+    end
+
+
+    # Get the robots.txt uri for uri.
+    def robots_uri(uri)
+        unless uri.relative?
+            ruri = uri.dup
+            ruri.path = '/robots.txt'
+            ruri
+        end
+    end
+
+
+    def sort_difftext!(difftext)
+        difftext.sort! do |a, b|
+            aa = a[0]
+            bb = b[0]
+            get(aa, :title, aa).downcase <=> get(bb, :title, bb).downcase
+        end
+    end
+
+
+    def file_url(filename)
+        # filename = File.join(File.basename(File.dirname(filename)), File.basename(filename))
+        # "file://#{encode(filename, ':/')}"
+        filename = call_cmd(get_optionvalue(:global, :file_url), [filename], filename)
+        encode(filename, ':/')
+    end
+
+
+    def encode(text, chars='')
+        text.gsub(/[^a-zA-Z0-9,._#{chars}-]/) {|t| '%%%02x' % t[0]}
+    end
+
+
+    def decode(text)
+        text.gsub(/%(..)/) {|t| "%c" % $1.hex}
+    end
+
+
+    def output_suffix(outformat)
+        outformat ||= @output_format[0]
+        @suffix[outformat] || outformat
+    end
+
+
+    def output_basename
+        @profiles.join(',')
+    end
+
+
+    def get_outfile(outformat=nil)
+        @outfile[outformat] || File.join(@cfgdir, "#{output_basename}.#{output_suffix(outformat)}")
+    end
+
+
+    def view_output_general(outfile)
+        if @view
+            system((@view % outfile))
+        end
+    end
+    alias :view_output_html :view_output_general
+    alias :view_output_text :view_output_general
+    alias :view_output_rss :view_output_general
+
+end
+
+
+
+
+# Local Variables:
+# revisionRx: REVISION\s\+=\s\+\'
+# End: