iudex-worker 1.0.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-04-04)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,17 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+bin/iudex-worker-fg
+config/config.rb
+init/iudex-worker
+lib/iudex-worker/base.rb
+lib/iudex-worker.rb
+lib/iudex-worker/agent.rb
+lib/iudex-worker/fetch_helper.rb
+lib/iudex-worker/filter_chain_factory.rb
+lib/iudex-worker/prioritizer.rb
+test/setup.rb
+test/test_agent.rb
+test/test_filter_chain_factory.rb
+test/test_prioritizer.rb

data/README.rdoc

@@ -0,0 +1,25 @@
+= iudex-worker
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-worker gem provides a worker deamon for feed/page
+processing.
+
+== License
+
+Copyright (c) 2008-2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You may
+obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,41 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-worker/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.2'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-worker', Iudex::Worker::VERSION, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+
+  h.extra_deps += [ [ 'iudex-core',     '~> 1.0.0' ],
+                    [ 'rjack-logback',  '~> 1.0'   ],
+                    [ 'iudex-da',       '~> 1.0.0' ],
+                    [ 'iudex-rome',     '~> 1.0.0' ],
+                    [ 'iudex-html',     '~> 1.0.0' ],
+                    [ 'iudex-simhash',  '~> 1.0.0' ],
+                    [ 'iudex-httpclient-3', '~> 1.0.0' ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',   '>= 1.7.1', '< 2.1' ] ]
+end
+
+task :chk_hist_vers do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :chk_init_v do
+  t.test_line_match( 'init/iudex-worker', /^gem.+#{t.name}/, /= #{t.version}/ )
+end
+task :chk_hist_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :chk_hist_vers, :chk_init_v                 ]
+task :tag  => [ :chk_hist_vers, :chk_init_v, :chk_hist_date ]
+task :push => [                              :chk_hist_date ]
+
+t.define_tasks

data/bin/iudex-worker-fg

@@ -0,0 +1,50 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'optparse'
+
+module IudexBinScript
+
+  require 'rubygems'
+  require 'rjack-logback'
+
+  include RJack
+  Logback.config_console( :mdc => "uhash", :thread => true )
+
+  require 'iudex-worker'
+  include Iudex
+
+  Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+  OptionParser.new do |opts|
+    opts.on( "-v", "--version", "Display version" ) do |file|
+      puts "iudex-worker: #{ Worker::VERSION }"
+      exit 1
+    end
+    opts.on( "-d", "--debug", "Enable verbose DEBUG logging" ) do |file|
+      Logback[ 'iudex' ].level = Logback::DEBUG
+    end
+    Hooker.register_config( opts )
+  end.parse!
+
+  Worker::Agent.new.run
+
+end

data/config/config.rb

@@ -0,0 +1,46 @@
+
+RJack::Logback[ 'iudex.filter.core.FilterChain.agent' ].level =
+  RJack::Logback::DEBUG
+
+Iudex.configure do |c|
+
+  threads = 3
+
+  c.setup_connect_props do
+    { :ds_pool  => { :max_active => threads / 3 * 2,
+                     :max_idle   => threads / 3 },
+      :loglevel => 1 }
+  end
+
+  c.setup_http_client_3 do |mgr|
+    mgr.manager_params.max_total_connections = threads * 10
+  end
+
+  c.setup_visit_executor do |vx|
+    vx.max_threads = threads
+    vx.min_host_delay = 100 #ms
+  end
+
+  c.setup_work_poller do |wp|
+    wp.min_order_remaining_ratio = 0.30
+    wp.max_check_interval = 100 #ms
+    wp.min_poll_interval = 2_000 #ms
+  end
+
+  c.setup_filter_factory do |ff|
+
+    def ff.barc_writer
+      bw = super
+      bw.do_compress = false
+      bw
+    end
+
+    def ff.barc_directory
+      bdir = super
+      bdir.target_length = 2 * ( 1024 ** 2 )
+      bdir
+    end
+
+  end
+
+end

data/init/iudex-worker

@@ -0,0 +1,46 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#. hashdot.profile         += daemon
+#. hashdot.pid_file         = ./iudex-worker.pid
+#. hashdot.io_redirect.file = ./iudex-worker.log
+#. hashdot.vm.options += -Xmx1g
+#. hashdot.vm.options += -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rubygems'
+
+gem( "iudex-worker", "= 1.0.0" )
+
+module IudexInitScript
+
+  require 'rjack-logback'
+  include RJack
+  Logback.config_console( :full => true, :thread => true, :mdc => "uhash" )
+
+  require 'iudex-worker'
+  include Iudex
+
+  Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+  if File.exist?( './config.rb' )
+    Hooker.load_file( './config.rb' )
+  end
+
+  Worker::Agent.new.run
+
+end

data/lib/iudex-worker/agent.rb

@@ -0,0 +1,87 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da'
+require 'iudex-da/key_helper'
+require 'iudex-da/pool_data_source_factory'
+
+require 'iudex-httpclient-3'
+
+require 'iudex-worker'
+require 'iudex-worker/filter_chain_factory'
+
+require 'hooker'
+
+module Iudex
+  module Worker
+
+    class Agent
+      include Iudex::DA
+      include Iudex::Filter::KeyHelper
+      include Iudex::Core
+      include Iudex::Worker
+      include Gravitext::HTMap
+
+      def initialize
+        Hooker.apply( [ :iudex, :worker ], self )
+      end
+
+      def poll_keys
+        [ :url, :type, :priority, :next_visit_after, :last_visit, :etag ]
+      end
+
+      # Note this can/is used to override factory in derived classes.
+      def filter_chain_factory
+        FilterChainFactory.new( 'agent' )
+      end
+
+      def run
+        Hooker.with( :iudex ) do
+          dsf = PoolDataSourceFactory.new
+          data_source = dsf.create
+
+          cmapper = ContentMapper.new( keys( poll_keys ) )
+          wpoller = WorkPoller.new( data_source, cmapper )
+          Hooker.apply( :work_poller, wpoller )
+
+          mgr = HTTPClient3.create_manager
+          mgr.start
+          http_client = HTTPClient3::HTTPClient3.new( mgr.client )
+
+          fcf = filter_chain_factory
+          fcf.http_client = http_client
+          fcf.data_source = data_source
+
+          Hooker.apply( :filter_factory, fcf )
+
+          fcf.filter do |chain|
+            vexec = VisitExecutor.new( chain, wpoller )
+            Hooker.apply( :visit_executor, vexec )
+
+            Hooker.log_not_applied # All hooks should be used by now
+
+            vexec.start
+            vexec.join    #Run until interrupted
+          end # fcf closes
+
+          mgr.shutdown
+          dsf.close
+        end
+      end
+    end
+
+  end
+end

data/lib/iudex-worker/base.rb

@@ -0,0 +1,21 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module Worker
+    VERSION = '1.0.0'
+  end
+end

data/lib/iudex-worker/fetch_helper.rb

@@ -0,0 +1,79 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-worker'
+
+module Iudex
+  module Worker
+
+    module FetchHelper
+      include Iudex::HTTP
+      include Iudex::Core::Filters
+
+      def create_content_fetcher( accept_types, receiver_sym )
+        cf = ContentFetcher.new( http_client, create_chain( receiver_sym ) )
+
+        alist = accept_list( accept_types )
+        cf.accepted_content_types = alist unless alist.include?( '*/*' )
+
+        headers = [ [ 'User-Agent', http_user_agent ],
+                    [ 'Accept',     accept_header( accept_types ) ] ]
+
+        cf.request_headers = headers.map { |kv| Header.new( *kv ) }
+
+        cf
+      end
+
+      def http_user_agent
+        ( "Mozilla/5.0 (compatible; " +
+          "Iudex #{Iudex::Worker::VERSION}; " +
+           "+http://gravitext.com/iudex)" )
+      end
+
+      def feed_mime_types
+        # List of accepted mime types grouped and order in descending
+        # order of preference.
+        [ %w[ application/atom+xml application/rss+xml ],
+          %w[ application/rdf+xml application/xml ],
+          %w[ text/xml ],
+          %w[ text/* ],
+          %w[ */* ] ]
+      end
+
+      def page_mime_types
+        [ %w[ application/xhtml+xml text/html ],
+          %w[ application/xml ],
+          %w[ text/* ] ]
+      end
+
+      def accept_header( types )
+        q = 1.0
+        ts = types.map do |tgrp|
+          tgrp = tgrp.map { |m| "#{m};q=#{q}" } if q < 1.0
+          q -= 0.1
+          tgrp
+        end
+        ts.flatten.join( ',' )
+      end
+
+      def accept_list( types )
+        types.flatten
+      end
+
+    end
+
+  end
+end

data/lib/iudex-worker/filter_chain_factory.rb

@@ -0,0 +1,201 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-filter'
+require 'iudex-filter/filter_chain_factory'
+
+require 'iudex-barc'
+
+require 'iudex-core'
+
+require 'iudex-da'
+require 'iudex-da/factory_helper'
+
+require 'iudex-rome'
+
+require 'iudex-html'
+require 'iudex-html/factory_helper'
+
+require 'iudex-simhash'
+require 'iudex-simhash/factory_helper'
+
+require 'iudex-worker'
+require 'iudex-worker/fetch_helper'
+require 'iudex-worker/prioritizer'
+
+module Iudex
+  module Worker
+
+    class FilterChainFactory < Iudex::Filter::Core::FilterChainFactory
+      include Iudex::Filter::Core
+      include Iudex::BARC
+      include Iudex::Core
+      include Iudex::Core::Filters
+      include Iudex::ROME
+
+      include Iudex::DA::Filters::FactoryHelper
+      include Iudex::HTML::Filters::FactoryHelper
+      include Iudex::SimHash::Filters::FactoryHelper
+      include FetchHelper
+
+      attr_accessor :http_client
+      attr_accessor :data_source
+
+      def initialize( name )
+        super
+        setup_reporters
+      end
+
+      def setup_reporters
+        add_summary_reporter
+        add_by_filter_reporter
+      end
+
+      def filters
+        [ UHashMDCSetter.new,
+          DefaultFilter.new,
+          super,
+          type_switch ].flatten
+      end
+
+      def listeners
+        super + [ MDCUnsetter.new( "uhash" ) ]
+      end
+
+      def type_map
+        { "FEED" => feed_fetcher,
+          "PAGE" => page_fetcher }
+      end
+
+      def type_switch( tmap = type_map )
+        create_switch( :type.to_k, tmap )
+      end
+
+      def feed_fetcher
+        [ create_content_fetcher( feed_mime_types, :feed_receiver ) ]
+      end
+
+      def page_fetcher
+        [ create_content_fetcher( page_mime_types, :page_receiver ) ]
+      end
+
+      def feed_receiver
+        [ RomeFeedParser.new,
+          DefaultFilter.new,
+          DateChangeFilter.new( false ),
+          feed_updater ]
+      end
+
+      def feed_updater
+        create_update_filter( keys( feed_update_keys ),
+                              :feed_post, :feed_ref_update, :feed_ref_new )
+      end
+
+      def feed_ref_new
+        [ UHashMDCSetter.new,
+          ref_common_cleanup,
+          Prioritizer.new( "feed-ref-new",
+                           :constant => 50,
+                           :min_next => 0.0 ) ].flatten
+      end
+
+      def feed_ref_update
+        [ UHashMDCSetter.new,
+          DateChangeFilter.new( true ),
+          ref_common_cleanup,
+          Prioritizer.new( "feed-ref-update",
+                           :constant => 10,
+                           :min_next => 0.0 ) ].flatten
+      end
+
+      # Note: *_post is run possibly twice, once for both base content
+      # map and referer map.
+      def feed_post
+        [ UHashMDCSetter.new,
+          ref_common_cleanup,
+          Prioritizer.new( "feed-post",
+                           :constant => 30,
+                           :visiting_now => true ),
+          last_visit_setter ].flatten
+      end
+
+      def ref_common_cleanup
+        [ ref_html_filters,
+          TextCtrlWSFilter.new( :title.to_k ),
+          FutureDateFilter.new( :pub_date.to_k ) ].flatten
+      end
+
+      def ref_html_filters
+        [ html_clean_filters( :title ),
+          html_clean_filters( :summary ),
+          html_clean_filters( :content ),
+          html_write_filter( :summary ),
+          html_write_filter( :content ) ].flatten
+      end
+
+      def feed_update_keys
+        page_update_keys + [ :title, :summary, :content ]
+      end
+
+      def page_receiver
+        [ html_clean_filters( :source ),
+          simhash_generator,
+          page_updater ].flatten
+      end
+
+      def barc_writer
+        bw = BARCWriter.new( barc_directory )
+        bw.do_compress = true
+        bw
+      end
+
+      def barc_directory
+        bdir = BARCDirectory.new( Java::java.io.File.new( "./barc" ) )
+        bdir
+      end
+
+      def page_updater
+        create_update_filter( keys( page_update_keys ), :page_post )
+      end
+
+      # Note: *_post is run possibly twice, once for both base content
+      # map and referer map.
+      def page_post
+        [ UHashMDCSetter.new,
+          barc_writer, # Not run in 302 referer case, since no SOURCE.
+          Prioritizer.new( "page-post",
+                           :constant => 0,
+                           :min_next => ( 30 * 60.0 ),
+                           :visiting_now => true ),
+          last_visit_setter ]
+      end
+
+      def page_update_keys
+        [ :uhash, :host, :url, :type,
+          :ref_pub_date, :pub_date,
+          :priority, :last_visit, :next_visit_after,
+          :status, :etag, :reason, :referer, :referent,
+          :cache_file, :cache_file_offset, :simhash ]
+      end
+
+      def last_visit_setter
+        Copier.new( *keys( :visit_start, :last_visit ) )
+      end
+
+    end
+
+  end
+end

data/lib/iudex-worker/prioritizer.rb

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module Worker
+
+    class Prioritizer < Iudex::Filter::FilterBase
+      include Math
+
+      attr_accessor :constant
+      attr_accessor :impedance
+      attr_accessor :min_next
+      attr_accessor :min_next_unmodified
+      attr_accessor :factors
+      attr_accessor :visiting_now
+
+      WWW_BEGINS = Time.utc( 1991, "aug", 6, 20,0,0 ) # WWW begins
+      MINUTE     = 60.0
+      HOUR       = 60.0 * 60.0
+
+      def initialize( name, opts = {} )
+        @name = name
+
+        @constant            = 0.0
+        @impedance           = 2.0
+        @min_next_unmodified =  5 * MINUTE
+        @min_next            = 10 * MINUTE
+        @visiting_now        = false
+
+        @factors = [ [ 30.0, :ref_change_rate ],
+                     [ -1.0, :log_pub_age ] ]
+
+        @log = RJack::SLF4J[ self.class ]
+
+        opts.each { |k,v| send( k.to_s + '=', v ) }
+        yield self if block_given?
+
+        @min_next_unmodified = [ @min_next_unmodified, @min_next ].min
+        @constant = @constant.to_f
+      end
+
+      def describe
+        [ @name, @constant, @min_next ]
+      end
+
+      def filter( map )
+
+        map.priority, delta = adjust( map, map.priority )
+
+        map.next_visit_after = ( as_time( map.visit_start ) + delta if delta )
+
+        true
+      end
+
+      def adjust( map, priority, delta = 0.0 )
+
+        old_priority = priority
+        memo = ( ( ( @constant != 0.0 ) ? [ @constant ] : [] ) if @log.debug? )
+
+        new_priority = @factors.inject( @constant ) do | p, (w,func)|
+          comp = ( w * send( func, map ) )
+          ( memo << "%.1f:%s" % [ comp.to_f, func ] ) if memo && comp != 0.0
+          p + comp
+        end
+
+        #FIXME: new_priority = [ 0.0, new_priority ].max
+
+        priority = ( ( ( priority || 0.0 ) * @impedance + new_priority ) /
+                     ( @impedance + 1 ) )
+
+        if map.last_visit || visiting_now
+          delta = ( map.status == 304 ) ? @min_next_unmodified : @min_next
+        else
+          delta = 0.0
+        end
+
+        @log.debug do
+          memo.join( ' + ' ) +
+            ( " :: %.1f -> %.1f = %.1f in %.1fs" %
+              ( [ old_priority, new_priority,
+                  priority, delta ].map { |f| f.to_f } ) )
+        end
+
+        [ priority, delta ]
+      end
+
+      def log_pub_age( map )
+        diff = sdiff( ( map.pub_date || WWW_BEGINS ), map.visit_start ) / MINUTE
+        diff = 1.0 / MINUTE if diff < 1.0 / MINUTE
+        ( log( diff ) - log( 1.0 / MINUTE ) )
+      end
+
+      # FIXME: Useful?
+      # def ref_pub_age( map )
+      #   map.visit_start - ( map.ref_pub_date || WWW_BEGINS )
+      # end
+
+      # References per hour, with updates rated at 1/4 a new reference.
+      def ref_change_rate( map )
+        s = since( map )
+        if s.nil? || s == 0.0
+          0.0
+        else
+          ( ( ( map.new_references || 0.0 ) +
+              ( map.updated_references || 0.0 ) / 4.0 ) /
+            s *
+            HOUR )
+        end
+      end
+
+      def since( map )
+        sdiff( map.last_visit || oldest( map.references ),
+               map.visit_start )
+      end
+
+      def oldest( refs )
+        ( refs.map { |r| r.pub_date }.compact.min ) if refs
+      end
+
+      def sdiff( prev, now )
+        diff = as_time( now ) - as_time( prev || WWW_BEGINS )
+        ( diff < 0.0 ) ? 0.0 : diff
+      end
+
+      # FIXME: Generalize?
+      def as_time( torj )
+        if torj.is_a?( Time )
+          torj
+        else
+          ms = torj.time
+          Time.at( ms / 1_000, ( ms % 1_000 ) * 1_000 ) # s, µs
+        end
+      end
+
+    end
+
+  end
+end

data/lib/iudex-worker.rb

@@ -0,0 +1,20 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+
+require 'iudex-worker/base'
+require 'iudex-worker/agent'

data/test/setup.rb

@@ -0,0 +1,34 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+RJack::Logback.config_console( :stderr => true )
+
+require 'minitest/unit'
+require 'minitest/autorun'
+
+# Make test output logging compatible: no partial lines.
+class TestOut
+  def print( *a ); $stdout.puts( *a ); end
+  def puts( *a );  $stdout.puts( *a ); end
+end
+MiniTest::Unit.output = TestOut.new

data/test/test_agent.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-worker'
+
+class TestAgent < MiniTest::Unit::TestCase
+  include Iudex::Worker
+  include RJack
+
+  def setup
+    Logback[ 'iudex.worker.FilterChainFactory' ].level = Logback::WARN
+  end
+
+  def teardown
+    Logback[ 'iudex.worker.FilterChainFactory' ].level = nil
+    Hooker.send( :clear )
+  end
+
+  def test_agent_default
+    assert_agent
+  end
+
+  def test_agent_with_sample_config
+    # Test out the sample config
+    Hooker.load_file( File.join( File.dirname( __FILE__ ),
+                                 '..', 'config', 'config.rb' ) )
+
+    assert_agent
+  end
+
+  def assert_agent
+
+    # Stub VisitExecutor.start to allow agent.run to return early.
+    Hooker.add( [ :iudex, :visit_executor ] ) do |vexec|
+      def vexec.start
+        #disable
+      end
+    end
+
+    agent = Agent.new
+    agent.run
+    pass
+
+    Hooker.check_not_applied do |*args|
+      flunk( "Hooks not applied: " + args.inspect )
+    end
+    pass
+
+  end
+
+end

data/test/test_filter_chain_factory.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+RJack::Logback.config_console( :stderr => true, :mdc => "uhash" )
+
+RJack::Logback[ 'iudex' ].level = RJack::Logback::DEBUG
+
+require 'iudex-httpclient-3'
+
+require 'iudex-da'
+require 'iudex-da/pool_data_source_factory'
+
+require 'iudex-worker'
+require 'iudex-worker/filter_chain_factory'
+
+class TestFilterChainFactory < MiniTest::Unit::TestCase
+  include Iudex
+  include Gravitext::HTMap
+
+  def test_filter
+    fcf = Worker::FilterChainFactory.new( "test" )
+
+    mgr = HTTPClient3.create_manager
+    mgr.start
+    fcf.http_client = HTTPClient3::HTTPClient3.new( mgr.client )
+
+    dsf = DA::PoolDataSourceFactory.new
+    fcf.data_source = dsf.create
+
+    fcf.filter do |chain|
+      # Run twice (assume new the first time, updates the second).
+      2.times do
+        content = UniMap.new
+        content.url = Core::VisitURL.normalize( "http://gravitext.com/atom.xml" )
+        content.type = "FEED"
+        content.priority = 1.0
+        assert( chain.filter( content ) )
+      end
+    end
+
+    mgr.shutdown
+    dsf.close
+  end
+
+end

data/test/test_prioritizer.rb

@@ -0,0 +1,105 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+RJack::Logback.config_console( :stderr => true, :mdc => "uhash" )
+
+RJack::Logback[ 'iudex' ].level = RJack::Logback::DEBUG
+
+require 'iudex-worker'
+require 'iudex-worker/prioritizer'
+
+class TestPrioritizer < MiniTest::Unit::TestCase
+  include Iudex::Worker
+  include Gravitext::HTMap
+  JDate = Java::java.util.Date
+
+  UniMap.define_accessors
+
+  def test_identity
+    m = new_map
+    p = Prioritizer.new( "test", :constant => 3.2,
+                         :factors => [], :impedance => 0 )
+
+    assert( p.filter( m ) )
+    assert_equal_fuzzy( 3.2, m.priority )
+    assert_equal( m.visit_start, m.next_visit_after )
+  end
+
+  def test_visiting_now
+    m = new_map
+    p = Prioritizer.new( "test", :visiting_now => true )
+
+    assert( p.filter( m ) )
+    assert_equal_fuzzy( m.visit_start.time/1000.0 + p.min_next,
+                        m.next_visit_after.time/1000.0 )
+  end
+
+  def test_oldest
+    map = new_map
+
+    times = [ Time.utc( 2010, "jul", 17, 19,0,0 ),
+              oldest = Time.utc( 2010, "jul", 17, 18,0,0 ),
+              Time.utc( 2010, "jul", 17, 20,0,0 ),
+              nil ]
+
+    map.references = times.map do |t|
+      ref = UniMap.new
+      ref.pub_date = t
+      ref
+    end
+
+    p = prioritizer
+    assert_equal( oldest, p.as_time( p.oldest( map.references ) ) )
+  end
+
+  def test_since_last
+    assert_equal( 60.0, prioritizer.since( one_minute_last_map ) )
+  end
+
+  def test_ref_change_rate
+    map = one_minute_last_map
+    map.new_references = 1
+    map.updated_references = 4
+    assert_equal_fuzzy( 120, prioritizer.ref_change_rate( map ) )
+  end
+
+  def one_minute_last_map
+    map = new_map
+    map.visit_start = start = JDate.new
+    map.last_visit = JDate.new( start.time - ( 1_000 * 60 ) )
+    map
+  end
+
+  def assert_equal_fuzzy( l, r )
+    assert( ( l - r ).abs < 0.1, "#{l} ~!= #{r}" )
+  end
+
+  def new_map
+    map = UniMap.new
+    map.visit_start = JDate.new
+    map
+  end
+
+  def prioritizer
+    Prioritizer.new( "test" )
+  end
+
+end

metadata

@@ -0,0 +1,182 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-worker
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-core
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: iudex-da
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: iudex-rome
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: iudex-html
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: iudex-simhash
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id006
+  - !ruby/object:Gem::Dependency 
+    name: iudex-httpclient-3
+    prerelease: false
+    requirement: &id007 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id007
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id008 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.7.1
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+    type: :development
+    version_requirements: *id008
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id009 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :development
+    version_requirements: *id009
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-worker gem provides a worker deamon for feed/page
+  processing.
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - iudex-worker-fg
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - bin/iudex-worker-fg
+  - config/config.rb
+  - init/iudex-worker
+  - lib/iudex-worker/base.rb
+  - lib/iudex-worker.rb
+  - lib/iudex-worker/agent.rb
+  - lib/iudex-worker/fetch_helper.rb
+  - lib/iudex-worker/filter_chain_factory.rb
+  - lib/iudex-worker/prioritizer.rb
+  - test/setup.rb
+  - test/test_agent.rb
+  - test/test_filter_chain_factory.rb
+  - test/test_prioritizer.rb
+has_rdoc: true
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-worker
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_agent.rb
+  - test/test_filter_chain_factory.rb
+  - test/test_prioritizer.rb