iudex-worker 1.0.0-java → 1.1.0-java

data/History.rdoc

@@ -1,2 +1,14 @@
+=== 1.1.0 (2011-11-13)
+* Update to iudex-core, -da, -rome, -html, -simhash ~> 1.1.0
+* Changes for VisitManager, VisitCounter, RedirectHandler, Revisitor
+* Use ContentTypeSet in ContentFetcher
+* Generalize Agent to for all three HTTP clients; all are now
+  optional/dev dependencies
+* Add iudex-char-detector ~> 1.1.0 dep and use CharDetectFilter in
+  FCF.page_receiver
+* Improved setup error logging in Agent
+* Enable :main listeners in filter_chain_factory
+* Update to minitest ~> 2.3
+
 === 1.0.0 (2011-04-04)
 * Initial release.

data/Manifest.txt

@@ -4,6 +4,8 @@ README.rdoc
 Rakefile
 bin/iudex-worker-fg
 config/config.rb
+config/config_async_http.rb
+config/config_jetty_http.rb
 init/iudex-worker
 lib/iudex-worker/base.rb
 lib/iudex-worker.rb

data/Rakefile

@@ -4,7 +4,7 @@ $LOAD_PATH << './lib'
 require 'iudex-worker/base'
 
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.2'
+gem     'rjack-tarpit', '~> 1.4'
 require 'rjack-tarpit'
 
 t = RJack::TarPit.new( 'iudex-worker', Iudex::Worker::VERSION, :java_platform )
@@ -12,16 +12,19 @@ t = RJack::TarPit.new( 'iudex-worker', Iudex::Worker::VERSION, :java_platform )
 t.specify do |h|
   h.developer( "David Kellum", "dek-oss@gravitext.com" )
 
-  h.extra_deps += [ [ 'iudex-core',     '~> 1.0.0' ],
+  h.extra_deps += [ [ 'iudex-core',     '~> 1.1.0' ],
                     [ 'rjack-logback',  '~> 1.0'   ],
-                    [ 'iudex-da',       '~> 1.0.0' ],
-                    [ 'iudex-rome',     '~> 1.0.0' ],
-                    [ 'iudex-html',     '~> 1.0.0' ],
-                    [ 'iudex-simhash',  '~> 1.0.0' ],
-                    [ 'iudex-httpclient-3', '~> 1.0.0' ] ]
+                    [ 'iudex-da',       '~> 1.1.0' ],
+                    [ 'iudex-rome',     '~> 1.1.0' ],
+                    [ 'iudex-html',     '~> 1.1.0' ],
+                    [ 'iudex-simhash',  '~> 1.1.0' ],
+                    [ 'iudex-char-detector', '~> 1.1.0' ] ]
 
   h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',   '>= 1.7.1', '< 2.1' ] ]
+  h.extra_dev_deps += [ [ 'minitest',   '~> 2.3' ],
+                        [ 'iudex-httpclient-3',     '~> 1.1.0' ],
+                        [ 'iudex-jetty-httpclient', '~> 1.1.0' ],
+                        [ 'iudex-async-httpclient', '~> 1.1.0' ] ]
 end
 
 task :chk_hist_vers do

data/bin/iudex-worker-fg

@@ -35,11 +35,11 @@ module IudexBinScript
   Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
 
   OptionParser.new do |opts|
-    opts.on( "-v", "--version", "Display version" ) do |file|
+    opts.on( "-v", "--version", "Display version" ) do
       puts "iudex-worker: #{ Worker::VERSION }"
       exit 1
     end
-    opts.on( "-d", "--debug", "Enable verbose DEBUG logging" ) do |file|
+    opts.on( "-d", "--debug", "Enable verbose DEBUG logging" ) do
       Logback[ 'iudex' ].level = Logback::DEBUG
     end
     Hooker.register_config( opts )

data/config/config.rb

@@ -7,7 +7,8 @@ Iudex.configure do |c|
   threads = 3
 
   c.setup_connect_props do
-    { :ds_pool  => { :max_active => threads / 3 * 2,
+    { :database => 'iudex_test',
+      :ds_pool  => { :max_active => threads / 3 * 2,
                      :max_idle   => threads / 3 },
       :loglevel => 1 }
   end
@@ -16,9 +17,15 @@ Iudex.configure do |c|
     mgr.manager_params.max_total_connections = threads * 10
   end
 
-  c.setup_visit_executor do |vx|
+  c.setup_visit_manager do |vx|
     vx.max_threads = threads
-    vx.min_host_delay = 100 #ms
+  end
+
+  c.setup_visit_queue do |q|
+    q.default_min_host_delay = 100 #ms
+    q.default_max_access_per_host = 1
+
+    q.configure_host( "gravitext.com", 100, 2 ) # 100ms, 2 connections
   end
 
   c.setup_work_poller do |wp|
@@ -30,15 +37,15 @@ Iudex.configure do |c|
   c.setup_filter_factory do |ff|
 
     def ff.barc_writer
-      bw = super
-      bw.do_compress = false
-      bw
+      super.tap do |w|
+        w.do_compress = false
+      end
     end
 
     def ff.barc_directory
-      bdir = super
-      bdir.target_length = 2 * ( 1024 ** 2 )
-      bdir
+      super.tap do |bdir|
+        bdir.target_length = 2 * ( 1024 ** 2 )
+      end
     end
 
   end

data/config/config_async_http.rb

@@ -0,0 +1,13 @@
+require 'iudex-async-httpclient'
+
+Iudex.configure do |c|
+
+  c.setup_async_httpclient do
+    { :connection_timeout_in_ms      =>  5_000,
+      :request_timeout_in_ms         => 10_000,
+      :idle_connection_timeout_in_ms =>  6_000,
+      :maximum_connections_total     =>    200,
+      :maximum_connections_per_host  =>      5 }
+  end
+
+end

data/config/config_jetty_http.rb

@@ -0,0 +1,11 @@
+require 'iudex-jetty-httpclient'
+
+Iudex.configure do |c|
+
+  c.setup_jetty_httpclient do
+    { :timeout => 20_000,
+      :max_connections_per_address => 2,
+      :max_queue_size_per_address => 20 }
+  end
+
+end

data/init/iudex-worker

@@ -24,7 +24,7 @@
 
 require 'rubygems'
 
-gem( "iudex-worker", "= 1.0.0" )
+gem( "iudex-worker", "= 1.1.0" )
 
 module IudexInitScript
 

data/lib/iudex-worker/agent.rb

@@ -18,8 +18,6 @@ require 'iudex-da'
 require 'iudex-da/key_helper'
 require 'iudex-da/pool_data_source_factory'
 
-require 'iudex-httpclient-3'
-
 require 'iudex-worker'
 require 'iudex-worker/filter_chain_factory'
 
@@ -36,6 +34,8 @@ module Iudex
       include Gravitext::HTMap
 
       def initialize
+        @log = RJack::SLF4J[ self.class ]
+        @http_manager = nil
         Hooker.apply( [ :iudex, :worker ], self )
       end
 
@@ -48,28 +48,66 @@ module Iudex
         FilterChainFactory.new( 'agent' )
       end
 
+      def http_client( executor )
+        if defined?( JettyHTTPClient )
+          @log.info "Setting up JettyHTTPClient"
+          JettyHTTPClient.create_client.tap do |c|
+            c.executor = executor
+            c.start
+          end
+        elsif defined?( AsyncHTTPClient )
+          @log.info "Setting up AsyncHTTPClient"
+          AsyncHTTPClient.create_client( :executor_service => executor )
+        else
+          gem     'iudex-httpclient-3', '~> 1.1.0'
+          require 'iudex-httpclient-3'
+          @log.info "Setting up HTTPClient3"
+          @http_manager = HTTPClient3.create_manager
+          @http_manager.start
+          HTTPClient3::HTTPClient3.new( @http_manager.client )
+        end
+      end
+
+      def visit_manager( wpoller )
+        vexec = VisitManager.new( wpoller )
+        Hooker.apply( [ :iudex, :visit_manager ], vexec )
+      end
+
+      def work_poller( data_source )
+        cmapper = ContentMapper.new( keys( poll_keys ) )
+        wpoller = WorkPoller.new( data_source, cmapper )
+
+        visit_q = Hooker.apply( [ :iudex, :visit_queue ], VisitQueue.new )
+
+        wpoller.visit_queue_factory = VisitQueueFactory.new( visit_q )
+
+        Hooker.apply( [ :iudex, :work_poller ], wpoller )
+      end
+
       def run
         Hooker.with( :iudex ) do
           dsf = PoolDataSourceFactory.new
           data_source = dsf.create
 
-          cmapper = ContentMapper.new( keys( poll_keys ) )
-          wpoller = WorkPoller.new( data_source, cmapper )
-          Hooker.apply( :work_poller, wpoller )
+          wpoller = work_poller( data_source )
+          vexec = visit_manager( wpoller )
+          vexec.start_executor
 
-          mgr = HTTPClient3.create_manager
-          mgr.start
-          http_client = HTTPClient3::HTTPClient3.new( mgr.client )
+          hclient = http_client( vexec.executor )
 
           fcf = filter_chain_factory
-          fcf.http_client = http_client
+          fcf.http_client = hclient
           fcf.data_source = data_source
+          fcf.visit_counter = vexec
+
+          # FilterChain's executor is the same executor, unless using
+          # HTTPClient3, where executor is best not used
+          fcf.executor = vexec.executor unless @http_manager
 
           Hooker.apply( :filter_factory, fcf )
 
           fcf.filter do |chain|
-            vexec = VisitExecutor.new( chain, wpoller )
-            Hooker.apply( :visit_executor, vexec )
+            vexec.filter_chain = chain
 
             Hooker.log_not_applied # All hooks should be used by now
 
@@ -77,10 +115,15 @@ module Iudex
             vexec.join    #Run until interrupted
           end # fcf closes
 
-          mgr.shutdown
+          hclient.close if hclient.respond_to?( :close )
+          @http_manager.shutdown if @http_manager
+
           dsf.close
         end
+      rescue => e
+        @log.error( "On run: ", e )
       end
+
     end
 
   end

data/lib/iudex-worker/base.rb

@@ -16,6 +16,6 @@
 
 module Iudex
   module Worker
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
   end
 end

data/lib/iudex-worker/fetch_helper.rb

@@ -23,11 +23,17 @@ module Iudex
       include Iudex::HTTP
       include Iudex::Core::Filters
 
-      def create_content_fetcher( accept_types, receiver_sym )
-        cf = ContentFetcher.new( http_client, create_chain( receiver_sym ) )
+      def create_content_fetcher( accept_types, receiver, listener = nil )
+        cf = ContentFetcher.new( http_client,
+                                 visit_counter,
+                                 create_chain( receiver, nil, listener ) )
+
+        cf.executor = executor if executor
 
         alist = accept_list( accept_types )
-        cf.accepted_content_types = alist unless alist.include?( '*/*' )
+        unless alist.include?( '*/*' )
+          cf.accepted_content_types = ContentTypeSet.new( alist )
+        end
 
         headers = [ [ 'User-Agent', http_user_agent ],
                     [ 'Accept',     accept_header( accept_types ) ] ]

data/lib/iudex-worker/filter_chain_factory.rb

@@ -26,6 +26,8 @@ require 'iudex-da/factory_helper'
 
 require 'iudex-rome'
 
+require 'iudex-char-detector'
+
 require 'iudex-html'
 require 'iudex-html/factory_helper'
 
@@ -45,6 +47,7 @@ module Iudex
       include Iudex::Core
       include Iudex::Core::Filters
       include Iudex::ROME
+      include Iudex::CharDetector
 
       include Iudex::DA::Filters::FactoryHelper
       include Iudex::HTML::Filters::FactoryHelper
@@ -53,6 +56,8 @@ module Iudex
 
       attr_accessor :http_client
       attr_accessor :data_source
+      attr_accessor :visit_counter
+      attr_accessor :executor
 
       def initialize( name )
         super
@@ -60,15 +65,14 @@ module Iudex
       end
 
       def setup_reporters
-        add_summary_reporter
-        add_by_filter_reporter
+        # Use default, preserved for overrides
       end
 
       def filters
         [ UHashMDCSetter.new,
           DefaultFilter.new,
           super,
-          type_switch ].flatten
+          type_switch ]
       end
 
       def listeners
@@ -76,8 +80,8 @@ module Iudex
       end
 
       def type_map
-        { "FEED" => feed_fetcher,
-          "PAGE" => page_fetcher }
+        { "FEED" => [ feed_fetcher, :main ],
+          "PAGE" => [ page_fetcher, :main ] }
       end
 
       def type_switch( tmap = type_map )
@@ -85,15 +89,17 @@ module Iudex
       end
 
       def feed_fetcher
-        [ create_content_fetcher( feed_mime_types, :feed_receiver ) ]
+        [ create_content_fetcher( feed_mime_types, :feed_receiver, :main ) ]
       end
 
       def page_fetcher
-        [ create_content_fetcher( page_mime_types, :page_receiver ) ]
+        [ create_content_fetcher( page_mime_types, :page_receiver, :main ) ]
       end
 
       def feed_receiver
-        [ RomeFeedParser.new,
+        [ RedirectHandler.new,
+          Revisitor.new( visit_counter ),
+          RomeFeedParser.new,
           DefaultFilter.new,
           DateChangeFilter.new( false ),
           feed_updater ]
@@ -109,7 +115,7 @@ module Iudex
           ref_common_cleanup,
           Prioritizer.new( "feed-ref-new",
                            :constant => 50,
-                           :min_next => 0.0 ) ].flatten
+                           :min_next => 0.0 ) ]
       end
 
       def feed_ref_update
@@ -118,24 +124,32 @@ module Iudex
           ref_common_cleanup,
           Prioritizer.new( "feed-ref-update",
                            :constant => 10,
-                           :min_next => 0.0 ) ].flatten
-      end
-
-      # Note: *_post is run possibly twice, once for both base content
-      # map and referer map.
+                           :min_next => 0.0 ) ]
+      end
+
+      # Filters to apply for feed update.
+      #
+      # Notes:
+      #
+      # * This is run possibly twice, for both base content map and
+      #   referer map if present.
+      #
+      # * If this is an update then these filters act on a *new* map,
+      #   thus any changes made here will not be visible after exit
+      #   from the update_filter.
       def feed_post
         [ UHashMDCSetter.new,
           ref_common_cleanup,
           Prioritizer.new( "feed-post",
                            :constant => 30,
                            :visiting_now => true ),
-          last_visit_setter ].flatten
+          last_visit_setter ]
       end
 
       def ref_common_cleanup
         [ ref_html_filters,
           TextCtrlWSFilter.new( :title.to_k ),
-          FutureDateFilter.new( :pub_date.to_k ) ].flatten
+          FutureDateFilter.new( :pub_date.to_k ) ]
       end
 
       def ref_html_filters
@@ -143,7 +157,7 @@ module Iudex
           html_clean_filters( :summary ),
           html_clean_filters( :content ),
           html_write_filter( :summary ),
-          html_write_filter( :content ) ].flatten
+          html_write_filter( :content ) ]
       end
 
       def feed_update_keys
@@ -151,9 +165,12 @@ module Iudex
       end
 
       def page_receiver
-        [ html_clean_filters( :source ),
+        [ RedirectHandler.new,
+          Revisitor.new( visit_counter ),
+          CharDetectFilter.new,
+          html_clean_filters( :source ),
           simhash_generator,
-          page_updater ].flatten
+          page_updater ]
       end
 
       def barc_writer
@@ -171,8 +188,16 @@ module Iudex
         create_update_filter( keys( page_update_keys ), :page_post )
       end
 
-      # Note: *_post is run possibly twice, once for both base content
-      # map and referer map.
+      # Filters to apply during page update
+      #
+      # Notes:
+      #
+      # * This is run possibly twice, for both base content map and
+      #   referer map if present.
+      #
+      # * If this is an update then these filters act on a *new* map,
+      #   thus any changes made here will not be visible after exit
+      #   from the update_filter.
       def page_post
         [ UHashMDCSetter.new,
           barc_writer, # Not run in 302 referer case, since no SOURCE.
@@ -184,7 +209,7 @@ module Iudex
       end
 
       def page_update_keys
-        [ :uhash, :host, :url, :type,
+        [ :uhash, :domain, :url, :type,
           :ref_pub_date, :pub_date,
           :priority, :last_visit, :next_visit_after,
           :status, :etag, :reason, :referer, :referent,

data/test/setup.rb

@@ -21,7 +21,7 @@ $LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
 
 require 'rubygems'
 require 'rjack-logback'
-RJack::Logback.config_console( :stderr => true )
+RJack::Logback.config_console( :stderr => true, :mdc => "uhash" )
 
 require 'minitest/unit'
 require 'minitest/autorun'

data/test/test_agent.rb

@@ -27,6 +27,7 @@ class TestAgent < MiniTest::Unit::TestCase
 
   def setup
     Logback[ 'iudex.worker.FilterChainFactory' ].level = Logback::WARN
+    Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
   end
 
   def teardown
@@ -46,11 +47,33 @@ class TestAgent < MiniTest::Unit::TestCase
     assert_agent
   end
 
+  def test_agent_with_sample_config_jetty_http
+    Hooker.load_file( File.join( File.dirname( __FILE__ ),
+                                 '..', 'config', 'config_jetty_http.rb' ) )
+
+    assert_agent
+  ensure
+    # Hack to avoid interference in test of other configs, given
+    # require iudex-jetty-httpclient. This only works once, but
+    # thats enough for testing.
+    Iudex.send( :remove_const, :JettyHTTPClient )
+  end
+
+  def test_agent_with_sample_config_async_http
+    Hooker.load_file( File.join( File.dirname( __FILE__ ),
+                                 '..', 'config', 'config_async_http.rb' ) )
+
+    assert_agent
+  ensure
+    # Same hack as above.
+    Iudex.send( :remove_const, :AsyncHTTPClient )
+  end
+
   def assert_agent
 
-    # Stub VisitExecutor.start to allow agent.run to return early.
-    Hooker.add( [ :iudex, :visit_executor ] ) do |vexec|
-      def vexec.start
+    # Stub VisitManager.start to allow agent.run to return early.
+    Hooker.add( [ :iudex, :visit_manager ] ) do |vm|
+      def vm.start
         #disable
       end
     end

data/test/test_filter_chain_factory.rb

@@ -19,10 +19,6 @@
 
 require File.join( File.dirname( __FILE__ ), "setup" )
 
-RJack::Logback.config_console( :stderr => true, :mdc => "uhash" )
-
-RJack::Logback[ 'iudex' ].level = RJack::Logback::DEBUG
-
 require 'iudex-httpclient-3'
 
 require 'iudex-da'
@@ -30,34 +26,70 @@ require 'iudex-da/pool_data_source_factory'
 
 require 'iudex-worker'
 require 'iudex-worker/filter_chain_factory'
+require 'iudex-httpclient-3'
 
 class TestFilterChainFactory < MiniTest::Unit::TestCase
   include Iudex
   include Gravitext::HTMap
 
+  def setup
+    RJack::Logback[ 'iudex' ].level = RJack::Logback::DEBUG
+  end
+
+  def teardown
+    RJack::Logback[ 'iudex' ].level = nil
+  end
+
+  import 'iudex.core.VisitCounter'
+  class TestVisitCounter
+    include VisitCounter
+
+    attr_reader :released
+
+    def initialize
+      super()
+      @released = []
+    end
+
+    def release( acquired, newOrder )
+      @released << acquired
+    end
+  end
+
   def test_filter
     fcf = Worker::FilterChainFactory.new( "test" )
 
     mgr = HTTPClient3.create_manager
     mgr.start
     fcf.http_client = HTTPClient3::HTTPClient3.new( mgr.client )
+    fcf.visit_counter = counter = TestVisitCounter.new
 
     dsf = DA::PoolDataSourceFactory.new
     fcf.data_source = dsf.create
 
     fcf.filter do |chain|
       # Run twice (assume new the first time, updates the second).
-      2.times do
-        content = UniMap.new
-        content.url = Core::VisitURL.normalize( "http://gravitext.com/atom.xml" )
-        content.type = "FEED"
-        content.priority = 1.0
-        assert( chain.filter( content ) )
+
+      order = UniMap.new.tap do |o|
+        o.url = Core::VisitURL.normalize( "http://gravitext.com/atom.xml" )
+        o.type = "FEED"
+        o.priority = 1.0
       end
+
+      orders = [ order, order.clone ]
+
+      orders.each do |o|
+        assert( chain.filter( o ) )
+      end
+
+      # Note this only works timing wise because of blocking
+      # HTTPClient.
+      assert_equal( orders, counter.released )
     end
 
     mgr.shutdown
     dsf.close
+
   end
 
 end

metadata

@@ -2,7 +2,7 @@
 name: iudex-worker
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.0
+  version: 1.1.0
 platform: java
 authors: 
   - David Kellum
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-04 00:00:00 -07:00
-default_executable: 
+date: 2011-11-13 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: iudex-core
@@ -21,7 +20,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id001
   - !ruby/object:Gem::Dependency 
@@ -43,7 +42,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id003
   - !ruby/object:Gem::Dependency 
@@ -54,7 +53,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id004
   - !ruby/object:Gem::Dependency 
@@ -65,7 +64,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id005
   - !ruby/object:Gem::Dependency 
@@ -76,18 +75,18 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id006
   - !ruby/object:Gem::Dependency 
-    name: iudex-httpclient-3
+    name: iudex-char-detector
     prerelease: false
     requirement: &id007 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id007
   - !ruby/object:Gem::Dependency 
@@ -96,25 +95,55 @@ dependencies:
     requirement: &id008 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.7.1
-        - - <
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "2.1"
+            version: "2.3"
     type: :development
     version_requirements: *id008
   - !ruby/object:Gem::Dependency 
-    name: rjack-tarpit
+    name: iudex-httpclient-3
     prerelease: false
     requirement: &id009 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.3.0
+            version: 1.1.0
     type: :development
     version_requirements: *id009
+  - !ruby/object:Gem::Dependency 
+    name: iudex-jetty-httpclient
+    prerelease: false
+    requirement: &id010 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :development
+    version_requirements: *id010
+  - !ruby/object:Gem::Dependency 
+    name: iudex-async-httpclient
+    prerelease: false
+    requirement: &id011 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :development
+    version_requirements: *id011
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id012 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.4.0
+    type: :development
+    version_requirements: *id012
 description: |-
   Iudex is a general purpose web crawler and feed processor in
   ruby/java. The iudex-worker gem provides a worker deamon for feed/page
@@ -136,6 +165,8 @@ files:
   - Rakefile
   - bin/iudex-worker-fg
   - config/config.rb
+  - config/config_async_http.rb
+  - config/config_jetty_http.rb
   - init/iudex-worker
   - lib/iudex-worker/base.rb
   - lib/iudex-worker.rb
@@ -147,7 +178,7 @@ files:
   - test/test_agent.rb
   - test/test_filter_chain_factory.rb
   - test/test_prioritizer.rb
-has_rdoc: true
+  - .gemtest
 homepage: http://github.com/dekellum/iudex
 licenses: []
 
@@ -172,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: iudex-worker
-rubygems_version: 1.5.1
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java