iudex-worker 1.3.2-java → 1.4.0-java

data/History.rdoc

@@ -1,3 +1,17 @@
+=== 1.4.0 (2013-10-29)
+* Add Agent.raise_on_run flag for testing
+* Ensure executor/VisitManager shutdown on failed Agent.run_safe (for
+  example on filter chain factory errors.) Jetty 9 uses executor
+  threads at startup so this is now important.
+* FilterChainFactory.last_visit_setter and page/feed keys extended to
+  conditially include handling of reserved, instance fields based on
+  set (by Agent) WorkPoller reserve?, instance properties as configured.
+* Add config example of WorkPoller do_reserve and instance settings.
+* Add VisitQueue.max_access_total config eaxample.
+* Update config_jetty_http.rb sample for client 1.4.0 (jetty 9.x)
+* Upgrade to iudex-* ~> 1.4.0 dependencies
+* Upgrade to minitest ~> 4.7.4 (dev)
+
 === 1.3.2 (2012-11-8)
 * Upgrade/narrow to iudex-core, -da, -rome, -html, -simhash ~> 1.3.0
 * FetchHelper.create_content_fetcher now prefers an options Hash,

data/README.rdoc

@@ -11,7 +11,7 @@ processing.
 
 == License
 
-Copyright (c) 2008-2012 David Kellum
+Copyright (c) 2008-2013 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You may

data/bin/iudex-worker-fg

@@ -2,7 +2,7 @@
 # -*- ruby -*-
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/config/config.rb

@@ -22,6 +22,7 @@ Iudex.configure do |c|
   end
 
   c.setup_visit_queue do |q|
+    q.max_access_total = threads * 8
     q.config(                             :rate =>  5.0, :cons => 1 )
     q.config( :domain => "gravitext.com", :rate => 10.0, :cons => 2 )
   end
@@ -30,6 +31,9 @@ Iudex.configure do |c|
     wp.min_order_remaining_ratio = 0.30
     wp.max_check_interval = 100 #ms
     wp.min_poll_interval = 2_000 #ms
+    wp.do_reserve = true
+    wp.instance = 'solo'
+    wp.instance_unreserve
   end
 
   c.setup_filter_factory do |ff|

data/config/config_jetty_http.rb

@@ -3,9 +3,12 @@ require 'iudex-jetty-httpclient'
 Iudex.configure do |c|
 
   c.setup_jetty_httpclient do
-    { :timeout => 20_000,
-      :max_connections_per_address => 2,
-      :max_queue_size_per_address => 20 }
+    { :timeout         => 35_000,
+      :connect_timeout => 12_000,
+      :idle_timeout    => 20_000,
+      :max_connections_per_destination => 2,
+      :max_requests_queued_per_destination => 20,
+      :follow_redirects => false }
   end
 
 end

data/init/iudex-worker

@@ -7,7 +7,7 @@
 #. hashdot.vm.options += -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -24,7 +24,7 @@
 
 require 'rubygems'
 
-gem( "iudex-worker", "= 1.3.2" )
+gem( "iudex-worker", "= 1.4.0" )
 
 module IudexInitScript
 

data/lib/iudex-worker.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-worker/agent.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -33,9 +33,12 @@ module Iudex
       include Iudex::Worker
       include Gravitext::HTMap
 
+      attr_accessor :raise_on_run
+
       def initialize
         @log = RJack::SLF4J[ self.class ]
         @http_manager = nil
+        @raise_on_run = false
         Hooker.apply( [ :iudex, :worker ], self )
       end
 
@@ -57,7 +60,9 @@ module Iudex
           end
         elsif defined?( AsyncHTTPClient.create_client )
           @log.info "Setting up AsyncHTTPClient"
-          AsyncHTTPClient.create_client( :executor_service => executor )
+          opts = {}
+          opts[ :executor_service ] = executor if executor
+          AsyncHTTPClient.create_client( opts )
         else
           gem     'iudex-httpclient-3', '~> 1.2.b'
           require 'iudex-httpclient-3'
@@ -104,6 +109,7 @@ module Iudex
         fcf.http_client = hclient
         fcf.data_source = data_source
         fcf.visit_counter = vexec
+        fcf.work_poller = wpoller
 
         # FilterChain's executor is the same executor, unless using
         # HTTPClient3, where executor is best not used
@@ -117,14 +123,25 @@ module Iudex
           Hooker.log_not_applied # All hooks should be used by now
 
           vexec.start
-          vexec.join # Run until interrupted
+          veref, vexec = vexec, nil
+          veref.join # Run until interrupted
         end # fcf closes
 
       rescue => e
-        @log.error( "On run: ", e )
+        if @raise_on_run
+          raise e
+        else
+          @log.error( "On run: ", e )
+        end
       ensure
         hclient.close if hclient && hclient.respond_to?( :close )
         @http_manager.shutdown if @http_manager
+
+        # Jetty 9 for example, uses executor thread on start, so
+        # executor shutdown is required for early termination
+        # (i.e. when fcf.filter raises)
+        vexec.shutdown if vexec
+
         dsf.close if dsf
       end
 

data/lib/iudex-worker/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,6 +16,6 @@
 
 module Iudex
   module Worker
-    VERSION = '1.3.2'
+    VERSION = '1.4.0'
   end
 end

data/lib/iudex-worker/fetch_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-worker/filter_chain_factory.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -58,6 +58,7 @@ module Iudex
       attr_accessor :data_source
       attr_accessor :visit_counter
       attr_accessor :executor
+      attr_accessor :work_poller
 
       def initialize( name )
         super
@@ -197,14 +198,20 @@ module Iudex
 
       def page_update_keys
         [ :uhash, :domain, :url, :type,
+          ( :reserved if work_poller && work_poller.reserve? ),
+          ( :instance if work_poller && work_poller.instance ),
           :ref_pub_date, :pub_date,
           :priority, :last_visit, :next_visit_after,
           :status, :etag, :reason, :referer, :referent,
-          :cache_file, :cache_file_offset, :simhash ]
+          :cache_file, :cache_file_offset, :simhash ].compact
       end
 
       def last_visit_setter
-        Copier.new( *keys( :visit_start, :last_visit ) )
+        resv = work_poller && work_poller.reserve?
+        inst = work_poller && work_poller.instance
+        [ Copier.new( *keys( :visit_start, :last_visit ) ),
+          ( Setter.new( :reserved.to_k, nil )  if resv ),
+          ( Setter.new( :instance.to_k, inst ) if inst ) ]
       end
 
     end

data/lib/iudex-worker/prioritizer.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_agent.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -69,29 +69,39 @@ class TestAgent < MiniTest::Unit::TestCase
     Iudex.send( :remove_const, :AsyncHTTPClient )
   end
 
+  class TestBadness < StandardError
+  end
+
   def test_agent_graceful_shutdown_on_fcf_error
     Hooker.add( [ :iudex, :filter_factory ] ) do |fcf|
       def fcf.filters
         super
-        raise "Test Badness in FCF"
+        raise TestBadness, "Test Badness in FCF"
       end
     end
 
     agent = Agent.new
-    agent.run
-    pass # returns
+    agent.raise_on_run = true
+    assert_raises( TestBadness ) do
+      agent.run
+    end
   end
 
   def assert_agent
 
-    # Stub VisitManager.start to allow agent.run to return early.
+    # Stub VisitManager.start/start_executor to allow agent.run to
+    # return early.
     Hooker.add( [ :iudex, :visit_manager ] ) do |vm|
       def vm.start
         #disable
       end
+      def vm.start_executor
+        #disable
+      end
     end
 
     agent = Agent.new
+    agent.raise_on_run = true
     agent.run
     pass
 

data/test/test_filter_chain_factory.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_prioritizer.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

metadata

@@ -2,14 +2,14 @@
 name: iudex-worker
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.2
+  version: 1.4.0
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-08 00:00:00.000000000 Z
+date: 2013-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
@@ -17,13 +17,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -33,13 +33,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -49,13 +49,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -65,13 +65,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -81,13 +81,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -97,13 +97,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -129,13 +129,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   prerelease: false
   type: :development
@@ -145,13 +145,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :development
@@ -161,13 +161,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :development
@@ -177,13 +177,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :development