iudex-worker 1.3.1-java → 1.3.2-java

data/History.rdoc

@@ -1,3 +1,13 @@
+=== 1.3.2 (2012-11-8)
+* Upgrade/narrow to iudex-core, -da, -rome, -html, -simhash ~> 1.3.0
+* FetchHelper.create_content_fetcher now prefers an options Hash,
+  exposing settings for :types, :client, :user_agent,
+  :request_headers, etc. and passing additional options to
+  create_chain (:filters, etc.)
+* Various updates to FilterChainFactory and config sample using new
+  options, etc.
+* Upgrade to logback ~> 1.5
+
 === 1.3.1 (2012-10-25)
 * Improve shutdown reliability by adding Agent.run_safe with ensure'd
   close calls.

data/config/config.rb

@@ -22,10 +22,8 @@ Iudex.configure do |c|
   end
 
   c.setup_visit_queue do |q|
-    q.default_min_host_delay = 100 #ms
-    q.default_max_access_per_host = 1
-
-    q.configure_host( "gravitext.com", 100, 2 ) # 100ms, 2 connections
+    q.config(                             :rate =>  5.0, :cons => 1 )
+    q.config( :domain => "gravitext.com", :rate => 10.0, :cons => 2 )
   end
 
   c.setup_work_poller do |wp|

data/init/iudex-worker

@@ -24,7 +24,7 @@
 
 require 'rubygems'
 
-gem( "iudex-worker", "= 1.3.1" )
+gem( "iudex-worker", "= 1.3.2" )
 
 module IudexInitScript
 

data/lib/iudex-worker/base.rb

@@ -16,6 +16,6 @@
 
 module Iudex
   module Worker
-    VERSION = '1.3.1'
+    VERSION = '1.3.2'
   end
 end

data/lib/iudex-worker/fetch_helper.rb

@@ -23,26 +23,79 @@ module Iudex
       include Iudex::HTTP
       include Iudex::Core::Filters
 
-      def create_content_fetcher( accept_types, receiver, listener = nil )
-        cf = ContentFetcher.new( http_client,
-                                 visit_counter,
-                                 create_chain( receiver, nil, listener ) )
-
-        cf.executor = executor if executor
-
-        alist = accept_list( accept_types )
+      # Create a ContentFetcher including a filter chain to receive
+      # the fetch result.
+      #
+      # === Options
+      #
+      # Options support literal values, or a Proc, Method, or a Symbol
+      # to self send unless otherwise noted.
+      #
+      # :types:: An Array or table of Mime types use Accept header in
+      #          default :request_headers and to restrict returned
+      #          results on. (Default: #page_mime_types)
+      #
+      # :client:: The Java::iudex.http.HTTPClient implementation to
+      #           use (Default: :http_client)
+      #
+      # :user_agent:: The HTTP User-Agent for default
+      #               :request_headers. Proc's will receive the
+      #               options Hash as parameter (Default: #http_user_agent)
+      #
+      # :visit_counter:: The Java::iudex.core.VisitCounter
+      #                  implementation. (Default: :visit_counter)
+      #
+      # :executor:: The java.util.concurrent.Executor to use for
+      #             running the receiver filter chain. (Default: :executor)
+      #
+      # :request_headers:: HTTP Request headers as Array<iudex.http.Header>
+      #                    (Default: #http_request_headers)
+      #
+      # All options (including the required :filters option, and
+      # :listener with default :main) are also passed to
+      # self.create_chain for creating the receiver filter chain
+      #
+      # The positional parameters equivalent to ( :types, :filters,
+      # :listener ) as defined above are also supported, but
+      # deprecated.
+      #
+      def create_content_fetcher( *args )
+        opts = args.last.is_a?( Hash ) ? args.pop.dup : {}
+
+        opts[ :types ]    ||= args.shift
+        opts[ :filters  ] ||= args.shift
+        opts[ :listener ] ||= args.shift
+
+        opts = { :types           => :page_mime_types,
+                 :listener        => :main,
+                 :client          => :http_client,
+                 :user_agent      => :http_user_agent,
+                 :visit_counter   => :visit_counter,
+                 :executor        => :executor,
+                 :request_headers => :http_request_headers
+               }.merge( opts )
+
+        cf = ContentFetcher.new( call_if( opts[ :client ] ),
+                                 call_if( opts[ :visit_counter ] ),
+                                 create_chain( opts ) )
+
+        cf.executor = call_if( opts[ :executor ] )
+
+        alist = accept_list( call_if( opts[ :types ] ) )
         unless alist.include?( '*/*' )
           cf.accepted_content_types = ContentTypeSet.new( alist )
         end
 
-        headers = [ [ 'User-Agent', http_user_agent ],
-                    [ 'Accept',     accept_header( accept_types ) ] ]
-
-        cf.request_headers = headers.map { |kv| Header.new( *kv ) }
-
+        cf.request_headers = call_if( opts[ :request_headers ], opts )
         cf
       end
 
+      def http_request_headers( opts )
+        [ [ 'User-Agent', call_if( opts[ :user_agent ] ) ],
+          [ 'Accept',     accept_header( call_if( opts[ :types ] ) ) ]
+        ].map { |kv| Header.new( *kv ) }
+      end
+
       def http_user_agent
         ( "Mozilla/5.0 (compatible; " +
           "Iudex #{Iudex::Worker::VERSION}; " +
@@ -79,6 +132,16 @@ module Iudex
         types.flatten
       end
 
+      def call_if( v, *args )
+        if v.is_a?( Proc ) || v.is_a?( Method )
+          v.call( *args )
+        elsif v.is_a?( Symbol )
+          send( v, *args )
+        else
+          v
+        end
+      end
+
     end
 
   end

data/lib/iudex-worker/filter_chain_factory.rb

@@ -89,11 +89,13 @@ module Iudex
       end
 
       def feed_fetcher
-        [ create_content_fetcher( feed_mime_types, :feed_receiver, :main ) ]
+        [ create_content_fetcher( :types => :feed_mime_types,
+                                  :filters => :feed_receiver ) ]
       end
 
       def page_fetcher
-        [ create_content_fetcher( page_mime_types, :page_receiver, :main ) ]
+        [ create_content_fetcher( :types => :page_mime_types,
+                                  :filters => :page_receiver ) ]
       end
 
       def feed_receiver
@@ -106,8 +108,11 @@ module Iudex
       end
 
       def feed_updater
-        create_update_filter( feed_update_keys,
-                              :feed_post, :feed_ref_update, :feed_ref_new )
+        create_update_filter( :fields        => feed_update_keys,
+                              :on_content    => :feed_post,
+                              :on_referer    => :feed_post,
+                              :on_ref_update => :feed_ref_update,
+                              :on_ref_new    => :feed_ref_new )
       end
 
       def feed_ref_new
@@ -127,16 +132,6 @@ module Iudex
                            :min_next => 0.0 ) ]
       end
 
-      # Filters to apply for feed update.
-      #
-      # Notes:
-      #
-      # * This is run possibly twice, for both base content map and
-      #   referer map if present.
-      #
-      # * If this is an update then these filters act on a *new* map,
-      #   thus any changes made here will not be visible after exit
-      #   from the update_filter.
       def feed_post
         [ UHashMDCSetter.new,
           ref_common_cleanup,
@@ -185,19 +180,11 @@ module Iudex
       end
 
       def page_updater
-        create_update_filter( page_update_keys, :page_post )
-      end
-
-      # Filters to apply during page update
-      #
-      # Notes:
-      #
-      # * This is run possibly twice, for both base content map and
-      #   referer map if present.
-      #
-      # * If this is an update then these filters act on a *new* map,
-      #   thus any changes made here will not be visible after exit
-      #   from the update_filter.
+        create_update_filter( :fields     => page_update_keys,
+                              :on_content => :page_post,
+                              :on_referer => :page_post )
+      end
+
       def page_post
         [ UHashMDCSetter.new,
           barc_writer, # Not run in 302 referer case, since no SOURCE.

metadata

@@ -2,34 +2,28 @@
 name: iudex-worker
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.1
+  version: 1.3.2
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-10-25 00:00:00.000000000 Z
+date: 2012-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -53,21 +47,15 @@ dependencies:
   name: iudex-rome
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -75,21 +63,15 @@ dependencies:
   name: iudex-html
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -97,21 +79,15 @@ dependencies:
   name: iudex-simhash
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -119,21 +95,15 @@ dependencies:
   name: iudex-char-detector
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -143,13 +113,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
     none: false
   prerelease: false
   type: :runtime
@@ -173,21 +143,15 @@ dependencies:
   name: iudex-httpclient-3
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :development
@@ -195,21 +159,15 @@ dependencies:
   name: iudex-jetty-httpclient
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :development
@@ -217,21 +175,15 @@ dependencies:
   name: iudex-async-httpclient
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :development