iudex-da 1.3.2-java → 1.3.3-java

data/History.rdoc

@@ -1,3 +1,23 @@
+=== 1.3.3 (2012-11-8)
+* FactoryHelper.create_update_filter now prefers an options Hash
+  exposing greater control over what is updated and how. In
+  particular, :on_referer can be independently set.
+* Add (Base)Transformer, ContentUpdater, UpdateFilter support for a
+  distinct REFERER filter chain. Now content, referer, and references
+  updates are all optional.
+* BaseTransformer.merge now augments the updated map with the current
+  (database) contents instead of creating a temporary map on which
+  UpdateFilter chain mutations would be discarded.  This change makes
+  it consistent with either new or updated content.
+* Add new options Hash syntax and :type support to
+  WorkPoller.domain_union
+* Fix WorkPoller uhash_slice range calculation for ruby 1.8
+* Intern :type values on read in ContentMapper
+* Upgrade/narrow to iudex-core ~> 1.3.0 (incl. gravitext-util ~> 1.7.0)
+* Upgrade to logback ~> 1.5 (dev)
+* Add WorkPoller logging and consolidate log from
+  GenericWorkPollStrategy.
+
 === 1.3.2 (2012-10-25)
 * Add migration to make index_next_visit partial over non-null
   next_visit_after rows. (Index rebuild may take a while.)

data/Manifest.txt

@@ -28,4 +28,4 @@ test/test_migrate.rb
 test/test_pool_factory.rb
 test/test_url_model.rb
 test/test_work_poller.rb
-lib/iudex-da/iudex-da-1.3.2.jar
+lib/iudex-da/iudex-da-1.3.3.jar

data/lib/iudex-da/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module DA
-    VERSION = '1.3.2'
+    VERSION = '1.3.3'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-da/factory_helper.rb

@@ -31,22 +31,88 @@ module Iudex
           @data_source ||= PoolDataSourceFactory.new.create
         end
 
-        # Create UpdateFilter given fields and filter list factory
-        # methods
-        def create_update_filter( fields,
-                                  post_sym = nil,
-                                  update_sym = nil,
-                                  new_sym = nil )
-
-          f = UpdateFilter.new( data_source, field_mapper( fields ) )
-          create_chain( update_sym ) { |c| f.update_ref_filter = c }
-          create_chain( new_sym )    { |c| f.new_ref_filter    = c }
-          create_chain( post_sym )   { |c| f.content_filter    = c }
+        # Create an UpdateFilter given the provided options.
+        #
+        # === Options
+        #
+        # :fields:: The Array of fields (Symbol or Key) to (re-)read
+        #           from the database and update. The required :uhash
+        #           field is included automatically.
+        #
+        # :max_retries:: Maximum number of retries not including the
+        #                initial attempt, in case of a database
+        #                conflict (Default: 3)
+        #
+        # :isolation_level:: A transaction isolation constant as
+        #                    defined in java.sql.Connection
+        #                    (Default: REPEATABLE_READ 0x04)
+        #
+        # :on_content:: Filter option for current (content) UniMap.
+        #
+        # :on_ref_update:: Filter option for REFERENCES that are
+        #                  already existing in the database
+        #
+        # :on_ref_new:: Filter option for REFERENCES that are new (not
+        #               found in db).
+        #
+        # :on_referer:: Filter option for the REFERER to the current
+        #               content.
+        #
+        # The positional parameters equivalent to
+        # ( :fields, :on_content, :on_ref_update, :on_ref_new )
+        # as defined above are also supported but deprecated.
+        #
+        # === Filter options
+        #
+        # Each of the on_* filter options defined above may take a
+        # value of a Filter, Array of filters or a Symbol value. The
+        # following symbol values are special, all other Symbol values
+        # are passed as the :filters option to create_chain, and is
+        # interpreted as a method name.
+        #
+        # :merge:: The default behavior of merging the in-memory state
+        #          with the current database state. This is equivalent
+        #          to providing a NoOpFilter.
+        #
+        # :ignore:: Do not update this element. This is equivalent to
+        #           providing a filter that always rejects, but faster
+        #           since it need not read the value from the db.
+        #
+        def create_update_filter( *args )
+          opts = args.last.is_a?( Hash ) ? args.pop.dup : {}
+
+          opts[ :fields ]        ||= args.shift #deprecated
+          opts[ :on_content ]    ||= args.shift
+          opts[ :on_ref_update ] ||= args.shift
+          opts[ :on_ref_new    ] ||= args.shift
+
+          f = UpdateFilter.new( data_source, field_mapper( opts[ :fields ] ) )
+          updater_chain( opts[:on_ref_update]) { |c| f.update_ref_filter = c }
+          updater_chain( opts[ :on_ref_new ] ) { |c| f.new_ref_filter    = c }
+          updater_chain( opts[ :on_content ] ) { |c| f.content_filter    = c }
+          updater_chain( opts[ :on_referer ] ) { |c| f.referer_filter    = c }
+
+          f.max_retries     = opts[ :max_retries ]   if opts[ :max_retries ]
+          f.isolation_level = opts[:isolation_level] if opts[:isolation_level]
+
           f
         end
 
-        def create_read_filter( fields = [] )
-          f = ReadFilter.new( data_source, field_mapper( fields ) )
+        # Create a ReadFilter given the provided options.
+        #
+        # === Options
+        #
+        # :fields:: The Array of fields (Symbol or Key) to read from
+        #           the database.
+        #
+        # The positional parameters equivalent to ( :fields ) as
+        # defined above is also supported but deprecated.
+        #
+        def create_read_filter( *args )
+          opts = args.last.is_a?( Hash ) ? args.pop.dup : {}
+          opts[ :fields ]        ||= args.shift #deprecated
+
+          ReadFilter.new( data_source, field_mapper( opts[ :fields ] ) )
         end
 
         def field_mapper( fields )
@@ -54,6 +120,18 @@ module Iudex
           ContentMapper.new( fields )
         end
 
+        def updater_chain( v, &block )
+          if v.is_a?( Iudex::Filter::Filter )
+            block.call( v )
+          elsif v == :merge
+            block.call( UpdateFilter::DEFAULT_MERGE )
+          elsif v == :ignore
+            block.call( nil )
+          else
+            create_chain( :filters => v, &block )
+          end
+        end
+
       end
 
     end

data/lib/iudex-da/work_poller.rb

@@ -90,19 +90,33 @@ module Iudex::DA
         age_coef_2 && age_coef_2 > 0.0 )
     end
 
-    # An Array of [ domain, max_urls ] pairs where each domain is a
-    # unique reqistration-level, normalized lower-case domain. A nil
-    # domain applies to all domains not covered by another
-    # row. Without a nil domain row, work is limited to the explicit
-    # domains listed. If provided these max_urls values are used
-    # instead of top level #max_urls. Domain depth should most likely
-    # be avoided if this feature is used. (default: [], off)
+    # A table of option rows as defined below. A nil/unspecified
+    # domain and type row applies to all domains/types not covered by
+    # another row. Without such a row, work is limited to the explicit
+    # domains/types listed.
+    #
+    # ==== Options
+    #
+    # :domain:: The registration-level, normalized lower-case domain
+    #           value.
+    #
+    # :type:: An (upper-case) TYPE value to be AND'd with a domain
+    #         domain or may appear on its own, applying to all
+    #         unconfigured domains.
+    #
+    # :max:: The maximum number of visit urls to obtain in one poll
+    #        (instead of the top level #max_urls.) A zero max_urls
+    #        value excludes this domain/type (efficiently).
+    #
+    # Also a [ domain, max ] alternative syntax is currently supported
+    # but deprecated.
+    #
     attr_accessor :domain_union
 
     # An array containing a zero-based position and a total number of
     # evenly divided segments within the range of possible uhash
     # values. If set only work with uhashes in the designated range
-    # will be polled. Note that the uhash is indepedent of domain,
+    # will be polled. Note that the uhash is independent of domain,
     # being a hash on the entire URL. (default: nil, off)
     attr_accessor :uhash_slice
 
@@ -136,6 +150,11 @@ module Iudex::DA
       @data_source = data_source
     end
 
+    # Override GenericWorkPollStrategy
+    def log
+      @log.java_logger
+    end
+
     # Override GenericWorkPollStrategy
     def pollWorkImpl( visit_queue )
       visit_queue.add_all( poll )
@@ -147,6 +166,7 @@ module Iudex::DA
     # Raises SQLException
     def poll
       query, params = generate_query
+      @log.debug { "Poll query: #{query}; #{params.inspect}" }
       reader.select( query, *params )
     end
 
@@ -156,6 +176,16 @@ module Iudex::DA
       end
     end
 
+    def domain_union=( table )
+      @domain_union = table.map do | *args |
+        args = args.flatten.dup
+        opts = args.last.is_a?( Hash ) ? args.pop.dup : {}
+        opts[ :domain ] ||= args.shift
+        opts[ :max ]    ||= args.shift
+        opts
+      end
+    end
+
     def generate_query
       criteria = [ "next_visit_after <= now()" ]
 
@@ -172,18 +202,34 @@ module Iudex::DA
         params = [ max_urls ]
       else
         subqueries = []
-        @domain_union.each do | domain, dmax |
-          next if dmax == 0
+        @domain_union.each do | opts |
+          opts = opts.dup
+          opts[ :max ]    ||= @max_urls
+
+          next if opts[ :max ] == 0
+
           c = criteria.dup
-          if domain.nil?
-            c += @domain_union.map { |nd,_| nd }.
+          if opts[ :domain ].nil?
+            c += @domain_union.map { |r| r[ :domain ] }.
                                compact.
+                               uniq.
                                map { |nd| "domain != '#{nd}'" }
           else
-            c << "domain = '#{domain}'"
+            c << "domain = '#{opts[ :domain ]}'"
           end
+
+          if opts[ :type ].nil?
+            c += @domain_union.select { |r| r[ :domain ] == opts[ :domain ] }.
+                               map { |r| r[ :type ] }.
+                               compact.
+                               uniq.
+                               map { |nt| "type != '#{nt}'" }
+          elsif opts[ :type ]
+            c << "type = '#{opts[ :type ]}'"
+          end
+
           subqueries << generate_query_inner( c )
-          params << dmax
+          params << opts[ :max ]
         end
         if subqueries.size == 1
           query = subqueries.first
@@ -291,7 +337,7 @@ module Iudex::DA
       high = ( period * (pos+1) ).round if (pos+1) < segments
 
       [ low, high ].map do |i|
-        URL64_ORDER[ i / 64 ] + URL64_ORDER[ i % 64 ] if i
+        URL64_ORDER[ i / 64 ].chr + URL64_ORDER[ i % 64 ].chr if i
       end
     end
 

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-da</artifactId>
   <packaging>jar</packaging>
-  <version>1.3.2</version>
+  <version>1.3.3</version>
   <name>Iudex Data Access</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.2.1</version>
+    <version>1.3.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -20,7 +20,7 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.2.1,1.3.999)</version>
+      <version>[1.3.0,1.3.999)</version>
     </dependency>
 
     <dependency>

data/test/test_work_poller.rb

@@ -33,13 +33,13 @@ class TestWorkPoller < MiniTest::Unit::TestCase
 
   URLS = [ [ "http://foo.gravitext.com/bar/1", 11 ],
            [ "http://hometown.com/33",         10 ],
-           [ "http://gravitext.com/2",          9 ] ]
+           [ "http://gravitext.com/2",          9, "ALT" ] ]
 
   def setup
     Url.truncate
 
-    URLS.each do | u, p |
-      Url.create( :visit_url => u, :priority => p, :type => "PAGE"  )
+    URLS.each do | u, p, t |
+      Url.create( :visit_url => u, :priority => p, :type => t || "PAGE"  )
     end
 
     @factory = PoolDataSourceFactory.new( :loglevel => 4 )
@@ -124,17 +124,78 @@ class TestWorkPoller < MiniTest::Unit::TestCase
   end
 
   def test_poll_domain_union_2
-    poller.domain_union = [ [ 'gravitext.com', 15000 ],
-                            [ nil, 10000 ] ]
+    poller.domain_union = [ { :domain => 'gravitext.com', :max => 15000 },
+                            {                             :max => 10000 } ]
 
     result = poller.poll
     assert_equal( 3, result.size )
   end
 
   def test_poll_domain_union_3
-    poller.domain_union = [ [ 'gravitext.com', 1 ],
-                            [ 'hometown.com', 1 ],
-                            [ nil, 3 ] ]
+    poller.domain_union = [ { :domain => 'gravitext.com', :max => 1 },
+                            { :domain => 'hometown.com',  :max => 1 },
+                            {                             :max => 3 } ]
+
+    result = poller.poll
+    assert_equal( 2, result.size )
+  end
+
+  def test_poll_domain_union_type_1
+    poller.domain_union = [
+      { :domain => 'gravitext.com', :type => 'ALT', :max => 15000 } ]
+
+    result = poller.poll
+    assert_equal( 1, result.size )
+  end
+
+  def test_poll_domain_union_type_2
+    poller.domain_union = [
+      { :domain => 'gravitext.com', :type => 'ALT', :max => 1 },
+      { :domain => 'gravitext.com',                 :max => 1 } ]
+
+    result = poller.poll
+    assert_equal( 2, result.size )
+  end
+
+  def test_poll_domain_union_type_3
+    poller.domain_union = [
+      { :domain => 'gravitext.com', :type => 'ALT', :max => 1 },
+      { :domain => 'gravitext.com', :type => 'NOT', :max => 1 },
+      { :domain => 'gravitext.com',                 :max => 1 } ]
+
+    result = poller.poll
+    assert_equal( 2, result.size )
+  end
+
+  def test_poll_domain_union_type_4
+    poller.domain_union = [
+      { :domain => 'gravitext.com', :type => 'ALT', :max => 1 },
+      { :domain => 'gravitext.com', :type => 'NOT', :max => 1 },
+      { :domain => 'gravitext.com',                 :max => 1 },
+      { :domain =>  'hometown.com',                 :max => 1 } ]
+
+    result = poller.poll
+    assert_equal( 3, result.size )
+  end
+
+  def test_poll_domain_union_type_5
+    poller.domain_union = [ { :type => 'ALT', :max => 15000 } ]
+
+    result = poller.poll
+    assert_equal( 1, result.size )
+  end
+
+  def test_poll_domain_union_type_6
+    poller.domain_union = [ { :type => 'ALT', :max => 2 },
+                            {                 :max => 3 } ]
+
+    result = poller.poll
+    assert_equal( 3, result.size )
+  end
+
+  def test_poll_domain_union_type_7
+    poller.domain_union = [ { :type => 'ALT', :max => 2 },
+                            {                 :max => 1 } ]
 
     result = poller.poll
     assert_equal( 2, result.size )

metadata

@@ -2,34 +2,28 @@
 name: iudex-da
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.2
+  version: 1.3.3
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-10-25 00:00:00.000000000 Z
+date: 2012-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.3.0
     none: false
   prerelease: false
   type: :runtime
@@ -119,13 +113,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
     none: false
   prerelease: false
   type: :development
@@ -188,7 +182,7 @@ files:
 - test/test_pool_factory.rb
 - test/test_url_model.rb
 - test/test_work_poller.rb
-- lib/iudex-da/iudex-da-1.3.2.jar
+- lib/iudex-da/iudex-da-1.3.3.jar
 homepage: http://iudex.gravitext.com
 licenses: []
 post_install_message: