iudex-da 1.3.3-java → 1.4.0-java

data/History.rdoc

@@ -1,3 +1,31 @@
+=== 1.4.0 (2013-10-29)
+* Add experimental WorkPoller.do_reserve support, which utilizes
+  PostgreSQL 9.1+ writable Common Table Expressions (CTEs) to
+  efficiently(?) set a reserved column and instance (identifier) on
+  poll that guards against polling by another or the same iudex-worker
+  process. This enables multiple workers to cover the same urls/orders
+  as well as concurrent polling. Previously only an instance unique,
+  uhash_slice single-tenant partitioning scheme was supported for
+  multi-worker distribution.
+* Add experimental support for concurrent (no discard) WorkPoller (in
+  collaboration with iudex-core 1.4.0 VisitManager), enabled when set
+  do_reserve = true, do_discard = false.
+  * When enabled, polled max_urls is adjusted by current order_count
+    for subsequent polls.
+  * New max_reserved_time will force discard of possibly long reserved
+    tenure based on time since the queue was last empty.
+  * New max_discard_ratio will also force a discard if
+    order_count/max_urls exceeds this ratio.
+* Add iudex-da-unreserve utility for cleaning reservations left due to
+  failure, based on age.
+* Add DAKeys INSTANCE (String identifier) and RESERVED (date)
+* Add migrations for instance and reserved columns; removal of
+  reserved from index_next_visit (profile)
+* Upgrade to dbutils ~> 1.5.0, rjack-jdbc-postgres ~> 9.2.1002,
+  sequel ~> 3.46 (but not upcoming 4.x)
+* Upgrade to iudex-* ~> 1.4.0 dependencies
+* Upgrade to minitest ~> 4.7.4 (dev)
+
 === 1.3.3 (2012-11-8)
 * FactoryHelper.create_update_filter now prefers an options Hash
   exposing greater control over what is updated and how. In

data/Manifest.txt

@@ -6,12 +6,16 @@ pom.xml
 bin/iudex-da-generate-test-data
 bin/iudex-da-import
 bin/iudex-da-simhash-dump
+bin/iudex-da-unreserve
 bin/iudex-migrate
 config/config.rb
 db/20111012173757_base.rb
 db/20120930173600_uhash_collation_order.rb
+db/20130419090000_instance_column.rb
+db/20130419095500_reserved_column.rb
 db/index_next_visit/21500000000101_add_index_next_visit.rb
 db/index_next_visit/21500000000110_index_next_visit_partial.rb
+db/index_next_visit/21500000000120_index_next_visit_not_reserved.rb
 db/simhash/21500000000001_add_simhash_index.rb
 lib/iudex-da/base.rb
 lib/iudex-da.rb
@@ -28,4 +32,4 @@ test/test_migrate.rb
 test/test_pool_factory.rb
 test/test_url_model.rb
 test/test_work_poller.rb
-lib/iudex-da/iudex-da-1.3.3.jar
+lib/iudex-da/iudex-da-1.4.0.jar

data/README.rdoc

@@ -16,7 +16,7 @@ meta-data store and work priority queue.
 
 == License
 
-Copyright (c) 2008-2012 David Kellum
+Copyright (c) 2008-2013 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You may

data/bin/iudex-da-generate-test-data

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-import

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-simhash-dump

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-unreserve

@@ -0,0 +1,107 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2013 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'optparse'
+
+module IudexBinScript
+
+  require 'rjack-logback'
+  include RJack
+  Logback.config_console
+
+  require 'iudex-core'
+  require 'iudex-da/base'
+  require 'iudex-da/config'
+  include Iudex
+
+  Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+  interval = '60 minutes'
+
+  # Apply all config before including ar
+  OptionParser.new do |opts|
+    opts.banner = <<END
+Usage: iudex-da-unreserve [options]
+END
+    opts.on( "-s", "--set name=value", String,
+             "Set connect prop (ex: database=iudex)" ) do |nv|
+      name,value = nv.split('=').map { |t| t.strip }
+      Hooker.add( [ :iudex, :connect_props ] ) do
+        { name.to_sym => value }
+      end
+    end
+    opts.on( "-p", "--profile NAME", String,
+             "Add a migration profile (ex: simhash)" ) do |p|
+      Hooker.add( [ :iudex, :migration_profiles ] ) do |profiles|
+        profiles << p
+      end
+    end
+    opts.on( "-d", "--debug" ) do
+      Logback[ 'iudex.da' ].level = :debug
+      Hooker.add( [ :iudex, :connect_props ] ) { { :log => true } }
+    end
+    opts.on( "-v", "--version", "Display version and exit" ) do
+      puts "iudex-da: #{DA::VERSION}"
+      exit 1
+    end
+    opts.on( "-l", "--long",  "Log in long form, to STDERR" ) do
+      Logback.config_console( :full => true, :stderr => true )
+    end
+    opts.on( "-a", "--age AGE",
+             "Age to unreserve, in PG interval syntax " +
+             "(default: '60 minutes')" ) do |age|
+      interval = age
+    end
+    Hooker.register_config( opts )
+  end.parse!
+
+  require 'iudex-da/orm'
+
+  class Runner
+    include Iudex::DA
+
+    def unreserve( age )
+      ORM.db.transaction( :isolation => :repeatable,
+                          :retry_on => [ Sequel::SerializationFailure ] ) do
+        ds = ORM::db.fetch <<-SQL
+          WITH unreserve AS (
+            UPDATE urls
+            SET reserved = NULL
+            WHERE reserved < ( now() - interval '#{age}' )
+            RETURNING instance )
+          SELECT count(*),instance FROM unreserve
+          GROUP BY instance
+          ORDER BY count DESC
+        SQL
+
+        total = ds.inject( 0 ) do |m,row|
+          puts( "%30s %7d" % [ row[:instance], row[:count] ] )
+          m + row[:count]
+        end
+        puts( "%30s %7d" % [ 'TOTAL', total ] ) if total > 0
+
+      end
+    end
+
+  end
+
+  Runner.new.unreserve( interval )
+end

data/bin/iudex-migrate

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -29,7 +29,7 @@ module IudexBinScript
 
   # Note: Avoid loading iudex-da with its jar dependency which would
   # make it hard to boostrap the db from source alone.
-  # Instead load only nessary core, base, config, and ar (post config):
+  # Instead load only necessary core, base, config, and orm (post config):
   require 'iudex-core'
   require 'iudex-da/base'
   require 'iudex-da/config'

data/db/20111012173757_base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20120930173600_uhash_collation_order.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20130419090000_instance_column.rb

@@ -0,0 +1,27 @@
+#--
+# Copyright (c) 2008-2013 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+Sequel.migration do
+  change do
+    alter_table( :urls ) do
+      # A short string uniquely identifying the iudex worker instance
+      # that has last operated on this URL (for example processed, or
+      # in future: reserved). A hostname or IP may be a suitable
+      # value.
+      add_column :instance, String
+    end
+  end
+end

data/db/20130419095500_reserved_column.rb

@@ -0,0 +1,23 @@
+#--
+# Copyright (c) 2008-2013 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+Sequel.migration do
+  change do
+    alter_table( :urls ) do
+      add_column :reserved, "timestamp with time zone"
+    end
+  end
+end

data/db/index_next_visit/21500000000101_add_index_next_visit.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/index_next_visit/21500000000110_index_next_visit_partial.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/index_next_visit/21500000000120_index_next_visit_not_reserved.rb

@@ -0,0 +1,35 @@
+#--
+# Copyright (c) 2008-2013 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+Sequel.migration do
+
+  up do
+    alter_table( :urls ) do
+      drop_index( :next_visit_after )
+      add_index( :next_visit_after,
+                 :where => 'next_visit_after IS NOT NULL AND reserved IS NULL' )
+    end
+  end
+
+  down do
+    alter_table( :urls ) do
+      drop_index( :next_visit_after )
+      add_index( :next_visit_after,
+                 :where => 'next_visit_after is not null' )
+    end
+  end
+
+end

data/db/simhash/21500000000001_add_simhash_index.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,7 +16,7 @@
 
 module Iudex
   module DA
-    VERSION = '1.3.3'
+    VERSION = '1.4.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-da/config.rb

@@ -1,6 +1,6 @@
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/factory_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/importer.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/key_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/models.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/orm.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -17,7 +17,7 @@
 require 'rjack-slf4j'
 require 'iudex-da/config'
 require 'sequel'
-require 'jdbc/postgres'
+require 'rjack-jdbc-postgres'
 require 'hooker'
 
 Sequel.extension :migration

data/lib/iudex-da/pool_data_source_factory.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -17,7 +17,7 @@
 require 'iudex-da'
 require 'rjack-slf4j'
 require 'java'
-require 'jdbc/postgres'
+require 'rjack-jdbc-postgres'
 
 module Iudex::DA
 

data/lib/iudex-da/work_poller.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -70,10 +70,53 @@ module Iudex::DA
     # priority order (default: false)
     attr_writer :do_domain_group
 
+    # If set true, UPDATE reserved date (and instance, if specified)
+    # (Default: false)
+    attr_writer :do_reserve
+
+    # If set true, discards old queue at every poll, even if
+    # do_reserve could make queue re-fill a safe operation.
+    # (Default: true)
+    attr_writer :do_discard
+
+    # The maximum ratio of current to max_urls where the old queue
+    # will be discarded as a safety to avoid starvation (Default: 0.667)
+    attr_accessor :max_discard_ratio
+
+    # The maximum amount of time in milliseconds that the oldest order
+    # can remain reserved before a discard is required. This is only
+    # relevant when do_reserve is true and do_discard is set false,
+    # and typically would be set as a multiple of max_poll_interval
+    # (ms). Note that max_poll_interval is interpreted as the worst
+    # case next discard opportunity for this purpose. The next poll
+    # made to an empty queue, either by prior discard or completion,
+    # resets the time tracking.  (Default: nil, off)
+    def max_reserved_time
+      @max_reserved_time_s && ( @max_reserved_time_s * 1000.0 ).round
+    end
+
+    def max_reserved_time=( ms )
+      @max_reserved_time_s = ms / 1000.0
+    end
+
+    attr_reader :max_reserved_time_s
+
     def domain_group?
       @do_domain_group
     end
 
+    def reserve?
+      @do_reserve
+    end
+
+    def discard?
+      @do_discard
+    end
+
+    # String uniquely identifying this worker instance. Only used here
+    # with do_reserve.
+    attr_accessor :instance
+
     # First age coefficient. If set > 0.0, adjust priority by the
     # equation:
     #
@@ -125,10 +168,16 @@ module Iudex::DA
 
       @domain_depth_coef  = nil
       @do_domain_group    = false
+      @do_reserve         = false
+      @do_discard         = true
+      @instance           = nil
 
       @max_priority_urls  =    nil
       @max_domain_urls    = 10_000
       @max_urls           = 50_000
+      @max_discard_ratio  = 2.0/3.0
+      @max_reserved_time_s = nil
+      @last_none_reserved  = Time.now
 
       @age_coef_1         = 0.2
       @age_coef_2         = 0.1
@@ -138,7 +187,6 @@ module Iudex::DA
       @uhash_slice        = nil
 
       @log = RJack::SLF4J[ self.class ]
-      #FIXME: Add accessor for log in GenericWorkPollStrategy
 
       keys( :url, :priority, :next_visit_after ).each do |k|
         unless mapper.fields.include?( k )
@@ -157,22 +205,68 @@ module Iudex::DA
 
     # Override GenericWorkPollStrategy
     def pollWorkImpl( visit_queue )
-      visit_queue.add_all( poll )
+      visit_queue.add_all( poll( visit_queue.order_count ) )
     rescue SQLException => x
       @log.error( "On poll: ", x )
     end
 
     # Poll work and return as List<UniMap>
     # Raises SQLException
-    def poll
-      query, params = generate_query
-      @log.debug { "Poll query: #{query}; #{params.inspect}" }
-      reader.select( query, *params )
+    def poll( current_urls = 0 )
+      @last_none_reserved = Time.now if max_reserved_time_s && current_urls == 0
+      query = generate_query( current_urls )
+      @log.debug { "Poll query: #{query}" }
+      reader.select_with_retry( query )
+    end
+
+    # Override GenericWorkPollStrategy
+    def shouldReplaceQueue( visit_queue )
+      ( !reserve? || discard? ||
+        ( ( visit_queue.order_count.to_f / max_urls ) > max_discard_ratio ) ||
+        ( max_reserved_time_s && next_reserve_time > max_reserved_time_s ) )
+    end
+
+    def next_reserve_time( now = Time.now )
+      now - @last_none_reserved + ( max_poll_interval / 1000.0 )
+    end
+
+    # Override GenericWorkPollStrategy to discard old VisitQueue
+    # contents when do_reserve is enabled.
+    def discard( visit_queue )
+      if reserve? && visit_queue.order_count > 0
+        orders = visit_queue.hosts.inject( [] ) do |a, hq|
+          a.concat( hq.orders.to_a )
+        end
+        if orders.length > 0
+          n = reader.unreserve( orders )
+          @log.info { "Unreserved #{n} orders on discard" }
+        end
+      end
+    rescue SQLException => x
+      @log.error( "On discard: ", x )
+    end
+
+    # Unreserve any orders that are reserved by the current instance.
+    # No-op unless do_reserve and instance are set.
+    def instance_unreserve
+      if reserve? && instance
+        n = reader.update( <<-SQL )
+          UPDATE urls
+          SET reserved = NULL
+          WHERE reserved IS NOT NULL AND
+                instance = '#{instance}'
+        SQL
+        @log.info { "Unreserved #{n} orders for instance #{instance}" }
+        n
+      end
+    rescue SQLException => x
+      @log.error( "On instance_unreserve: ", x )
     end
 
     def reader
       @reader ||= ContentReader.new( @data_source, @mapper ).tap do |r|
         r.priority_adjusted = aged_priority?
+        r.max_retries = 10
       end
     end
 
@@ -186,25 +280,33 @@ module Iudex::DA
       end
     end
 
-    def generate_query
+    def domain_union?
+      !@domain_union.empty?
+    end
+
+    def generate_query( current_urls )
       criteria = [ "next_visit_after <= now()" ]
 
+      criteria << "reserved IS NULL" if reserve?
+
       if uhash_slice
         min, max = url64_range( *uhash_slice )
         criteria << "uhash > ( '#{min}' COLLATE \"C\" )" if min
         criteria << "uhash < ( '#{max}' COLLATE \"C\" )" if max
       end
 
-      params = []
-
-      if @domain_union.empty?
-        query = generate_query_inner( criteria )
-        params = [ max_urls ]
+      unless domain_union?
+        query = generate_query_inner( criteria, ( max_urls - current_urls ) )
       else
         subqueries = []
         @domain_union.each do | opts |
           opts = opts.dup
-          opts[ :max ]    ||= @max_urls
+          if opts[ :max ]
+            opts[ :max ] = ( opts[ :max ] * ( max_urls - current_urls ) /
+                             max_urls.to_f ).floor
+          else
+            opts[ :max ] = ( max_urls - current_urls )
+          end
 
           next if opts[ :max ] == 0
 
@@ -228,8 +330,7 @@ module Iudex::DA
             c << "type = '#{opts[ :type ]}'"
           end
 
-          subqueries << generate_query_inner( c )
-          params << opts[ :max ]
+          subqueries << generate_query_inner( c, opts[ :max ] )
         end
         if subqueries.size == 1
           query = subqueries.first
@@ -238,17 +339,20 @@ module Iudex::DA
         end
       end
 
+      query = wrap_with_update( fields, query ) if reserve?
+
       query = wrap_domain_group_query( fields, query ) if domain_group?
 
       query = query.gsub( /\s+/, ' ').strip
 
-      [ query, params ]
+      query
     end
 
-    def generate_query_inner( criteria )
+    def generate_query_inner( criteria, max_urls )
 
       query = filter_query(
-            fields( ( :domain if domain_depth? || domain_group? ) ),
+            fields( ( :domain if domain_depth? || domain_group? ),
+                    ( :uhash if reserve? ) ),
             ( max_priority_urls if domain_depth? ),
             criteria )
 
@@ -260,7 +364,7 @@ module Iudex::DA
       limit_priority = domain_depth? ? :adj_priority : :priority
       query += <<-SQL
         ORDER BY #{limit_priority} DESC
-        LIMIT ?
+        LIMIT #{max_urls}
       SQL
 
       query
@@ -288,7 +392,7 @@ module Iudex::DA
       if aged_priority?
         flds = flds.dup
         i = flds.index( :priority ) || flds.size
-        flds[ i ] = <<-SQL
+        flds[ i ] = <<-SQL.strip
           ( priority +
             #{age_coef_1}::REAL *
                   SQRT( #{age_coef_2}::REAL *
@@ -311,6 +415,24 @@ module Iudex::DA
       sql
     end
 
+    def wrap_with_update( flds, sub )
+      sflds = [ "reserved = now()" ]
+      sflds << "instance = '#{instance}'" if instance
+
+      # Use ..FOR UPDATE unless not supported by query specific
+      # options with PostgreSQL <= 9.1
+      sub += " FOR UPDATE" unless domain_depth? || domain_union?
+
+      <<-SQL
+        WITH work AS ( #{sub} ),
+        reserve AS (
+          UPDATE urls
+          SET #{clist sflds}
+          WHERE uhash IN ( SELECT uhash FROM work ) )
+        SELECT #{clist flds} FROM work
+      SQL
+    end
+
     def wrap_domain_group_query( flds, sub )
       <<-SQL
         SELECT #{clist flds}

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-da</artifactId>
   <packaging>jar</packaging>
-  <version>1.3.3</version>
+  <version>1.4.0</version>
   <name>Iudex Data Access</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -20,13 +20,13 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.3.0,1.3.999)</version>
+      <version>[1.4.0,1.4.999)</version>
     </dependency>
 
     <dependency>
       <groupId>commons-dbutils</groupId>
       <artifactId>commons-dbutils</artifactId>
-      <version>1.4</version>
+      <version>1.5</version>
     </dependency>
 
     <dependency>
@@ -38,7 +38,7 @@
     <dependency>
       <groupId>commons-pool</groupId>
       <artifactId>commons-pool</artifactId>
-      <version>[1.5.4,1.5.6]</version>
+      <version>[1.5.7,1.5.999]</version>
     </dependency>
 
     <dependency>

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_migrate.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/test/test_pool_factory.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_url_model.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/test/test_work_poller.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -25,6 +25,7 @@ require 'iudex-da/pool_data_source_factory'
 require 'iudex-da/models'
 
 class TestWorkPoller < MiniTest::Unit::TestCase
+  include Iudex::Core
   include Iudex::Filter::KeyHelper
   include Iudex::DA
   include Iudex::DA::ORM
@@ -66,6 +67,31 @@ class TestWorkPoller < MiniTest::Unit::TestCase
     assert_equal( 3, pos )
   end
 
+  def test_poll_with_reserve
+    poller.do_reserve = true
+    poller.max_urls = 2
+    poller.instance = 'test'
+
+    polled = poller.poll
+    polled.each_with_index do |map,i|
+      assert_equal( URLS[ i ][ 0 ], map.url.url )
+    end
+    assert_equal( 2, polled.size )
+    reserved = polled
+
+    polled = poller.poll
+    assert_equal( 1, polled.size )
+    assert_equal( URLS[2][0], polled.first.url.url )
+    reserved += polled
+
+    RJack::Logback[ 'iudex.da.WorkPoller' ].with_level( :warn ) do
+      poller.discard( VisitQueue.new.tap { |q| q.add_all( reserved ) } )
+    end
+    poller.max_urls = 3
+
+    assert_equal( 3, poller.poll.size )
+  end
+
   def test_poll_with_max_priority_urls
     poller.max_priority_urls = 4
 
@@ -89,6 +115,25 @@ class TestWorkPoller < MiniTest::Unit::TestCase
     assert_equal( 3, pos )
   end
 
+  def test_poll_with_domain_depth_reserve
+    poller.domain_depth_coef = 0.125
+    poller.max_priority_urls = 4
+    poller.do_reserve = true
+    poller.instance = 'test'
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( URLS[ pos ][ 0 ], map.url.url )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+    RJack::Logback[ 'iudex.da.WorkPoller' ].with_level( :warn ) do
+      assert_equal( 3, poller.instance_unreserve )
+    end
+    assert_equal( 3, poller.poll.size )
+    assert_equal( 0, poller.poll.size )
+  end
+
   def test_poll_with_domain_depth_only
     poller.domain_depth_coef = 0.125
     poller.age_coef_1        = 0.0
@@ -131,6 +176,15 @@ class TestWorkPoller < MiniTest::Unit::TestCase
     assert_equal( 3, result.size )
   end
 
+  def test_poll_domain_union_2_reserve
+    poller.do_reserve = true
+    poller.domain_union = [ { :domain => 'gravitext.com', :max => 15000 },
+                            {                             :max => 10000 } ]
+
+    assert_equal( 3, poller.poll.size )
+    assert_equal( 0, poller.poll.size )
+  end
+
   def test_poll_domain_union_3
     poller.domain_union = [ { :domain => 'gravitext.com', :max => 1 },
                             { :domain => 'hometown.com',  :max => 1 },

metadata

@@ -2,14 +2,14 @@
 name: iudex-da
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.3
+  version: 1.4.0
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-08 00:00:00.000000000 Z
+date: 2013-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
@@ -17,13 +17,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -33,29 +33,29 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.40.0
+        version: '3.46'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.40.0
+        version: '3.46'
     none: false
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: jdbc-postgres
+  name: rjack-jdbc-postgres
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 9.1.901
+        version: 9.2.1002
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 9.1.901
+        version: 9.2.1002
     none: false
   prerelease: false
   type: :runtime
@@ -81,13 +81,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
     none: false
   prerelease: false
   type: :runtime
@@ -97,13 +97,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   prerelease: false
   type: :development
@@ -146,6 +146,7 @@ executables:
 - iudex-da-generate-test-data
 - iudex-da-import
 - iudex-da-simhash-dump
+- iudex-da-unreserve
 - iudex-migrate
 extensions: []
 extra_rdoc_files:
@@ -160,12 +161,16 @@ files:
 - bin/iudex-da-generate-test-data
 - bin/iudex-da-import
 - bin/iudex-da-simhash-dump
+- bin/iudex-da-unreserve
 - bin/iudex-migrate
 - config/config.rb
 - db/20111012173757_base.rb
 - db/20120930173600_uhash_collation_order.rb
+- db/20130419090000_instance_column.rb
+- db/20130419095500_reserved_column.rb
 - db/index_next_visit/21500000000101_add_index_next_visit.rb
 - db/index_next_visit/21500000000110_index_next_visit_partial.rb
+- db/index_next_visit/21500000000120_index_next_visit_not_reserved.rb
 - db/simhash/21500000000001_add_simhash_index.rb
 - lib/iudex-da/base.rb
 - lib/iudex-da.rb
@@ -182,7 +187,7 @@ files:
 - test/test_pool_factory.rb
 - test/test_url_model.rb
 - test/test_work_poller.rb
-- lib/iudex-da/iudex-da-1.3.3.jar
+- lib/iudex-da/iudex-da-1.4.0.jar
 homepage: http://iudex.gravitext.com
 licenses: []
 post_install_message: