iudex-da 1.4.0-java → 1.5.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 78b67b192c7ddc6613cab0a4f4946d2d78c5d96c
+  data.tar.gz: 9b240415f16b58d8c264caf9a34a1b791208758b
+SHA512:
+  metadata.gz: 2f49c63c044a60a37d589737998e743c742db233b02bf14ffa02f7a47ceeafa128000ea72a5759fafc3cff12d9425c5cb67c1669a17a28b34b177a833958c5c9
+  data.tar.gz: b4c7fe664325e13b5f5c9d77653aedd4930bedb0c800c095c00a08bcd5d2dec9f48af9b467bfe736a9173e7b67920e0c1e97f2add895e69882891a93800bc9b9

data/History.rdoc

@@ -1,3 +1,19 @@
+=== 1.5.0 (2014-3-27)
+Note: this release is at this time considered experimental or an "alpha"
+by comparison to 1.4.0
+* Upgrade to jdbc-postgres ~> 9.3.1100 (also tested 9.3.1101)
+* Upgrade to sequel ~> 4.8.0
+* Expand to >= 1.4.0, < 1.7 iudex-core dependency
+* Add WorkPoller.isolation_level accessor
+* Add WorkPoller.max_retries accessor
+* Auto-compute WorkPoller max_urls from set domain_union
+* Add exponential backoff delay option to ContentReader.handleError
+  retries, setter in WorkPoller
+* Add explicit LOCK mode option to URLs table in ContentReader, setter
+  in WorkPoller
+* Add null check to WorkPoller select_with_retry call (may occur when
+  retries exhausted).
+
 === 1.4.0 (2013-10-29)
 * Add experimental WorkPoller.do_reserve support, which utilizes
   PostgreSQL 9.1+ writable Common Table Expressions (CTEs) to

data/Manifest.txt

@@ -32,4 +32,4 @@ test/test_migrate.rb
 test/test_pool_factory.rb
 test/test_url_model.rb
 test/test_work_poller.rb
-lib/iudex-da/iudex-da-1.4.0.jar
+lib/iudex-da/iudex-da-1.5.0.jar

data/README.rdoc

@@ -16,7 +16,7 @@ meta-data store and work priority queue.
 
 == License
 
-Copyright (c) 2008-2013 David Kellum
+Copyright (c) 2008-2014 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You may

data/bin/iudex-da-generate-test-data

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-import

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-simhash-dump

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-da-unreserve

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-migrate

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20111012173757_base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20120930173600_uhash_collation_order.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20130419090000_instance_column.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/20130419095500_reserved_column.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/index_next_visit/21500000000101_add_index_next_visit.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/index_next_visit/21500000000110_index_next_visit_partial.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/index_next_visit/21500000000120_index_next_visit_not_reserved.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/db/simhash/21500000000001_add_simhash_index.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,7 +16,7 @@
 
 module Iudex
   module DA
-    VERSION = '1.4.0'
+    VERSION = '1.5.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-da/config.rb

@@ -1,6 +1,6 @@
 
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/factory_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/importer.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/key_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-da/models.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/orm.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/pool_data_source_factory.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-da/work_poller.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -58,7 +58,8 @@ module Iudex::DA
     alias :max_host_urls= :max_domain_urls=
 
     # The limit of urls to obtain in a single poll (across all
-    # domains) (default: 50_000)
+    # domains). May be overridden by #domain_union=
+    # (default: 50_000)
     attr_accessor :max_urls
 
     # A secondary limit on the number of urls to consider, taking the
@@ -148,13 +149,13 @@ module Iudex::DA
     #         unconfigured domains.
     #
     # :max:: The maximum number of visit urls to obtain in one poll
-    #        (instead of the top level #max_urls.) A zero max_urls
-    #        value excludes this domain/type (efficiently).
+    #        (instead of the top level #max_urls.) A zero value
+    #        efficiently excludes this domain/type.
     #
     # Also a [ domain, max ] alternative syntax is currently supported
     # but deprecated.
     #
-    attr_accessor :domain_union
+    attr_reader :domain_union
 
     # An array containing a zero-based position and a total number of
     # evenly divided segments within the range of possible uhash
@@ -163,6 +164,25 @@ module Iudex::DA
     # being a hash on the entire URL. (default: nil, off)
     attr_accessor :uhash_slice
 
+    # Max-retries not including the initial try (Default: 10)
+    attr_accessor :max_retries
+
+    # The transaction isolation level to use for work polling
+    # See {java.sql.Connection constants}[http://docs.oracle.com/javase/7/docs/api/constant-values.html#java.sql.Connection.TRANSACTION_READ_COMMITTED]
+    # (Default: TRANSACTION_REPEATABLE_READ (4))
+    attr_accessor :isolation_level
+
+    # URLS table explicit lock mode (String) for work polling,
+    # i.e. 'EXCLUSIVE'. Try this if deadlocks/contention are otherwise causing work
+    # poll starvation or excessive retries.
+    # (Default: nil -> implicit locking via isolation level only)
+    attr_accessor :lock_mode
+
+    # Maximum first-retry back-off delay in seconds (Float). Actual
+    # delay is a random value from 0..(value * 2^tries) for each
+    # retry.  Default: 0.0 : no delay
+    attr_accessor :backoff
+
     def initialize( data_source, mapper )
       super()
 
@@ -178,6 +198,9 @@ module Iudex::DA
       @max_discard_ratio  = 2.0/3.0
       @max_reserved_time_s = nil
       @last_none_reserved  = Time.now
+      @max_retries        = 10
+      @isolation_level    = 4
+      @backoff            = 0.0
 
       @age_coef_1         = 0.2
       @age_coef_2         = 0.1
@@ -205,7 +228,8 @@ module Iudex::DA
 
     # Override GenericWorkPollStrategy
     def pollWorkImpl( visit_queue )
-      visit_queue.add_all( poll( visit_queue.order_count ) )
+      res = poll( visit_queue.order_count )
+      visit_queue.add_all( res ) if res
     rescue SQLException => x
       @log.error( "On poll: ", x )
     end
@@ -266,7 +290,10 @@ module Iudex::DA
     def reader
       @reader ||= ContentReader.new( @data_source, @mapper ).tap do |r|
         r.priority_adjusted = aged_priority?
-        r.max_retries = 10
+        r.max_retries = max_retries
+        r.isolation_level = isolation_level
+        r.lock_mode = lock_mode if lock_mode
+        r.backoff = ( backoff * 1_000 ).round if backoff > 0.0
       end
     end
 
@@ -275,9 +302,12 @@ module Iudex::DA
         args = args.flatten.dup
         opts = args.last.is_a?( Hash ) ? args.pop.dup : {}
         opts[ :domain ] ||= args.shift
-        opts[ :max ]    ||= args.shift
+        opts[ :max ]    ||= args.shift || @max_urls
         opts
       end
+      @max_urls = @domain_union.inject( 0 ) do |m,r|
+        m + r[:max]
+      end
     end
 
     def domain_union?
@@ -301,13 +331,8 @@ module Iudex::DA
         subqueries = []
         @domain_union.each do | opts |
           opts = opts.dup
-          if opts[ :max ]
-            opts[ :max ] = ( opts[ :max ] * ( max_urls - current_urls ) /
-                             max_urls.to_f ).floor
-          else
-            opts[ :max ] = ( max_urls - current_urls )
-          end
-
+          opts[ :max ] = ( opts[ :max ] * ( max_urls - current_urls ) /
+                           max_urls.to_f ).floor
           next if opts[ :max ] == 0
 
           c = criteria.dup

data/pom.xml

@@ -5,7 +5,7 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-da</artifactId>
   <packaging>jar</packaging>
-  <version>1.4.0</version>
+  <version>1.5.0</version>
   <name>Iudex Data Access</name>
 
   <parent>
@@ -20,7 +20,7 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.4.0,1.4.999)</version>
+      <version>[1.4.0,1.7)</version>
     </dependency>
 
     <dependency>

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_migrate.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/test/test_pool_factory.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_url_model.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/test/test_work_poller.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2013 David Kellum
+# Copyright (c) 2008-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

metadata

@@ -1,30 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: iudex-da
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 1.4.0
+  version: 1.5.0
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-10-30 00:00:00.000000000 Z
+date: 2014-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
-    none: false
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.7'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
-    none: false
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.7'
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -33,14 +36,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.46'
-    none: false
+        version: 4.8.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.46'
-    none: false
+        version: 4.8.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -49,14 +50,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 9.2.1002
-    none: false
+        version: 9.3.1100
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 9.2.1002
-    none: false
+        version: 9.3.1100
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -66,13 +65,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.4.0
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.4.0
-    none: false
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -82,13 +79,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.0
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.0
-    none: false
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -98,13 +93,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 4.7.4
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 4.7.4
-    none: false
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -114,13 +107,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
-    none: false
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -129,14 +120,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.0'
-    none: false
+        version: '2.1'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.0'
-    none: false
+        version: '2.1'
   prerelease: false
   type: :development
 description: Iudex is a general purpose web crawler and feed processor in ruby/java. The iudex-da gem provides a PostgreSQL-based content meta-data store and work priority queue.
@@ -187,9 +176,11 @@ files:
 - test/test_pool_factory.rb
 - test/test_url_model.rb
 - test/test_work_poller.rb
-- lib/iudex-da/iudex-da-1.4.0.jar
+- lib/iudex-da/iudex-da-1.5.0.jar
 homepage: http://iudex.gravitext.com
-licenses: []
+licenses:
+- Apache-2.0
+metadata: {}
 post_install_message:
 rdoc_options:
 - --main
@@ -198,27 +189,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: 2
-  none: false
+      version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2
-  none: false
 requirements: []
 rubyforge_project:
-rubygems_version: 1.8.24
+rubygems_version: 2.1.9
 signing_key:
-specification_version: 3
+specification_version: 4
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java.
 test_files: []
-...