iudex-da 1.2.1-java → 1.3.0-java

data/History.rdoc

@@ -1,3 +1,33 @@
+=== 1.3.0 (2012-10-4)
+* Expand to iudex-core [1.2.1,1.4)
+* Rewrite of WorkPoller:
+  * Now in Ruby, for ease of customization.
+  * max_priority_urls now defaults off for common cases
+  * do_domain_group now selectable (default off, commonly unneeded)
+  * domain_depth_coef (deprecated host_depth_divisor reciprocal) now
+    defaults nil (domain window partitioning off)
+  * Added new priority aging feature (priority increases with current
+    age of next_visit_after), on by default.
+  * Added new domain_union feature, which allows domain filtering,
+    independent size control, and faster polling, as compared with
+    general domain depth prioritization, when a limited set of domains
+    is used.
+  * Added new uhash_slice feature, which allows selecting a specific
+    segment of the overall uhash space, and scaling to multiple worker
+    instances (and IPs) with a shared database. This form of sharding
+    offers best pseudo-random balancing of URLs at the expense of
+    single-domain-instance politeness control.
+* Replace use of Activerecord (for migrations and tests) with Sequel:
+  * Smaller deps, faster load times (particularly under jruby)
+  * Better Primary Key, composite VisitURL, type field support (models
+    Url).
+  * Provide Activerecord -> Sequel migration support (from a complete
+    iudex-da 1.2.1 db); consolidate prior migrations to a single base
+    and profile migrations.
+* Add new (Sequel) migration to set "C" locale collate (aka ASCII sort
+  order) for uhash (required for uhash_slice, see above.) iudex-da now
+  depends on PostgreSQL 9.1 for COLLATE support.
+
 === 1.2.1 (2012-9-15)
 * Upgrade to logback ~> 1.2 (dev)
 

data/Manifest.txt

@@ -8,29 +8,23 @@ bin/iudex-da-import
 bin/iudex-da-simhash-dump
 bin/iudex-migrate
 config/config.rb
-db/0010_base_urls.rb
-db/0020_add_feed_metadata.rb
-db/0021_more_feed_text.rb
-db/0030_add_priority.rb
-db/0040_add_visit_after.rb
-db/0050_add_cache_location.rb
-db/0060_url_indexes.rb
-db/0070_add_created_at.rb
-db/0080_add_simhash.rb
-db/0081_remove_simhash_index.rb
-db/0110_host_to_domain.rb
-db/index_next_visit/0100_add_index_next_visit.rb
-db/simhash/0085_add_simhash_index.rb
+db/20111012173757_base.rb
+db/20120930173600_uhash_collation_order.rb
+db/index_next_visit/21500000000101_add_index_next_visit.rb
+db/simhash/21500000000001_add_simhash_index.rb
 lib/iudex-da/base.rb
 lib/iudex-da.rb
-lib/iudex-da/ar.rb
 lib/iudex-da/config.rb
 lib/iudex-da/factory_helper.rb
 lib/iudex-da/importer.rb
 lib/iudex-da/key_helper.rb
+lib/iudex-da/models.rb
+lib/iudex-da/orm.rb
 lib/iudex-da/pool_data_source_factory.rb
+lib/iudex-da/work_poller.rb
 test/setup.rb
 test/test_migrate.rb
-test/test_poll_work.rb
 test/test_pool_factory.rb
-lib/iudex-da/iudex-da-1.2.1.jar
+test/test_url_model.rb
+test/test_work_poller.rb
+lib/iudex-da/iudex-da-1.3.0.jar

data/bin/iudex-migrate

@@ -56,7 +56,8 @@ END
       end
     end
     opts.on( "-d", "--debug" ) do
-      Logback[ 'iudex.da' ].level = Logback::DEBUG
+      Logback[ 'iudex.da' ].level = :debug
+      Hooker.add( [ :iudex, :connect_props ] ) { { :log => true } }
     end
     opts.on( "-v", "--version", "Display version and exit" ) do
       puts "iudex-da: #{DA::VERSION}"
@@ -65,8 +66,11 @@ END
     Hooker.register_config( opts )
   end.parse!
 
-  require 'iudex-da/ar'
+  require 'iudex-da/orm'
 
-  DA::migrate( ARGV[0] && ARGV[0].to_i )
+  target = ARGV[0] && ARGV[0].to_i
+  opts = {}
+  opts[ :target ] = target if target
+  DA::ORM::migrate( opts )
 
 end

data/db/20111012173757_base.rb

@@ -0,0 +1,117 @@
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+Sequel.migration do
+  change do
+
+    # The main/only urls table, in matching a default-setup iudex
+    # 1.1.0-1.2.1.
+    create_table( :urls ) do
+
+      twtz = "timestamp with time zone"
+      now  = Sequel::CURRENT_TIMESTAMP
+
+      String    :uhash,            :null => false
+      # 23 byte ASCII PRIMARY KEY SHA-1 hash fragment of URL
+
+      String    :url,              :null => false
+      # Complete normalized url (exactly as used for uhash)
+
+      String    :domain,           :null => false
+      # Registration level domain from url host
+
+      String    :type,             :null => false
+      # FEED, PAGE, ROBOTS, SITEMAP
+      # Potentially speculative (i.e. "PAGE" before visited)
+      # FIXME: Or REDIRECT here instead of status?
+
+      String    :etag
+      # HTTP ETag header used for subsequent conditional GET
+      # Should only be on 200 and related HTTP status, not redirect
+
+      DateTime  :last_visit,       :type => twtz
+      # Time of last visit (and thus last type,status,reason,etc.)
+
+      Integer   :status
+      # HTTP status code or special (negative) status mapping
+      # null     : Not yet visited
+      #  -1      : Connection Failed
+      # 4xx      : Permanent Failures
+      # 5xx      : Transient server error
+      # 200      : Success
+      # 304      : Not Modified
+      # 301,302  : Redirect
+      # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+
+      TrueClass :pass
+      # null     : Not yet processed (i.e. visit failed)
+      # false    : Rejected by processing (for reason), DELETE required
+      # true     : Fully Processed
+
+      String    :reason
+      # null      : None
+      # DUPE      : Duplicate of referent
+      # rejection filter (intended as key)
+
+      String    :referent
+      # null      : None
+      # uhash of url this is refering to
+      # (includes status:REDIRECT, reason:DUPE, etc.)
+
+      String    :referer
+      # null      : None
+      # uhash of url this was refered from. (i.e. the feed URL)
+
+      String    :title
+      # PAGE,FEED title
+
+      DateTime  :ref_pub_date,     :type => twtz
+      # (Latest) published date as provided from feed (may be ahead of
+      # or set before pub_date, below).
+
+      DateTime  :pub_date,         :type => twtz
+      # (Latest) published date as processed
+
+      String    :summary
+      # (Feed) summary
+
+      String    :content
+      # (Feed) content
+
+      Float     :priority,         :type => "real", :default => 0.0, :null => false
+      # Prioritization of next visit, range -INF,+INF
+
+      DateTime  :next_visit_after, :type => twtz,   :default => now
+      # null: never visit (terminal result)
+      # Don't visit again before the specified date.
+
+      Integer   :cache_file
+      # 32-bit file number
+
+      Bignum    :cache_file_offset
+      # 64-bit byte offset within file
+
+      DateTime  :created_at,       :type => twtz,   :default => now
+      # When inserted
+
+      Bignum    :simhash
+      # A simhash signature as a signed 8-byte long (should be
+      # compatible with java long).
+
+      primary_key [ :uhash ]
+    end
+  end
+end

data/db/{0070_add_created_at.rb → 20120930173600_uhash_collation_order.rb}

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -14,15 +14,21 @@
 # permissions and limitations under the License.
 #++
 
-class AddCreatedAt < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'created_at', 'timestamp with time zone' )
-    execute 'ALTER TABLE urls ALTER COLUMN created_at SET DEFAULT now()'
+Sequel.migration do
+  up do
+    run <<-DDL
+     ALTER TABLE urls
+       ALTER COLUMN uhash
+       SET DATA TYPE text COLLATE "C"
+    DDL
+    run "REINDEX INDEX urls_pkey"
   end
-
-  def self.down
-    remove_column( 'urls', 'created_at' )
+  down do
+    run <<-DDL
+     ALTER TABLE urls
+       ALTER COLUMN uhash
+       SET DATA TYPE text
+    DDL
+    run "REINDEX INDEX urls_pkey"
   end
-
 end

data/db/{simhash/0085_add_simhash_index.rb → index_next_visit/21500000000101_add_index_next_visit.rb}

@@ -14,14 +14,10 @@
 # permissions and limitations under the License.
 #++
 
-class AddSimhashIndex < ActiveRecord::Migration
-
-  def self.up
-    add_index( 'urls', [ 'simhash' ] )
-  end
-
-  def self.down
-    remove_index( 'urls', 'simhash'  )
+Sequel.migration do
+  change do
+    alter_table( :urls ) do
+      add_index( :next_visit_after )
+    end
   end
-
 end

data/db/{0081_remove_simhash_index.rb → simhash/21500000000001_add_simhash_index.rb}

@@ -14,14 +14,10 @@
 # permissions and limitations under the License.
 #++
 
-class RemoveSimhashIndex < ActiveRecord::Migration
-
-  def self.up
-    remove_index( 'urls', 'simhash'  )
-  end
-
-  def self.down
-    add_index( 'urls', [ 'simhash' ] )
+Sequel.migration do
+  change do
+    alter_table( :urls ) do
+      add_index( :simhash )
+    end
   end
-
 end

data/lib/iudex-da.rb

@@ -29,8 +29,8 @@ module Iudex
 
     require "#{LIB_DIR}/iudex-da-#{VERSION}.jar"
 
-    import 'iudex.da.WorkPoller'
     import 'iudex.da.ContentMapper'
+    import 'iudex.da.ContentReader'
 
     module Filters
       import 'iudex.da.filters.UpdateFilter'
@@ -39,3 +39,5 @@ module Iudex
 
   end
 end
+
+require 'iudex-da/work_poller.rb'

data/lib/iudex-da/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module DA
-    VERSION = '1.2.1'
+    VERSION = '1.3.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-da/config.rb

@@ -18,10 +18,10 @@
 module Iudex
 
   module DA
-    # Default database connection configuration for both ActiveRecord
-    # (migrations, testing) and PoolDataSourceFactory.
+    # Default database connection configuration for both Sequel
+    # (migrations, testing) and JDBC PoolDataSourceFactory.
     CONFIG = {
-      :adapter  => 'jdbcpostgresql',
+      :adapter  => 'jdbc:postgresql',
       :host     => 'localhost',
       :database => 'iudex_test',
       :username => 'iudex',

data/lib/iudex-da/models.rb

@@ -0,0 +1,66 @@
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da/orm'
+
+# Ensure setup has been run, as Sequel::Model needs the database
+# connected for schema at model class creation
+Iudex::DA::ORM.db
+
+module Iudex::DA::ORM
+
+  # Url model (for urls table). Usage note: ORM::setup must be called
+  # before this can be loaded.
+  class Url < ::Sequel::Model
+
+    VisitURL = Iudex::Core::VisitURL
+
+    plugin :composition
+
+    composition( :visit_url,
+                 :composer => proc { VisitURL.trust( url ) },
+                 :decomposer => proc {
+                   if v = compositions[ :visit_url ]
+                     self.url    = v.url
+                     self.uhash  = v.uhash
+                     self.domain = v.domain
+                   end
+                 } )
+
+    def visit_url=( vurl )
+      vurl = VisitURL.normalize( vurl ) unless vurl.is_a?( VisitURL )
+      super( vurl )
+    end
+
+    def self.find_by_url( vurl )
+      vurl = VisitURL.normalize( vurl ) unless vurl.is_a?( VisitURL )
+      self[ vurl.uhash ]
+    end
+
+    # Specifically include type accessors to avoid deprecation warnings for
+    # old ruby method.
+
+    def type
+      self[ :type ]
+    end
+
+    def type=( t )
+      self[ :type ] = t
+    end
+
+  end
+
+end

data/lib/iudex-da/orm.rb

@@ -0,0 +1,183 @@
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rjack-slf4j'
+require 'iudex-da/config'
+require 'sequel'
+require 'jdbc/postgres'
+require 'hooker'
+
+Sequel.extension :migration
+
+module Iudex::DA
+
+  module ORM
+
+    class << self
+
+      # The Sequel::Database instance. #setup is called if necessary.
+      def db
+        setup unless @db
+        @db
+      end
+
+      # Setup the ORM (Sequel) connection given CONFIG defaults, any
+      # passed opts, and connect_props config hooks.
+      def setup( opts = {} )
+
+        @db.disconnect if @db
+
+        log = RJack::SLF4J[ "iudex.da.sequel" ]
+        conf = CONFIG.merge( opts )
+        conf = Hooker.merge( [ :iudex, :connect_props ], conf )
+
+        conf[ :loggers ] = [ log ] if conf[ :log ]
+
+        cstr = ( "%s://%s/%s?%s" %
+                 [ conf[ :adapter ],
+                   [ conf[ :host ], conf[ :port ] ].compact.join( ':' ),
+                   conf[ :database ],
+                   params( conf ) ] )
+
+        log.info { "Connecting: #{cstr}" }
+        log.debug { "Full Params: #{ conf.inspect }" }
+
+        @db = Sequel.connect( cstr, conf )
+
+      end
+
+      # Migrate the DB given opts, including :target version.  For
+      # backward compatibility, opts may be a single Integer,
+      # interpreted as the :target version.  Setup must be called
+      # beforehand.
+      # See also opts for #migrate_ar_to_sequel
+      def migrate( opts = {} )
+        opts = {} if opts.nil?
+        opts = { :target => opts } if opts.is_a?( Integer )
+        raise "setup must be run before migrate" unless db
+        profiles = Hooker.apply( [ :iudex, :migration_profiles ],
+                                 opts[ :profiles ] || [] )
+
+        migrate_ar_to_sequel( opts )
+
+        pm = ProfileMigrator.new( db, profiles, opts )
+        pm.run
+      end
+
+      # Migrate from a iudex [1.1.0,1.3) database managed by
+      # activerecord to a 1.3.x database managed by Sequel. No-op if
+      # already Sequel.
+      # === Options
+      # :ar_to_sequel_migrations:: Hash<Integer,String> AR migration
+      #                            number to sequel filename
+      #                            (timestamp) map for extensions
+      #                            supported externally to iudex-da.
+      def migrate_ar_to_sequel( opts )
+
+        columns = ( db.table_exists?( :schema_migrations ) &&
+                    db.schema( :schema_migrations ).map { |sr| sr[0] } )
+
+        if columns == [ :version ] # Old format AR schema_migrations
+          db.transaction do
+            versions = db.from( :schema_migrations ).
+              map { |r| r[ :version ].to_i }
+
+            if ( versions & AR_REQUIRED ) != AR_REQUIRED
+              missing = AR_REQUIRED - ( versions & AR_REQUIRED )
+              raise( ARNotComplete,
+                     "Missing AR migrations #{missing.inspect}; " +
+                     "Use 'iudex-migrate _1.2.1_' first" )
+            end
+
+            migrations_map = AR_TO_SEQUEL_MIGRATIONS.
+              merge( opts[ :ar_to_sequel_migrations ] || {} )
+
+            db.drop_table( :schema_migrations )
+            db.create_table( :schema_migrations ) do
+              String :filename, :null => false
+              primary_key [ :filename ]
+            end
+
+            sm = db[:schema_migrations]
+            sm.insert( :filename => '20111012173757_base.rb' )
+
+            migrations_map.each do | version, filename |
+              sm.insert( :filename => filename ) if versions.include?( version )
+            end
+          end
+        end
+      end
+
+      def params( opts )
+        pms = {}
+
+        u = opts[ :username ]
+        pms[ :user ] = u if u
+
+        p = opts[ :password ]
+        pms[ :password ] = p if p
+
+        pms.sort.map { |*p| p.join( '=' ) }.join( '&' )
+      end
+
+    end
+
+    AR_TO_SEQUEL_MIGRATIONS = {
+       85 => '21500000000001_add_simhash_index.rb',
+      100 => '21500000000101_add_index_next_visit.rb'
+    }
+    AR_REQUIRED = [ 10, 20, 21, 30, 40, 50, 60, 70, 80, 81, 110 ]
+
+    ARNotComplete = Class.new(StandardError)
+
+    @db = nil
+
+    # Custom migrator handling "profile" directories (optional
+    # migrations)
+    class ProfileMigrator < Sequel::TimestampMigrator
+
+      def initialize( db, profiles, opts )
+
+        base = File.join( LIB_DIR, '..', '..', 'db' )
+        paths = [ base ]
+
+        paths += profiles.compact.map do |p|
+          p = p.to_s
+          if p =~ %r{^/}
+            p
+          else
+            File.join( base, p )
+          end
+        end
+
+        @pattern = if paths.size > 1
+                     '{' + paths.join( ',' ) + '}'
+                   else
+                     paths.first
+                   end
+
+        super( db, base, opts )
+      end
+
+      def get_migration_files
+        Dir.glob( "#{@pattern}/[0-9]*_*.rb" )
+      end
+
+    end
+
+  end
+
+end