iudex-da 1.0.0-java → 1.1.0-java

data/History.rdoc

@@ -1,2 +1,12 @@
+=== 1.1.0 (2011-11-13)
+* Update to iudex-core ~> 1.1.0
+* Update to activerecord ~> 3.0.10, pg ~> 9.0.801, adapter ~> 1.1.3
+* Expand commons-pool pom dep to [1.5.4, 1.5.6]
+* Migrate host field to domain (registration level), mapper/writer
+  domain support, :domain in WorkPoller, drop :host key.
+* Add migration profile support, move simhash index to simhash
+  profile, add index_next_visit migration profile
+* Update to minitest ~> 2.3
+
 === 1.0.0 (2011-04-04)
 * Initial release.

data/Manifest.txt

@@ -17,6 +17,9 @@ db/0050_add_cache_location.rb
 db/0060_url_indexes.rb
 db/0070_add_created_at.rb
 db/0080_add_simhash.rb
+db/0081_remove_simhash_index.rb
+db/index_next_visit/0100_add_index_next_visit.rb
+db/simhash/0085_add_simhash_index.rb
 lib/iudex-da/base.rb
 lib/iudex-da.rb
 lib/iudex-da/ar.rb
@@ -29,4 +32,4 @@ test/setup.rb
 test/test_migrate.rb
 test/test_poll_work.rb
 test/test_pool_factory.rb
-lib/iudex-da/iudex-da-1.0.0.jar
+lib/iudex-da/iudex-da-1.1.0.jar

data/Rakefile

@@ -4,7 +4,7 @@ $LOAD_PATH << './lib'
 require 'iudex-da/base'
 
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.2'
+gem     'rjack-tarpit', '~> 1.4'
 require 'rjack-tarpit'
 
 t = RJack::TarPit.new( 'iudex-da',
@@ -14,15 +14,15 @@ t = RJack::TarPit.new( 'iudex-da',
 t.specify do |h|
   h.developer( "David Kellum", "dek-oss@gravitext.com" )
 
-  h.extra_deps += [ [ 'iudex-core',            '~> 1.0.0'  ],
-                    [ 'activerecord',          '~> 2.3.10' ],
-                    [ 'jdbc-postgres',         '>= 8.4.702',  '< 9.1' ],
-                    [ 'activerecord-jdbcpostgresql-adapter', '~> 1.1.0' ],
+  h.extra_deps += [ [ 'iudex-core',            '~> 1.1.0'   ],
+                    [ 'activerecord',          '~> 3.0.10'  ],
+                    [ 'jdbc-postgres',         '~> 9.0.801' ],
+                    [ 'activerecord-jdbcpostgresql-adapter', '~> 1.1.3' ],
                     [ 'rjack-commons-dbcp',    '~> 1.4.0'  ],
                     [ 'rjack-commons-dbutils', '~> 1.3.0'  ] ]
 
   h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',          '>= 1.7.1', '< 2.1' ],
+  h.extra_dev_deps += [ [ 'minitest',          '~> 2.3' ],
                         [ 'rjack-logback',     '~> 1.0' ] ]
 end
 
@@ -42,8 +42,4 @@ task :gem  => [ :check_pom_version, :check_history_version
 task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
 task :push => [                                             :check_history_date ]
 
-# Disable verbose warnings, which are a bit much with ActiveRecord
-# 2.3.x at least.
-Hoe::RUBY_FLAGS.sub!( /\-w(\s|$)/, '-W1\1' )
-
 t.define_tasks

data/bin/iudex-da-generate-test-data

@@ -115,7 +115,7 @@ OptionParser.new do |opts|
   opts.on( "-d", "--debug" ) do
     RJack::Logback[ 'iudex.da' ].level = RJack::Logback::DEBUG
   end
-  opts.on( "-v", "--version", "Display version and exit" ) do |file|
+  opts.on( "-v", "--version", "Display version and exit" ) do
     puts "iudex-da: #{Iudex::DA::VERSION}"
     exit 1
   end

data/bin/iudex-da-import

@@ -49,7 +49,7 @@ END
     opts.on( "-d", "--debug" ) do
       Logback[ 'iudex' ].level = Logback::DEBUG
     end
-    opts.on( "-v", "--version", "Display version and exit" ) do |file|
+    opts.on( "-v", "--version", "Display version and exit" ) do
       puts "iudex-da: #{Iudex::DA::VERSION}"
       exit 1
     end

data/bin/iudex-da-simhash-dump

@@ -54,7 +54,7 @@ END
       opts.on( "-d", "--debug" ) do
         Logback[ 'iudex.da' ].level = Logback::DEBUG
       end
-      opts.on( "-v", "--version", "Display version and exit" ) do |file|
+      opts.on( "-v", "--version", "Display version and exit" ) do
         puts "iudex-da: #{Iudex::DA::VERSION}"
         exit 1
       end

data/bin/iudex-migrate

@@ -49,10 +49,16 @@ END
         { name.to_sym => value }
       end
     end
+    opts.on( "-p", "--profile NAME", String,
+             "Add a migration profile (ex: simhash)" ) do |p|
+      Hooker.add( [ :iudex, :migration_profiles ] ) do |profiles|
+        profiles << p
+      end
+    end
     opts.on( "-d", "--debug" ) do
       Logback[ 'iudex.da' ].level = Logback::DEBUG
     end
-    opts.on( "-v", "--version", "Display version and exit" ) do |file|
+    opts.on( "-v", "--version", "Display version and exit" ) do
       puts "iudex-da: #{DA::VERSION}"
       exit 1
     end

data/config/config.rb

@@ -11,4 +11,9 @@ Iudex.configure do |c|
       :loglevel => 2 }
   end
 
+  # Add optional migration profiles
+  c.setup_migration_profiles do |profiles|
+    profiles += [ :simhash :index_next_visit ]
+  end
+
 end

data/db/0081_remove_simhash_index.rb

@@ -0,0 +1,27 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class RemoveSimhashIndex < ActiveRecord::Migration
+
+  def self.up
+    remove_index( 'urls', 'simhash'  )
+  end
+
+  def self.down
+    add_index( 'urls', [ 'simhash' ] )
+  end
+
+end

data/db/index_next_visit/0100_add_index_next_visit.rb

@@ -0,0 +1,27 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddIndexNextVisit < ActiveRecord::Migration
+
+  def self.up
+    add_index( 'urls', 'next_visit_after' )
+  end
+
+  def self.down
+    remove_index( 'urls', 'next_visit_after' )
+  end
+
+end

data/db/simhash/0085_add_simhash_index.rb

@@ -0,0 +1,27 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddSimhashIndex < ActiveRecord::Migration
+
+  def self.up
+    add_index( 'urls', [ 'simhash' ] )
+  end
+
+  def self.down
+    remove_index( 'urls', 'simhash'  )
+  end
+
+end

data/lib/iudex-da/ar.rb

@@ -33,9 +33,13 @@ module Iudex::DA
   setup #FIXME: Require explicit setup for use?
 
   def migrate( target_version = nil )
-    ActiveRecord::Migrator.migrate( File.join( LIB_DIR, '..', '..', 'db' ),
-                                    target_version )
-    #FIXME: Support additional migration directories?
+    base = File.join( LIB_DIR, '..', '..', 'db' )
+
+    profiles = Hooker.apply( [ :iudex, :migration_profiles ], [] )
+
+    ext = profiles.compact.map { |p| "/#{p}" }.join(',')
+    base += "{#{ext},}" unless ext.empty?
+    ActiveRecord::Migrator.migrate( base, target_version )
   end
 
   module_function :migrate

data/lib/iudex-da/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module DA
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-da/importer.rb

@@ -66,7 +66,7 @@ module Iudex::DA
     end
 
     def import_keys
-      [ :uhash, :host, :url, :type, :priority, :next_visit_after ]
+      [ :uhash, :domain, :url, :type, :priority, :next_visit_after ]
     end
 
     def template_map

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-da</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.0</version>
+  <version>1.1.0</version>
   <name>Iudex Data Access</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -29,7 +29,7 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.0,1.1)</version>
+      <version>[1.1,1.2)</version>
     </dependency>
 
     <dependency>
@@ -47,13 +47,13 @@
     <dependency>
       <groupId>commons-pool</groupId>
       <artifactId>commons-pool</artifactId>
-      <version>[1.5.4,1.5.5]</version>
+      <version>[1.5.4,1.5.6]</version>
     </dependency>
 
     <dependency>
       <groupId>org.postgresql</groupId>
       <artifactId>postgresql-jdbc4</artifactId>
-      <version>[8.4.702,9.1)</version>
+      <version>[8.4.702,9.0.9999]</version>
       <scope>test</scope>
     </dependency>
 

data/test/test_migrate.rb

@@ -26,16 +26,49 @@ class TestMigrate < MiniTest::Unit::TestCase
   include Iudex::DA
   include RJack
 
-  def test_up_down_up
-    Logback[ 'iudex.da.ActiveRecord' ].level = Logback::WARN
+  VERBOSE = ! ( ARGV & %w[ -v --verbose ] ).empty?
 
-    ActiveRecord::Migration.suppress_messages do
+  def setup
+    unless VERBOSE
+      Logback[ 'iudex.da.ActiveRecord' ].level = Logback::WARN
+    end
+  end
+
+  def teardown
+    Hooker.send( :clear )
+    suppress_messages? { migrate }
+    Logback[ 'iudex.da.ActiveRecord' ].level = nil
+  end
+
+  def test_default
+    check_up_down
+  end
+
+  def test_simhash_profile
+    Hooker.add( [ :iudex, :migration_profiles ] ) { |p| p << :simhash }
+    check_up_down
+  end
+
+  def test_next_visit_profile
+    Hooker.add( [ :iudex, :migration_profiles ] ) { |p| p << :index_next_visit }
+    check_up_down
+  end
+
+  def check_up_down
+    suppress_messages? do
       migrate
+      pass
       migrate( 0 )
-      migrate
+      pass
     end
+  end
 
-    Logback[ 'iudex.da.ActiveRecord' ].level = nil
+  def suppress_messages?( &block )
+    if VERBOSE
+      block.call
+    else
+      ActiveRecord::Migration.suppress_messages( &block )
+    end
   end
 
 end

data/test/test_poll_work.rb

@@ -29,15 +29,15 @@ class TestPollWork < MiniTest::Unit::TestCase
   def setup
     Url.delete_all
 
-    hosts = [ 'foo.org', 'other.net', 'gravitext.com', 'one.at' ]
+    domains = [ 'foo.org', 'other.net', 'gravitext.com', 'one.at' ]
     count = 0
-    hosts.each do |host|
+    domains.each do |domain|
       (5..15).each do |val|
         url = Url.create! do |u|
           u.priority = ( val.to_f / 10.0 ) + (count.to_f / 50.0)
-          vurl = VisitURL.normalize( "http://#{host}/#{u.priority}" )
+          vurl = VisitURL.normalize( "http://#{domain}/#{u.priority}" )
           u.type = "FEED"
-          u.host = vurl.host
+          u.domain = vurl.domain
           u.url = vurl.to_s
           u.uhash = vurl.uhash
           u.next_visit_after = Time.now
@@ -51,38 +51,38 @@ class TestPollWork < MiniTest::Unit::TestCase
     Url.delete_all
   end
 
-  # Query to get new work, with limits on work per host, and total
+  # Query to get new work, with limits on work per domain, and total
   # work (in descending piority order)
   def test_poll
     query = <<END
-SELECT url, host, type, priority
+SELECT url, domain, type, priority
 FROM ( SELECT *, row_number() OVER ( ORDER BY priority DESC ) as ppos
-       FROM ( SELECT *, row_number() OVER ( PARTITION BY host
+       FROM ( SELECT *, row_number() OVER ( PARTITION BY domain
                                             ORDER BY priority DESC ) AS hpos
               FROM urls
               WHERE next_visit_after <= now() ) AS subh
        WHERE hpos <= ? ) AS subp
 WHERE ppos <= ?
-ORDER BY host, priority DESC;
+ORDER BY domain, priority DESC;
 END
     res = Url.find_by_sql( [ query, 5, 18 ] )
 
-    def check_host_subset( byhost )
-      assert( byhost.length <= 5 )
-      byhost.each_cons(2) { |p,n| assert( p.priority >= n.priority ) }
+    def check_domain_subset( bydomain )
+      assert( bydomain.length <= 5 )
+      bydomain.each_cons(2) { |p,n| assert( p.priority >= n.priority ) }
     end
 
     assert( res.length <= 18 )
-    byhost = []
+    bydomain = []
     res.each do |u|
-      if byhost.empty? || byhost.last.host == u.host
-        byhost << u
+      if bydomain.empty? || bydomain.last.domain == u.domain
+        bydomain << u
       else
-        check_host_subset( byhost )
-        byhost = []
+        check_domain_subset( bydomain )
+        bydomain = []
       end
     end
-    check_host_subset( byhost ) unless byhost.empty?
+    check_domain_subset( bydomain ) unless bydomain.empty?
 
   end
 
@@ -93,7 +93,7 @@ END
 CREATE TEMPORARY TABLE mod_urls
   ( uhash text,
     url text,
-    host text );
+    domain text );
 END
     # ON COMMIT DROP;
 
@@ -110,14 +110,14 @@ END
         vurl = VisitURL.normalize( "http://gravitext.com/#{priority}" )
 
         sql = "INSERT into mod_urls VALUES ('%s','%s','%s')" %
-          [ vurl.uhash, vurl.to_s, vurl.host ]
+          [ vurl.uhash, vurl.to_s, vurl.domain ]
         Url.connection.execute( sql )
           # u.next_visit_after = Time.now
         count += 1
       end
       insert_query = <<END
-INSERT INTO urls (uhash,url,host,type,priority)
-  ( SELECT uhash,url,host,'FEEDX',4.78 FROM mod_urls
+INSERT INTO urls (uhash,url,domain,type,priority)
+  ( SELECT uhash,url,domain,'FEEDX',4.78 FROM mod_urls
     WHERE uhash NOT IN ( SELECT uhash FROM urls ) );
 END
       Url.connection.execute( insert_query )

data/test/test_pool_factory.rb

@@ -50,7 +50,7 @@ class TestPoolFactory < MiniTest::Unit::TestCase
   end
 
   def test_query
-    assert( ! @data_source.nil? )
+    refute( @data_source.nil? )
     qrun = QueryRunner.new( @data_source )
     qrun.query( "SELECT url FROM urls WHERE uhash IN ('uRlU1h_YL-NvooSv2i98Rd3', 'notthere' );",
                 TestHandler.new )

metadata

@@ -2,7 +2,7 @@
 name: iudex-da
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.0
+  version: 1.1.0
 platform: java
 authors: 
   - David Kellum
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-04 00:00:00 -07:00
-default_executable: 
+date: 2011-11-13 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: iudex-core
@@ -21,7 +20,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id001
   - !ruby/object:Gem::Dependency 
@@ -32,7 +31,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 2.3.10
+            version: 3.0.10
     type: :runtime
     version_requirements: *id002
   - !ruby/object:Gem::Dependency 
@@ -41,12 +40,9 @@ dependencies:
     requirement: &id003 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 8.4.702
-        - - <
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "9.1"
+            version: 9.0.801
     type: :runtime
     version_requirements: *id003
   - !ruby/object:Gem::Dependency 
@@ -57,7 +53,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.1.0
+            version: 1.1.3
     type: :runtime
     version_requirements: *id004
   - !ruby/object:Gem::Dependency 
@@ -88,12 +84,9 @@ dependencies:
     requirement: &id007 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.7.1
-        - - <
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "2.1"
+            version: "2.3"
     type: :development
     version_requirements: *id007
   - !ruby/object:Gem::Dependency 
@@ -115,7 +108,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.3.0
+            version: 1.4.0
     type: :development
     version_requirements: *id009
 description: |-
@@ -155,6 +148,9 @@ files:
   - db/0060_url_indexes.rb
   - db/0070_add_created_at.rb
   - db/0080_add_simhash.rb
+  - db/0081_remove_simhash_index.rb
+  - db/index_next_visit/0100_add_index_next_visit.rb
+  - db/simhash/0085_add_simhash_index.rb
   - lib/iudex-da/base.rb
   - lib/iudex-da.rb
   - lib/iudex-da/ar.rb
@@ -167,8 +163,8 @@ files:
   - test/test_migrate.rb
   - test/test_poll_work.rb
   - test/test_pool_factory.rb
-  - lib/iudex-da/iudex-da-1.0.0.jar
-has_rdoc: true
+  - lib/iudex-da/iudex-da-1.1.0.jar
+  - .gemtest
 homepage: http://github.com/dekellum/iudex
 licenses: []
 
@@ -193,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: iudex-da
-rubygems_version: 1.5.1
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java