iudex-da 1.2.1-java → 1.3.0-java

data/test/test_url_model.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-da'
+require 'iudex-da/models'
+
+class TestUrlModel < MiniTest::Unit::TestCase
+  include Iudex::DA
+  include Iudex::DA::ORM
+
+  def setup
+    Url.truncate
+  end
+
+  def test_round_trip
+    urls = [ "http://foo.gravitext.com/bar/1",
+             "http://gravitext.com/2",
+             "http://hometown.com/33" ]
+
+    urls.each do | u, p |
+      Url.create( :visit_url => u, :type => "PAGE"  )
+    end
+
+    assert_equal( 2, Url.where( :domain => 'gravitext.com' ).count )
+    assert_equal( 1, Url.where( :domain => 'hometown.com' ).count )
+
+    sample = Url.find_by_url( urls[ 0 ] )
+    assert_equal( urls[ 0 ], sample.url )
+    assert_equal( 'PAGE', sample.type )
+
+    refute( Url.find_by_url( "http://spunk" ) )
+  end
+
+end

data/test/test_work_poller.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-da'
+require 'iudex-da/key_helper'
+require 'iudex-da/pool_data_source_factory'
+require 'iudex-da/models'
+
+class TestWorkPoller < MiniTest::Unit::TestCase
+  include Iudex::Filter::KeyHelper
+  include Iudex::DA
+  include Iudex::DA::ORM
+
+  Gravitext::HTMap::UniMap.define_accessors
+
+  URLS = [ [ "http://foo.gravitext.com/bar/1", 11 ],
+           [ "http://hometown.com/33",         10 ],
+           [ "http://gravitext.com/2",          9 ] ]
+
+  def setup
+    Url.truncate
+
+    URLS.each do | u, p |
+      Url.create( :visit_url => u, :priority => p, :type => "PAGE"  )
+    end
+
+    @factory = PoolDataSourceFactory.new( :loglevel => 4 )
+    @data_source = @factory.create
+    @mapper = ContentMapper.new( keys( :url, :type, :priority,
+                                       :next_visit_after ) )
+    @poller = WorkPoller.new( @data_source, @mapper )
+  end
+
+  def teardown
+    @factory.close
+    @date_source = nil
+    @poller = nil
+  end
+
+  attr_reader :poller
+
+  def test_default_poll
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( URLS[ pos ][ 0 ], map.url.url )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+  end
+
+  def test_poll_with_max_priority_urls
+    poller.max_priority_urls = 4
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( URLS[ pos ][ 0 ], map.url.url )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+  end
+
+  def test_poll_with_domain_depth
+    poller.domain_depth_coef = 0.125
+    poller.max_priority_urls = 4
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( URLS[ pos ][ 0 ], map.url.url )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+  end
+
+  def test_poll_with_domain_depth_only
+    poller.domain_depth_coef = 0.125
+    poller.age_coef_1        = 0.0
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( URLS[ pos ][ 0 ], map.url.url )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+  end
+
+  def test_poll_with_domain_group
+    poller.do_domain_group = true
+
+    urls = [ [ "http://foo.gravitext.com/bar/1", 11 ],
+             [ "http://gravitext.com/2",          9 ],
+             [ "http://hometown.com/33",         10 ] ]
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( urls[ pos ][ 0 ], map.url.url, "pos #{pos}" )
+      pos += 1
+    end
+    assert_equal( 3, pos )
+  end
+
+
+  def test_poll_domain_union_1
+    poller.domain_union = [ [ 'gravitext.com', 15000 ] ]
+
+    result = poller.poll
+    assert_equal( 2, result.size )
+  end
+
+  def test_poll_domain_union_2
+    poller.domain_union = [ [ 'gravitext.com', 15000 ],
+                            [ nil, 10000 ] ]
+
+    result = poller.poll
+    assert_equal( 3, result.size )
+  end
+
+  def test_poll_domain_union_3
+    poller.domain_union = [ [ 'gravitext.com', 1 ],
+                            [ 'hometown.com', 1 ],
+                            [ nil, 3 ] ]
+
+    result = poller.poll
+    assert_equal( 2, result.size )
+  end
+
+  def test_poll_uhash_slice
+    poller.uhash_slice = [ 4, 5 ]
+
+    urls = [ [ "http://hometown.com/33",         10 ] ]
+
+    pos = 0
+    poller.poll.each do |map|
+      assert_equal( urls[ pos ][ 0 ], map.url.url, "pos #{pos}" )
+      pos += 1
+    end
+    assert_equal( 1, pos )
+  end
+
+end

metadata

@@ -1,199 +1,224 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: iudex-da
-version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 1.2.1
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 1.3.0
 platform: java
-authors: 
-  - David Kellum
-autorequire: 
+authors:
+- David Kellum
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2012-09-15 00:00:00 Z
-dependencies: 
-  - !ruby/object:Gem::Dependency 
-    name: iudex-core
-    version_requirements: &id001 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.2.1
-    requirement: *id001
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: activerecord
-    version_requirements: &id002 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 3.1.3
-    requirement: *id002
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: jdbc-postgres
-    version_requirements: &id003 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 9.1.901
-    requirement: *id003
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: activerecord-jdbcpostgresql-adapter
-    version_requirements: &id004 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.2.2
-    requirement: *id004
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: rjack-commons-dbcp
-    version_requirements: &id005 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.4.0
-    requirement: *id005
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: rjack-commons-dbutils
-    version_requirements: &id006 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: 1.4.0
-    requirement: *id006
-    prerelease: false
-    type: :runtime
-  - !ruby/object:Gem::Dependency 
-    name: minitest
-    version_requirements: &id007 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "2.3"
-    requirement: *id007
-    prerelease: false
-    type: :development
-  - !ruby/object:Gem::Dependency 
-    name: rjack-logback
-    version_requirements: &id008 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "1.2"
-    requirement: *id008
-    prerelease: false
-    type: :development
-  - !ruby/object:Gem::Dependency 
-    name: rjack-tarpit
-    version_requirements: &id009 !ruby/object:Gem::Requirement 
-      none: false
-      requirements: 
-        - - ~>
-          - !ruby/object:Gem::Version 
-            version: "2.0"
-    requirement: *id009
-    prerelease: false
-    type: :development
+date: 2012-10-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: iudex-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: sequel
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.40.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.40.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jdbc-postgres
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 9.1.901
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 9.1.901
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: rjack-commons-dbcp
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: rjack-commons-dbutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: minitest
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rjack-logback
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rjack-tarpit
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    none: false
+  prerelease: false
+  type: :development
 description: Iudex is a general purpose web crawler and feed processor in ruby/java. The iudex-da gem provides a PostgreSQL-based content meta-data store and work priority queue.
-email: 
-  - dek-oss@gravitext.com
-executables: 
-  - iudex-da-generate-test-data
-  - iudex-da-import
-  - iudex-da-simhash-dump
-  - iudex-migrate
+email:
+- dek-oss@gravitext.com
+executables:
+- iudex-da-generate-test-data
+- iudex-da-import
+- iudex-da-simhash-dump
+- iudex-migrate
 extensions: []
-
-extra_rdoc_files: 
-  - History.rdoc
-  - README.rdoc
-files: 
-  - History.rdoc
-  - Manifest.txt
-  - README.rdoc
-  - Rakefile
-  - pom.xml
-  - bin/iudex-da-generate-test-data
-  - bin/iudex-da-import
-  - bin/iudex-da-simhash-dump
-  - bin/iudex-migrate
-  - config/config.rb
-  - db/0010_base_urls.rb
-  - db/0020_add_feed_metadata.rb
-  - db/0021_more_feed_text.rb
-  - db/0030_add_priority.rb
-  - db/0040_add_visit_after.rb
-  - db/0050_add_cache_location.rb
-  - db/0060_url_indexes.rb
-  - db/0070_add_created_at.rb
-  - db/0080_add_simhash.rb
-  - db/0081_remove_simhash_index.rb
-  - db/0110_host_to_domain.rb
-  - db/index_next_visit/0100_add_index_next_visit.rb
-  - db/simhash/0085_add_simhash_index.rb
-  - lib/iudex-da/base.rb
-  - lib/iudex-da.rb
-  - lib/iudex-da/ar.rb
-  - lib/iudex-da/config.rb
-  - lib/iudex-da/factory_helper.rb
-  - lib/iudex-da/importer.rb
-  - lib/iudex-da/key_helper.rb
-  - lib/iudex-da/pool_data_source_factory.rb
-  - test/setup.rb
-  - test/test_migrate.rb
-  - test/test_poll_work.rb
-  - test/test_pool_factory.rb
-  - lib/iudex-da/iudex-da-1.2.1.jar
+extra_rdoc_files:
+- History.rdoc
+- README.rdoc
+files:
+- History.rdoc
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- pom.xml
+- bin/iudex-da-generate-test-data
+- bin/iudex-da-import
+- bin/iudex-da-simhash-dump
+- bin/iudex-migrate
+- config/config.rb
+- db/20111012173757_base.rb
+- db/20120930173600_uhash_collation_order.rb
+- db/index_next_visit/21500000000101_add_index_next_visit.rb
+- db/simhash/21500000000001_add_simhash_index.rb
+- lib/iudex-da/base.rb
+- lib/iudex-da.rb
+- lib/iudex-da/config.rb
+- lib/iudex-da/factory_helper.rb
+- lib/iudex-da/importer.rb
+- lib/iudex-da/key_helper.rb
+- lib/iudex-da/models.rb
+- lib/iudex-da/orm.rb
+- lib/iudex-da/pool_data_source_factory.rb
+- lib/iudex-da/work_poller.rb
+- test/setup.rb
+- test/test_migrate.rb
+- test/test_pool_factory.rb
+- test/test_url_model.rb
+- test/test_work_poller.rb
+- lib/iudex-da/iudex-da-1.3.0.jar
 homepage: http://iudex.gravitext.com
 licenses: []
-
-post_install_message: 
-rdoc_options: 
-  - --main
-  - README.rdoc
-require_paths: 
-  - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+post_install_message:
+rdoc_options:
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2
   none: false
-  requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 2
-        segments: 
-          - 0
-        version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2
   none: false
-  requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 2
-        segments: 
-          - 0
-        version: "0"
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.8.15
-signing_key: 
+rubyforge_project:
+rubygems_version: 1.8.24
+signing_key:
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java.
 test_files: []
-
+...