iudex-da 1.0.0-java

data/test/test_migrate.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-da'
+require 'iudex-da/ar'
+
+class TestMigrate < MiniTest::Unit::TestCase
+  include Iudex::DA
+  include RJack
+
+  def test_up_down_up
+    Logback[ 'iudex.da.ActiveRecord' ].level = Logback::WARN
+
+    ActiveRecord::Migration.suppress_messages do
+      migrate
+      migrate( 0 )
+      migrate
+    end
+
+    Logback[ 'iudex.da.ActiveRecord' ].level = nil
+  end
+
+end

data/test/test_poll_work.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-core'
+require 'iudex-da/ar'
+
+class TestPollWork < MiniTest::Unit::TestCase
+  include Iudex::DA
+  import 'iudex.core.VisitURL'
+
+  def setup
+    Url.delete_all
+
+    hosts = [ 'foo.org', 'other.net', 'gravitext.com', 'one.at' ]
+    count = 0
+    hosts.each do |host|
+      (5..15).each do |val|
+        url = Url.create! do |u|
+          u.priority = ( val.to_f / 10.0 ) + (count.to_f / 50.0)
+          vurl = VisitURL.normalize( "http://#{host}/#{u.priority}" )
+          u.type = "FEED"
+          u.host = vurl.host
+          u.url = vurl.to_s
+          u.uhash = vurl.uhash
+          u.next_visit_after = Time.now
+          count += 1
+        end
+      end
+    end
+  end
+
+  def teardown
+    Url.delete_all
+  end
+
+  # Query to get new work, with limits on work per host, and total
+  # work (in descending piority order)
+  def test_poll
+    query = <<END
+SELECT url, host, type, priority
+FROM ( SELECT *, row_number() OVER ( ORDER BY priority DESC ) as ppos
+       FROM ( SELECT *, row_number() OVER ( PARTITION BY host
+                                            ORDER BY priority DESC ) AS hpos
+              FROM urls
+              WHERE next_visit_after <= now() ) AS subh
+       WHERE hpos <= ? ) AS subp
+WHERE ppos <= ?
+ORDER BY host, priority DESC;
+END
+    res = Url.find_by_sql( [ query, 5, 18 ] )
+
+    def check_host_subset( byhost )
+      assert( byhost.length <= 5 )
+      byhost.each_cons(2) { |p,n| assert( p.priority >= n.priority ) }
+    end
+
+    assert( res.length <= 18 )
+    byhost = []
+    res.each do |u|
+      if byhost.empty? || byhost.last.host == u.host
+        byhost << u
+      else
+        check_host_subset( byhost )
+        byhost = []
+      end
+    end
+    check_host_subset( byhost ) unless byhost.empty?
+
+  end
+
+  def test_insert
+
+    Url.transaction do
+      sql = <<END
+CREATE TEMPORARY TABLE mod_urls
+  ( uhash text,
+    url text,
+    host text );
+END
+    # ON COMMIT DROP;
+
+      Url.connection.execute( sql ) #FIXME: auto-commit mode?
+
+      # Url.set_table_name "mod_urls"
+
+      count = ( 11 * 2 )
+      (5..20).each do |val|
+#        url = Url.create! do |u|
+        priority = ( val.to_f / 10.0 ) + (count.to_f / 50.0)
+          # u.priority =
+          # u.type = "FEEDX"
+        vurl = VisitURL.normalize( "http://gravitext.com/#{priority}" )
+
+        sql = "INSERT into mod_urls VALUES ('%s','%s','%s')" %
+          [ vurl.uhash, vurl.to_s, vurl.host ]
+        Url.connection.execute( sql )
+          # u.next_visit_after = Time.now
+        count += 1
+      end
+      insert_query = <<END
+INSERT INTO urls (uhash,url,host,type,priority)
+  ( SELECT uhash,url,host,'FEEDX',4.78 FROM mod_urls
+    WHERE uhash NOT IN ( SELECT uhash FROM urls ) );
+END
+      Url.connection.execute( insert_query )
+
+      Url.connection.execute( "DROP TABLE mod_urls;" )
+
+      # Url.set_table_name "urls"
+    end
+
+  end
+
+end

data/test/test_pool_factory.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-core'
+require 'iudex-da/ar'
+
+require 'iudex-da'
+require 'iudex-da/pool_data_source_factory'
+
+class TestPoolFactory < MiniTest::Unit::TestCase
+  include Iudex::DA
+  import 'org.apache.commons.dbutils.ResultSetHandler'
+  import 'org.apache.commons.dbutils.QueryRunner'
+
+  def setup
+    @factory = PoolDataSourceFactory.new( :loglevel => 2 )
+    @data_source = @factory.create
+  end
+
+  def teardown
+    @factory.close
+    @data_source = nil
+  end
+
+  class TestHandler
+    include ResultSetHandler
+    def handle( rs )
+      while rs.next
+        p [ rs.string( 'url' ) ]
+      end
+      nil
+    end
+  end
+
+  def test_query
+    assert( ! @data_source.nil? )
+    qrun = QueryRunner.new( @data_source )
+    qrun.query( "SELECT url FROM urls WHERE uhash IN ('uRlU1h_YL-NvooSv2i98Rd3', 'notthere' );",
+                TestHandler.new )
+  end
+
+end

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-da
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-core
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: activerecord
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 2.3.10
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: jdbc-postgres
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 8.4.702
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "9.1"
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: activerecord-jdbcpostgresql-adapter
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :runtime
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: rjack-commons-dbcp
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.4.0
+    type: :runtime
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: rjack-commons-dbutils
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :runtime
+    version_requirements: *id006
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id007 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.7.1
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+    type: :development
+    version_requirements: *id007
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id008 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :development
+    version_requirements: *id008
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id009 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :development
+    version_requirements: *id009
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-da gem provides a PostgreSQL-based content
+  meta-data store and work priority queue.
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - iudex-da-generate-test-data
+  - iudex-da-import
+  - iudex-da-simhash-dump
+  - iudex-migrate
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - bin/iudex-da-generate-test-data
+  - bin/iudex-da-import
+  - bin/iudex-da-simhash-dump
+  - bin/iudex-migrate
+  - config/config.rb
+  - db/0010_base_urls.rb
+  - db/0020_add_feed_metadata.rb
+  - db/0021_more_feed_text.rb
+  - db/0030_add_priority.rb
+  - db/0040_add_visit_after.rb
+  - db/0050_add_cache_location.rb
+  - db/0060_url_indexes.rb
+  - db/0070_add_created_at.rb
+  - db/0080_add_simhash.rb
+  - lib/iudex-da/base.rb
+  - lib/iudex-da.rb
+  - lib/iudex-da/ar.rb
+  - lib/iudex-da/config.rb
+  - lib/iudex-da/factory_helper.rb
+  - lib/iudex-da/importer.rb
+  - lib/iudex-da/key_helper.rb
+  - lib/iudex-da/pool_data_source_factory.rb
+  - test/setup.rb
+  - test/test_migrate.rb
+  - test/test_poll_work.rb
+  - test/test_pool_factory.rb
+  - lib/iudex-da/iudex-da-1.0.0.jar
+has_rdoc: true
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-da
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_migrate.rb
+  - test/test_poll_work.rb
+  - test/test_pool_factory.rb