iudex-da 1.0.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-04-04)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,32 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+bin/iudex-da-generate-test-data
+bin/iudex-da-import
+bin/iudex-da-simhash-dump
+bin/iudex-migrate
+config/config.rb
+db/0010_base_urls.rb
+db/0020_add_feed_metadata.rb
+db/0021_more_feed_text.rb
+db/0030_add_priority.rb
+db/0040_add_visit_after.rb
+db/0050_add_cache_location.rb
+db/0060_url_indexes.rb
+db/0070_add_created_at.rb
+db/0080_add_simhash.rb
+lib/iudex-da/base.rb
+lib/iudex-da.rb
+lib/iudex-da/ar.rb
+lib/iudex-da/config.rb
+lib/iudex-da/factory_helper.rb
+lib/iudex-da/importer.rb
+lib/iudex-da/key_helper.rb
+lib/iudex-da/pool_data_source_factory.rb
+test/setup.rb
+test/test_migrate.rb
+test/test_poll_work.rb
+test/test_pool_factory.rb
+lib/iudex-da/iudex-da-1.0.0.jar

data/README.rdoc

@@ -0,0 +1,30 @@
+= iudex-da
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-da gem provides a PostgreSQL-based content
+meta-data store and work priority queue.
+
+== Postgresql Setup
+
+ % createuser iudex
+ % createdb iudex_test -O iudex
+
+== License
+
+Copyright (c) 2008-2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You may
+obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,49 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-da/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.2'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-da',
+                       Iudex::DA::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+
+  h.extra_deps += [ [ 'iudex-core',            '~> 1.0.0'  ],
+                    [ 'activerecord',          '~> 2.3.10' ],
+                    [ 'jdbc-postgres',         '>= 8.4.702',  '< 9.1' ],
+                    [ 'activerecord-jdbcpostgresql-adapter', '~> 1.1.0' ],
+                    [ 'rjack-commons-dbcp',    '~> 1.4.0'  ],
+                    [ 'rjack-commons-dbutils', '~> 1.3.0'  ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',          '>= 1.7.1', '< 2.1' ],
+                        [ 'rjack-logback',     '~> 1.0' ] ]
+end
+
+file 'Manifest.txt' => "lib/#{t.name}/base.rb"
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+# Disable verbose warnings, which are a bit much with ActiveRecord
+# 2.3.x at least.
+Hoe::RUBY_FLAGS.sub!( /\-w(\s|$)/, '-W1\1' )
+
+t.define_tasks

data/bin/iudex-da-generate-test-data

@@ -0,0 +1,127 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+
+require 'rjack-logback'
+RJack::Logback.config_console( :thread => true )
+
+require 'iudex-da'
+require 'iudex-da/pool_data_source_factory'
+require 'optparse'
+
+class Generator
+  include Iudex::DA
+  include Gravitext::HTMap
+
+  import 'iudex.core.VisitURL'
+  import 'iudex.core.ContentKeys'
+  import 'org.apache.commons.dbutils.ResultSetHandler'
+  import 'org.apache.commons.dbutils.QueryRunner'
+  import 'iudex.da.ContentMapper'
+  import 'iudex.da.ContentWriter'
+
+  # 26^3 or 17,576 hosts
+  def hosts
+    h = []
+    r = ('a'..'z')
+    r.each do |i|
+      r.each do |j|
+        r.each do |k|
+          h << i.to_s + j + k + ".com"
+        end
+      end
+    end
+    h
+  end
+
+  def initialize
+    @factory = PoolDataSourceFactory.new
+    @data_source = @factory.create
+
+    @kmap = ContentMapper.new( [ ContentMapper::UHASH,
+                                 ContentMapper::HOST,
+                                 ContentKeys::URL,
+                                 ContentKeys::TYPE,
+                                 ContentKeys::PRIORITY,
+                                 ContentKeys::NEXT_VISIT_AFTER ] )
+
+    UniMap.define_accessors
+  end
+
+  def clear
+    qrun = QueryRunner.new( @data_source )
+    qrun.update( "DELETE from urls;" )
+  end
+
+  def write( parts = 3 )
+    writer = ContentWriter.new( @data_source, @kmap )
+    hs = hosts
+    sl = hs.length / parts
+    threads = []
+    while( ( h = hs.slice!( 0, sl ) ).length > 0 )
+      threads << Thread.new( h, writer ) do |hi, out|
+        now = Time.now
+        batch = []
+        hi.each do |host|
+          (rand(50) + 1).times do |i|
+            c = UniMap.new
+            c.priority = ( rand * 9.99 + 0.01 )
+            c.url = VisitURL.normalize( "http://#{host}/#{i}" )
+            c.type = "FEED"
+            c.next_visit_after = now + ( rand(3) * 60 * 60 * 24 * 100 ) # 2/3 are in future
+            batch << c
+            if batch.length >= 10_000
+              out.write( batch )
+              batch.clear
+            end
+          end
+        end
+        out.write( batch ) unless batch.empty?
+      end
+    end
+    threads.each { |t| t.join }
+  end
+
+end
+
+Hooker.log_with { |m| RJack::SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+OptionParser.new do |opts|
+  opts.on( "-s", "--set name=value", String,
+             "Set connect prop (ex: database=iudex)" ) do |nv|
+    name,value = nv.split('=').map { |t| t.strip }
+    Hooker.add( [ :iudex, :connect_props ] ) do
+      { name.to_sym => value }
+    end
+  end
+  opts.on( "-d", "--debug" ) do
+    RJack::Logback[ 'iudex.da' ].level = RJack::Logback::DEBUG
+  end
+  opts.on( "-v", "--version", "Display version and exit" ) do |file|
+    puts "iudex-da: #{Iudex::DA::VERSION}"
+    exit 1
+  end
+  Hooker.register_config( opts )
+end.parse!
+
+g = Generator.new
+g.clear
+g.write

data/bin/iudex-da-import

@@ -0,0 +1,62 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+module IudexBinScript
+
+  require 'rubygems'
+  require 'optparse'
+
+  require 'rjack-logback'
+  include RJack
+  Logback.config_console( :mdc => "uhash" )
+
+  require 'iudex-da'
+  require 'iudex-da/importer'
+
+  include Iudex
+
+  Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+  OptionParser.new do |opts|
+    opts.banner = <<END
+iudex-da: #{Iudex::DA::VERSION}
+Usage: iudex-da-import [Options] ImportCSV...
+END
+    opts.on( "-s", "--set name=value", String,
+             "Set connect prop (ex: database=iudex)" ) do |nv|
+      name,value = nv.split('=').map { |t| t.strip }
+      Hooker.add( [ :iudex, :connect_props ] ) do
+        { name.to_sym => value }
+      end
+    end
+    opts.on( "-d", "--debug" ) do
+      Logback[ 'iudex' ].level = Logback::DEBUG
+    end
+    opts.on( "-v", "--version", "Display version and exit" ) do |file|
+      puts "iudex-da: #{Iudex::DA::VERSION}"
+      exit 1
+    end
+    Hooker.register_config( opts )
+  end.parse!
+
+  importer = DA::Importer.new
+  importer.import_files( ARGV )
+
+end

data/bin/iudex-da-simhash-dump

@@ -0,0 +1,112 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+
+require 'rjack-logback'
+RJack::Logback.config_console
+
+require 'iudex-da/pool_data_source_factory'
+require 'optparse'
+
+class Iudex::DA::SimHashDumper
+  import 'org.apache.commons.dbutils.ResultSetHandler'
+  import 'org.apache.commons.dbutils.QueryRunner'
+
+  include RJack
+
+  def initialize
+    @log = SLF4J[ self.class ]
+  end
+
+  def parse_options
+    Hooker.log_with { |m| @log.info( m.rstrip ) }
+
+    # Apply all config before including ar
+    parser = OptionParser.new do |opts|
+      opts.banner = <<END
+Usage: iudex-da-simhash-dump [options] <output-file>
+END
+      opts.on( "-s", "--set name=value", String,
+               "Set connect prop (ex: database=iudex)" ) do |nv|
+        name,value = nv.split('=').map { |t| t.strip }
+        Hooker.add( [ :iudex, :connect_props ] ) do
+          { name.to_sym => value }
+        end
+      end
+      opts.on( "-d", "--debug" ) do
+        Logback[ 'iudex.da' ].level = Logback::DEBUG
+      end
+      opts.on( "-v", "--version", "Display version and exit" ) do |file|
+        puts "iudex-da: #{Iudex::DA::VERSION}"
+        exit 1
+      end
+      Hooker.register_config( opts )
+    end
+    parser.parse!
+  end
+
+  def run
+    parse_options
+
+    unless ARGV.empty?
+      @factory = Iudex::DA::PoolDataSourceFactory.new
+      @data_source = @factory.create
+      dump
+    end
+  end
+
+  class UniWriter
+    include ResultSetHandler
+    def initialize( log, fout )
+      @fout = fout
+      @log = log
+    end
+    def handle( rset )
+      cnt = 0
+      while rset.next
+        @fout.puts rset.get_string( 1 )
+        cnt += 1
+        @log.info( "Wrote #{cnt} simhashes." ) if ( cnt % 100000 ) == 0
+      end
+      @log.info( "Wrote #{cnt} simhashes total." )
+    end
+  end
+
+  def dump( file = ARGV.first )
+
+    File.open( file, 'w' ) do |fout|
+      qrun = QueryRunner.new( @data_source )
+      hdlr = UniWriter.new( @log, fout )
+
+      qrun.query( <<-"SQL", hdlr )
+        SELECT lpad( to_hex( simhash ), 16, '0' ) AS sh
+        FROM urls
+        WHERE simhash IS NOT NULL AND status IN (200, 304)
+        ORDER BY last_visit ASC;
+      SQL
+
+      #FIXME: WHERE pass (accepted state) instead of status?
+    end
+  end
+
+end
+
+Iudex::DA::SimHashDumper.new.run

data/bin/iudex-migrate

@@ -0,0 +1,66 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'optparse'
+
+module IudexBinScript
+
+  require 'rjack-logback'
+  include RJack
+  Logback.config_console
+
+  # Note: Avoid loading iudex-da with its jar dependency which would
+  # make it hard to boostrap the db from source alone.
+  # Instead load only nessary core, base, config, and ar (post config):
+  require 'iudex-core'
+  require 'iudex-da/base'
+  require 'iudex-da/config'
+  include Iudex
+
+  Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+  # Apply all config before including ar
+  OptionParser.new do |opts|
+    opts.banner = <<END
+Usage: iudex-migrate [options] [target-migration-number]
+END
+    opts.on( "-s", "--set name=value", String,
+             "Set connect prop (ex: database=iudex)" ) do |nv|
+      name,value = nv.split('=').map { |t| t.strip }
+      Hooker.add( [ :iudex, :connect_props ] ) do
+        { name.to_sym => value }
+      end
+    end
+    opts.on( "-d", "--debug" ) do
+      Logback[ 'iudex.da' ].level = Logback::DEBUG
+    end
+    opts.on( "-v", "--version", "Display version and exit" ) do |file|
+      puts "iudex-da: #{DA::VERSION}"
+      exit 1
+    end
+    Hooker.register_config( opts )
+  end.parse!
+
+  require 'iudex-da/ar'
+
+  DA::migrate( ARGV[0] && ARGV[0].to_i )
+
+end