iudex-da 1.0.0-java

data/config/config.rb

@@ -0,0 +1,14 @@
+
+Iudex.configure do |c|
+
+  # Set DA connection properties
+  c.setup_connect_props do
+    { :host     => 'localhost',
+      :database => 'iudex_test',
+      :username => 'iudex',
+      :ds_pool     => { :max_active => 4,
+                        :max_idle   => 2 },
+      :loglevel => 2 }
+  end
+
+end

data/db/0010_base_urls.rb

@@ -0,0 +1,84 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+# http://api.rubyonrails.org/classes/ActiveRecord/Migration.html
+# http://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/TableDefinition.html
+
+# Base Urls table schema
+class BaseUrls < ActiveRecord::Migration
+  def self.up
+    create_table( 'urls', :id => false ) {}
+
+    add_column( 'urls', 'uhash', :text, :null => false )
+    # 23 byte ASCII PRIMARY KEY SHA-1 hash fragment of URL
+    # (Note :limit not useful.)
+
+    add_column( 'urls', 'url', :text,  :null => false )
+    # Complete normalized url (exactly as used for uhash)
+
+    add_column( 'urls', 'host', :text, :null => false )
+    # Normalized host portion of URL
+
+    add_column( 'urls', 'type', :text, :null => false )
+    # FEED, PAGE, ROBOTS, SITEMAP
+    # Potentially speculative (i.e. "PAGE" before visited)
+    # FIXME: Or REDIRECT here instead of status?
+
+    add_column( 'urls', 'etag', :text )
+    # HTTP ETag header used for subsequent conditional GET
+    # Should only be on 200 and related HTTP status, not redirect
+
+    add_column( 'urls', 'last_visit', 'timestamp with time zone' )
+    # Time of last visit (and thus last type,status,reason,etc.)
+
+    add_column( 'urls', 'status', :integer )
+    # HTTP status code or special (negative) status mapping
+    # null     : Not yet visited
+    #  -1      : Connection Failed
+    # 4xx      : Permanent Failures
+    # 5xx      : Transient server error
+    # 200      : Success
+    # 304      : Not Modified
+    # 301,302  : Redirect
+    # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+    # Compare to: http://crawler.archive.org/articles/user_manual/glossary.html#statuscodes
+
+    add_column( 'urls', 'pass', :boolean )
+    # null     : Not yet processed (i.e. visit failed)
+    # false    : Rejected by processing (for reason), DELETE required
+    # true     : Fully Processed
+
+    add_column( 'urls', 'reason', :text )
+    # null      : None
+    # DUPE      : Duplicate of referent
+    # rejection filter (intended as key)
+
+    add_column( 'urls', 'referent', :text )
+    # null      : None
+    # uhash of url this is refering to
+    # (includes status:REDIRECT, reason:DUPE, etc.)
+
+    add_column( 'urls', 'referer', :text )
+    # null      : None
+    # uhash of url this was refered from. (i.e. the feed URL)
+
+    execute( "ALTER TABLE urls ADD PRIMARY KEY (uhash)" )
+  end
+
+  def self.down
+    drop_table 'urls'
+  end
+end

data/db/0020_add_feed_metadata.rb

@@ -0,0 +1,37 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddFeedMetadata < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'title', :text )
+    # PAGE,FEED title
+
+    add_column( 'urls', 'ref_pub_date', 'timestamp with time zone' )
+    # (Latest) published date as provided from feed (may be ahead of
+    # or set before pub_date, below).
+
+    add_column( 'urls', 'pub_date', 'timestamp with time zone' )
+    # (Latest) published date as processed
+  end
+
+  def self.down
+    remove_column( 'urls', 'title' )
+    remove_column( 'urls', 'ref_pub_date' )
+    remove_column( 'urls', 'pub_date' )
+  end
+
+end

data/db/0021_more_feed_text.rb

@@ -0,0 +1,29 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class MoreFeedText < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'summary', :text )
+    add_column( 'urls', 'content', :text )
+  end
+
+  def self.down
+    remove_column( 'urls', 'summary' )
+    remove_column( 'urls', 'content' )
+  end
+
+end

data/db/0030_add_priority.rb

@@ -0,0 +1,28 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddPriority < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls',  'priority', 'real', :null => false, :default => 0.0 )
+    # Prioritization of next visit, range -INF,+INF
+  end
+
+  def self.down
+    remove_column( 'urls', 'priority' )
+  end
+
+end

data/db/0040_add_visit_after.rb

@@ -0,0 +1,30 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddVisitAfter < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'next_visit_after', 'timestamp with time zone' )
+    execute 'ALTER TABLE urls ALTER COLUMN next_visit_after SET DEFAULT now()'
+    # null: never visit (terminal result)
+    # Don't visit again before the specified date.
+  end
+
+  def self.down
+    remove_column( 'urls', 'next_visit_after' )
+  end
+
+end

data/db/0050_add_cache_location.rb

@@ -0,0 +1,32 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddCacheLocation < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'cache_file', :integer, :limit => 4 )
+    # 32-bit file number
+
+    add_column( 'urls', 'cache_file_offset', :integer, :limit => 8 )
+    # 64-bit byte offset within file
+  end
+
+  def self.down
+    remove_column 'urls',   'cache_file'
+    remove_column 'urls',   'cache_file_offset'
+  end
+
+end

data/db/0060_url_indexes.rb

@@ -0,0 +1,41 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+# http://api.rubyonrails.org/classes/ActiveRecord/Migration.html
+# http://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/TableDefinition.html
+
+# Indexes for urls table
+class UrlIndexes < ActiveRecord::Migration
+  def self.up
+    # FIXME:  Disabled for now. Which are helpful?
+
+    # add_index( 'urls', [ 'host' ] )
+    # Used by (obsolesced) LIMIT/sub-query based work poll
+
+    # add_index( 'urls', [ 'priority' ] )
+    # FIXME: Consider partial index, e.g. WHERE next_visit_after IS NOT NULL?
+    # FIXME: Consider a combined index 'host', 'priority'?
+
+    # add_index( 'urls', [ 'next_visit_after' ] )
+    # Used by (obsolesced) LIMIT/sub-query based work poll
+  end
+
+  def self.down
+    # remove_index( 'urls', 'host' )
+    # remove_index( 'urls', 'priority' )
+    # remove_index( 'urls', 'next_visit_after' )
+  end
+end

data/db/0070_add_created_at.rb

@@ -0,0 +1,28 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddCreatedAt < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'created_at', 'timestamp with time zone' )
+    execute 'ALTER TABLE urls ALTER COLUMN created_at SET DEFAULT now()'
+  end
+
+  def self.down
+    remove_column( 'urls', 'created_at' )
+  end
+
+end

data/db/0080_add_simhash.rb

@@ -0,0 +1,33 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+class AddSimhash < ActiveRecord::Migration
+
+  def self.up
+    add_column( 'urls', 'simhash', :integer, :limit => 8 )
+    # A simhash signature as a signed 8-byte long (should be
+    # compatible with java long).
+
+    add_index( 'urls', [ 'simhash' ] )
+    # And its index
+  end
+
+  def self.down
+    remove_index( 'urls', 'simhash'  )
+    remove_column( 'urls', 'simhash' )
+  end
+
+end

data/lib/iudex-da.rb

@@ -0,0 +1,40 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+
+require 'rjack-commons-dbcp'
+require 'rjack-commons-dbutils'
+
+require 'iudex-da/base'
+require 'iudex-da/config'
+
+require 'java'
+
+module Iudex
+  module DA
+
+    require "#{LIB_DIR}/iudex-da-#{VERSION}.jar"
+
+    import 'iudex.da.WorkPoller'
+    import 'iudex.da.ContentMapper'
+
+    module Filters
+      import 'iudex.da.filters.UpdateFilter'
+    end
+
+  end
+end

data/lib/iudex-da/ar.rb

@@ -0,0 +1,48 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rjack-slf4j'
+require 'iudex-da/config'
+require 'active_record'
+require 'hooker'
+
+module Iudex::DA
+
+  def self.setup
+    log = RJack::SLF4J[ "iudex.da.ActiveRecord" ]
+    conf = Hooker.merge( [ :iudex, :connect_props ], CONFIG )
+    log.info { "Connecting: #{ conf.inspect }" }
+
+    ActiveRecord::Base.logger = log
+    ActiveRecord::Base.establish_connection( conf )
+  end
+
+  setup #FIXME: Require explicit setup for use?
+
+  def migrate( target_version = nil )
+    ActiveRecord::Migrator.migrate( File.join( LIB_DIR, '..', '..', 'db' ),
+                                    target_version )
+    #FIXME: Support additional migration directories?
+  end
+
+  module_function :migrate
+
+  class Url < ActiveRecord::Base
+    set_primary_key :uhash
+    set_inheritance_column :object_type # since "type" used already
+  end
+
+end