iudex-da 1.2.1-java → 1.3.0-java

data/db/0010_base_urls.rb

@@ -1,84 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-# http://api.rubyonrails.org/classes/ActiveRecord/Migration.html
-# http://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/TableDefinition.html
-
-# Base Urls table schema
-class BaseUrls < ActiveRecord::Migration
-  def self.up
-    create_table( 'urls', :id => false ) {}
-
-    add_column( 'urls', 'uhash', :text, :null => false )
-    # 23 byte ASCII PRIMARY KEY SHA-1 hash fragment of URL
-    # (Note :limit not useful.)
-
-    add_column( 'urls', 'url', :text,  :null => false )
-    # Complete normalized url (exactly as used for uhash)
-
-    add_column( 'urls', 'host', :text, :null => false )
-    # Normalized host portion of URL
-
-    add_column( 'urls', 'type', :text, :null => false )
-    # FEED, PAGE, ROBOTS, SITEMAP
-    # Potentially speculative (i.e. "PAGE" before visited)
-    # FIXME: Or REDIRECT here instead of status?
-
-    add_column( 'urls', 'etag', :text )
-    # HTTP ETag header used for subsequent conditional GET
-    # Should only be on 200 and related HTTP status, not redirect
-
-    add_column( 'urls', 'last_visit', 'timestamp with time zone' )
-    # Time of last visit (and thus last type,status,reason,etc.)
-
-    add_column( 'urls', 'status', :integer )
-    # HTTP status code or special (negative) status mapping
-    # null     : Not yet visited
-    #  -1      : Connection Failed
-    # 4xx      : Permanent Failures
-    # 5xx      : Transient server error
-    # 200      : Success
-    # 304      : Not Modified
-    # 301,302  : Redirect
-    # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-    # Compare to: http://crawler.archive.org/articles/user_manual/glossary.html#statuscodes
-
-    add_column( 'urls', 'pass', :boolean )
-    # null     : Not yet processed (i.e. visit failed)
-    # false    : Rejected by processing (for reason), DELETE required
-    # true     : Fully Processed
-
-    add_column( 'urls', 'reason', :text )
-    # null      : None
-    # DUPE      : Duplicate of referent
-    # rejection filter (intended as key)
-
-    add_column( 'urls', 'referent', :text )
-    # null      : None
-    # uhash of url this is refering to
-    # (includes status:REDIRECT, reason:DUPE, etc.)
-
-    add_column( 'urls', 'referer', :text )
-    # null      : None
-    # uhash of url this was refered from. (i.e. the feed URL)
-
-    execute( "ALTER TABLE urls ADD PRIMARY KEY (uhash)" )
-  end
-
-  def self.down
-    drop_table 'urls'
-  end
-end

data/db/0020_add_feed_metadata.rb

@@ -1,37 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddFeedMetadata < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'title', :text )
-    # PAGE,FEED title
-
-    add_column( 'urls', 'ref_pub_date', 'timestamp with time zone' )
-    # (Latest) published date as provided from feed (may be ahead of
-    # or set before pub_date, below).
-
-    add_column( 'urls', 'pub_date', 'timestamp with time zone' )
-    # (Latest) published date as processed
-  end
-
-  def self.down
-    remove_column( 'urls', 'title' )
-    remove_column( 'urls', 'ref_pub_date' )
-    remove_column( 'urls', 'pub_date' )
-  end
-
-end

data/db/0021_more_feed_text.rb

@@ -1,29 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class MoreFeedText < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'summary', :text )
-    add_column( 'urls', 'content', :text )
-  end
-
-  def self.down
-    remove_column( 'urls', 'summary' )
-    remove_column( 'urls', 'content' )
-  end
-
-end

data/db/0030_add_priority.rb

@@ -1,28 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddPriority < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls',  'priority', 'real', :null => false, :default => 0.0 )
-    # Prioritization of next visit, range -INF,+INF
-  end
-
-  def self.down
-    remove_column( 'urls', 'priority' )
-  end
-
-end

data/db/0040_add_visit_after.rb

@@ -1,30 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddVisitAfter < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'next_visit_after', 'timestamp with time zone' )
-    execute 'ALTER TABLE urls ALTER COLUMN next_visit_after SET DEFAULT now()'
-    # null: never visit (terminal result)
-    # Don't visit again before the specified date.
-  end
-
-  def self.down
-    remove_column( 'urls', 'next_visit_after' )
-  end
-
-end

data/db/0050_add_cache_location.rb

@@ -1,32 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddCacheLocation < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'cache_file', :integer, :limit => 4 )
-    # 32-bit file number
-
-    add_column( 'urls', 'cache_file_offset', :integer, :limit => 8 )
-    # 64-bit byte offset within file
-  end
-
-  def self.down
-    remove_column 'urls',   'cache_file'
-    remove_column 'urls',   'cache_file_offset'
-  end
-
-end

data/db/0060_url_indexes.rb

@@ -1,41 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-# http://api.rubyonrails.org/classes/ActiveRecord/Migration.html
-# http://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/TableDefinition.html
-
-# Indexes for urls table
-class UrlIndexes < ActiveRecord::Migration
-  def self.up
-    # FIXME:  Disabled for now. Which are helpful?
-
-    # add_index( 'urls', [ 'host' ] )
-    # Used by (obsolesced) LIMIT/sub-query based work poll
-
-    # add_index( 'urls', [ 'priority' ] )
-    # FIXME: Consider partial index, e.g. WHERE next_visit_after IS NOT NULL?
-    # FIXME: Consider a combined index 'host', 'priority'?
-
-    # add_index( 'urls', [ 'next_visit_after' ] )
-    # Used by (obsolesced) LIMIT/sub-query based work poll
-  end
-
-  def self.down
-    # remove_index( 'urls', 'host' )
-    # remove_index( 'urls', 'priority' )
-    # remove_index( 'urls', 'next_visit_after' )
-  end
-end

data/db/0080_add_simhash.rb

@@ -1,33 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddSimhash < ActiveRecord::Migration
-
-  def self.up
-    add_column( 'urls', 'simhash', :integer, :limit => 8 )
-    # A simhash signature as a signed 8-byte long (should be
-    # compatible with java long).
-
-    add_index( 'urls', [ 'simhash' ] )
-    # And its index
-  end
-
-  def self.down
-    remove_index( 'urls', 'simhash'  )
-    remove_column( 'urls', 'simhash' )
-  end
-
-end

data/db/0110_host_to_domain.rb

@@ -1,36 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class HostToDomain < ActiveRecord::Migration
-
-  def self.up
-    # Move host to domain in place. This is intended to be the
-    # normalized registration level domain as provided by
-    # VisitURL.domain. Existing host values will often not match the
-    # RL domain, but the usage as a WorkPoller grouping does not
-    # strictly require this. Furthermore it would be very costly to
-    # rewrite a large database to correct domain values.  Start with a
-    # clean or custom migrated database if this consistency is
-    # important for your needs.
-    rename_column( 'urls', 'host', 'domain' )
-    #Equiv: add_column( 'urls', 'domain', :text, :null => false )
-  end
-
-  def self.down
-    rename_column( 'urls', 'domain', 'host' )
-  end
-
-end

data/db/index_next_visit/0100_add_index_next_visit.rb

@@ -1,27 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-class AddIndexNextVisit < ActiveRecord::Migration
-
-  def self.up
-    add_index( 'urls', 'next_visit_after' )
-  end
-
-  def self.down
-    remove_index( 'urls', 'next_visit_after' )
-  end
-
-end

data/lib/iudex-da/ar.rb

@@ -1,66 +0,0 @@
-#--
-# Copyright (c) 2008-2012 David Kellum
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you
-# may not use this file except in compliance with the License.  You
-# may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-#++
-
-require 'rjack-slf4j'
-require 'iudex-da/config'
-require 'active_record'
-require 'hooker'
-
-module Iudex::DA
-
-  def self.setup
-    log = RJack::SLF4J[ "iudex.da.ActiveRecord" ]
-    conf = Hooker.merge( [ :iudex, :connect_props ], CONFIG )
-    log.info { "Connecting: #{ conf.inspect }" }
-
-    ActiveRecord::Base.logger = log
-    ActiveRecord::Base.establish_connection( conf )
-  end
-
-  setup #FIXME: Require explicit setup for use?
-
-  def migrate( target_version = nil )
-    base = File.join( LIB_DIR, '..', '..', 'db' )
-    paths = [ base ]
-
-    profiles = Hooker.apply( [ :iudex, :migration_profiles ], [] )
-
-    paths += profiles.compact.map do |p|
-      p = p.to_s
-      if p =~ %r{^/}
-        p
-      else
-        File.join( base, p )
-      end
-    end
-
-    pattern = if paths.size > 1
-                '{' + paths.join( ',' ) + '}'
-              else
-                paths.first
-              end
-
-    ActiveRecord::Migrator.migrate( pattern, target_version )
-  end
-
-  module_function :migrate
-
-  class Url < ActiveRecord::Base
-    set_primary_key :uhash
-    set_inheritance_column :object_type # since "type" used already
-  end
-
-end