iudex-da 1.0.0-java

data/lib/iudex-da/base.rb

@@ -0,0 +1,23 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module DA
+    VERSION = '1.0.0'
+
+    LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
+  end
+end

data/lib/iudex-da/config.rb

@@ -0,0 +1,31 @@
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+
+  module DA
+    # Default database connection configuration for both ActiveRecord
+    # (migrations, testing) and PoolDataSourceFactory.
+    CONFIG = {
+      :adapter  => 'jdbcpostgresql',
+      :host     => 'localhost',
+      :database => 'iudex_test',
+      :username => 'iudex',
+      :pool     => 10 }
+  end
+
+end

data/lib/iudex-da/factory_helper.rb

@@ -0,0 +1,53 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da'
+require 'iudex-da/key_helper'
+require 'iudex-da/pool_data_source_factory'
+
+module Iudex
+  module DA
+    module Filters
+
+      # Mixin FilterChainFactory helper methods
+      module FactoryHelper
+        include Iudex::Filter::KeyHelper
+
+        # Lazy initialize DataSource
+        def data_source
+          @data_source ||= PoolDataSourceFactory.new.create
+        end
+
+        # Create UpdateFilter given fields and filter list factory
+        # methods
+        def create_update_filter( fields = [],
+                                  post_sym = nil,
+                                  update_sym = nil,
+                                  new_sym = nil )
+          fields = ( keys( :uhash ) + fields ).uniq
+
+          f = UpdateFilter.new( data_source, ContentMapper.new( fields ) )
+          create_chain( update_sym ) { |c| f.update_ref_filter = c }
+          create_chain( new_sym )    { |c| f.new_ref_filter    = c }
+          create_chain( post_sym )   { |c| f.content_filter    = c }
+          f
+        end
+
+      end
+
+    end
+  end
+end

data/lib/iudex-da/importer.rb

@@ -0,0 +1,91 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da'
+require 'iudex-da/key_helper'
+require 'iudex-da/pool_data_source_factory'
+
+module Iudex::DA
+
+  class Importer
+    include Iudex::Core
+    include Gravitext::HTMap
+
+    include Iudex::Filter::KeyHelper
+
+    import 'iudex.da.BaseTransformer'
+    import 'iudex.da.ContentUpdater'
+
+    def initialize()
+      @dsf = PoolDataSourceFactory.new
+      UniMap.define_accessors
+
+      Hooker.apply( [ :iudex, :importer ], self )
+    end
+
+    def import_files( files = ARGV )
+      files.each do |fname|
+        open( fname, "r" ) do |fin|
+          import( fin )
+        end
+      end
+    end
+
+    def import( input )
+
+      cmapper = ContentMapper.new( keys( import_keys ) )
+      transformer = BaseTransformer.new
+      updater = ContentUpdater.new( @dsf.create, cmapper, transformer )
+
+      tmpl = template_map
+      batch = []
+
+      input.each do |line|
+        umap = tmpl.clone
+        parse_to( line, umap )
+        batch << umap
+        if batch.length >= 1_000
+          updater.update( batch )
+          batch.clear
+         end
+      end
+      updater.update( batch ) unless batch.empty?
+    end
+
+    def import_keys
+      [ :uhash, :host, :url, :type, :priority, :next_visit_after ]
+    end
+
+    def template_map
+      umap = UniMap.new
+      umap.type = "FEED"
+      umap.next_visit_after = Time.now
+      umap.priority = 1.0
+      umap
+    end
+
+    def parse_to( line, umap )
+      fields = line.split( ',' )
+      umap.url = VisitURL.normalize( fields[0] )
+      umap
+    end
+
+    def close
+      @dsf.close if @dsf
+      @dsf = nil
+    end
+  end
+end

data/lib/iudex-da/key_helper.rb

@@ -0,0 +1,33 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da'
+require 'iudex-filter/key_helper'
+
+module Iudex
+  module Filter
+    module KeyHelper
+
+      # Override to lookup matching Key in ContentMapper::LOGICAL_KEYS
+      # or normal UniMap::KEY_SPACE
+      def self.lookup_key( name )
+        Iudex::DA::ContentMapper::LOGICAL_KEYS.get( name ) or
+          lookup_key_space( name )
+      end
+
+    end
+  end
+end

data/lib/iudex-da/pool_data_source_factory.rb

@@ -0,0 +1,108 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-da'
+require 'rjack-slf4j'
+require 'java'
+require 'jdbc/postgres'
+
+module Iudex::DA
+
+  # Factory for a DataSource using commons-dbcp and postgres driver
+  class PoolDataSourceFactory
+    import 'java.io.PrintWriter'
+    import 'java.sql.DriverManager'
+    import 'java.util.Properties'
+    import 'java.util.regex.Pattern'
+    import 'org.apache.commons.dbcp.DriverManagerConnectionFactory'
+    import 'org.apache.commons.dbcp.PoolableConnectionFactory'
+    import 'org.apache.commons.dbcp.PoolingDataSource'
+    import 'org.apache.commons.pool.impl.GenericObjectPool'
+    import 'iudex.util.LogWriter'
+
+    attr_accessor :data_source
+
+    def initialize( in_props = {} )
+      @props = Hooker.merge( [ :iudex, :connect_props ],
+                             CONFIG.merge( in_props ) )
+
+      # Tweeks specific for Java datasource/pool
+      @props[ :user ] ||= @props[ :username ]
+      @props.delete( :username )
+
+      @props[ :loglevel ] ||= 1
+
+      RJack::SLF4J[ 'iudex.da.PoolDataSourceFactory' ].info do
+        "Init properties: #{@props.inspect}"
+      end
+      load_driver
+    end
+
+    def create
+      con_factory = create_connection_factory
+      @con_pool = create_connection_pool( con_factory )
+      @data_source = PoolingDataSource.new( @con_pool )
+    end
+
+    def close
+      @con_pool.close
+      @con_pool = @data_source = nil
+    end
+
+    def load_driver
+      import 'org.postgresql.Driver'
+      lw = LogWriter.new( 'iudex.da.Driver' )
+      # Remove postgres time stamp, trailing whitespace.
+      lw.remove_pattern =
+        Pattern.compile( '(^\d\d:\d\d:\d\d\.\d\d\d\s\(\d\)\s)|(\s+$)' )
+      DriverManager::set_log_writer( PrintWriter.new( lw, true ) )
+    end
+
+    def create_connection_factory
+      uri = "jdbc:postgresql://%s/%s" % [ @props[ :host ], @props[ :database ] ]
+
+      jprops = Properties.new
+      @props.each { |k,v| jprops.set_property( k.to_s, v.to_s ) }
+
+      DriverManagerConnectionFactory.new( uri, jprops )
+    end
+
+    def create_connection_pool( con_factory )
+      con_pool = GenericObjectPool.new( nil )
+
+      con_count = @props[ :pool ]
+      if con_count
+        con_pool.max_active = con_count
+        con_pool.max_idle = con_count
+      end
+
+      props = @props[ :ds_pool ]
+      if props
+        props.each { |k,v| con_pool.send( k.to_s + '=', v ) }
+      end
+
+      # This sets self on con_pool
+      PoolableConnectionFactory.new( con_factory,
+                                     con_pool,
+                                     nil, #stmtPoolFactory
+                                     nil, #validationQuery
+                                     false, #read_only_default
+                                     true ) #auto_commit_default
+      con_pool
+    end
+  end
+
+end

data/pom.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-da</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0.0</version>
+  <name>Iudex Data Access</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.0</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <repositories>
+    <repository>
+      <id>maven2-repository.dev.java.net</id>
+      <name>Java.net Repository for Maven</name>
+      <url>http://download.java.net/maven/2/</url>
+      <layout>default</layout>
+    </repository>
+  </repositories>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-core</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-dbutils</groupId>
+      <artifactId>commons-dbutils</artifactId>
+      <version>1.3</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-dbcp</groupId>
+      <artifactId>commons-dbcp</artifactId>
+      <version>1.4</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-pool</groupId>
+      <artifactId>commons-pool</artifactId>
+      <version>[1.5.4,1.5.5]</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql-jdbc4</artifactId>
+      <version>[8.4.702,9.1)</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

data/test/setup.rb

@@ -0,0 +1,34 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+RJack::Logback.config_console( :stderr => true )
+
+require 'minitest/unit'
+require 'minitest/autorun'
+
+# Make test output logging compatible: no partial lines.
+class TestOut
+  def print( *a ); $stdout.puts( *a ); end
+  def puts( *a );  $stdout.puts( *a ); end
+end
+MiniTest::Unit.output = TestOut.new