iudex-core 1.0.0-java

data/build/tld_set_generator.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'erb'
+
+# Generator for TLDSets.java from input effective_tld_name.dat
+# See http://publicsuffix.org/
+class TLDSetGenerator
+
+  attr_reader :tlds, :tld_parents, :reg_exceptions
+
+  BASEDIR = File.dirname( __FILE__ )
+
+  INPUT_DAT = File.join( BASEDIR, 'effective_tld_name.dat' )
+
+  JAVA_OUT  = File.join( BASEDIR, '..', 'src',
+                         'main', 'java', 'iudex', 'core', 'TLDSets.java' )
+
+  def run( tld_file = INPUT_DAT, java_file = JAVA_OUT )
+    parse( tld_file )
+    generate_java( java_file )
+  end
+
+  def parse( tld_file )
+    @tlds           = []
+    @tld_parents    = []
+    @reg_exceptions = []
+
+    open( tld_file, "r" ) do |fin|
+      fin.each do |line|
+        case line
+        when %r{^\s*//}, /^\s*$/
+          # ignore comment, empty lines
+        when /^\s*([^\s\*\!]+)\s*$/
+          @tlds << $1
+        when /^\s*\*\.([^\s\*\!]+)\s*$/
+          @tld_parents << $1
+        when /^\s*\!([^\s\*\!]+)\s*$/
+          @reg_exceptions << $1
+        else
+          raise "Parse ERROR: line [#{line}]"
+        end
+      end
+    end
+
+    [ @tlds, @tld_parents, @reg_exceptions ]
+  end
+
+  def generate_java( java_file )
+    erb_file = File.join( BASEDIR, 'TLDSets.java.erb' )
+    template = ERB.new( IO.read( erb_file ), nil, '%' )
+
+    open( java_file, 'w' ) do |fout|
+      fout << template.result( binding )
+    end
+  end
+
+  def format_list( list )
+    all = list.map { |d| '"' + d + '"' }.join( ", " )
+    out = ""
+    until( all.empty? )
+      out << ' ' * 8
+      if all.length > 71
+        i = all.rindex( ',', 71 )
+        out << all.slice!( 0..i )
+        all.lstrip!
+      else
+        out << all
+        all = ""
+      end
+      out << "\n" unless all.empty?
+    end
+    out
+  end
+
+end
+
+if $0 == __FILE__
+  TLDSetGenerator.new.run( *ARGV )
+end

data/config/config.rb

@@ -0,0 +1,12 @@
+
+Iudex.configure do |c|
+
+  c.setup_bogus { raise "Shouldn't call" }
+
+  c.setup_visit_executor do |vx|
+    vx.max_threads             = 10
+    vx.min_host_delay          = 2_000
+    vx.max_shutdown_wait       = 19_000
+  end
+
+end

data/lib/iudex-core.rb

@@ -0,0 +1,49 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'gravitext-util'
+require 'iudex-http'
+require 'iudex-filter'
+require 'iudex-barc'
+
+require 'iudex-core/base'
+
+require 'iudex-core/config'
+
+require 'java'
+
+module Iudex
+  module Core
+    require "#{LIB_DIR}/iudex-core-#{VERSION}.jar"
+
+    import 'iudex.core.ContentKeys'
+    import 'iudex.core.ContentSource'
+    import 'iudex.core.VisitExecutor'
+    import 'iudex.core.VisitURL'
+
+    module Filters
+      import 'iudex.core.filters.BARCWriter'
+      import 'iudex.core.filters.ContentFetcher'
+      import 'iudex.core.filters.DateChangeFilter'
+      import 'iudex.core.filters.DefaultFilter'
+      import 'iudex.core.filters.FutureDateFilter'
+      import 'iudex.core.filters.RLDomainFilter'
+      import 'iudex.core.filters.TextCtrlWSFilter'
+      import 'iudex.core.filters.UHashMDCSetter'
+    end
+
+  end
+end

data/lib/iudex-core/base.rb

@@ -0,0 +1,23 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module Core
+    VERSION = '1.0.0'
+
+    LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
+  end
+end

data/lib/iudex-core/config.rb

@@ -0,0 +1,51 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rjack-slf4j'
+require 'optparse'
+require 'hooker'
+
+Hooker.log_with { |m| RJack::SLF4J[ 'iudex' ].info( m.rstrip ) }
+
+module Iudex
+
+  # Apply configuration from block
+  def self.configure( &block )
+    Hooker.with( :iudex, &block )
+  end
+
+  module Core
+
+    # <b>DEPRECATED:</b> Extensible module defining top level
+    # configuration blocks.
+    module Config
+
+      # <b>DEPRECATED:</b> Parse options using an OptionParser,
+      # defining (-c)onfig option, and yielding to block for further
+      # option handling.
+      def self.parse_options( args = ARGV, &block )
+        warn( "DEPRECATED parse_options called from #{caller.first.to_s}\n" +
+              "Use Hooker.register_config and OptionParser.new instead." )
+        parser = OptionParser.new do |opts|
+          Hooker.register_config( opts )
+          block.call( opts ) if block
+        end
+        parser.parse!
+      end
+
+    end
+  end
+end

data/pom.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-core</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0.0</version>
+  <name>Iudex Core System</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.0</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.gravitext</groupId>
+      <artifactId>gravitext-util</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-filter</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-http</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-barc</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

data/test/setup.rb

@@ -0,0 +1,40 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+require 'minitest/unit'
+require 'minitest/autorun'
+
+module TestSetup
+  include RJack
+  Logback.config_console( :stderr => true )
+  if ARGV.include?( '--verbose' ) || ARGV.include?( '-v' )
+    Logback.root.level = Logback::DEBUG
+  end
+end
+
+# Make test output logging compatible: no partial lines.
+class TestOut
+  def print( *a ); $stdout.puts( *a ); end
+  def puts( *a );  $stdout.puts( *a ); end
+end
+MiniTest::Unit.output = TestOut.new

data/test/test_content_fetcher.rb

@@ -0,0 +1,200 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+module TestHTTPMocks
+  include Iudex::Filter
+
+  class MockHTTPClient
+    include Iudex::HTTP::HTTPClient
+    def create_session
+      MockSession.new
+    end
+
+    def request( session, handler )
+      session.execute( handler )
+    end
+  end
+
+  WEAK_ETAG = 'W/"weak-etag"'
+
+  class MockSession < Iudex::HTTP::HTTPSession
+    import 'com.gravitext.util.ByteBufferInputStream'
+    import 'java.nio.ByteBuffer'
+    include Iudex::HTTP
+
+    def requestHeaders
+      [ ]
+    end
+
+    def responseHeaders
+      [ Header.new( "ETag", WEAK_ETAG ) ]
+    end
+
+    def responseCode
+      200
+    end
+
+    def responseStream
+      ByteBufferInputStream.new( ByteBuffer::wrap( "".to_java_bytes ) )
+    end
+
+    def statusText
+      "status text"
+    end
+
+    def execute( handler )
+      handler.handle_success( self )
+    end
+  end
+
+  class TestReceiver < FilterBase
+    def initialize( &block )
+      @block = block
+      @log = RJack::SLF4J[ self.class ]
+    end
+
+    def filter( out )
+      pretty_log( out )
+      @block.call( out )
+    end
+
+    def pretty_log( out )
+      @log.debug do
+        p = 0
+        rep = out.to_s.gsub( /{/ ) do
+          "\n" + ( '  ' * (p += 1) ) + "{ "
+        end
+        rep.gsub( /}/, " }" )
+      end
+    end
+  end
+
+end
+
+class TestContentFetcher < MiniTest::Unit::TestCase
+  include Iudex::Core
+  include Iudex::Core::Filters
+  include Iudex::Filter::Core
+  include Gravitext::HTMap
+
+  UniMap.define_accessors
+
+  include TestHTTPMocks
+
+  def setup
+    @fetcher = nil
+  end
+
+  DEFAULT_URL   = "http://gravitext.com/test"
+
+  def test_simple
+    inp = create_content
+    fetch( inp ) do |out|
+      assert_equal( DEFAULT_URL, out.url.to_s )
+      assert_equal( 200, out.status )
+      assert_equal( WEAK_ETAG, out.etag )
+      assert( out.source )
+    end
+  end
+
+  def test_304
+    client = MockHTTPClient.new
+    def client.request( session, handler )
+      handler.handle_error( session, 304 )
+    end
+    fetch( create_content, client ) do |out|
+      assert_equal( DEFAULT_URL, out.url.to_s )
+      assert_equal( 304, out.status )
+      assert_nil( out.etag )
+      assert_nil( out.source )
+    end
+  end
+
+  REDIRECT_URL  = "http://gravitext.com/redirect#foo"
+  REDIRECT_NORM = "http://gravitext.com/redirect"
+
+  def test_redirect
+    client = MockHTTPClient.new
+    def client.create_session
+      s = MockSession.new
+      def s.execute( handler )
+        self.url = REDIRECT_URL
+        super
+      end
+      s
+    end
+    fetch( create_content, client ) do |out|
+      assert_equal( REDIRECT_NORM, out.url.to_s )
+      assert_equal( 200, out.status )
+
+      ref = out.referer
+
+      assert_equal( DEFAULT_URL, ref.url.to_s )
+      assert_equal( 302, ref.status )
+      assert_equal( REDIRECT_NORM, ref.referent.url.to_s )
+    end
+  end
+
+  import "java.net.UnknownHostException"
+  import "java.io.IOException"
+
+  def test_connect_error
+    client = MockHTTPClient.new
+    def client.create_session
+      s = MockSession.new
+      def s.execute( handler )
+        handler.handle_exception( self,
+                                  UnknownHostException.new( "foobar.com" ) )
+      end
+      def s.responseCode
+        nil
+      end
+      def s.responseHeaders
+        nil
+      end
+      s
+    end
+    fetch( create_content, client ) do |out|
+      assert_equal( -1, out.status )
+      assert_nil( out.response_headers )
+      assert( out.reason =~ /UnknownHostException/ )
+    end
+  end
+
+  def fetch( content, client = MockHTTPClient.new, &block )
+    rec = TestReceiver.new( &block )
+    cf = ContentFetcher.new( client,
+                             FilterChain.new( "test-rec", [ rec ] ) )
+    cf.filter( content )
+  end
+
+  def create_content( url = DEFAULT_URL )
+    content = UniMap.new
+    content.url = visit_url( url )
+    content
+  end
+
+  def visit_url( url )
+    VisitURL.normalize( url )
+  end
+
+end