iudex-core 1.0.0-java → 1.1.0-java

data/test/test_redirect_handler.rb

@@ -0,0 +1,170 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+class TestRedirectHandler < MiniTest::Unit::TestCase
+  include Iudex::HTTP
+  include Iudex::Core
+  include Iudex::Core::Filters
+  include Gravitext::HTMap
+
+  UniMap.define_accessors
+
+  def test_first_redirect
+    order = new_order
+    redirect( order )
+    do_filter( order )
+
+    assert_equal( "http://www/1", order.url.to_s )
+    assert_equal( 301, order.status )
+    assert_equal( 1.0, order.priority )
+
+    order = revisit( order )
+    assert_nil( order.status )
+    assert_equal( 1.5, order.priority )
+
+    assert_equal( "http://www/2", order.url.to_s )
+    assert_equal( "http://www/1", order.last.url.to_s )
+
+    assert_equal( "http://www/1", order.referer.url.to_s )
+    assert_equal( "http://www/2", order.referer.referent.url.to_s )
+  end
+
+  def test_second_redirect
+    order = new_order
+    redirect( order, 301, 2 )
+    do_filter( order )
+
+    order = revisit( order )
+    redirect( order, 302, 3 )
+    do_filter( order )
+
+    order = revisit( order )
+
+    assert_equal( 2.0, order.priority )
+
+    assert_equal( "http://www/3", order.url.to_s )
+    assert_equal( "http://www/2", order.last.url.to_s )
+    assert_equal( "http://www/1", order.referer.url.to_s )
+    assert_equal( "http://www/3", order.referer.referent.url.to_s )
+  end
+
+  def test_third_redirect
+    order = new_order
+    redirect( order, 301, 2 )
+    do_filter( order )
+
+    order = revisit( order )
+    redirect( order, 302, 3 )
+    do_filter( order )
+
+    order = revisit( order )
+    redirect( order, 303, 4 )
+    do_filter( order )
+
+    order = revisit( order )
+
+    assert_equal( 2.5, order.priority )
+
+    assert_equal( "http://www/4", order.url.to_s )
+    assert_equal( "http://www/3", order.last.url.to_s )
+    assert_equal( "http://www/2", order.last.last.url.to_s )
+    assert_equal( "http://www/1", order.referer.url.to_s )
+    assert_equal( "http://www/4", order.referer.referent.url.to_s )
+  end
+
+  def test_self_redirect
+    order = new_order
+    redirect( order, 307, 1 )
+    do_filter( order )
+
+    assert_equal( "http://www/1", order.url.to_s )
+    assert_equal( HTTPSession::REDIRECT_LOOP, order.status )
+    assert_equal( 1.0, order.priority )
+    assert_nil( order.revisit_order )
+  end
+
+  def test_missing_location
+    order = new_order
+    order.status = 307
+    do_filter( order )
+
+    assert_equal( "http://www/1", order.url.to_s )
+    assert_equal( HTTPSession::MISSING_REDIRECT_LOCATION, order.status )
+    assert_equal( 1.0, order.priority )
+    assert_nil( order.revisit_order )
+  end
+
+  def test_redirect_loop
+    order = new_order
+    redirect( order, 301, 2 )
+    do_filter( order )
+
+    order = revisit( order )
+    redirect( order, 302, 3 )
+    do_filter( order )
+
+    order = revisit( order )
+    redirect( order, 303, 1 ) #Eat our tail
+    do_filter( order )
+
+    assert_equal( "http://www/3", order.url.to_s )
+    assert_equal( HTTPSession::REDIRECT_LOOP, order.status )
+    assert_nil( order.revisit_order )
+  end
+
+  def test_max_path
+    order = new_order
+    (2..5).each do |i|
+      redirect( order, 302, i )
+      do_filter( order, 3 )
+      order = revisit( order ) || break
+    end
+
+    assert_equal( "http://www/3", order.url.to_s )
+    assert_equal( HTTPSession::MAX_REDIRECTS_EXCEEDED, order.status )
+    assert_nil( order.revisit_order )
+  end
+
+  def do_filter( order, max_path = 4 )
+    orig_url = order.url
+    handler = RedirectHandler.new
+    handler.max_path = max_path
+    handler.filter( order )
+  end
+
+  def revisit( order )
+    order.remove( ContentKeys::REVISIT_ORDER )
+  end
+
+  def new_order( i = 1 )
+    UniMap.new.tap do |o|
+      o.url = VisitURL.normalize( "http://www/#{i}" )
+      o.priority = 1.0
+    end
+  end
+
+  def redirect( o, s = 301, r = 2 )
+    o.status = 301
+    o.response_headers = [ Header.new( "Location", "http://WWW/#{r}" ) ]
+  end
+
+end

data/test/test_visit_manager.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+class TestVisitManager < MiniTest::Unit::TestCase
+  include Gravitext::HTMap
+  include Iudex::Filter
+  include Iudex::Filter::Core
+  include Iudex::Core
+
+  import 'iudex.core.GenericWorkPollStrategy'
+
+  import 'java.util.concurrent.Executors'
+  import 'java.util.concurrent.TimeUnit'
+  import 'java.util.concurrent.CountDownLatch'
+
+  UniMap.define_accessors
+
+  def setup
+    @latch = CountDownLatch.new( 20 )
+
+    @manager = VisitManager.new( TestWorkPoller.new )
+
+    test_filter = fltr do |order|
+      @scheduler.schedule( proc { @manager.release( order, nil ) },
+                           rand( 20_000 ), TimeUnit::MICROSECONDS )
+      @latch.countDown();
+    end
+
+    @manager.filter_chain = FilterChain.new( "test", [ test_filter ] )
+
+    @scheduler = Executors::new_scheduled_thread_pool( 1 )
+  end
+
+  def teardown
+    @scheduler.shutdown_now if @scheduler
+    @manager.shutdown if @manager
+  end
+
+  def test
+    @manager.start
+    pass
+    assert( @latch.await( 5, TimeUnit::SECONDS ) )
+    @manager.shutdown
+    pass
+  end
+
+  class TestWorkPoller < GenericWorkPollStrategy
+    include Gravitext::HTMap
+    include Iudex::Core
+
+    def initialize
+      super()
+      self.min_poll_interval = 5
+      self.max_check_interval = 21
+      self.max_poll_interval = 130 #ms
+      @batch = 0
+    end
+
+    def pollWorkImpl( visit_q )
+      @batch += 1
+
+      [ %w[ h2 a 2.2 ],
+        %w[ h2 b 2.1 ],
+        %w[ h2 c 2.0 ],
+        %w[ h3 a 3.2 ],
+        %w[ h3 b 3.1 ],
+        %w[ h3 c 3.0 ],
+        %w[ h1 a 1.2 ],
+        %w[ h1 b 1.1 ] ].each do |h,i,p|
+
+        visit_q.add( order( h, @batch, i, p ) )
+      end
+    end
+
+    def order( host, batch, i, p )
+      UniMap.new.tap do |o|
+        o.url = visit_url( "http://#{host}.com/#{batch}/#{i}" )
+        o.priority = p.to_f
+      end
+    end
+
+    def visit_url( url )
+      VisitURL.normalize( url )
+    end
+
+  end
+
+end

data/test/test_visit_queue.rb

@@ -0,0 +1,268 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+class TestVisitQueue < MiniTest::Unit::TestCase
+  include Iudex::Core
+  include Gravitext::HTMap
+
+  import 'java.util.concurrent.Executors'
+  import 'java.util.concurrent.TimeUnit'
+  import 'java.lang.Runnable'
+
+  UniMap.create_key( 'vtest_input' )
+
+  UniMap.define_accessors
+
+  def setup
+    @visit_q = VisitQueue.new
+    @visit_q.default_min_host_delay = 50 #ms
+    @scheduler = Executors::new_scheduled_thread_pool( 2 )
+  end
+
+  def teardown
+    @scheduler.shutdown_now if @scheduler
+  end
+
+  def test_priority
+    orders = [ %w[ h a 1.3 ],
+               %w[ h b 1.1 ],
+               %w[ h c 1.2 ] ].map do |oinp|
+      order( oinp )
+    end
+
+    @visit_q.add_all( orders )
+
+    orders.sort { |p,n| n.priority <=> p.priority }.each do |o|
+      assert_equal( o.vtest_input, acquire_order )
+    end
+
+    assert_queue_empty
+  end
+
+  def add_common_orders
+    [ %w[   h2 a 2.2 100 ],
+      %w[ w.h2 b 2.1 ],
+      %w[   h2 c 2.0 ],
+      %w[   h3 a 3.2 130 ],
+      %w[   h3 b 3.1 130 ],
+      %w[ m.h3 c 3.0 ],
+      %w[   h1 a 1.2 ],
+      %w[   h1 b 1.1 ] ].each do |oinp|
+
+      @visit_q.add( order( oinp ) )
+    end
+
+    assert_equal( 3, @visit_q.host_count, "host count" )
+    assert_equal( 8, @visit_q.order_count, "order count" )
+  end
+
+  def test_hosts_acquire
+    add_common_orders
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h1 b 1.1 ],
+                 %w[ w.h2 b 2.1 ],
+                 %w[   h3 b 3.1 ],
+                 %w[   h2 c 2.0 ],
+                 %w[ m.h3 c 3.0 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    assert_queue_empty
+  end
+
+  def test_configure
+    @visit_q.configure_host( 'h2.com', 75, 2 )
+
+    [ %w[   h2 a 2.2 ],
+      %w[ w.h2 b 2.1 ],
+      %w[   h3 a 3.2 ],
+      %w[   h3 b 3.1 ],
+      %w[   h1 a 1.2 ],
+      %w[   h1 b 1.1 ] ].each do |oinp|
+
+      @visit_q.add( order( oinp ) )
+
+    end
+    assert_equal( 3, @visit_q.host_count, "host count" )
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h3 b 3.1 ],
+                 %w[   h1 b 1.1 ],
+                 %w[ w.h2 b 2.1 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    assert_queue_empty
+  end
+
+  def test_multi_access_2
+    @visit_q.default_max_access_per_host = 2
+    add_common_orders
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h3 b 3.1 ],
+                 %w[ w.h2 b 2.1 ],
+                 %w[   h1 b 1.1 ],
+                 %w[   h2 c 2.0 ],
+                 %w[ m.h3 c 3.0 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    assert_queue_empty
+  end
+
+  def test_multi_access_3
+    @visit_q.default_max_access_per_host = 3
+    add_common_orders
+
+    expected = [ %w[   h3 a 3.2 ],
+                 %w[   h2 a 2.2 ],
+                 %w[   h1 a 1.2 ],
+                 %w[   h3 b 3.1 ],
+                 %w[ w.h2 b 2.1 ],
+                 %w[   h1 b 1.1 ],
+                 %w[ m.h3 c 3.0 ],
+                 %w[   h2 c 2.0 ] ]
+
+    p = 0
+    expected.each do |o|
+      assert_equal( o, acquire_order, p += 1 )
+    end
+
+    assert_queue_empty
+  end
+
+  def test_interleaved
+    @visit_q.default_max_access_per_host = 2
+    @visit_q.default_min_host_delay = 3 #ms
+    @visit_q.configure_host( 'h2.com', 1, 4 )
+
+    512.times do |i|
+      @visit_q.add( order( [ %w[ h1 h2 ][rand( 2 )], i, 5 * rand ] ) )
+    end
+
+    c = @visit_q.order_count
+    added = 0
+
+    while c > 0
+      o = @visit_q.acquire( 300 )
+      flunk( "acquire returned null" ) unless o
+      c -= 1
+      @scheduler.schedule( ReleaseJob.new( @visit_q, o ),
+                           rand( 20_000 ), TimeUnit::MICROSECONDS )
+
+      while ( added < 1024 ) && ( rand(3) != 1 )
+        added += 1
+        c += 1
+        j = Job.new( added ) do | i, p |
+          @visit_q.add( order( [ %w[ h1 h2 ][rand( 2 )], i, 5 * rand ] ) )
+        end
+        @scheduler.schedule( j, rand( 20_000 ), TimeUnit::MICROSECONDS )
+      end
+
+    end
+
+    assert_queue_empty
+  end
+
+  def assert_queue_empty
+    @scheduler.shutdown
+    @scheduler.await_termination( 2, TimeUnit::SECONDS )
+    @scheduler = nil
+    assert_equal( 0, @visit_q.order_count, "order count" )
+    assert_equal( 0, @visit_q.host_count,  "host count" )
+  end
+
+  def acquire_order
+    o = @visit_q.acquire( 200 )
+    if o
+      o.vtest_input.tap do |i|
+        delay = ( i[3] || 20 ).to_i
+        @scheduler.schedule( ReleaseJob.new( @visit_q, o ),
+                             delay,
+                             TimeUnit::MILLISECONDS )
+      end.slice( 0..2 )
+    end
+  end
+
+  def order( args )
+    host, c, p = args
+    UniMap.new.tap do |o|
+      o.url = visit_url( "http://#{host}.com/#{c}" )
+      o.priority = p.to_f
+      o.vtest_input = args
+    end
+  end
+
+  def visit_url( url )
+    VisitURL.normalize( url )
+  end
+
+  LOG = RJack::SLF4J[ self ]
+
+  class ReleaseJob
+    include Runnable
+
+    def initialize( visit_q, order )
+      super()
+      @visit_q = visit_q
+      @order = order
+    end
+
+    def run
+      @visit_q.release( @order, nil )
+    rescue => e
+      LOG.error( e )
+    end
+  end
+
+  class Job
+    include Runnable
+
+    def initialize( *args, &block )
+      @block = block
+      @args = args
+    end
+    def run
+      @block.call( *@args )
+    rescue => e
+      LOG.error( e )
+    end
+  end
+
+end

data/test/test_visit_url.rb

@@ -0,0 +1,150 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+class TestVisitURL < MiniTest::Unit::TestCase
+  include Iudex::Core
+
+# def setup; end
+# def teardown end
+
+  def test_normalize_basic
+
+    sets = [ %w[ http://h.c/foo http://h.c/foo
+                                http://h.c//foo
+                                http://h.c/foo#anchor
+                                HTTP://H.C:80/foo
+                                HTTP://h.c/bar/../foo
+                                http://h.c/./foo
+                                http://h.c./foo
+                                http://h.c/foo? ],
+
+             %w[ http://h.c/    http://h.c ],
+
+             %w[ http://h.c/?x=a%26b http://h.c/?x=a%26b ],
+
+             [ "http://h.c/foo", " \thttp://h.c/foo\n\r\t" ],
+             [ "http://h.c/foo?q=a+b",   "http://h.c/foo?q=a+b" ],
+             [ "http://h.c/foo?q=a%20b", "http://h.c/foo?q=a b",
+                                         "http://h.c/foo?q=a  b",
+                                         "HTTP://h.c/foo?q=a%20b#anchor",
+                                         "http://h.c/foo?q=a\t b#anchor\t" ] ]
+
+    sets.each do |tset|
+      expected = VisitURL.normalize( tset.shift )
+      tset.each do |raw|
+        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+      end
+    end
+  end
+
+  def test_normalize_utf8
+
+    sets = [ %w[ http://h.c/f%C5%8Do HTTP://h.c/fōo ] ]
+
+    sets.each do |tset|
+      expected = VisitURL.normalize( tset.shift )
+      tset.each do |raw|
+        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+      end
+    end
+  end
+
+  def test_normalize_escape_case
+    skip( "Escape normalizations not implemented" )
+
+    sets = [ %w[ http://h.c/?x=a%3Ab http://h.c/?x=a%3ab ],
+             %w[ http://h.c/%C2      http://h.c/%C2
+                                     http://h.c/%c2 ],
+             %w[ http://h.c/foo%20bar HTTP://h.c/%66oo%20bar ],
+             %w[ http://h.c/a%5Bb%5D http://h.c/a[b] ] ]
+
+    sets.each do |tset|
+      expected = VisitURL.normalize( tset.shift )
+      tset.each do |raw|
+        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+      end
+    end
+  end
+
+  def test_normalize_idn
+    skip( "IDN normalization not implemented" )
+
+    sets = [ %w[ http://xn--bcher-kva.ch/ http://Bücher.ch ] ]
+
+    sets.each do |tset|
+      expected = VisitURL.normalize( tset.shift )
+      tset.each do |raw|
+        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+      end
+    end
+  end
+
+  def test_uhash
+    h = VisitURL.normalize( "http://gravitext.com/" ).uhash
+    assert_equal( "8dOml647JKxoA1vSNdi3WAK", h.to_s )
+
+    h = VisitURL.normalize( "http://gravitext.com/x/y" ).uhash
+    assert_equal( "0pRfQvGEzGRMQ-RgFbytf7l", h.to_s )
+  end
+
+  def test_domain_hash
+    d = VisitURL.hash_domain( "gravitext.com" );
+    assert_equal( "VdYKPM", d.to_s )
+
+    d = VisitURL.hash_domain( "other.com" );
+    assert_equal( "ZleSiQ", d.to_s )
+  end
+
+  def test_resolve
+
+    sets = [ %w[ http://h.c/        http://h.c/foo ] << "",
+             %w[ http://h.c/        http://h.c/    ] << "",
+             %w[ http://h.c/        http://h.c/foo ] << " ",
+
+             %w[ http://h.c/        http://h.c/foo      .   ],
+             %w[ http://h.c/bar     http://h.c/foo     /bar ],
+             %w[ http://h.c/bar     http://h.c/foo      bar ],
+             %w[ http://h.c/bar     http://h.c/foo?q=1  bar ],
+             %w[ http://h.c/bar     http://h.c/foo/x/y /bar ],
+             %w[ http://h.c/foo/bar http://h.c/foo/x/y ../bar ],
+             %w[ http://h.c/foo/bar http://h.c/foo/     bar ],
+
+             %w[ http://h.c/a%20b/c%20d http://h.c/a%20b/f ] << "c d",
+
+             %w[ http://h.c/bar?q=1     http://h.c/foo      bar?q=1 ],
+             %w[ http://h.c/bar?q=1     http://h.c/foo/    /bar?q=1 ],
+             %w[ http://h.c/bar?q=1     http://h.c/foo?x=2  bar?q=1 ],
+             %w[ http://h.c/foo/bar?q=1 http://h.c/foo/     bar?q=1 ],
+             %w[ http://h.c/foo/bar?q=1 http://h.c/foo/   ./bar?q=1 ] ]
+
+    sets.each do |e,b,r|
+      expected = VisitURL.normalize( e )
+      base = VisitURL.normalize( b )
+      resolved = base.resolve( r )
+
+      assert_equal( expected.to_s, resolved.to_s, [ e,b,r ].inspect )
+    end
+
+  end
+
+end