iudex-core 1.1.0-java → 1.2.b.0-java

data/History.rdoc

@@ -1,3 +1,10 @@
+=== 1.2.b.0 (2012-3-4)
+* VisitURL now throws SyntaxException on missing scheme, host, bad
+  ports and some other URL malformations.
+* MojiBakeMapper optimization, no copy when nothing found.
+* Upgrade to gravitext-util ~> 1.6.b (beta)
+* Upgrade to tarpit ~> 2.0, bundler Gemfile, gemspec (dev)
+
 === 1.1.0 (2011-11-13)
 * Update to iudex-filter,http,barc ~> 1.1.0
 * ContentFetcher updates for iudex-http changes

data/Manifest.txt

@@ -23,4 +23,4 @@ test/test_redirect_handler.rb
 test/test_visit_manager.rb
 test/test_visit_queue.rb
 test/test_visit_url.rb
-lib/iudex-core/iudex-core-1.1.0.jar
+lib/iudex-core/iudex-core-1.2.b.0.jar

data/README.rdoc

@@ -10,7 +10,7 @@ does not contain such facilities as database-backed state management.
 
 == License
 
-Copyright (c) 2008-2011 David Kellum
+Copyright (c) 2008-2012 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You

data/Rakefile

@@ -1,45 +1,10 @@
 # -*- ruby -*-
 
-$LOAD_PATH << './lib'
-require 'iudex-core/base'
-
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.4'
+require 'bundler/setup'
 require 'rjack-tarpit'
 
-t = RJack::TarPit.new( 'iudex-core',
-                       Iudex::Core::VERSION,
-                       :no_assembly, :java_platform )
-
-t.specify do |h|
-  h.developer( "David Kellum", "dek-oss@gravitext.com" )
-  h.extra_deps += [ [ 'rjack-slf4j',    '~> 1.6.1'  ],
-                    [ 'hooker',         '~> 1.0.0'  ],
-                    [ 'gravitext-util', '~> 1.5.1'  ],
-                    [ 'iudex-filter',   '~> 1.1.0'  ],
-                    [ 'iudex-http',     '~> 1.1.0'  ],
-                    [ 'iudex-barc',     '~> 1.1.0'  ] ]
-
-  h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
-                        [ 'rjack-logback', '~> 1.0' ] ]
-end
-
-file 'Manifest.txt' => "lib/#{t.name}/base.rb"
-
-task :check_pom_version do
-  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
-end
-task :check_history_version do
-  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
-end
-task :check_history_date do
-  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
-end
-
-task :gem  => [ :check_pom_version, :check_history_version                      ]
-task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
-task :push => [                                             :check_history_date ]
+RJack::TarPit.new( 'iudex-core' ).define_tasks
 
 file 'src/main/java/iudex/core/TLDSets.java' => FileList.new( "build/*" ) do
   require 'build/tld_set_generator'
@@ -57,5 +22,3 @@ task :refresh_tld_dat do
   sh( "curl http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/effective_tld_names.dat?raw=1" +
       " -o build/effective_tld_name.dat" )
 end
-
-t.define_tasks

data/bin/iudex-test-config

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/bin/iudex-url-norm

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/build/TLDSets.java.erb

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2011 David Kellum
+ * Copyright (c) 2008-2012 David Kellum
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License.  You may

data/build/effective_tld_name.dat

@@ -15,7 +15,7 @@
 //
 // The Initial Developer of the Original Code is
 // Jo Hermans <jo.hermans@gmail.com>.
-// Portions created by the Initial Developer are Copyright (C) 2007
+// Portions created by the Initial Developer are Copyright (c) 2008-2012 David Kellum
 // the Initial Developer. All Rights Reserved.
 //
 // Contributor(s):

data/build/tld_set_generator.rb

@@ -2,7 +2,7 @@
 # -*- ruby -*-
 
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-core/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -16,7 +16,7 @@
 
 module Iudex
   module Core
-    VERSION = '1.1.0'
+    VERSION = '1.2.b.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-core/config.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-core/mojibake.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/iudex-core.rb

@@ -1,6 +1,6 @@
 
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-core</artifactId>
   <packaging>jar</packaging>
-  <version>1.1.0</version>
+  <version>1.2.b.0</version>
   <name>Iudex Core System</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.1</version>
+    <version>1.2.b.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -30,19 +30,19 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-filter</artifactId>
-      <version>[1.1,1.2)</version>
+      <version>[1.2,1.2.9999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-http</artifactId>
-      <version>[1.1,1.2)</version>
+      <version>[1.2,1.2.9999)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-barc</artifactId>
-      <version>[1.1,1.2)</version>
+      <version>[1.2,1.2.9999)</version>
     </dependency>
 
     <dependency>

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -14,27 +14,33 @@
 # permissions and limitations under the License.
 #++
 
-#### General test setup: LOAD_PATH, logging, console output ####
-
-ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
-$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+#### General test setup, logging, console output ####
 
 require 'rubygems'
-require 'rjack-logback'
+require 'bundler/setup'
+
 require 'minitest/unit'
 require 'minitest/autorun'
 
+require 'rjack-logback'
+
 module TestSetup
   include RJack
-  Logback.config_console( :stderr => true )
-  if ARGV.include?( '--verbose' ) || ARGV.include?( '-v' )
+  Logback.config_console( :stderr => true, :thread => true )
+
+  if ( ARGV & %w[ -v --verbose --debug ] ).empty?
+
+    # Make test output logging compatible: no partial lines.
+    class TestOut
+      def print( *a ); $stdout.puts( *a ); end
+      def puts( *a );  $stdout.puts( *a ); end
+    end
+    MiniTest::Unit.output = TestOut.new
+
+  else
     Logback.root.level = Logback::DEBUG
   end
-end
 
-# Make test output logging compatible: no partial lines.
-class TestOut
-  def print( *a ); $stdout.puts( *a ); end
-  def puts( *a );  $stdout.puts( *a ); end
+  ARGV.delete( '--debug' )
+
 end
-MiniTest::Unit.output = TestOut.new

data/test/test_content_fetcher.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_content_source.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_log_writer.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_mojibake.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_redirect_handler.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_visit_manager.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_visit_queue.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_visit_url.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -40,6 +40,11 @@ class TestVisitURL < MiniTest::Unit::TestCase
 
              %w[ http://h.c/    http://h.c ],
 
+             %w[ http://127.0.0.1/ http://127.0.0.1:80 ],
+
+             %w[ https://h.c/   httpS://h.c:443/
+                                httpS://h.c:443 ],
+
              %w[ http://h.c/?x=a%26b http://h.c/?x=a%26b ],
 
              [ "http://h.c/foo", " \thttp://h.c/foo\n\r\t" ],
@@ -50,9 +55,9 @@ class TestVisitURL < MiniTest::Unit::TestCase
                                          "http://h.c/foo?q=a\t b#anchor\t" ] ]
 
     sets.each do |tset|
-      expected = VisitURL.normalize( tset.shift )
+      expected = tset.shift
       tset.each do |raw|
-        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+        assert_equal( expected, VisitURL.normalize( raw ).to_s )
       end
     end
   end
@@ -62,11 +67,80 @@ class TestVisitURL < MiniTest::Unit::TestCase
     sets = [ %w[ http://h.c/f%C5%8Do HTTP://h.c/fōo ] ]
 
     sets.each do |tset|
-      expected = VisitURL.normalize( tset.shift )
+      expected = tset.shift
+      tset.each do |raw|
+        assert_equal( expected, VisitURL.normalize( raw ).to_s )
+      end
+    end
+  end
+
+  def test_bad_urls
+    bads =   [ '',
+               ' ',
+               '.',
+               ':',
+               '\\',
+               '\/' ] +
+           %w[ bogus
+               bogus:
+               bogus:/
+               bogus:/bar
+               http
+               http:
+               http:/
+               http://
+               http:///
+               http:///path/
+               http://\[h/
+               http://h\]/
+               http://::/
+               http://[:]/
+               http://wonkie\biz
+               http://wonkie/biz\ness
+               https://h.c:-33/
+               https://h.c:0/
+               https://h.c:65537/ ]
+
+    bads.each do |raw|
+      begin
+        flunk "[#{raw}] normalized to [#{VisitURL.normalize( raw )}]"
+      rescue NativeException => e
+        if e.cause.is_a?( VisitURL::SyntaxException )
+          pass
+        else
+          raise e
+        end
+      end
+    end
+
+  end
+
+  def test_ipv6
+    # Demonstrate validity from a URI perspective, but likely want to
+    # post-filter these.
+    # http://www.ietf.org/rfc/rfc2732.txt
+    sets = [
+      %w[ http://[fedc:ba98:7654:3210:fedc:ba98:7654:3210]/index.html
+          http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html ],
+      %w[ http://[1080:0:0:0:8:800:200c:417a]/index.html ],
+      %w[ http://[3ffe:2a00:100:7031::1]/
+          http://[3ffe:2a00:100:7031::1] ],
+      %w[ http://[1080::8:800:200c:417a]/foo ],
+      %w[ http://[::]/ ], #FIXME: Unspecified or multicast
+      %w[ http://[::192.9.5.5]/ipng ],
+      %w[ http://[::ffff:129.144.52.38]/index.html
+          http://[::FFFF:129.144.52.38]:80/index.html ],
+      %w[ http://[2010:836b:4179::836b:4179]/
+          http://[2010:836B:4179::836B:4179] ] ]
+
+    sets.each do |tset|
+      expected = tset.shift
+      tset = [ expected ] if tset.empty? #identity test
       tset.each do |raw|
-        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+        assert_equal( expected, VisitURL.normalize( raw ).to_s )
       end
     end
+
   end
 
   def test_normalize_escape_case
@@ -79,9 +153,9 @@ class TestVisitURL < MiniTest::Unit::TestCase
              %w[ http://h.c/a%5Bb%5D http://h.c/a[b] ] ]
 
     sets.each do |tset|
-      expected = VisitURL.normalize( tset.shift )
+      expected = tset.shift
       tset.each do |raw|
-        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+        assert_equal( expected, VisitURL.normalize( raw ).to_s )
       end
     end
   end
@@ -92,9 +166,9 @@ class TestVisitURL < MiniTest::Unit::TestCase
     sets = [ %w[ http://xn--bcher-kva.ch/ http://Bücher.ch ] ]
 
     sets.each do |tset|
-      expected = VisitURL.normalize( tset.shift )
+      expected = tset.shift
       tset.each do |raw|
-        assert_equal( expected.to_s, VisitURL.normalize( raw ).to_s )
+        assert_equal( expected, VisitURL.normalize( raw ).to_s )
       end
     end
   end
@@ -138,11 +212,9 @@ class TestVisitURL < MiniTest::Unit::TestCase
              %w[ http://h.c/foo/bar?q=1 http://h.c/foo/   ./bar?q=1 ] ]
 
     sets.each do |e,b,r|
-      expected = VisitURL.normalize( e )
       base = VisitURL.normalize( b )
       resolved = base.resolve( r )
-
-      assert_equal( expected.to_s, resolved.to_s, [ e,b,r ].inspect )
+      assert_equal( e, resolved.to_s, [ e,b,r ].inspect )
     end
 
   end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: iudex-core
 version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 1.1.0
+  prerelease: 4
+  version: 1.2.b.0
 platform: java
 authors: 
   - David Kellum
@@ -10,111 +10,108 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-13 00:00:00 Z
+date: 2012-03-05 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: rjack-slf4j
-    prerelease: false
-    requirement: &id001 !ruby/object:Gem::Requirement 
+    version_requirements: &id001 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
             version: 1.6.1
+    requirement: *id001
+    prerelease: false
     type: :runtime
-    version_requirements: *id001
   - !ruby/object:Gem::Dependency 
     name: hooker
-    prerelease: false
-    requirement: &id002 !ruby/object:Gem::Requirement 
+    version_requirements: &id002 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
             version: 1.0.0
+    requirement: *id002
+    prerelease: false
     type: :runtime
-    version_requirements: *id002
   - !ruby/object:Gem::Dependency 
     name: gravitext-util
-    prerelease: false
-    requirement: &id003 !ruby/object:Gem::Requirement 
+    version_requirements: &id003 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.5.1
+            version: 1.6.b
+    requirement: *id003
+    prerelease: false
     type: :runtime
-    version_requirements: *id003
   - !ruby/object:Gem::Dependency 
     name: iudex-filter
-    prerelease: false
-    requirement: &id004 !ruby/object:Gem::Requirement 
+    version_requirements: &id004 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.1.0
+            version: 1.2.b
+    requirement: *id004
+    prerelease: false
     type: :runtime
-    version_requirements: *id004
   - !ruby/object:Gem::Dependency 
     name: iudex-http
-    prerelease: false
-    requirement: &id005 !ruby/object:Gem::Requirement 
+    version_requirements: &id005 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.1.0
+            version: 1.2.b
+    requirement: *id005
+    prerelease: false
     type: :runtime
-    version_requirements: *id005
   - !ruby/object:Gem::Dependency 
     name: iudex-barc
-    prerelease: false
-    requirement: &id006 !ruby/object:Gem::Requirement 
+    version_requirements: &id006 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.1.0
+            version: 1.2.b
+    requirement: *id006
+    prerelease: false
     type: :runtime
-    version_requirements: *id006
   - !ruby/object:Gem::Dependency 
     name: minitest
-    prerelease: false
-    requirement: &id007 !ruby/object:Gem::Requirement 
+    version_requirements: &id007 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
             version: "2.3"
+    requirement: *id007
+    prerelease: false
     type: :development
-    version_requirements: *id007
   - !ruby/object:Gem::Dependency 
     name: rjack-logback
-    prerelease: false
-    requirement: &id008 !ruby/object:Gem::Requirement 
+    version_requirements: &id008 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
             version: "1.0"
+    requirement: *id008
+    prerelease: false
     type: :development
-    version_requirements: *id008
   - !ruby/object:Gem::Dependency 
     name: rjack-tarpit
-    prerelease: false
-    requirement: &id009 !ruby/object:Gem::Requirement 
+    version_requirements: &id009 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.4.0
+            version: "2.0"
+    requirement: *id009
+    prerelease: false
     type: :development
-    version_requirements: *id009
-description: |-
-  Iudex is a general purpose web crawler and feed processor in
-  ruby/java. The iudex-core gem contains core facilities and notably,
-  does not contain such facilities as database-backed state management.
+description: Iudex is a general purpose web crawler and feed processor in ruby/java. The iudex-core gem contains core facilities and notably, does not contain such facilities as database-backed state management.
 email: 
   - dek-oss@gravitext.com
 executables: 
@@ -123,7 +120,6 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
-  - Manifest.txt
   - History.rdoc
   - README.rdoc
 files: 
@@ -152,8 +148,7 @@ files:
   - test/test_visit_manager.rb
   - test/test_visit_queue.rb
   - test/test_visit_url.rb
-  - lib/iudex-core/iudex-core-1.1.0.jar
-  - .gemtest
+  - lib/iudex-core/iudex-core-1.2.b.0.jar
 homepage: http://github.com/dekellum/iudex
 licenses: []
 
@@ -168,26 +163,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 2
+        segments: 
+          - 0
         version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.3.1
 requirements: []
 
-rubyforge_project: iudex-core
-rubygems_version: 1.8.9
+rubyforge_project: 
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
-summary: Iudex is a general purpose web crawler and feed processor in ruby/java
-test_files: 
-  - test/test_visit_manager.rb
-  - test/test_content_fetcher.rb
-  - test/test_visit_url.rb
-  - test/test_mojibake.rb
-  - test/test_log_writer.rb
-  - test/test_visit_queue.rb
-  - test/test_redirect_handler.rb
-  - test/test_content_source.rb
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java.
+test_files: []
+