iudex-core 1.0.0-java → 1.1.0-java

data/History.rdoc

@@ -1,2 +1,23 @@
+=== 1.1.0 (2011-11-13)
+* Update to iudex-filter,http,barc ~> 1.1.0
+* ContentFetcher updates for iudex-http changes
+* New MojiBakeFilter, MojiBakeMapper with config table loading support
+* Replaced VisitExecutor with asynchronous client compatible
+  VisitManager
+* Visit/HostQueue acquire/release for concurrecy and per host settings
+* VisitQueue uses VisitURL.domain (registration level) host keys
+* New VisitCounter interface
+* New RedirectHandler and Revisitor filters for direct redirect
+  handling with filter access
+* VisitURL.resolve for redirect support
+* Drop now redundant RLDomainFilter and RL_DOMAIN key
+* Add domain to iudex-url-norm output
+* Add encoding confidence map to ContentSource (supports encoding
+  detection)
+* Add U+2060 WORD JOINER to Characters.ctrlWS list
+* Update to minitest ~> 2.3
+* Update to gravitext-util ~> 1.5.1 (for UniMap.toString)
+* Update TLDSets based on upstream 9411dffc948b (2011-09-02)
+
 === 1.0.0 (2011-04-04)
 * Initial release.

data/Manifest.txt

@@ -9,10 +9,18 @@ build/TLDSets.java.erb
 build/effective_tld_name.dat
 build/tld_set_generator.rb
 config/config.rb
+config/mojibake
 lib/iudex-core/base.rb
 lib/iudex-core.rb
 lib/iudex-core/config.rb
+lib/iudex-core/mojibake.rb
 test/setup.rb
 test/test_content_fetcher.rb
+test/test_content_source.rb
 test/test_log_writer.rb
-lib/iudex-core/iudex-core-1.0.0.jar
+test/test_mojibake.rb
+test/test_redirect_handler.rb
+test/test_visit_manager.rb
+test/test_visit_queue.rb
+test/test_visit_url.rb
+lib/iudex-core/iudex-core-1.1.0.jar

data/Rakefile

@@ -4,7 +4,7 @@ $LOAD_PATH << './lib'
 require 'iudex-core/base'
 
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.2'
+gem     'rjack-tarpit', '~> 1.4'
 require 'rjack-tarpit'
 
 t = RJack::TarPit.new( 'iudex-core',
@@ -15,13 +15,13 @@ t.specify do |h|
   h.developer( "David Kellum", "dek-oss@gravitext.com" )
   h.extra_deps += [ [ 'rjack-slf4j',    '~> 1.6.1'  ],
                     [ 'hooker',         '~> 1.0.0'  ],
-                    [ 'gravitext-util', '~> 1.5.0'  ],
-                    [ 'iudex-filter',   '~> 1.0.0'  ],
-                    [ 'iudex-http',     '~> 1.0.0'  ],
-                    [ 'iudex-barc',     '~> 1.0.0'  ] ]
+                    [ 'gravitext-util', '~> 1.5.1'  ],
+                    [ 'iudex-filter',   '~> 1.1.0'  ],
+                    [ 'iudex-http',     '~> 1.1.0'  ],
+                    [ 'iudex-barc',     '~> 1.1.0'  ] ]
 
   h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',      '>= 1.7.1', '< 2.1' ],
+  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
                         [ 'rjack-logback', '~> 1.0' ] ]
 end
 

data/bin/iudex-test-config

@@ -34,7 +34,7 @@ module IudexBinScript
   Hooker.log_with { |m| SLF4J[ 'iudex' ].info( m.rstrip ) }
 
   OptionParser.new do |opts|
-    opts.on( "-v", "--version", "Display version" ) do |file|
+    opts.on( "-v", "--version", "Display version" ) do
       puts "iudex-core: #{ Core::VERSION }"
     end
     Hooker.register_config( opts )

data/bin/iudex-url-norm

@@ -34,14 +34,14 @@ module IudexBinScript
 
   OptionParser.new do |opts|
     opts.banner = "Usage: iudex-url-norm [options] [UrlsFile]..."
-    opts.on( "-v", "--version", "Display version" ) do |file|
+    opts.on( "-v", "--version", "Display version" ) do
       puts "iudex-core: #{Core::VERSION}"
       exit 1
     end
     opts.on_tail( "-h", "--help", "Show help and exit" ) do
       puts opts
       puts
-      puts( "Write uhash and normalized URLs to STDOUT, " +
+      puts( "Write uhash, domain, and normalized URLs to STDOUT, " +
             "from UrlsFile(s) or STDIN." )
       exit 1
     end
@@ -62,8 +62,8 @@ module IudexBinScript
 
   def self.process( fin )
     fin.each do |url|
-      vurl = Core::VisitURL.normalize( url.chomp )
-      puts vurl.uhash + ' ' + vurl.to_s
+      vurl = Core::VisitURL.normalize( url )
+      puts '%23s %24s %s' % [ vurl.uhash, vurl.domain, vurl ]
     end
   end