iudex-html 1.0.0-java → 1.1.0-java

data/History.rdoc

@@ -1,2 +1,9 @@
+=== 1.1.0 (2011-11-13)
+* Update to iudex-core ~> 1.1.0
+* Add NekoHTMLParser charset expansion and ContentSource encoding
+  confidence support
+* Add MojiBakeCleaner tree filter, helper support
+* Update to minitest ~> 2.3
+
 === 1.0.0 (2011-04-04)
 * Initial release.

data/Manifest.txt

@@ -21,4 +21,4 @@ test/test_other_tree_filters.rb
 test/test_parse_filter.rb
 test/test_tree_walker.rb
 test/test_word_counters.rb
-lib/iudex-html/iudex-html-1.0.0.jar
+lib/iudex-html/iudex-html-1.1.0.jar

data/Rakefile

@@ -4,7 +4,7 @@ $LOAD_PATH << './lib'
 require 'iudex-html/base'
 
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.2'
+gem     'rjack-tarpit', '~> 1.4'
 require 'rjack-tarpit'
 
 t = RJack::TarPit.new( 'iudex-html',
@@ -13,12 +13,12 @@ t = RJack::TarPit.new( 'iudex-html',
 
 t.specify do |h|
   h.developer( "David Kellum", "dek-oss@gravitext.com" )
-  h.extra_deps += [ [ 'iudex-core',        '~> 1.0.0'  ],
+  h.extra_deps += [ [ 'iudex-core',        '~> 1.1.0'  ],
                     [ 'rjack-nekohtml',    '~> 1.9.14' ],
                     [ 'gravitext-xmlprod', '~> 1.4.0'  ] ]
 
   h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',      '>= 1.7.1', '< 2.1' ],
+  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
                         [ 'rjack-logback', '~> 1.0' ] ]
 end
 

data/lib/iudex-html.rb

@@ -46,9 +46,24 @@ module Iudex
         import 'iudex.html.tree.filters.CharactersNormalizer'
         import 'iudex.html.tree.filters.EmptyInlineRemover'
         import 'iudex.html.tree.filters.MetaSkipFilter'
+        import 'iudex.html.tree.filters.MojiBakeCleaner'
         import 'iudex.html.tree.filters.WordCounter'
         import 'iudex.html.tree.filters.WordyCounter'
         import 'iudex.html.tree.filters.XmpToPreConverter'
+
+        # Re-open iudex.html.tree.filter.MojiBakeCleaner to add config file
+        # based initialization.
+        class MojiBakeCleaner
+          include Iudex::Core
+
+          # Alt constructor taking a configuration file in `mojibake
+          # -t` format.
+          def initialize( config_file = :default )
+            args = Array( config_file ) - [ :default ]
+            super( *MojiBake.load_config( *args ) )
+          end
+        end
+
       end
     end
 

data/lib/iudex-html/base.rb

@@ -16,6 +16,6 @@
 
 module Iudex
   module HTML
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
   end
 end

data/lib/iudex-html/factory_helper.rb

@@ -59,6 +59,7 @@ module Iudex
           [ XmpToPreConverter.new,    # Before CharactersNormalizer
             CSSDisplayFilter.new,     # Before AttributeCleaner
             AttributeCleaner.new,
+            MojiBakeCleaner.new,
             CharactersNormalizer.new,
             EmptyInlineRemover.new,   # Depth
             WordCounter.new,          # Depth; only for count deps?

data/pom.xml

@@ -3,13 +3,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-html</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.0</version>
+  <version>1.1.0</version>
   <name>Iudex HTML parsing/filtering and text extraction</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -18,7 +18,7 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.0,1.1)</version>
+      <version>[1.1,1.2)</version>
     </dependency>
 
     <dependency>

data/test/html_test_helper.rb

@@ -93,7 +93,7 @@ module HTMLTestHelper
 
   def content( html, charset = "UTF-8" )
     map = UniMap.new
-    map.source = HTMLUtils::source( html.to_java_bytes, "UTF-8" )
+    map.source = HTMLUtils::source( html.to_java_bytes, charset )
     map
   end
 

data/test/test_other_tree_filters.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
 #.hashdot.profile += jruby-shortlived
 
 #--
@@ -86,9 +87,9 @@ HTML
     assert( f.has_display_none( '{display: none}' ) ) #lenient
     assert( f.has_display_none( 'other:foo; DISPLAY:NONE;' ) )
 
-    assert( ! f.has_display_none( 'display: block' ) )
-    assert( ! f.has_display_none( 'other-display: none' ) )
-    assert( ! f.has_display_none( 'display: nonetheless' ) )
+    refute( f.has_display_none( 'display: block' ) )
+    refute( f.has_display_none( 'other-display: none' ) )
+    refute( f.has_display_none( 'display: nonetheless' ) )
   end
 
   def test_css_display_filter
@@ -114,6 +115,13 @@ HTML
     assert_transform( html, XmpToPreConverter.new )
   end
 
+  def test_mojibake_cleaner
+    html = { :in  => "<div><p>ascii</p> ° </div>",
+             :out => "<div><p>ascii</p> ~° </div>" }
+
+    assert_transform( html, MojiBakeCleaner.new )
+  end
+
   def cut_atts( html, *atts )
     atts.each do |att|
       html = html.gsub( / #{att}="[^"]+"/, '' )

data/test/test_parse_filter.rb

@@ -32,20 +32,20 @@ class TestParseFilter < MiniTest::Unit::TestCase
   end
 
   def test_marked
-    assert( ! marked?( "" ) )
-    assert( ! marked?( "simple" ) )
-    assert( ! marked?( "<simple" ) )
-    assert( ! marked?( "x < y" ) )
-    assert( ! marked?( "AT&T" ) )
-    assert( ! marked?( "AT & T;" ) )
+    refute( marked?( "" ) )
+    refute( marked?( "simple" ) )
+    refute( marked?( "<simple" ) )
+    refute( marked?( "x < y" ) )
+    refute( marked?( "AT&T" ) )
+    refute( marked?( "AT & T;" ) )
 
-    assert(   marked?( "<a>" ) )
-    assert(   marked?( "Words &copy; 2010" ) )
-    assert(   marked?( "&#xf43e;" ) )
-    assert(   marked?( "&#2028;" ) )
-    assert(   marked?( "<![CDATA[simple]]>" ) )
-    assert(   marked?( "<![CDATA[simple" ) )
-    assert(   marked?( "<!-- comment -->" ) )
+    assert( marked?( "<a>" ) )
+    assert( marked?( "Words &copy; 2010" ) )
+    assert( marked?( "&#xf43e;" ) )
+    assert( marked?( "&#2028;" ) )
+    assert( marked?( "<![CDATA[simple]]>" ) )
+    assert( marked?( "<![CDATA[simple" ) )
+    assert( marked?( "<!-- comment -->" ) )
   end
 
   def marked?( text )
@@ -54,6 +54,7 @@ class TestParseFilter < MiniTest::Unit::TestCase
 
   def test_markup
     tests = [ [ "simple",               0, "simple" ],
+              [ "AT&T",                 0, "AT&T" ],
               [ "<i>inner</i>",         1, nil ],
               [ "&lt;i>inner&lt;/i>",   2, nil ],
               [    "<!--ignore-->text", 1, "text" ],
@@ -61,6 +62,21 @@ class TestParseFilter < MiniTest::Unit::TestCase
               [ "&lt;",                 1, "<" ],
               [ "&amp;lt;",             2, "<" ] ]
 
+    @filter.min_parse = 0
+    tests.each do | input, count, out |
+      map = UniMap.new
+      map.title = input
+      assert_equal( count, @filter.parse_loop( map ), input )
+      assert_equal( out, map.title && map.title.to_s )
+    end
+  end
+
+  def test_markup_nochange
+    tests = [ [ "simple",               1, "simple" ],
+              [ "AT&T",                 1, "AT&T" ],
+              [ "<i>AT&T</i>",          1, nil ] ]
+
+    @filter.min_parse = 1
     tests.each do | input, count, out |
       map = UniMap.new
       map.title = input

metadata

@@ -2,7 +2,7 @@
 name: iudex-html
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.0
+  version: 1.1.0
 platform: java
 authors: 
   - David Kellum
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-04 00:00:00 -07:00
-default_executable: 
+date: 2011-11-13 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: iudex-core
@@ -21,7 +20,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.0.0
+            version: 1.1.0
     type: :runtime
     version_requirements: *id001
   - !ruby/object:Gem::Dependency 
@@ -52,12 +51,9 @@ dependencies:
     requirement: &id004 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.7.1
-        - - <
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "2.1"
+            version: "2.3"
     type: :development
     version_requirements: *id004
   - !ruby/object:Gem::Dependency 
@@ -79,7 +75,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.3.0
+            version: 1.4.0
     type: :development
     version_requirements: *id006
 description: |-
@@ -120,8 +116,8 @@ files:
   - test/test_parse_filter.rb
   - test/test_tree_walker.rb
   - test/test_word_counters.rb
-  - lib/iudex-html/iudex-html-1.0.0.jar
-has_rdoc: true
+  - lib/iudex-html/iudex-html-1.1.0.jar
+  - .gemtest
 homepage: http://github.com/dekellum/iudex
 licenses: []
 
@@ -146,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: iudex-html
-rubygems_version: 1.5.1
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: Iudex is a general purpose web crawler and feed processor in ruby/java