iudex-html 1.1.0-java → 1.2.b.0-java

data/History.rdoc

@@ -1,3 +1,14 @@
+=== 1.2.b.0 (2012-3-4)
+* Upgrade to gravitext-xmlprod ~> 1.5.b
+* Fix duplicate attributes from Neko, last value wins.
+* Use Element.to_xml( :implied_ns ) from xmlprod 1.5.b.2 in tests.
+* Add StAX-based HTMLStAXConsumer for faster parsing of pre-cleaned,
+  trusted XHTML. Add Tree.parse ruby utility.  Unrecognized tags are
+  constructed as BANNED.
+* Add iudex-html-perftest (neko vs stax)
+* Add basic iudex-html-clean utility for ad hoc testing
+* Upgrade to tarpit ~> 2.0, bundler Gemfile, gemspec (dev)
+
 === 1.1.0 (2011-11-13)
 * Update to iudex-core ~> 1.1.0
 * Add NekoHTMLParser charset expansion and ContentSource encoding

data/Manifest.txt

@@ -3,6 +3,8 @@ Manifest.txt
 README.rdoc
 Rakefile
 pom.xml
+bin/iudex-html-clean
+bin/iudex-html-perftest
 build/HTML.java.erb
 build/attributes
 build/java_generate.rb
@@ -11,6 +13,7 @@ lib/iudex-html/base.rb
 lib/iudex-html.rb
 lib/iudex-html/factory_helper.rb
 test/html_test_helper.rb
+test/reddit.xhtml
 test/setup.rb
 test/test_characters_normalizer.rb
 test/test_extract_filter.rb
@@ -19,6 +22,7 @@ test/test_html_parser.rb
 test/test_other_filters.rb
 test/test_other_tree_filters.rb
 test/test_parse_filter.rb
+test/test_stax_parser.rb
 test/test_tree_walker.rb
 test/test_word_counters.rb
-lib/iudex-html/iudex-html-1.1.0.jar
+lib/iudex-html/iudex-html-1.2.b.0.jar

data/README.rdoc

@@ -10,7 +10,7 @@ filtering, exracting text and links.
 
 == License
 
-Copyright (c) 2010-2011 David Kellum
+Copyright (c) 2008-2012 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You

data/Rakefile

@@ -1,42 +1,10 @@
 # -*- ruby -*-
 
-$LOAD_PATH << './lib'
-require 'iudex-html/base'
-
 require 'rubygems'
-gem     'rjack-tarpit', '~> 1.4'
+require 'bundler/setup'
 require 'rjack-tarpit'
 
-t = RJack::TarPit.new( 'iudex-html',
-                       Iudex::HTML::VERSION,
-                       :no_assembly, :java_platform )
-
-t.specify do |h|
-  h.developer( "David Kellum", "dek-oss@gravitext.com" )
-  h.extra_deps += [ [ 'iudex-core',        '~> 1.1.0'  ],
-                    [ 'rjack-nekohtml',    '~> 1.9.14' ],
-                    [ 'gravitext-xmlprod', '~> 1.4.0'  ] ]
-
-  h.testlib = :minitest
-  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
-                        [ 'rjack-logback', '~> 1.0' ] ]
-end
-
-file 'Manifest.txt' => [ 'pom.xml' ]
-
-task :check_pom_version do
-  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
-end
-task :check_history_version do
-  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
-end
-task :check_history_date do
-  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
-end
-
-task :gem  => [ :check_pom_version, :check_history_version                      ]
-task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
-task :push => [                                             :check_history_date ]
+RJack::TarPit.new( 'iudex-html' ).define_tasks
 
 file 'target/.tarpit' => [ 'src/main/java/iudex/html/HTML.java' ]
 
@@ -49,5 +17,3 @@ end
 task :clean do
   rm_f 'src/main/java/iudex/html/HTML.java'
 end
-
-t.define_tasks

data/bin/iudex-html-clean

@@ -0,0 +1,58 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'rjack-logback'
+
+require 'iudex-html'
+require 'iudex-filter/key_helper'
+
+require 'gravitext-xmlprod/extensions'
+
+require 'java'
+
+class HTMLCleaner
+  include Iudex::HTML
+  include Iudex::HTML::Tree
+  include Iudex::HTML::Filters
+  include Iudex::HTML::Tree::Filters
+
+  import 'iudex.html.HTMLUtils'
+  import 'iudex.html.tree.TreeWalker'
+
+  def run( input = ARGV.first )
+    source = HTMLUtils::source( IO.read( input ).to_java_bytes, "UTF-8" )
+    tree = HTMLUtils::parse( source )
+
+    tfc = TreeFilterChain.new( [ XmpToPreConverter.new,
+                                 CSSDisplayFilter.new,
+                                 AttributeCleaner.new,
+                                 MojiBakeCleaner.new,
+                                 CharactersNormalizer.new,
+                                 EmptyInlineRemover.new ] )
+
+    TreeWalker.walk_depth_first( tfc, tree )
+
+    puts tree.to_xml
+  end
+
+end
+
+HTMLCleaner.new.run

data/bin/iudex-html-perftest

@@ -0,0 +1,59 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'rjack-logback'
+
+require 'iudex-html'
+
+require 'gravitext-util'
+require 'gravitext-util/perftest'
+
+require 'java'
+
+class HTMLPerfTest
+  include Gravitext
+  include Gravitext::Concurrent
+
+  import 'iudex.html.tree.HTMLStAXUtils'
+  import 'iudex.html.HTMLUtils'
+
+  def run
+    PerfTest::Harness.new( create_tests ).execute
+  end
+
+  def create_tests
+    input = ARGV.first ||
+      File.join( File.dirname( __FILE__ ), '../test/reddit.xhtml' )
+    html = IO.read( input )
+
+    [ BlockTestFactory.new( 'neko' ) do
+        source = HTMLUtils::source( html.to_java_bytes, 'UTF-8' )
+        HTMLUtils::parse( source ).children.length
+      end,
+      BlockTestFactory.new( 'stax' ) do
+        Iudex::HTML::Tree.parse( html ).children.length
+      end
+    ]
+  end
+
+end
+
+HTMLPerfTest.new.run

data/build/HTML.java.erb

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010-2011 David Kellum
+ * Copyright (c) 2008-2012 David Kellum
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License.  You may

data/build/attributes

@@ -1,6 +1,6 @@
 # HTML Attributes
 #
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/build/java_generate.rb

@@ -2,7 +2,7 @@
 # -*- ruby -*-
 
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/build/tags

@@ -1,6 +1,6 @@
 # HTML Tags
 #
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-html/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -16,6 +16,6 @@
 
 module Iudex
   module HTML
-    VERSION = '1.1.0'
+    VERSION = '1.2.b.0'
   end
 end

data/lib/iudex-html/factory_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-html.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -16,9 +16,10 @@
 
 require 'iudex-core'
 require 'gravitext-xmlprod'
+require 'gravitext-xmlprod/extensions'
 require 'rjack-nekohtml'
 
-require 'iudex-html/base.rb'
+require 'iudex-html/base'
 
 require 'java'
 
@@ -39,6 +40,15 @@ module Iudex
     module Tree
       import 'iudex.html.tree.HTMLTreeKeys'
       import 'iudex.html.tree.TreeFilterChain'
+      import 'iudex.html.tree.HTMLStAXConsumer'
+      import 'javax.xml.stream.XMLStreamException'
+
+      # Parse the input String using HTMLStAXConsumer, HTMLTag.
+      # Raises XMLStreamException on parse error
+      def self.parse( input )
+        Gravitext::XMLProd::XMLHelper.
+          stax_parse_string( input, HTMLStAXConsumer.new )
+      end
 
       module Filters
         import 'iudex.html.tree.filters.AttributeCleaner'

data/pom.xml

@@ -3,13 +3,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-html</artifactId>
   <packaging>jar</packaging>
-  <version>1.1.0</version>
+  <version>1.2.b.0</version>
   <name>Iudex HTML parsing/filtering and text extraction</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.1</version>
+    <version>1.2.b.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -18,13 +18,13 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.1,1.2)</version>
+      <version>[1.2,1.2.9999)</version>
     </dependency>
 
     <dependency>
       <groupId>com.gravitext</groupId>
       <artifactId>gravitext-xmlprod</artifactId>
-      <version>[1.4,1.5)</version>
+      <version>[1.5,1.5.9999)</version>
     </dependency>
 
     <dependency>

data/test/html_test_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -19,16 +19,19 @@ require 'iudex-html/factory_helper'
 
 require 'iudex-filter/key_helper'
 
+require 'gravitext-xmlprod/extensions'
+
 module HTMLTestHelper
 
   include Gravitext::HTMap
   UniMap.define_accessors
 
+  include Gravitext::XMLProd
+
   include Iudex::Filter::Core
   include Iudex::HTML::Filters::FactoryHelper
 
   import 'com.gravitext.xml.tree.TreeUtils'
-  import 'com.gravitext.xml.producer.Indentor'
 
   import 'iudex.html.HTML'
   import 'iudex.html.HTMLUtils'
@@ -54,8 +57,7 @@ module HTMLTestHelper
 
   def assert_doc( html, root )
     html = compress( html )
-    assert_equal( html,
-                  TreeUtils::produceString( root, Indentor::COMPRESSED ) )
+    assert_equal( html, root.to_xml )
   end
 
   def assert_fragment( html, root, remove_padding = false )
@@ -64,8 +66,7 @@ module HTMLTestHelper
 
   def assert_fragment_ws( html, root, remove_padding = false )
     html = html.gsub( /~+/, '' ) if remove_padding
-    assert_equal( html,
-      HTMLUtils::produceFragmentString( root, Indentor::COMPRESSED ) )
+    assert_equal( html, root.to_xml( :implied_ns => [ HTML::NS_XHTML ] ) )
   end
 
   def assert_transform( html, filter = nil, func = :walk_depth_first )