iudex-html 1.0.0-java

data/lib/iudex-html.rb

@@ -0,0 +1,56 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+require 'gravitext-xmlprod'
+require 'rjack-nekohtml'
+
+require 'iudex-html/base.rb'
+
+require 'java'
+
+module Iudex
+  module HTML
+    require "iudex-html/iudex-html-#{VERSION}.jar"
+
+    import 'iudex.html.HTMLKeys'
+
+    module Filters
+      import 'iudex.html.filters.ExtractFilter'
+      import 'iudex.html.filters.HTMLParseFilter'
+      import 'iudex.html.filters.HTMLTreeFilter'
+      import 'iudex.html.filters.HTMLWriteFilter'
+      import 'iudex.html.filters.TitleExtractor'
+    end
+
+    module Tree
+      import 'iudex.html.tree.HTMLTreeKeys'
+      import 'iudex.html.tree.TreeFilterChain'
+
+      module Filters
+        import 'iudex.html.tree.filters.AttributeCleaner'
+        import 'iudex.html.tree.filters.CSSDisplayFilter'
+        import 'iudex.html.tree.filters.CharactersNormalizer'
+        import 'iudex.html.tree.filters.EmptyInlineRemover'
+        import 'iudex.html.tree.filters.MetaSkipFilter'
+        import 'iudex.html.tree.filters.WordCounter'
+        import 'iudex.html.tree.filters.WordyCounter'
+        import 'iudex.html.tree.filters.XmpToPreConverter'
+      end
+    end
+
+  end
+end

data/lib/iudex-html/base.rb

@@ -0,0 +1,21 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module HTML
+    VERSION = '1.0.0'
+  end
+end

data/lib/iudex-html/factory_helper.rb

@@ -0,0 +1,95 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-html'
+require 'iudex-filter/key_helper'
+
+module Iudex
+  module HTML
+    module Filters
+      module FactoryHelper
+        include Iudex::Core # ContentSource
+        include Iudex::HTML::Tree
+        include Iudex::HTML::Tree::Filters
+
+        # Create html parse and clean filters
+        # Expected usage:
+        #   PAGE: html_clean_filters( :source  )
+        #   FEED: html_clean_filters( :title   )
+        #   FEED: html_clean_filters( :summary )
+        #   FEED: html_clean_filters( :content )
+        #
+        def html_clean_filters( src_key, tree_key = nil )
+
+          tree_key = "#{src_key}_tree".to_sym unless tree_key
+          src_key, tree_key = src_key.to_k, tree_key.to_k
+
+          filters = []
+          filters << html_parse_filter( src_key, tree_key )
+
+          #FIXME: PAGE: filters << TitleExtractor.new, or after?
+
+          # FIXME: if src is text, last filter
+          # filters << TextCtrlWSFilter.new( ContentKeys::TITLE )
+
+          tfc = TreeFilterChain.new( html_tree_filters )
+
+          filters << HTMLTreeFilter.new( tree_key, tfc,
+                                         HTMLTreeFilter::Order::DEPTH_FIRST )
+
+          #FIXME: First block extractor back to text key?
+
+          filters
+        end
+
+        def html_tree_filters
+          [ XmpToPreConverter.new,    # Before CharactersNormalizer
+            CSSDisplayFilter.new,     # Before AttributeCleaner
+            AttributeCleaner.new,
+            CharactersNormalizer.new,
+            EmptyInlineRemover.new,   # Depth
+            WordCounter.new,          # Depth; only for count deps?
+            WordyCounter.new ]        # Depth; only with cleaners/simhash?
+        end
+
+        def html_parse_filter( src_key, tree_key = nil )
+
+          tree_key = "#{src_key}_tree".to_sym unless tree_key
+          src_key, tree_key = src_key.to_k, tree_key.to_k
+
+          if( src_key.value_type == ContentSource.java_class )
+            HTMLParseFilter.new( src_key, nil, tree_key )
+          else
+            HTMLParseFilter.new( src_key, tree_key )
+          end
+        end
+
+        # Expected usage:
+        #   FEED: html_write_filter( :summary )
+        def html_write_filter( key1, key2 = nil )
+
+          tree_key, out_key = if key2
+                                [ key1, key2 ]
+                              else
+                                [ "#{key1}_tree".to_sym, key1 ]
+                              end
+
+          HTMLWriteFilter.new( tree_key.to_k, out_key.to_k )
+        end
+      end
+    end
+  end
+end

data/pom.xml

@@ -0,0 +1,51 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-html</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0.0</version>
+  <name>Iudex HTML parsing/filtering and text extraction</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.0</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-core</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.gravitext</groupId>
+      <artifactId>gravitext-xmlprod</artifactId>
+      <version>[1.4,1.5)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>net.sourceforge.nekohtml</groupId>
+      <artifactId>nekohtml</artifactId>
+      <version>1.9.14</version>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

data/test/html_test_helper.rb

@@ -0,0 +1,100 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-html'
+require 'iudex-html/factory_helper'
+
+require 'iudex-filter/key_helper'
+
+module HTMLTestHelper
+
+  include Gravitext::HTMap
+  UniMap.define_accessors
+
+  include Iudex::Filter::Core
+  include Iudex::HTML::Filters::FactoryHelper
+
+  import 'com.gravitext.xml.tree.TreeUtils'
+  import 'com.gravitext.xml.producer.Indentor'
+
+  import 'iudex.html.HTML'
+  import 'iudex.html.HTMLUtils'
+  import 'iudex.html.tree.TreeFilterChain'
+  import 'iudex.html.tree.TreeWalker'
+
+  def parse( html, charset = "UTF-8" )
+    HTMLUtils::parse( source( html, charset ) )
+  end
+
+  def parseFragment( html, charset = "UTF-8" )
+    inner( HTMLUtils::parseFragment( source( html, charset ) ) )
+  end
+
+  def inner( tree )
+    c = tree.children
+    if ( c.size == 1 && c[0].element? )
+      c[0]
+    else
+      tree
+    end
+  end
+
+  def assert_doc( html, root )
+    html = compress( html )
+    assert_equal( html,
+                  TreeUtils::produceString( root, Indentor::COMPRESSED ) )
+  end
+
+  def assert_fragment( html, root, remove_padding = false )
+    assert_fragment_ws( compress( html ), root, remove_padding )
+  end
+
+  def assert_fragment_ws( html, root, remove_padding = false )
+    html = html.gsub( /~+/, '' ) if remove_padding
+    assert_equal( html,
+      HTMLUtils::produceFragmentString( root, Indentor::COMPRESSED ) )
+  end
+
+  def assert_transform( html, filter = nil, func = :walk_depth_first )
+    tree = parseFragment( html[ :in ] )
+    action = TreeWalker.send( func, filter, tree ) if func && filter
+    assert_fragment( html[ :out ], tree, true )
+    action
+  end
+
+  def source( html, charset = "UTF-8" )
+    HTMLUtils::source( compress( html ).to_java_bytes, charset )
+  end
+
+  def compress( html )
+    html.gsub( /\n\s*/, '' )
+  end
+
+  def filter_chain( filters, mode = :whole )
+    pf = html_parse_filter( :source )
+    pf.parse_as_fragment = true if mode == :fragment
+    filters = Array( filters )
+    filters.unshift( pf )
+    FilterChain.new( "test", filters )
+  end
+
+  def content( html, charset = "UTF-8" )
+    map = UniMap.new
+    map.source = HTMLUtils::source( html.to_java_bytes, "UTF-8" )
+    map
+  end
+
+end

data/test/setup.rb

@@ -0,0 +1,38 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+test_dir = File.dirname( __FILE__ )
+
+ldir = File.join( test_dir, "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+RJack::Logback.config_console( :stderr => true )
+
+require 'minitest/unit'
+require 'minitest/autorun'
+
+require File.join( test_dir, 'html_test_helper.rb' )
+
+# Make test output logging compatible: no partial lines.
+# class TestOut
+#   def print( *a ); $stdout.puts( *a ); end
+#   def puts( *a );  $stdout.puts( *a ); end
+# end
+# MiniTest::Unit.output = TestOut.new

data/test/test_characters_normalizer.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+class TestCharactersNormalizer < MiniTest::Unit::TestCase
+  include HTMLTestHelper
+
+  include Iudex::HTML
+  include Iudex::HTML::Filters
+  include Iudex::HTML::Tree
+  include Iudex::HTML::Tree::Filters
+  include Iudex::Core
+
+  Order = HTMLTreeFilter::Order
+
+  def test_simple_block
+    # Note: '~' is padding removed in compare
+
+    html = { :in  => "<p> x  y </p>",
+             :out => "<p>~x ~y~</p>" }
+
+    assert_normalize( html )
+  end
+
+  def test_simple_inline
+    html = { :in  => "<i> x  y </i>",
+             :out => "<i> x ~y </i>" }
+
+    assert_normalize( html )
+  end
+
+  def test_mixed_inline
+    html = { :in  => "<p> x  y <i>z </i> </p>",
+             :out => "<p>~x ~y <i>z </i>~</p>" }
+
+    assert_normalize( html )
+  end
+
+  def test_empty
+    html = { :in  => "<div><p> </p>  <p>foo</p> </div>",
+             :out => "<div><p/>~~~~~~<p>foo</p>~</div>" }
+
+    assert_normalize( html )
+  end
+
+  def test_pre
+    html = { :in  => "<div> x <pre>  \0x\n <a>  y </a></pre> </div>",
+             :out => "<div>~x~<pre>  ~ x\n <a>  y </a></pre>~</div>" }
+
+    assert_normalize( html )
+  end
+
+  def assert_normalize( html )
+    [ Order::BREADTH_FIRST, Order::DEPTH_FIRST ].each do |order|
+      map = content( html[ :in ] )
+      tfc = TreeFilterChain.new( [ CharactersNormalizer.new ] )
+      tf = HTMLTreeFilter.new( :source_tree.to_k, tfc, order )
+      chain = filter_chain( tf, :fragment )
+      assert( chain.filter( map ) )
+      assert_fragment_ws( html[ :out ], inner( map.source_tree ), true )
+    end
+  end
+
+end