iudex-html 1.0.0-java

data/test/test_parse_filter.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+class TestParseFilter < MiniTest::Unit::TestCase
+  include HTMLTestHelper
+  include Gravitext::HTMap
+  include Iudex::Core
+  include Iudex::HTML
+  include Iudex::HTML::Filters
+
+  def setup
+    @filter = html_parse_filter( :title )
+    @filter.min_parse = 0
+  end
+
+  def test_marked
+    assert( ! marked?( "" ) )
+    assert( ! marked?( "simple" ) )
+    assert( ! marked?( "<simple" ) )
+    assert( ! marked?( "x < y" ) )
+    assert( ! marked?( "AT&T" ) )
+    assert( ! marked?( "AT & T;" ) )
+
+    assert(   marked?( "<a>" ) )
+    assert(   marked?( "Words &copy; 2010" ) )
+    assert(   marked?( "&#xf43e;" ) )
+    assert(   marked?( "&#2028;" ) )
+    assert(   marked?( "<![CDATA[simple]]>" ) )
+    assert(   marked?( "<![CDATA[simple" ) )
+    assert(   marked?( "<!-- comment -->" ) )
+  end
+
+  def marked?( text )
+    @filter.text_marked( text )
+  end
+
+  def test_markup
+    tests = [ [ "simple",               0, "simple" ],
+              [ "<i>inner</i>",         1, nil ],
+              [ "&lt;i>inner&lt;/i>",   2, nil ],
+              [    "<!--ignore-->text", 1, "text" ],
+              [ "&lt;!--ignore-->text", 2, "text" ],
+              [ "&lt;",                 1, "<" ],
+              [ "&amp;lt;",             2, "<" ] ]
+
+    tests.each do | input, count, out |
+      map = UniMap.new
+      map.title = input
+      assert_equal( count, @filter.parse_loop( map ), input )
+      assert_equal( out, map.title && map.title.to_s )
+    end
+  end
+
+end

data/test/test_tree_walker.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+class TestTreeWalker < MiniTest::Unit::TestCase
+  include HTMLTestHelper
+
+  import 'iudex.html.tree.TreeFilter'
+  Action = TreeFilter::Action
+
+  DROP_HTML = {
+    :in  => "<div>one<p>foo</p><br/> two</div>",
+    :out => "<div>one~~~~~~~~~~<br/> two</div>" }
+  # Note: ~~~ is padding removed in compare
+
+  def test_drop
+    filter = TagFilter.new( HTML::P, Action::DROP )
+    [ :walk_depth_first, :walk_breadth_first ].each do |order|
+      assert_transform( DROP_HTML, filter, order )
+    end
+  end
+
+  SKIP_HTML = {
+    :in  => "<div>first<b>drop</b><span><b>not dropped</b></span></div>",
+    :out => "<div>first~~~~~~~~~~~<span><b>not dropped</b></span></div>" }
+
+  def test_skip
+    chain = TreeFilterChain.new( [ TagFilter.new( HTML::SPAN, Action::SKIP ),
+                                   TagFilter.new( HTML::B,    Action::DROP ) ] )
+    assert_transform( SKIP_HTML, chain, :walk_breadth_first )
+  end
+
+  TERM_HTML = {
+    :in  => "<div><span>first</span><b>term</b><span><b>not</b></span></div>",
+    :out => "<div>~~~~~~~~~~~~~~~~~~<b>term</b><span><b>not</b></span></div>" }
+
+  def test_terminate
+    chain = TreeFilterChain.new( [ TagFilter.new( HTML::B, Action::TERMINATE ),
+                                   TagFilter.new( HTML::SPAN, Action::DROP ) ] )
+    [ :walk_depth_first, :walk_breadth_first ].each do |order|
+      assert_equal( Action::TERMINATE,
+                    assert_transform( TERM_HTML, chain, order ) )
+    end
+  end
+
+  FOLD_HTML = {
+    :in  => "<div>first <b>drop</b> <span> remain <b>drop</b> </span> </div>",
+    :out => "<div>first ~~~~~~~~~~~ ~~~~~~ remain ~~~~~~~~~~~ ~~~~~~~ </div>" }
+
+  def test_fold
+    chain = TreeFilterChain.new( [ TagFilter.new( HTML::SPAN, Action::FOLD ),
+                                   TagFilter.new( HTML::B,    Action::DROP ) ] )
+
+    [ :walk_breadth_first, :walk_depth_first ].each do |order|
+      assert_transform( FOLD_HTML, chain, order )
+    end
+  end
+
+  class TagFilter
+    include TreeFilter
+
+    def initialize( tag, action )
+      @tag = tag
+      @action = action
+    end
+
+    def filter( node )
+      elm = node.as_element
+      if( elm && elm.tag == @tag )
+        @action
+      else
+        Action::CONTINUE
+      end
+    end
+  end
+
+end

data/test/test_word_counters.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+class TestWordCounters < MiniTest::Unit::TestCase
+  include HTMLTestHelper
+  include Iudex::HTML::Tree
+  include Iudex::HTML::Tree::Filters
+
+  def test_counts
+    tset = [ [ "",                                            0, 0 ],
+             [ "<div><span> </span></div>",                   0, 0 ],
+
+             [ "a b",                                         2, 2 ],
+             [ "<div>a b</div>",                              2, 2 ],
+             [ "<div><span>a b</span></div>",                 2, 2 ],
+             [ "<div><span>a b</span><span>c d</span></div>", 4, 4 ],
+
+             [ "<div>a b <span><a name='foo'>c</a> d</span> e f</div>", 6, 6 ],
+
+             [ "<div><a href='foo'>a b</a></div>",
+               2, 2 * 0.25 ],
+
+             [ "<div><div>a b</div><div>c d</div></div>",
+               4,       ( 2*2 + 2*2 ) / 4.0 ],
+
+             [ "<div><div>a b</div><div>c d</div><div> </div></div>",
+               4,       ( 2*2 + 2*2 + 0*0 ) / 4.0 ],
+
+             [ "<div>a <div>a b</div><div>c d</div></div>",
+               5, 1.0 + ( 2*2 + 2*2 ) / 5.0 ],
+
+             [ "<div>a <div>a b c</div><div>c d</div></div>",
+               6, 1.0 + ( 3*3 + 2*2 ) / 6.0 ],
+
+             [ "<div><p>a b</p><p>c d</p></div><div>e f g</div>",
+               7,       ( 2*4 + 3*3 ) / 7.0 ] ]
+
+    chain = TreeFilterChain.new( [ WordCounter.new, WordyCounter.new ] )
+
+    tset.each do |html, word_count, wordiness|
+      tree = parse( html )
+      TreeWalker::walk_depth_first( chain, tree )
+
+      assert_equal( word_count,
+                    tree.get( HTMLTreeKeys::WORD_COUNT ),
+                    "word_count for: " + html )
+
+      assert_in_delta( wordiness,
+                       tree.get( HTMLTreeKeys::WORDINESS ),
+                       1e-4,
+                       " wordiness for: " + html )
+    end
+  end
+
+  def test_doc
+    html = <<HTML
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+  <title>Iūdex</title>
+  <style>style</style>
+ </head>
+ <body>
+  <p>Iūdex test.</p>
+ </body>
+</html>
+HTML
+    tree = parse( html, "UTF-8" )
+    chain = TreeFilterChain.new( [ MetaSkipFilter.new,
+                                   WordCounter.new,
+                                   WordyCounter.new ] )
+    TreeWalker::walk_depth_first( chain, tree )
+    assert_equal( 2, tree.get( HTMLTreeKeys::WORD_COUNT ) );
+    assert_equal( 2, tree.get( HTMLTreeKeys::WORDINESS ) );
+  end
+
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-html
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-core
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rjack-nekohtml
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.9.14
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: gravitext-xmlprod
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.4.0
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.7.1
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+    type: :development
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :development
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :development
+    version_requirements: *id006
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-html gem contains filters for HTML parsing,
+  filtering, exracting text and links.
+email: 
+  - dek-oss@gravitext.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - build/HTML.java.erb
+  - build/attributes
+  - build/java_generate.rb
+  - build/tags
+  - lib/iudex-html/base.rb
+  - lib/iudex-html.rb
+  - lib/iudex-html/factory_helper.rb
+  - test/html_test_helper.rb
+  - test/setup.rb
+  - test/test_characters_normalizer.rb
+  - test/test_extract_filter.rb
+  - test/test_factory_helper.rb
+  - test/test_html_parser.rb
+  - test/test_other_filters.rb
+  - test/test_other_tree_filters.rb
+  - test/test_parse_filter.rb
+  - test/test_tree_walker.rb
+  - test/test_word_counters.rb
+  - lib/iudex-html/iudex-html-1.0.0.jar
+has_rdoc: true
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-html
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_factory_helper.rb
+  - test/test_other_filters.rb
+  - test/test_characters_normalizer.rb
+  - test/test_word_counters.rb
+  - test/test_extract_filter.rb
+  - test/test_tree_walker.rb
+  - test/test_other_tree_filters.rb
+  - test/test_html_parser.rb
+  - test/test_parse_filter.rb