RubyGems - iudex-html - Versions diffs - 1.1.0-java → 1.2.b.0-java - Mend

iudex-html 1.1.0-java → 1.2.b.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (30) hide show

data/History.rdoc +11 -0
data/Manifest.txt +5 -1
data/README.rdoc +1 -1
data/Rakefile +2 -36
data/bin/iudex-html-clean +58 -0
data/bin/iudex-html-perftest +59 -0
data/build/HTML.java.erb +1 -1
data/build/attributes +1 -1
data/build/java_generate.rb +1 -1
data/build/tags +1 -1
data/lib/iudex-html/base.rb +2 -2
data/lib/iudex-html/factory_helper.rb +1 -1
data/lib/iudex-html/{iudex-html-1.1.0.jar → iudex-html-1.2.b.0.jar} +0 -0
data/lib/iudex-html.rb +12 -2
data/pom.xml +4 -4
data/test/html_test_helper.rb +7 -6
data/test/reddit.xhtml +557 -0
data/test/setup.rb +18 -16
data/test/test_characters_normalizer.rb +1 -1
data/test/test_extract_filter.rb +1 -1
data/test/test_factory_helper.rb +1 -1
data/test/test_html_parser.rb +21 -1
data/test/test_other_filters.rb +1 -1
data/test/test_other_tree_filters.rb +1 -1
data/test/test_parse_filter.rb +1 -1
data/test/test_stax_parser.rb +78 -0
data/test/test_tree_walker.rb +1 -1
data/test/test_word_counters.rb +1 -1
metadata +43 -48
data/.gemtest +0 -0

data/test/setup.rb CHANGED Viewed

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -14,25 +14,27 @@
 # permissions and limitations under the License.
 #++
-#### General test setup: LOAD_PATH, logging, console output ####
-test_dir = File.dirname( __FILE__ )
-ldir = File.join( test_dir, "..", "lib" )
-$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+#### General test setup, logging, console output ####
 require 'rubygems'
-require 'rjack-logback'
-RJack::Logback.config_console( :stderr => true )
+require 'bundler/setup'
 require 'minitest/unit'
 require 'minitest/autorun'
-require File.join( test_dir, 'html_test_helper.rb' )
+require 'rjack-logback'
+module TestSetup
+  include RJack
+  Logback.config_console( :stderr => true, :thread => true )
+  if ( ARGV & %w[ -v --verbose --debug ] ).empty?
+    Logback.root.level = Logback::INFO
+  else
+    Logback.root.level = Logback::DEBUG
+  end
+  ARGV.delete( '--debug' )
+end
-# Make test output logging compatible: no partial lines.
-# class TestOut
-#   def print( *a ); $stdout.puts( *a ); end
-#   def puts( *a );  $stdout.puts( *a ); end
-# end
-# MiniTest::Unit.output = TestOut.new
+require File.join( File.dirname( __FILE__ ), 'html_test_helper.rb' )

data/test/test_characters_normalizer.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_extract_filter.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_factory_helper.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_html_parser.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -75,6 +75,26 @@ HTML
     assert_doc( HTML_SKIP_TAGS_SKIPPED, parse( HTML_SKIP_TAGS, "ISO-8859-1" ) )
   end
+  def test_attr_duplicates
+    input = <<-HTML
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head/>
+  <body>
+    <p class="foo" class="bar">hello</p>
+  </body>
+</html>
+HTML
+    output = <<-HTML
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+  <head/>
+  <body>
+    <p class="bar">hello</p>
+  </body>
+</html>
+HTML
+    assert_doc( output, parse( input ) )
+  end
   HTML_OUTSIDE = <<HTML
 before
 <html xmlns="http://www.w3.org/1999/xhtml">

data/test/test_other_filters.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_other_tree_filters.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_parse_filter.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_stax_parser.rb ADDED Viewed

@@ -0,0 +1,78 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+#--
+# Copyright (c) 2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+require File.join( File.dirname( __FILE__ ), "setup" )
+class TestStAXParser < MiniTest::Unit::TestCase
+  include HTMLTestHelper
+  import 'javax.xml.stream.XMLStreamException'
+  HTML_FULL = <<HTML
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+  <title>Iūdex</title>
+ </head>
+ <body>
+  <p>Iūdex test.</p>
+  <n:bogus xmlns:n="bogo">truely</n:bogus>
+ </body>
+</html>
+HTML
+  def test_with_non_html
+    root = parse( HTML_FULL, "UTF-8" )
+    assert( root.find( 'body' ).find( 'bogus' ).tag.banned? )
+    assert_doc( HTML_FULL, root )
+  end
+  HTML_CDATA = {
+    :in  => "<p><![CDATA[two]]></p>",
+    :out => "<p>two</p>" }
+  # Note: HTML parsers drop this instead.
+  def test_cdata
+    tree = parse( HTML_CDATA[ :in ] )
+    assert_fragment( HTML_CDATA[ :out ], tree )
+  end
+  def test_inline_nest
+    html = { :in  => "<div><i>begin <p>block</p> end.</i></div>",
+             :out => "<div><i>begin <p>block</p> end.</i></div>" }
+    tree = parse( html[ :in ] )
+    assert_fragment( html[ :out ], tree )
+  end
+  def test_invalid_error
+    assert_raises( XMLStreamException ) do
+      Iudex::HTML::Tree.parse( "" )
+    end
+    assert_raises( XMLStreamException ) do
+      Iudex::HTML::Tree.parse( "<doc><open></doc>" )
+    end
+  end
+  # Helper overrrides
+  def parse( html, charset = "UTF-8" )
+    Iudex::HTML::Tree.parse( compress( html ) )
+  end
+end

data/test/test_tree_walker.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_word_counters.rb CHANGED Viewed

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 #--
-# Copyright (c) 2010-2011 David Kellum
+# Copyright (c) 2008-2012 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

metadata CHANGED Viewed

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: iudex-html
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 1.1.0
+  prerelease: 4
+  version: 1.2.b.0
 platform: java
 authors:
   - David Kellum
@@ -10,86 +10,83 @@ autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-11-13 00:00:00 Z
+date: 2012-03-05 00:00:00 Z
 dependencies:
   - !ruby/object:Gem::Dependency
     name: iudex-core
-    prerelease: false
-    requirement: &id001 !ruby/object:Gem::Requirement
+    version_requirements: &id001 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
-            version: 1.1.0
+            version: 1.2.b
+    requirement: *id001
+    prerelease: false
     type: :runtime
-    version_requirements: *id001
   - !ruby/object:Gem::Dependency
     name: rjack-nekohtml
-    prerelease: false
-    requirement: &id002 !ruby/object:Gem::Requirement
+    version_requirements: &id002 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
             version: 1.9.14
+    requirement: *id002
+    prerelease: false
     type: :runtime
-    version_requirements: *id002
   - !ruby/object:Gem::Dependency
     name: gravitext-xmlprod
-    prerelease: false
-    requirement: &id003 !ruby/object:Gem::Requirement
+    version_requirements: &id003 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
-            version: 1.4.0
+            version: 1.5.b
+    requirement: *id003
+    prerelease: false
     type: :runtime
-    version_requirements: *id003
   - !ruby/object:Gem::Dependency
     name: minitest
-    prerelease: false
-    requirement: &id004 !ruby/object:Gem::Requirement
+    version_requirements: &id004 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
             version: "2.3"
+    requirement: *id004
+    prerelease: false
     type: :development
-    version_requirements: *id004
   - !ruby/object:Gem::Dependency
     name: rjack-logback
-    prerelease: false
-    requirement: &id005 !ruby/object:Gem::Requirement
+    version_requirements: &id005 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
             version: "1.0"
+    requirement: *id005
+    prerelease: false
     type: :development
-    version_requirements: *id005
   - !ruby/object:Gem::Dependency
     name: rjack-tarpit
-    prerelease: false
-    requirement: &id006 !ruby/object:Gem::Requirement
+    version_requirements: &id006 !ruby/object:Gem::Requirement
       none: false
       requirements:
         - - ~>
           - !ruby/object:Gem::Version
-            version: 1.4.0
+            version: "2.0"
+    requirement: *id006
+    prerelease: false
     type: :development
-    version_requirements: *id006
-description: |-
-  Iudex is a general purpose web crawler and feed processor in
-  ruby/java. The iudex-html gem contains filters for HTML parsing,
-  filtering, exracting text and links.
+description: Iudex is a general purpose web crawler and feed processor in ruby/java. The iudex-html gem contains filters for HTML parsing, filtering, exracting text and links.
 email:
   - dek-oss@gravitext.com
-executables: []
+executables:
+  - iudex-html-clean
+  - iudex-html-perftest
 extensions: []
 extra_rdoc_files:
-  - Manifest.txt
   - History.rdoc
   - README.rdoc
 files:
@@ -98,6 +95,8 @@ files:
   - README.rdoc
   - Rakefile
   - pom.xml
+  - bin/iudex-html-clean
+  - bin/iudex-html-perftest
   - build/HTML.java.erb
   - build/attributes
   - build/java_generate.rb
@@ -106,6 +105,7 @@ files:
   - lib/iudex-html.rb
   - lib/iudex-html/factory_helper.rb
   - test/html_test_helper.rb
+  - test/reddit.xhtml
   - test/setup.rb
   - test/test_characters_normalizer.rb
   - test/test_extract_filter.rb
@@ -114,10 +114,10 @@ files:
   - test/test_other_filters.rb
   - test/test_other_tree_filters.rb
   - test/test_parse_filter.rb
+  - test/test_stax_parser.rb
   - test/test_tree_walker.rb
   - test/test_word_counters.rb
-  - lib/iudex-html/iudex-html-1.1.0.jar
-  - .gemtest
+  - lib/iudex-html/iudex-html-1.2.b.0.jar
 homepage: http://github.com/dekellum/iudex
 licenses: []
@@ -132,27 +132,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
     - - ">="
       - !ruby/object:Gem::Version
+        hash: 2
+        segments:
+          - 0
         version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: "0"
+        version: 1.3.1
 requirements: []
-rubyforge_project: iudex-html
-rubygems_version: 1.8.9
+rubyforge_project:
+rubygems_version: 1.8.15
 signing_key:
 specification_version: 3
-summary: Iudex is a general purpose web crawler and feed processor in ruby/java
-test_files:
-  - test/test_factory_helper.rb
-  - test/test_other_filters.rb
-  - test/test_characters_normalizer.rb
-  - test/test_word_counters.rb
-  - test/test_extract_filter.rb
-  - test/test_tree_walker.rb
-  - test/test_other_tree_filters.rb
-  - test/test_html_parser.rb
-  - test/test_parse_filter.rb
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java.
+test_files: []

data/.gemtest DELETED Viewed

File without changes