iudex-html 1.3.0-java → 1.4.0-java

data/History.rdoc

@@ -1,3 +1,11 @@
+=== 1.4.0 (2013-10-29)
+* Fix meta-tag charset parse restart handling for case of empty
+  charset and conflicting/multiple combinations of Content-Type or
+  charset meta.
+* Upgrade to nekohtml ~> 1.9.18
+* Upgrade to iudex-* ~> 1.4.0 dependencies
+* Upgrade to minitest ~> 4.7.4 (dev)
+
 === 1.3.0 (2012-11-8)
 * Upgrade to gravitext-xmlprod ~> 1.7.0 (and gravitext-util ~> 1.7.0)
 * Upgrade to logback ~> 1.5 (dev)

data/Manifest.txt

@@ -25,4 +25,4 @@ test/test_parse_filter.rb
 test/test_stax_parser.rb
 test/test_tree_walker.rb
 test/test_word_counters.rb
-lib/iudex-html/iudex-html-1.3.0.jar
+lib/iudex-html/iudex-html-1.4.0.jar

data/README.rdoc

@@ -11,7 +11,7 @@ filtering, exracting text and links.
 
 == License
 
-Copyright (c) 2008-2012 David Kellum
+Copyright (c) 2008-2013 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You

data/bin/iudex-html-clean

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/bin/iudex-html-perftest

@@ -1,7 +1,7 @@
 #!/usr/bin/env jruby
 # -*- ruby -*-
 #--
-# Copyright (c) 2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/build/HTML.java.erb

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2012 David Kellum
+ * Copyright (c) 2008-2013 David Kellum
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License.  You may

data/build/attributes

@@ -1,6 +1,6 @@
 # HTML Attributes
 #
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/build/java_generate.rb

@@ -2,7 +2,7 @@
 # -*- ruby -*-
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/build/tags

@@ -1,6 +1,6 @@
 # HTML Tags
 #
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-html.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/iudex-html/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -16,6 +16,6 @@
 
 module Iudex
   module HTML
-    VERSION = '1.3.0'
+    VERSION = '1.4.0'
   end
 end

data/lib/iudex-html/factory_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/pom.xml

@@ -3,13 +3,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-html</artifactId>
   <packaging>jar</packaging>
-  <version>1.3.0</version>
+  <version>1.4.0</version>
   <name>Iudex HTML parsing/filtering and text extraction</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -18,7 +18,7 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-core</artifactId>
-      <version>[1.3.0,1.3.999)</version>
+      <version>[1.4.0,1.4.999)</version>
     </dependency>
 
     <dependency>
@@ -30,7 +30,7 @@
     <dependency>
       <groupId>net.sourceforge.nekohtml</groupId>
       <artifactId>nekohtml</artifactId>
-      <version>1.9.14</version>
+      <version>[1.9.18,1.9.999)</version>
     </dependency>
 
   </dependencies>

data/test/html_test_helper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/setup.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_characters_normalizer.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_extract_filter.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_factory_helper.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_html_parser.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -22,6 +22,7 @@ require File.join( File.dirname( __FILE__ ), "setup" )
 
 class TestHTMLParser < MiniTest::Unit::TestCase
   include HTMLTestHelper
+  import 'iudex.util.Charsets'
 
   HTML_META = <<HTML
 <html xmlns="http://www.w3.org/1999/xhtml">
@@ -48,11 +49,48 @@ HTML
     assert_doc( alt, parse( alt, "UTF-8" ) )
   end
 
+  def test_charset_missing
+    alt = HTML_META.sub( /; charset=utf-8/, '' )
+    assert_doc( alt, parse( alt, "UTF-8" ) )
+  end
+
   def test_meta_charset_rerun
     alt = HTML_META.sub( /<meta .*\/>/, '<meta charset="utf-8"/>' )
     assert_doc( alt, parse( alt, "ISO-8859-1" ) )
   end
 
+  def test_meta_charset_conflict
+    alt = HTML_META.sub( /(<head>)/, '<head><meta charset="ISO-8859-1"/>' )
+    assert_doc( alt, parse( alt, "ISO-8859-1" ) )
+  end
+
+  def test_meta_charset_conflict_2
+    alt = HTML_META.sub( /utf-8/, 'latin1' )
+    alt = HTML_META.sub( /(<\/head>)/, '<meta charset="UTF-8"/></head>' )
+    assert_doc( alt, parse( alt, "ISO-8859-1" ) )
+  end
+
+  HTML_META_2 = <<HTML
+<!DOCTYPE html>
+<head>
+<title>Page with skipped head tags</title>
+<meta name="description" content="">
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<script type="text/javascript" src="../js/swfobject.js"></script>
+<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+</head>
+<body>
+ <p>Iudex test.</p>
+</body>
+</html>
+HTML
+
+  def test_charset_conflict_3
+    src = source( HTML_META_2, "Windows-1252" )
+    src.set_default_encoding( Charsets::UTF_8, 0.10 )
+    assert( HTMLUtils::parse( src ) )
+  end
+
   HTML_SKIP_TAGS = <<HTML
 <html xmlns="http://www.w3.org/1999/xhtml">
  <head>

data/test/test_other_filters.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_other_tree_filters.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_parse_filter.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_stax_parser.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_tree_walker.rb

@@ -2,7 +2,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/test/test_word_counters.rb

@@ -3,7 +3,7 @@
 #.hashdot.profile += jruby-shortlived
 
 #--
-# Copyright (c) 2008-2012 David Kellum
+# Copyright (c) 2008-2013 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

metadata

@@ -2,14 +2,14 @@
 name: iudex-html
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.0
+  version: 1.4.0
 platform: java
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-08 00:00:00.000000000 Z
+date: 2013-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iudex-core
@@ -17,13 +17,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.4.0
     none: false
   prerelease: false
   type: :runtime
@@ -33,13 +33,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.9.14
+        version: 1.9.18
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.9.14
+        version: 1.9.18
     none: false
   prerelease: false
   type: :runtime
@@ -65,13 +65,13 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: 4.7.4
     none: false
   prerelease: false
   type: :development
@@ -145,7 +145,7 @@ files:
 - test/test_stax_parser.rb
 - test/test_tree_walker.rb
 - test/test_word_counters.rb
-- lib/iudex-html/iudex-html-1.3.0.jar
+- lib/iudex-html/iudex-html-1.4.0.jar
 homepage: http://iudex.gravitext.com
 licenses: []
 post_install_message: