iudex-html 1.2.b.0-java → 1.2.b.1-java

data/History.rdoc

@@ -1,3 +1,9 @@
+=== 1.2.b.1 (2012-5-31)
+* Add support for HTML 5 (draft) tags, attributes
+* Neko parser support for HTML 5 <meta charset>
+* Neko parser keeps non-HTML attributes when skipBanned = false
+* Add options, barc read support to iudex-html-clean
+
 === 1.2.b.0 (2012-3-4)
 * Upgrade to gravitext-xmlprod ~> 1.5.b
 * Fix duplicate attributes from Neko, last value wins.

data/Manifest.txt

@@ -25,4 +25,4 @@ test/test_parse_filter.rb
 test/test_stax_parser.rb
 test/test_tree_walker.rb
 test/test_word_counters.rb
-lib/iudex-html/iudex-html-1.2.b.0.jar
+lib/iudex-html/iudex-html-1.2.b.1.jar

data/bin/iudex-html-clean

@@ -19,40 +19,128 @@
 $LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
 
 require 'rubygems'
-require 'rjack-logback'
 
-require 'iudex-html'
-require 'iudex-filter/key_helper'
+module IudexBinScript
+  require 'rjack-logback'
+  include RJack
 
-require 'gravitext-xmlprod/extensions'
+  Logback.config_console( :stderr => true )
 
-require 'java'
+  require 'iudex-html'
+  require 'iudex-filter/key_helper'
 
-class HTMLCleaner
-  include Iudex::HTML
-  include Iudex::HTML::Tree
-  include Iudex::HTML::Filters
-  include Iudex::HTML::Tree::Filters
+  require 'gravitext-xmlprod/extensions'
 
-  import 'iudex.html.HTMLUtils'
-  import 'iudex.html.tree.TreeWalker'
+  require 'java'
 
-  def run( input = ARGV.first )
-    source = HTMLUtils::source( IO.read( input ).to_java_bytes, "UTF-8" )
-    tree = HTMLUtils::parse( source )
+  class HTMLCleaner
+    include Gravitext::XMLProd
+    include Iudex::Core
+    include Iudex::HTML
+    include Iudex::HTML::Tree
+    include Iudex::HTML::Filters
+    include Iudex::HTML::Tree::Filters
 
-    tfc = TreeFilterChain.new( [ XmpToPreConverter.new,
-                                 CSSDisplayFilter.new,
-                                 AttributeCleaner.new,
-                                 MojiBakeCleaner.new,
-                                 CharactersNormalizer.new,
-                                 EmptyInlineRemover.new ] )
+    import 'iudex.html.HTMLUtils'
+    import 'iudex.html.neko.NekoHTMLParser'
+    import 'iudex.html.tree.TreeWalker'
+    import 'iudex.http.Headers'
+    import 'iudex.util.Charsets'
 
-    TreeWalker.walk_depth_first( tfc, tree )
+    def initialize
+      @default_encoding = "UTF-8"
+      @trim_non_displayed = false
+      @indentor = Indentor::PRETTY
+    end
+
+    def run( args = ARGV )
+      files = parse_args( args )
+
+      files.each do |f|
+        if f =~ /\.barc$/
+          process_barc( f )
+        else
+          process_file( f )
+        end
+      end
+
+    end
+
+    def process_file( fname )
+
+      input = if fname == '-'
+                $stdin.read
+              else
+                IO.read( fname )
+              end
+
+      source = HTMLUtils::source( input.to_java_bytes, @default_encoding )
+      process( source )
+    end
+
+    def process_barc( bname )
+      require 'iudex-barc' #FIXME: Undeclared
+      barc_file = Iudex::BARC::BARCFile.new( java.io.File.new( bname ) )
+      barc_reader = barc_file.reader
+      while( rec = barc_reader.next )
+        next unless rec.type.chr == 'H'
+        source = ContentSource.new( rec.body_input_stream )
+        ctype = Headers.content_type( rec.response_headers )
+        if ctype && ctype.charset
+          enc = Charsets.lookup( ctype.charset )
+          source.set_default_encoding( enc ) if enc
+        end
+        process( source )
+      end
+    end
+
+    def process( source )
+      parser = NekoHTMLParser.new
+      parser.skip_banned = @trim_non_displayed
+
+      tree = parser.parse( source )
+
+      filters = [ XmpToPreConverter.new,
+                  ( [ CSSDisplayFilter.new,
+                      AttributeCleaner.new ] if @trim_non_displayed ),
+                  MojiBakeCleaner.new,
+                  CharactersNormalizer.new,
+                  EmptyInlineRemover.new ].flatten.compact
+
+      TreeWalker.walk_depth_first( TreeFilterChain.new( filters ),
+                                   tree )
+
+      puts tree.to_xml( :indentor => @indentor )
+    end
+
+    def parse_args( args = ARGV )
+      parser = OptionParser.new do |opts|
+        opts.banner =
+          "Usage: iudex-html-clean [options] (FILE|barc)...\n" +
+          "Options:\n"
+
+        opts.on( "-v", "--version", "Display version and exit" ) do
+          puts "iudex-html: #{Iudex::HTML::VERSION}"
+          exit 1
+        end
+
+        opts.on( "-t", "--trim-non-display",
+                 "Trim banned/non-displayed elements from output" ) do
+          @trim_non_displayed = true
+        end
+
+        opts.on( "-i", "--indentor NAME",
+                 "Specify indentor to use for output: " +
+                 "PRETTY (default), COMPACT" ) do |name|
+          @indentor = Indentor.const_get( name.upcase.to_sym )
+        end
+
+      end
+
+      parser.parse( args )
+    end
 
-    puts tree.to_xml
   end
 
+  HTMLCleaner.new.run
 end
-
-HTMLCleaner.new.run

data/build/attributes

@@ -21,6 +21,7 @@
 # Sources
 #  http://www.w3.org/TR/xhtml11/
 #  http://www.w3.org/TR/html4/
+#  http://dev.w3.org/html5/spec/
 #  http://www.w3schools.com/tags/ref_standardattributes.asp
 #  http://xhtml.com
 
@@ -28,6 +29,7 @@ CORE :: ALL except: base head html meta param script style title
 class       ,*CORE
 id          ,*CORE
 style       ,*CORE
+hidden      ,*CORE, hidden element
 title       ,CORE, extra title
 
 LANG :: ALL except: base br frame frameset hr iframe param
@@ -40,17 +42,17 @@ content     ,meta, text
 scheme      ,meta, format URI
 
 # Anchor and link attributes
-charset     ,a link, char_encoding of link
+charset     ,a link meta, char_encoding of link or (meta) document
 coords      ,*a, coordinates; i.e. image map
 hreflang    ,link, language_code of referent
 href        ,a base link, URL
-media       ,link
+media       ,a area link
 name        ,a, section_name anchor
 rel         ,a link
 rev         ,a link
 shape       ,*a
 target      ,*a *base *link
-type        ,link
+type        ,a link
 
 # Image and some frame attributes
 src         ,frame img

data/build/tags

@@ -17,6 +17,7 @@
 # Sources
 #  http://www.w3.org/TR/xhtml11/
 #  http://www.w3.org/TR/html4/
+#  http://dev.w3.org/html5/spec/
 #  http://www.w3schools.com/tags/default.asp
 #  http://xhtml.com/
 #
@@ -25,106 +26,120 @@
 # S :: In Strict HTML 4.01/XHTML 1.0
 # T :: In Transitional HTML 4.01/XHTML 1.0
 # F :: In frameset annex
+# 5 :: HTML5 new elements
 # D :: Deprecated
 # I :: Inline elements (Note <br/> is not labeled inline.)
 # M :: Metadata elements (content not visible text), i.e. head
 # B :: Banned/blacklisted elements from which text should not be extracted.
 
-a            ,  S T F   I    , anchor
-abbr         ,  S T F   I    , abbreviation
-acronym      ,  S T F   I    , acronym
-address      ,  S T F        , contact information for the author or owner
-applet       ,    T F D      , embedded applet
-area         ,E S T F        , area inside an image-map
-b            ,  S T F   I    , bold text
-base         ,E S T F     M  , default address or a default target for all links on a page
-basefont     ,E   T F D I M  , default font; color; or size for the text in a page
-bdo          ,  S T F   I    , the text direction
-big          ,  S T F   I    , big text
-blockquote   ,  S T F        , long quotation
-body         ,  S T F        , the document's body
-br           ,E S T F        , single line break
-button       ,  S T F   I   B, push button
-caption      ,  S T F        , table caption
-center       ,    T F D      , centered text
-cite         ,  S T F   I    , citation
-code         ,  S T F   I    , computer code text
-col          ,E S T F        , attribute values for one or more columns in a table
-colgroup     ,  S T F        , group of columns in a table for formatting
-dd           ,  S T F        , description of a term in a definition list
-del          ,  S T F   I    , deleted text
-dfn          ,  S T F   I    , definition term
-dir          ,    T F D      , directory list
-div          ,  S T F        , section in a document
-dl           ,  S T F        , definition list
-dt           ,  S T F        , term (an item) in a definition list
-em           ,  S T F   I    , emphasized text
-fieldset     ,  S T F       B, border around elements in a form
-font         ,    T F D I    , font; color; or size for text
-form         ,  S T F        , form for user input
-frame        ,E     F       B, window (a frame) in a frameset
-frameset     ,      F       B, set of frames
-h1           ,  S T F        , heading level 1
-h2           ,  S T F        , heading level 2
-h3           ,  S T F        , heading level 3
-h4           ,  S T F        , heading level 4
-h5           ,  S T F        , heading level 5
-h6           ,  S T F        , heading level 6
-head         ,  S T F     M  , information about the document
-hr           ,E S T F        , horizontal line
-html         ,  S T F        , document
-i            ,  S T F   I    , italic text
-iframe       ,    T F        , nline frame
-img          ,E S T F   I    , image
-input        ,E S T F   I   B, input control
-ins          ,  S T F   I    , inserted text
-isindex      ,    T F D      , searchable index related to a document
-kbd          ,  S T F   I    , keyboard text
-label        ,  S T F   I   B, label for an input element
-legend       ,  S T F       B, caption for a fieldset element
-li           ,  S T F        , list item
-link         ,E S T F     M  , the relationship between a document and an external resource
-map          ,  S T F   I    , image-map
-menu         ,    T F D      , menu list
-meta         ,E S T F     M  , metadata
-noframes     ,    T F       B, alternate content where frames not supported
-noscript     ,  S T F       B, alternate content script not supported
-object       ,  S T F   I   B, embedded object
-ol           ,  S T F        , ordered list
-optgroup     ,  S T F       B, group of related options in a select list
-option       ,  S T F       B, option in a select list
-p            ,  S T F        , paragraph
-param        ,E S T F        , parameter for an object
-pre          ,  S T F        , preformatted text
-q            ,  S T F   I    , short quotation
-rb           ,               , ruby base text
-rbc          ,               , ruby base container (complex)
-rp           ,               , ruby simple text container
-rt           ,               , ruby annotation text
-rtc          ,               , ruby text container (complex)
-ruby         ,          I    , ruby pronunciation aid
-s            ,    T F D I    , strikethrough text
-samp         ,  S T F   I    , sample computer code
-script       ,  S T F   I   B, client-side script
-select       ,  S T F   I   B, select list (drop-down list)
-small        ,  S T F   I    , small text
-span         ,  S T F   I    , section in a document
-strike       ,    T F D I    , strikethrough text
-strong       ,  S T F   I    , strong text
-style        ,  S T F       B, style information for a document
-sub          ,  S T F   I    , subscripted text
-sup          ,  S T F   I    , superscripted text
-table        ,  S T F        , table
-tbody        ,  S T F        , Groups the body content in a table
-td           ,  S T F        , cell in a table
-textarea     ,  S T F   I   B, multi-line text input control
-tfoot        ,  S T F        , Groups the footer content in a table
-th           ,  S T F        , header cell in a table
-thead        ,  S T F        , Groups the header content in a table
-title        ,  S T F     M  , the title of a document
-tr           ,  S T F        , row in a table
-tt           ,  S T F   I    , teletype text
-u            ,    T F D I    , underlined text
-ul           ,  S T F        , unordered list
-var          ,  S T F   I    , variable part of a text
-xmp          ,        D      , preformatted text
+a            ,  S T F 5   I    , anchor
+abbr         ,  S T F 5   I    , abbreviation
+acronym      ,  S T F     I    , acronym
+address      ,  S T F 5        , contact information for the author or owner
+applet       ,    T F   D      , embedded applet
+area         ,E S T F 5        , area inside an image-map
+article      ,        5        , Structure: An independent content element
+aside        ,        5        , Structure: Tengentially related content
+b            ,  S T F 5   I    , bold text
+base         ,E S T F 5     M  , default address or target for all links on a page
+basefont     ,E   T F   D I M  , default font; color; or size for the text in a page
+bdi          ,        5   I    , Text isolated from surrounding for BIDI formatting
+bdo          ,  S T F 5   I    , the text direction
+big          ,  S T F     I    , big text
+blockquote   ,  S T F 5        , long quotation
+body         ,  S T F 5        , the document's body
+br           ,E S T F 5        , single line break
+button       ,  S T F 5   I   B, push button
+caption      ,  S T F 5        , table caption
+center       ,    T F   D      , centered text
+cite         ,  S T F 5   I    , citation
+code         ,  S T F 5   I    , computer code text
+col          ,E S T F 5        , attribute values for one or more columns in a table
+colgroup     ,  S T F 5        , group of columns in a table for formatting
+dd           ,  S T F 5        , description of a term in a definition list
+del          ,  S T F 5   I    , deleted text
+dfn          ,  S T F 5   I    , definition term
+dir          ,    T F   D      , directory list
+div          ,  S T F 5        , section in a document
+dl           ,  S T F 5        , definition list
+dt           ,  S T F 5        , term (an item) in a definition list
+em           ,  S T F 5   I    , emphasized text
+fieldset     ,  S T F 5       B, border around elements in a form
+figcaption   ,        5        , Structure: A figure caption
+figure       ,        5        , Structure: Self contained content that can be moved.
+font         ,    T F   D I    , font; color; or size for text
+footer       ,        5        , Structure: A footer of a section
+form         ,  S T F 5        , form for user input
+frame        ,E     F         B, window (a frame) in a frameset
+frameset     ,      F         B, set of frames
+h1           ,  S T F 5        , heading level 1
+h2           ,  S T F 5        , heading level 2
+h3           ,  S T F 5        , heading level 3
+h4           ,  S T F 5        , heading level 4
+h5           ,  S T F 5        , heading level 5
+h6           ,  S T F 5        , heading level 6
+head         ,  S T F 5     M  , information about the document
+header       ,        5        , Structure: A header of a section
+hgroup       ,        5        , Structure: A group of headings
+hr           ,E S T F 5        , horizontal line
+html         ,  S T F 5        , document
+i            ,  S T F 5   I    , italic text
+iframe       ,    T F 5        , nline frame
+img          ,E S T F 5   I    , image
+input        ,E S T F 5   I   B, input control
+ins          ,  S T F 5   I    , inserted text
+isindex      ,    T F   D      , searchable index related to a document
+kbd          ,  S T F 5   I    , keyboard text
+label        ,  S T F 5   I   B, label for an input element
+legend       ,  S T F 5       B, caption for a fieldset element
+li           ,  S T F 5        , list item
+link         ,E S T F 5     M  , relationship between a document and an external resource
+map          ,  S T F 5   I    , image-map
+mark         ,        5   I    , Text marked/highlighted for reference purposes
+menu         ,    T F 5 D      , menu list
+meta         ,E S T F 5     M  , metadata
+nav          ,        5        , Structure: container for navigational links
+noframes     ,    T F         B, alternate content where frames not supported
+noscript     ,  S T F 5       B, alternate content script not supported
+object       ,  S T F 5   I   B, embedded object
+ol           ,  S T F 5        , ordered list
+optgroup     ,  S T F 5       B, group of related options in a select list
+option       ,  S T F 5       B, option in a select list
+p            ,  S T F 5        , paragraph
+param        ,E S T F 5        , parameter for an object
+pre          ,  S T F 5        , preformatted text
+q            ,  S T F 5   I    , short quotation
+rb           ,        5        , ruby base text
+rbc          ,        5        , ruby base container (complex)
+rp           ,        5        , ruby simple text container
+rt           ,        5        , ruby annotation text
+rtc          ,        5        , ruby text container (complex)
+ruby         ,        5   I    , ruby pronunciation aid
+s            ,    T F 5 D I    , strikethrough text
+samp         ,  S T F 5   I    , sample computer code
+script       ,  S T F 5   I   B, client-side script
+section      ,        5        , Structure: generic document/application section
+select       ,  S T F 5   I   B, select list (drop-down list)
+small        ,  S T F 5   I    , small text
+span         ,  S T F 5   I    , section in a document
+strike       ,    T F   D I    , strikethrough text
+strong       ,  S T F 5   I    , strong text
+style        ,  S T F 5       B, style information for a document
+sub          ,  S T F 5   I    , subscripted text
+sup          ,  S T F 5   I    , superscripted text
+table        ,  S T F 5        , table
+tbody        ,  S T F 5        , Groups the body content in a table
+td           ,  S T F 5        , cell in a table
+textarea     ,  S T F 5   I   B, multi-line text input control
+tfoot        ,  S T F 5        , Groups the footer content in a table
+th           ,  S T F 5        , header cell in a table
+thead        ,  S T F 5        , Groups the header content in a table
+time         ,        5   I    , A date or time
+title        ,  S T F 5     M  , the title of a document
+tr           ,  S T F 5        , row in a table
+tt           ,  S T F     I    , teletype text
+u            ,    T F 5 D I    , underlined text
+ul           ,  S T F 5        , unordered list
+var          ,  S T F 5   I    , variable part of a text
+wbr          ,E       5   I    , A line break opportunity
+xmp          ,          D      , preformatted text

data/lib/iudex-html/base.rb

@@ -16,6 +16,6 @@
 
 module Iudex
   module HTML
-    VERSION = '1.2.b.0'
+    VERSION = '1.2.b.1'
   end
 end

data/pom.xml

@@ -3,7 +3,7 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-html</artifactId>
   <packaging>jar</packaging>
-  <version>1.2.b.0</version>
+  <version>1.2.b.1</version>
   <name>Iudex HTML parsing/filtering and text extraction</name>
 
   <parent>
@@ -24,7 +24,7 @@
     <dependency>
       <groupId>com.gravitext</groupId>
       <artifactId>gravitext-xmlprod</artifactId>
-      <version>[1.5,1.5.9999)</version>
+      <version>[1.5.1,1.5.9999)</version>
     </dependency>
 
     <dependency>

data/test/test_html_parser.rb

@@ -48,6 +48,11 @@ HTML
     assert_doc( alt, parse( alt, "UTF-8" ) )
   end
 
+  def test_meta_charset_rerun
+    alt = HTML_META.sub( /<meta .*\/>/, '<meta charset="utf-8"/>' )
+    assert_doc( alt, parse( alt, "ISO-8859-1" ) )
+  end
+
   HTML_SKIP_TAGS = <<HTML
 <html xmlns="http://www.w3.org/1999/xhtml">
  <head>

metadata

@@ -2,7 +2,7 @@
 name: iudex-html
 version: !ruby/object:Gem::Version 
   prerelease: 4
-  version: 1.2.b.0
+  version: 1.2.b.1
 platform: java
 authors: 
   - David Kellum
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-05 00:00:00 Z
+date: 2012-06-01 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: iudex-core
@@ -41,7 +41,7 @@ dependencies:
       requirements: 
         - - ~>
           - !ruby/object:Gem::Version 
-            version: 1.5.b
+            version: 1.5.1
     requirement: *id003
     prerelease: false
     type: :runtime
@@ -117,7 +117,7 @@ files:
   - test/test_stax_parser.rb
   - test/test_tree_walker.rb
   - test/test_word_counters.rb
-  - lib/iudex-html/iudex-html-1.2.b.0.jar
+  - lib/iudex-html/iudex-html-1.2.b.1.jar
 homepage: http://github.com/dekellum/iudex
 licenses: []