iudex-html 1.0.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-04-04)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,24 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+build/HTML.java.erb
+build/attributes
+build/java_generate.rb
+build/tags
+lib/iudex-html/base.rb
+lib/iudex-html.rb
+lib/iudex-html/factory_helper.rb
+test/html_test_helper.rb
+test/setup.rb
+test/test_characters_normalizer.rb
+test/test_extract_filter.rb
+test/test_factory_helper.rb
+test/test_html_parser.rb
+test/test_other_filters.rb
+test/test_other_tree_filters.rb
+test/test_parse_filter.rb
+test/test_tree_walker.rb
+test/test_word_counters.rb
+lib/iudex-html/iudex-html-1.0.0.jar

data/README.rdoc

@@ -0,0 +1,25 @@
+= iudex-html
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-html gem contains filters for HTML parsing,
+filtering, exracting text and links.
+
+== License
+
+Copyright (c) 2010-2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,53 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-html/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.2'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-html',
+                       Iudex::HTML::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+  h.extra_deps += [ [ 'iudex-core',        '~> 1.0.0'  ],
+                    [ 'rjack-nekohtml',    '~> 1.9.14' ],
+                    [ 'gravitext-xmlprod', '~> 1.4.0'  ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',      '>= 1.7.1', '< 2.1' ],
+                        [ 'rjack-logback', '~> 1.0' ] ]
+end
+
+file 'Manifest.txt' => [ 'pom.xml' ]
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+file 'target/.tarpit' => [ 'src/main/java/iudex/html/HTML.java' ]
+
+file 'src/main/java/iudex/html/HTML.java' => FileList.new( "build/*" ) do
+  require 'build/java_generate'
+  puts "Generating HTML.java"
+  JavaGenerator.new.run
+end
+
+task :clean do
+  rm_f 'src/main/java/iudex/html/HTML.java'
+end
+
+t.define_tasks

data/build/HTML.java.erb

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2010-2011 David Kellum
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.  See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package iudex.html;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+
+import iudex.html.HTMLTag.Flag;
+import static iudex.html.HTMLTag.Flag.*;
+
+import com.gravitext.xml.producer.Namespace;
+import com.gravitext.xml.producer.Attribute;
+
+/**
+ * HTML Tag constants
+ * This class is GENERATED by java_generate.rb.
+ */
+public class HTML
+{
+    public static final Namespace NS_XHTML =
+        new Namespace( Namespace.DEFAULT, "http://www.w3.org/1999/xhtml" );
+
+    public static final Map<String,HTMLTag>   TAGS =
+        new HashMap<String,HTMLTag>( 127 );
+
+    public static final Map<String,Attribute> ATTRIBUTES =
+        new HashMap<String,Attribute>( 59 );
+
+    public static final List<Attribute> EMPTY_ATTS = Collections.emptyList();
+% attributes.each do |a|
+%   if a.desc
+
+    /**
+     * Attribute <%= a.name %>: <%= a.desc %>
+     */
+%   end
+    public static final Attribute <%= awidth( 'ATTR_' + const( a.name.upcase ), 5 ) %> = attr( <%= awidth( '"' + a.name + '"', 2 ) %> );
+% end
+
+% tags.each do |tag|
+%   targs = [ '"' + tag.name + '"' ]
+%   basic_atts = tag.basic_atts.map { |a| 'ATTR_' + const( a.name.upcase ) }
+%   targs << if basic_atts.empty?
+%              "EMPTY_ATTS"
+%            else
+%              "Arrays.asList( #{ basic_atts.join( ', ' ) } )"
+%            end
+%   targs += tag.flags
+%   if tag.desc
+    /**
+     * Tag &lt;<%= tag.name %>>: <%= tag.desc %>
+     */
+%   end
+    public static final HTMLTag <%= twidth( tag.name.upcase ) %> =
+        tag( <%= targs.join( ', ' ) %> );
+
+% end
+
+    private static HTMLTag tag( String name,
+                                List<Attribute> basicAtts,
+                                Flag...flags )
+    {
+        HTMLTag t = new HTMLTag( name, NS_XHTML, basicAtts, flags );
+        TAGS.put( t.name(), t );
+        return t;
+    }
+
+    private static Attribute attr( String name )
+    {
+        Attribute a = new Attribute( name, NS_XHTML );
+        ATTRIBUTES.put( a.name(), a );
+        return a;
+    }
+}

data/build/attributes

@@ -0,0 +1,82 @@
+# HTML Attributes
+#
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# Format:
+#  (CSV like) columns: name, tags, description
+#  Tags marked with asterisk (*): attribute is for style purposes only.
+#
+# Sources
+#  http://www.w3.org/TR/xhtml11/
+#  http://www.w3.org/TR/html4/
+#  http://www.w3schools.com/tags/ref_standardattributes.asp
+#  http://xhtml.com
+
+CORE :: ALL except: base head html meta param script style title
+class       ,*CORE
+id          ,*CORE
+style       ,*CORE
+title       ,CORE, extra title
+
+LANG :: ALL except: base br frame frameset hr iframe param
+dir         ,LANG, Text direction; ltr or rtl
+lang        ,LANG, language_code; also xml:lang
+
+# Meta tag attributes
+http-equiv  ,meta, HTTP Header name
+content     ,meta, text
+scheme      ,meta, format URI
+
+# Anchor and link attributes
+charset     ,a link, char_encoding of link
+coords      ,*a, coordinates; i.e. image map
+hreflang    ,link, language_code of referent
+href        ,a base link, URL
+media       ,link
+name        ,a, section_name anchor
+rel         ,a link
+rev         ,a link
+shape       ,*a
+target      ,*a *base *link
+type        ,link
+
+# Image and some frame attributes
+src         ,frame img
+alt         ,img
+height      ,img        *tr *th *td *iframe *object
+width       ,img *table *tr *th *td *iframe *object
+
+# Table specific attributes
+abbr        ,tr th
+align       ,table tr td th iframe object
+axis        ,tr th
+bgcolor     ,*table *tr *td *th
+border      ,*table
+cellpadding ,*table
+cellspacing ,*table
+char        ,tr td th
+charoff     ,tr td th
+colspan     ,tr td th
+frame       ,*table
+headers     ,tr td
+nowrap      ,*tr *td *th
+rowspan     ,tr td th
+rules       ,*table
+scope       ,tr td th
+summary     ,table
+valign      ,*tr *td
+
+# Purposefully omitted (will be dropped on parse)
+# -- The event attributes on*, onmouse*, onkey*

data/build/java_generate.rb

@@ -0,0 +1,139 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'erb'
+require 'ostruct'
+
+# Generator for HTML.java tags/attribute input configuration
+class JavaGenerator
+
+  attr_reader :tags, :attributes
+
+  BASEDIR = File.dirname( __FILE__ )
+
+  JAVA_OUT  = File.join( BASEDIR, '..', 'src',
+                         'main', 'java', 'iudex', 'html', 'HTML.java' )
+
+  def run( java_file = JAVA_OUT )
+    parse_tags
+    parse_attributes
+    map_basic_attributes
+    generate_java( java_file )
+  end
+
+  FLAGS = {
+    'D' => 'DEPRECATED',
+    'I' => 'INLINE',
+    'M' => 'METADATA',
+    'B' => 'BANNED' }
+
+  def parse_tags()
+    @tags = []
+
+    open( File.join( BASEDIR, 'tags' ), 'r' ) do |fin|
+      fin.each do |line|
+        case line
+        when /^\s*#/, /^\s*$/
+          # ignore comment, empty lines
+        when /^\s*[^\s,]+\s*,[^,]*,[^,]*$/
+          r = line.split(',').map { |c| c.strip }
+          r = r.compact.reject { |c| c.empty? }
+          flags = r[1].split(' ').map { |f| FLAGS[f] }.compact
+          @tags << OpenStruct.new( :name => r[0],
+                                   :flags => flags,
+                                   :desc => r[2] )
+        else
+          raise "Parse ERROR: line [#{line}]"
+        end
+      end
+    end
+
+    @tag_max_len = @tags.map { |t| t.name.length }.max
+    [ @tags ]
+  end
+
+  def parse_attributes()
+    @attributes = []
+    tagsets = {}
+
+    open( File.join( BASEDIR, 'attributes' ), 'r' ) do |fin|
+      fin.each do |line|
+        case line
+        when /^\s*#/, /^\s*$/
+          # ignore comment, empty lines
+        when /^\s*([A-Z]+)\s*::\s*ALL\s+except:(.*)$/
+          sname = $1
+          except = $2.split( ' ' ).compact.reject { |t| t.empty? }
+          tset = @tags.reject { |t| except.include?( t.name ) }
+          tset.map! { |t| t.name }
+          tagsets[sname] = tset
+        when /^\s*[^\s,]+\s*,/
+          r = line.split(',').map { |c| c.strip }
+          r = r.compact.reject { |c| c.empty? }
+          # FIXME: Handle attributes, desc.
+
+          btags = r[1].split(' ').compact.reject { |t| t.empty? || t =~ /^\*/ }
+          btags = btags.map { |t| tagsets[ t ] || t }.flatten
+
+          @attributes << OpenStruct.new( :name => r[0],
+                                         :basic_tags => btags,
+                                         :desc => r[2] )
+        else
+          raise "Parse ERROR: line [#{line}]"
+        end
+      end
+    end
+
+  def map_basic_attributes()
+    @tags.each do |tag|
+      tag.basic_atts =
+        @attributes.select { |attr| attr.basic_tags.include?( tag.name ) }
+    end
+  end
+
+    @attr_max_len = @attributes.map { |t| t.name.length }.max
+    [ @attributes ]
+  end
+
+  def twidth( val, extra = 0 )
+    val + ( ' ' * ( @tag_max_len - val.length + extra )  )
+  end
+
+  def awidth( val, extra = 0 )
+    val + ( ' ' * ( @attr_max_len - val.length + extra )  )
+  end
+
+  def const( val )
+    val.gsub( /\-/, '_' )
+  end
+
+  def generate_java( java_file )
+    erb_file = File.join( BASEDIR, 'HTML.java.erb' )
+    template = ERB.new( IO.read( erb_file ), nil, '%' )
+
+    open( java_file, 'w' ) do |fout|
+      fout << template.result( binding )
+    end
+  end
+
+end
+
+if $0 == __FILE__
+  JavaGenerator.new.run( *ARGV )
+end

data/build/tags

@@ -0,0 +1,130 @@
+# HTML Tags
+#
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# Sources
+#  http://www.w3.org/TR/xhtml11/
+#  http://www.w3.org/TR/html4/
+#  http://www.w3schools.com/tags/default.asp
+#  http://xhtml.com/
+#
+# Codes:
+# E :: Empty Tag
+# S :: In Strict HTML 4.01/XHTML 1.0
+# T :: In Transitional HTML 4.01/XHTML 1.0
+# F :: In frameset annex
+# D :: Deprecated
+# I :: Inline elements (Note <br/> is not labeled inline.)
+# M :: Metadata elements (content not visible text), i.e. head
+# B :: Banned/blacklisted elements from which text should not be extracted.
+
+a            ,  S T F   I    , anchor
+abbr         ,  S T F   I    , abbreviation
+acronym      ,  S T F   I    , acronym
+address      ,  S T F        , contact information for the author or owner
+applet       ,    T F D      , embedded applet
+area         ,E S T F        , area inside an image-map
+b            ,  S T F   I    , bold text
+base         ,E S T F     M  , default address or a default target for all links on a page
+basefont     ,E   T F D I M  , default font; color; or size for the text in a page
+bdo          ,  S T F   I    , the text direction
+big          ,  S T F   I    , big text
+blockquote   ,  S T F        , long quotation
+body         ,  S T F        , the document's body
+br           ,E S T F        , single line break
+button       ,  S T F   I   B, push button
+caption      ,  S T F        , table caption
+center       ,    T F D      , centered text
+cite         ,  S T F   I    , citation
+code         ,  S T F   I    , computer code text
+col          ,E S T F        , attribute values for one or more columns in a table
+colgroup     ,  S T F        , group of columns in a table for formatting
+dd           ,  S T F        , description of a term in a definition list
+del          ,  S T F   I    , deleted text
+dfn          ,  S T F   I    , definition term
+dir          ,    T F D      , directory list
+div          ,  S T F        , section in a document
+dl           ,  S T F        , definition list
+dt           ,  S T F        , term (an item) in a definition list
+em           ,  S T F   I    , emphasized text
+fieldset     ,  S T F       B, border around elements in a form
+font         ,    T F D I    , font; color; or size for text
+form         ,  S T F        , form for user input
+frame        ,E     F       B, window (a frame) in a frameset
+frameset     ,      F       B, set of frames
+h1           ,  S T F        , heading level 1
+h2           ,  S T F        , heading level 2
+h3           ,  S T F        , heading level 3
+h4           ,  S T F        , heading level 4
+h5           ,  S T F        , heading level 5
+h6           ,  S T F        , heading level 6
+head         ,  S T F     M  , information about the document
+hr           ,E S T F        , horizontal line
+html         ,  S T F        , document
+i            ,  S T F   I    , italic text
+iframe       ,    T F        , nline frame
+img          ,E S T F   I    , image
+input        ,E S T F   I   B, input control
+ins          ,  S T F   I    , inserted text
+isindex      ,    T F D      , searchable index related to a document
+kbd          ,  S T F   I    , keyboard text
+label        ,  S T F   I   B, label for an input element
+legend       ,  S T F       B, caption for a fieldset element
+li           ,  S T F        , list item
+link         ,E S T F     M  , the relationship between a document and an external resource
+map          ,  S T F   I    , image-map
+menu         ,    T F D      , menu list
+meta         ,E S T F     M  , metadata
+noframes     ,    T F       B, alternate content where frames not supported
+noscript     ,  S T F       B, alternate content script not supported
+object       ,  S T F   I   B, embedded object
+ol           ,  S T F        , ordered list
+optgroup     ,  S T F       B, group of related options in a select list
+option       ,  S T F       B, option in a select list
+p            ,  S T F        , paragraph
+param        ,E S T F        , parameter for an object
+pre          ,  S T F        , preformatted text
+q            ,  S T F   I    , short quotation
+rb           ,               , ruby base text
+rbc          ,               , ruby base container (complex)
+rp           ,               , ruby simple text container
+rt           ,               , ruby annotation text
+rtc          ,               , ruby text container (complex)
+ruby         ,          I    , ruby pronunciation aid
+s            ,    T F D I    , strikethrough text
+samp         ,  S T F   I    , sample computer code
+script       ,  S T F   I   B, client-side script
+select       ,  S T F   I   B, select list (drop-down list)
+small        ,  S T F   I    , small text
+span         ,  S T F   I    , section in a document
+strike       ,    T F D I    , strikethrough text
+strong       ,  S T F   I    , strong text
+style        ,  S T F       B, style information for a document
+sub          ,  S T F   I    , subscripted text
+sup          ,  S T F   I    , superscripted text
+table        ,  S T F        , table
+tbody        ,  S T F        , Groups the body content in a table
+td           ,  S T F        , cell in a table
+textarea     ,  S T F   I   B, multi-line text input control
+tfoot        ,  S T F        , Groups the footer content in a table
+th           ,  S T F        , header cell in a table
+thead        ,  S T F        , Groups the header content in a table
+title        ,  S T F     M  , the title of a document
+tr           ,  S T F        , row in a table
+tt           ,  S T F   I    , teletype text
+u            ,    T F D I    , underlined text
+ul           ,  S T F        , unordered list
+var          ,  S T F   I    , variable part of a text
+xmp          ,        D      , preformatted text