iudex-char-detector 1.1.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.1.0 (2011-11-13)
+* Initial release with Iudex 1.1.x.

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+bin/iudex-char-detect
+lib/iudex-char-detector/base.rb
+lib/iudex-char-detector.rb
+test/sample.html.ascii
+test/sample.html.iso
+test/sample.html.utf16
+test/sample.html.utf16le
+test/sample.html.utf8
+test/setup.rb
+test/test_char_detector.rb
+lib/iudex-char-detector/iudex-char-detector-1.1.0.jar

data/README.rdoc

@@ -0,0 +1,25 @@
+= iudex-char-detector
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-char-detector gem provides charset detection
+support.
+
+== License
+
+Copyright (c) 2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,40 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-char-detector/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.4'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-char-detector',
+                       Iudex::CharDetector::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+  h.extra_deps += [ [ 'iudex-core',        '~> 1.1.0'  ],
+                    [ 'rjack-icu',         '~> 4.8.0' ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',      '~> 2.3' ],
+                        [ 'rjack-logback', '~> 1.0' ] ]
+end
+
+file 'Manifest.txt' => [ 'pom.xml' ]
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+t.define_tasks

data/bin/iudex-char-detect

@@ -0,0 +1,117 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'optparse'
+
+class ChartDetectUtil
+
+  require 'rjack-logback'
+  include RJack
+
+  Logback.config_console( :level => Logback::INFO, :stderr => true )
+
+  require 'iudex-char-detector'
+  include Iudex
+
+  include Gravitext::HTMap
+  UniMap.define_accessors
+
+  include Iudex::Core
+  include Iudex::CharDetector
+
+  import 'java.nio.ByteBuffer'
+  import 'java.nio.charset.Charset'
+
+  def initialize
+    @default_encode = "windows-1252"
+  end
+
+  def run( args = ARGV )
+
+    p = OptionParser.new do |opts|
+      opts.banner = "Usage: iudex-char-detect [options] [TestFile]"
+      opts.on( "-v", "--version", "Display version" ) do
+        puts "iudex-char-detector: #{CharDetector::VERSION}"
+        exit 1
+      end
+      opts.on( "-d", "--debug" ) do
+        Logback[ 'iudex' ].level = Logback::DEBUG
+      end
+      opts.on_tail( "-e", "--encoding ENCODING", String,
+                    "Set default encoding (#{@default_encode})" ) do |enc|
+        @default_encode = enc
+      end
+      opts.on_tail( "-h", "--help", "Show help and exit" ) do
+        puts opts
+        puts
+        puts( "Detect charset of File and report encoding, confidence " +
+              "(independent of HTML features)" )
+        exit 1
+      end
+    end
+
+    p.parse!( args )
+
+    if args.empty?
+      show_detail( detect( $stdin ) )
+    else
+      max_name = args.map { |fn| fn.length }.max
+      args.each do |fname|
+        res = open( fname, "r" ) { |fin| detect( fin ) }
+        if ARGV.length > 1
+          res.unshift( fname )
+          puts "%-#{ max_name }s : %-12s (%5.4f)" % res
+        else
+          show_detail( res )
+        end
+      end
+    end
+
+  end
+
+  def detect( fin )
+    map = UniMap.new
+    bytes = ByteBuffer::wrap( fin.read.to_java_bytes )
+    map.source = ContentSource.new( bytes )
+    map.source.set_default_encoding( Charset::lookup( @default_encode ) )
+
+    df = CharDetectFilter.new
+    #FIXME: Option? df.max_detect_length =
+    df.filter( map )
+    s = map.source
+
+    [ s.default_encoding.name,
+      s.encoding_confidence,
+      s.encoding_confidences ]
+  end
+
+  def show_detail( res )
+    puts "%-12s (%.4f)" % res
+    puts "====================="
+    res[2].each do |enc,conf|
+      puts "%-12s (%.4f)" % [ enc, conf ]
+    end
+  end
+
+end
+
+ChartDetectUtil.new.run

data/lib/iudex-char-detector/base.rb

@@ -0,0 +1,21 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module CharDetector
+    VERSION = '1.1.0'
+  end
+end

data/lib/iudex-char-detector.rb

@@ -0,0 +1,30 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+require 'rjack-icu'
+
+require 'iudex-char-detector/base.rb'
+
+require 'java'
+
+module Iudex
+  module CharDetector
+    require "iudex-char-detector/iudex-char-detector-#{VERSION}.jar"
+
+    import 'iudex.chardetector.CharDetectFilter'
+  end
+end

data/pom.xml

@@ -0,0 +1,45 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-char-detector</artifactId>
+  <packaging>jar</packaging>
+  <version>1.1.0</version>
+  <name>Iudex charset detection support</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.1</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-core</artifactId>
+      <version>[1.1,1.2)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.ibm.icu</groupId>
+      <artifactId>icu4j</artifactId>
+      <version>[4.8,4.8.9999]</version>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

data/test/sample.html.ascii

@@ -0,0 +1,8 @@
+<html>
+ <head>
+  <title>Un documento electronica (titulo en ASCII)</title>
+ </head>
+ <body>
+  <p>De donde eres tu?</p>
+ </body>
+</html>

data/test/sample.html.iso

@@ -0,0 +1,8 @@
+<html>
+ <head>
+  <title>Un documento electronica (titulo en ASCII)</title>
+ </head>
+ <body>
+  <p>�De donde eres t�?</p>
+ </body>
+</html>

data/test/sample.html.utf8

@@ -0,0 +1,8 @@
+<html>
+ <head>
+  <title>Un documento electronica (titulo en ASCII)</title>
+ </head>
+ <body>
+  <p>¿De donde eres tú?</p>
+ </body>
+</html>

data/test/setup.rb

@@ -0,0 +1,39 @@
+#--
+# Copyright (c) 2010-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+test_dir = File.dirname( __FILE__ )
+
+ldir = File.join( test_dir, "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+RJack::Logback.config_console( :stderr => true )
+if ARGV.include?( '--verbose' ) || ARGV.include?( '-v' )
+  RJack::Logback.root.level = RJack::Logback::DEBUG
+end
+
+require 'minitest/unit'
+require 'minitest/autorun'
+
+# Make test output logging compatible: no partial lines.
+# class TestOut
+#   def print( *a ); $stdout.puts( *a ); end
+#   def puts( *a );  $stdout.puts( *a ); end
+# end
+# MiniTest::Unit.output = TestOut.new

data/test/test_char_detector.rb

@@ -0,0 +1,156 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-char-detector'
+
+class TestCharDetector < MiniTest::Unit::TestCase
+  include Gravitext::HTMap
+  UniMap.define_accessors
+
+  include Iudex::Core
+  include Iudex::CharDetector
+
+  import 'java.nio.ByteBuffer'
+  import 'java.nio.charset.Charset'
+  JString = Java::java.lang.String
+
+  SHORT_HTML = <<HTML
+<html>
+ <head>
+  <title>Un documento electronica (titulo en ASCII)</title>
+ </head>
+ <body>
+  <p>¿De donde eres tú?</p>
+ </body>
+</html>
+HTML
+
+  def test_find_nothing
+    df = CharDetectFilter.new
+    df.max_detect_length = 3
+
+    [ "", "a", "ascii" ].each do |ib|
+      assert_nil( df.find_detect_buffer( wrap( ib ) ), ib )
+    end
+  end
+
+  def test_find_something
+    df = CharDetectFilter.new
+    df.max_detect_length = 3
+
+    trials = [ %w[ á     á   ],
+               %w[ é.    é.  ],
+               %w[ ..ü   ..ü ],
+               %w[ ..ü0  ..ü ],
+               %w[ 0..í  ..í ],
+               %w[ 0..ó0 ..ó ] ]
+
+    trials.each do |ib,ob|
+      out = df.find_detect_buffer( wrap( encode_as( ib, "ISO-8859-1" ) ) )
+      assert( out, ob )
+      assert_equal( ob, JString.new( out, "ISO-8859-1" ).to_s, ob )
+    end
+
+  end
+
+  def test_ascii
+    map = detect_from( "", "UTF-8" )
+    assert_encoding( map.source, "UTF-8", 0.0 )
+
+    map = detect_from( "ascii", "UTF-8" )
+    assert_encoding( map.source, "UTF-8", 0.0 )
+  end
+
+  def test_html_utf8_as_default
+    map = detect_from( SHORT_HTML, "UTF-8" )
+    assert_encoding( map.source, "UTF-8", 0.80 )
+  end
+
+  def test_html_utf8_wrong_default
+    map = detect_from( SHORT_HTML, "UTF-8", "ISO-8859-1" )
+    assert_encoding( map.source, "UTF-8", 0.80 )
+  end
+
+  def test_html_iso_as_default
+    map = detect_from( SHORT_HTML, "ISO-8859-1" )
+    assert_encoding( map.source, "ISO-8859-1", 0.40 )
+  end
+
+  def test_html_iso_wrong_default
+    map = detect_from( SHORT_HTML, "ISO-8859-1", "UTF-8" )
+    assert_encoding( map.source, "ISO-8859-1", 0.40 )
+  end
+
+  def test_html_iso_from_windows
+    map = detect_from( SHORT_HTML, "windows-1252" )
+    assert_encoding( map.source, "ISO-8859-1", 0.40 )
+  end
+
+  def test_windows_default
+    map = detect_from( '“¿De donde eres tú?”', "windows-1252" )
+    assert_encoding( map.source, "windows-1252", 0.90 )
+  end
+
+  def test_windows_wrong_default
+    map = detect_from( '“¿De donde eres tú?”', "windows-1252", "UTF-8" )
+    assert_encoding( map.source, "windows-1252", 0.90 )
+  end
+
+  def test_mojibaked_utf8
+    map = detect_from( '“quoted”', "UTF-8" )
+    assert_encoding( map.source, "UTF-8", 0.99 )
+  end
+
+  def detect_from( bytes, enc, claimed_enc = nil )
+    map = content( encode_as( bytes, enc ), claimed_enc || enc )
+    df = CharDetectFilter.new
+    df.max_detect_length = SHORT_HTML.length - 20
+    assert( df.filter( map ) )
+    map
+  end
+
+  def assert_encoding( source, enc, min_confidence = 0.10 )
+    assert_equal( enc, source.default_encoding.name )
+    assert_operator( source.encoding_confidence, :>=, min_confidence )
+  end
+
+  def encode_as( bytes, encoding )
+    if encoding == "UTF-8"
+      bytes
+    else
+      bytes = bytes.to_java_bytes if bytes.respond_to?( :to_java_bytes )
+      JString.new( bytes, "UTF-8" ).bytes( encoding )
+    end
+  end
+
+  def content( bytes, charset = "UTF-8" )
+    map = UniMap.new
+    map.source = ContentSource.new( wrap( bytes ) )
+    map.source.set_default_encoding( Charset::lookup( charset ) )
+    map
+  end
+
+  def wrap( bytes )
+    bytes = bytes.to_java_bytes if bytes.respond_to?( :to_java_bytes )
+    ByteBuffer::wrap( bytes )
+  end
+
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-char-detector
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.1.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-13 00:00:00 Z
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-core
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rjack-icu
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 4.8.0
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "2.3"
+    type: :development
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :development
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.4.0
+    type: :development
+    version_requirements: *id005
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-char-detector gem provides charset detection
+  support.
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - iudex-char-detect
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - bin/iudex-char-detect
+  - lib/iudex-char-detector/base.rb
+  - lib/iudex-char-detector.rb
+  - test/sample.html.ascii
+  - test/sample.html.iso
+  - test/sample.html.utf16
+  - test/sample.html.utf16le
+  - test/sample.html.utf8
+  - test/setup.rb
+  - test/test_char_detector.rb
+  - lib/iudex-char-detector/iudex-char-detector-1.1.0.jar
+  - .gemtest
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-char-detector
+rubygems_version: 1.8.9
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_char_detector.rb