mojibake 1.0.0

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-6-21)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,10 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+bin/mojibake
+lib/mojibake/base.rb
+lib/mojibake.rb
+lib/mojibake/mapper.rb
+test/test.txt
+test/test_mojibake.rb

data/README.rdoc

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+= MojiBake
+
+* http://github.com/dekellum/mojibake
+
+== Description
+
+Mojibake occurs in English most frequently due to misinterpreting and
+bad-transcoding between Windows-1252, ISO-8859-1, and UTF-8. This
+module provides a mojibake sequence to original character mapping
+table, and utility to recover mojibake’d text.
+
+Testing has been with English but other Latin based languages, where
+Windows-1252 is in the wild, should also benefit.
+
+== Dependencies
+
+Requires the String Encoding support of ruby 1.9+ (tested 1.9.2p180
+Linux) or jruby 1.6+ (tested 1.6.2, Linux).
+
+== Synopsis
+
+ gem install mojibake
+
+ require 'mojibake'
+ mapper = MojiBake::Mapper.new
+ mapper.recover( '“quotedâ€�' ) #=> '“quoted”'
+
+Or via cli:
+
+ mojibake -h
+
+List the mojibake mapping table (output in UTF-8):
+
+ mojibake -t
+
+Recover from a text file:
+
+ mojibake input.txt
+
+== License
+
+Copyright (c) 2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,35 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.3.2'
+require 'rjack-tarpit'
+
+require 'mojibake/base'
+
+t = RJack::TarPit.new( 'mojibake', MojiBake::VERSION )
+
+t.specify do |h|
+  h.developer( 'David Kellum', 'dek-oss@gravitext.com' )
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest', '>= 2.1', '< 2.4' ] ]
+
+  h.url = 'http://github.com/dekellum/mojibake'
+end
+
+# Version/date consistency checks:
+
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{ t.version } / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_history_version  ]
+task :tag  => [ :check_history_version, :check_history_date ]
+task :push => [ :check_history_version, :check_history_date ]
+
+t.define_tasks

data/bin/mojibake

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+#.hashdot.args.pre = --1.9
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname( __FILE__ ), "..", "lib" )
+
+require 'mojibake'
+require 'optparse'
+
+module Script
+
+  mapper = MojiBake::Mapper.new
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: mojibake [options] [InputFile]"
+    opts.on( "-v", "--version", "Display version and exit" ) do
+      puts "mojibake: #{MojiBake::VERSION}"
+      exit 1
+    end
+    opts.on( "--no-windows-1252",
+             "Don't include miscodings from Windows-1252" ) do
+      mapper.map_windows_1252 = false
+    end
+    opts.on( "--no-iso-8859-1",
+             "Don't include miscodings from ISO-8859-1" ) do
+      mapper.map_iso_8859_1 = false
+    end
+    opts.on( "--no-permutations",
+             "Don't include ISO/Windows permutations" ) do
+      mapper.map_permutations = false
+    end
+    opts.on_tail( "-t", "--table",
+                  "Write MojiBake Mapper table (UTF-8)" ) do
+      puts mapper.table
+      exit 1
+    end
+    opts.on_tail( "-r", "--regex",
+                  "Display MojiBake Mapper regex (UTF-8) and exit" ) do
+      puts mapper.regexp.inspect
+      exit 1
+    end
+    opts.on_tail( "-h", "--help", "Show help and exit" ) do
+      puts opts
+      puts
+      puts( "Recover InputFile and writes to STDOUT." )
+      exit 1
+    end
+  end.parse!
+
+  input_file = ARGV.shift
+  if input_file
+    $stdout.write( mapper.recover( IO.read( input_file ).encode( 'UTF-8' ) ) )
+  end
+
+end

data/lib/mojibake.rb

@@ -0,0 +1,22 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+if ( RUBY_VERSION.split( '.' ).map { |d| d.to_i } <=> [ 1, 9 ] ) < 0
+  raise "Requires ruby ~> 1.9 for String.encode support"
+end
+
+require 'mojibake/base'
+require 'mojibake/mapper'

data/lib/mojibake/base.rb

@@ -0,0 +1,19 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module MojiBake
+  VERSION = "1.0.0"
+end

data/lib/mojibake/mapper.rb

@@ -0,0 +1,187 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module MojiBake
+
+  # Creates a Map from mojibake sequences to recovered/original
+  # characters.
+  class Mapper
+
+    W252 = Encoding::WINDOWS_1252
+    ISO8 = Encoding::ISO_8859_1
+    UTF8 = Encoding::UTF_8
+
+    # The 8-bit high-order characters assigned in Windows-1252, as UTF8.
+    # This is actually a superset of ISO-8859-1 high order set,
+    # including in particular, punctuation characters like EM DASH and
+    # RIGHT DOUBLE QUOTATION MARK. These are the most common problem
+    # chars in English and probably most latin languages.
+    HIGH_ORDER_CHARS =
+      ( ( 0x80..0xFF ).to_a - [ 0x81, 0x8D, 0x8F, 0x90, 0x9D ] ).
+      map { |i| i.chr( W252 ).encode( UTF8 ) }.
+      sort
+
+    # Additional Unicode codepoints of mojibake potential, like alt
+    # whitespace, C1 control characters, and BOMs.
+    INTEREST_CODEPOINTS =
+      [ (0x0080..0x009F).to_a, # ISO/Unicode C1 control codes.
+        0x00A0,                # NO-BREAK SPACE
+        (0x2000..0x200B).to_a, # EN QUAD ... ZERO WIDTH SPACE
+        0x2060,                # WORD JOINER
+        0xfeff,                # ZERO WIDTH SPACE, BYTE-ORDER-MARK (BOM)
+        0xfffd,                # REPLACEMENT CHARACTER
+        0xfffe ].              # UNASSIGNED, BAD BOM
+      flatten.
+      sort
+
+    INTEREST_CHARS = INTEREST_CODEPOINTS.map { |c| c.chr( UTF8 ) }
+
+    # Mojibake candidate characters in reverse; HIGH_ORDER_CHARS and
+    # lowest codepoints have highest precedence.
+    CANDIDATE_CHARS = ( HIGH_ORDER_CHARS + INTEREST_CHARS ).reverse
+
+    # Include Windows-1252 transcodes in map (default: true)
+    attr_accessor :map_windows_1252
+
+    # Include ISO-8859-1 transcodes in map (default: true)
+    attr_accessor :map_iso_8859_1
+
+    # Include permutations between ISO-8859-1 and Windows-1252
+    # (default: true).  This covers ambiguities of C1 control codes.
+    attr_accessor :map_permutations
+
+    def initialize( options = {} )
+      @map_windows_1252 = true
+      @map_iso_8859_1   = true
+      @map_permutations = true
+
+      options.map { |k,v| send( k.to_s + '=', v ) }
+    end
+
+    # Return Hash of mojibake UTF-8 2-3 character sequences to original
+    # UTF-8 (recovered) characters
+    def hash
+      @hash ||= CANDIDATE_CHARS.inject( {} ) do |h,c|
+
+        # Mis-interpret as ISO-8859-1, and encode back to UTF-8
+        moji_8 = c.encode( UTF8, ISO8 )
+        h[moji_8] = c if @map_iso_8859_1
+
+        # Mis-interpret as Windows-1252, and encode back to UTF-8
+        moji_w = c.encode( UTF8, W252, :undef => :replace )
+        h[moji_w] = c if @map_windows_1252
+
+        if @map_permutations
+          # Also add permutations of unassigned Windows-1252 chars to
+          # the 8bit equivalent.
+          i = 0
+          moji_w.each_codepoint do |cp|
+            if cp == 0xFFFD
+              moji_n = moji_w.dup
+              moji_n[i] = moji_8[i]
+              h[moji_n] = c
+            end
+            i += 1
+          end
+        end
+
+        h
+      end
+    end
+
+    # Return pretty table formatting of hash (array of lines)
+    def table
+      lines = [ "# -*- coding: utf-8 -*- mojibake: #{MojiBake::VERSION}" ]
+      lines << regexp.inspect
+      lines << ""
+      lines << "Moji\tUNICODE  \tOrg\tCODE"
+      lines << "+----\t---- ---- ----\t-----\t---+"
+      lines += hash.sort.map do |moji,c|
+        "[%s]\t%s\t[%s]\t%s" %
+          [ moji, codepoints_hex( moji ), c, codepoints_hex( c ) ]
+      end
+      lines
+    end
+
+    # A Regexp that will match any of the mojibake sequences, as
+    # found in hash.keys.
+    def regexp
+      @regexp ||= Regexp.new( tree_flatten( char_tree( hash.keys ) ) )
+    end
+
+    # Recover original characters from input using regexp, recursively.
+    def recover( input, recursive = true )
+      output = input.gsub( regexp ) { |moji| hash[moji] }
+
+      # Only recurse if requested and substituted something (output
+      # shorter) in this run.
+      if recursive && ( output.length < input.length )
+        recover( output )
+      else
+        output
+      end
+    end
+
+    def char_tree( seqs )
+      seqs.inject( {} ) do |h,seq|
+        seq.chars.inject( h ) do |hs,c|
+          hs[c] ||= {}
+        end
+        h
+      end
+    end
+
+    def tree_flatten( tree )
+      cs = tree.sort.map do |k,v|
+        o = regex_encode( k )
+        unless v.empty?
+          c = tree_flatten( v )
+          o << if c =~ /^\[.*\]$/ || v.length == 1
+                 c
+               else
+                 '(' + c + ')'
+               end
+        end
+        o
+      end
+      if cs.find { |o| o =~ /[()|\[\]]/ }
+        cs.join( '|' ).force_encoding( "UTF-8" )
+        #FIXME: Join looses encoding so force, jruby bug?
+      else
+        if cs.length > 1
+          '[' + cs.inject(:+) + ']'
+        else
+          cs.first
+        end
+      end
+    end
+
+    # Unicode hex dump of codepoints
+    def codepoints_hex( s )
+      s.codepoints.map { |i| sprintf( "%04X", i ) }.join( ' ' )
+    end
+
+    def regex_encode( c )
+      i = c.codepoints.next #only one
+      if INTEREST_CODEPOINTS.include?( i )
+        sprintf( '\u%04X', i )
+      else
+        Regexp.escape( c )
+      end
+    end
+
+  end
+end

data/test/test.txt

@@ -0,0 +1,12 @@
+-*- coding: utf-8 -*-
+Source: http://en.wikipedia.org/wiki/Mojibake
+
+== English ==
+
+Mojibake in English texts generally occurs in punctuation, such as em
+dashes (—), en dashes (–), and curly quotes (“, ”), but rarely in
+character text, since most encodings agree with ASCII on the encoding
+of the English alphabet. For example, the pound sign "£" will appear
+as "£" if it was encoded by the sender as UTF-8 but interpreted by
+the recipient as CP1252 or ISO 8859-1. If iterated, this can lead to
+"£", "£", etc.

data/test/test_mojibake.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#.hashdot.args.pre = --1.9
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'minitest/unit'
+require 'minitest/autorun'
+
+require 'mojibake'
+
+class TestMojiBake < MiniTest::Unit::TestCase
+  include MojiBake
+
+  def setup
+    @mapper = Mapper.new
+  end
+
+  TEST_TREE = { "a" => { "b" => { "c" => {},
+                                  "d" => {} } },
+                "d" => { "b" => { "f" => {} } } }
+
+  def test_init_options
+    assert_equal( true, Mapper.new.map_iso_8859_1 )
+    m = Mapper.new( :map_iso_8859_1 => false )
+    assert_equal( false, m.map_iso_8859_1 )
+  end
+
+  def test_char_tree
+    assert_equal( TEST_TREE,
+                  @mapper.char_tree( [ "abc", "abd", "dbf" ] ) )
+  end
+
+  def test_tree_flaten
+    assert_equal( "ab[cd]|dbf",
+                  @mapper.tree_flatten( TEST_TREE ) )
+  end
+
+  def test_regexp
+    re = Regexp.new( @mapper.tree_flatten( TEST_TREE ) )
+    assert_match( re, "abc" )
+    assert_match( re, "abd" )
+    assert_match( re, "dbf" )
+
+    refute_match( re, "ab" )
+    refute_match( re, "abf" )
+
+    assert_equal(  "xbf" , "abdbf".gsub( re, 'x' ) )
+    assert_equal(  "dbf" , "abdbf".gsub( re, 'd' ) )
+  end
+
+  def test_nomatch_recover
+    assert_equal( '', @mapper.recover( '' ) )
+    assert_equal( 'ascii', @mapper.recover( 'ascii' ) )
+    assert_equal( 'Â', @mapper.recover( 'Â' ) )
+  end
+
+  def test_simple_recover
+    assert_equal( '[°]', @mapper.recover( '[°]' ) )
+    assert_equal( '“quoted”', @mapper.recover( '“quotedâ€�' ) )
+    assert_equal( '“quoted”', @mapper.recover( '“quoted”' ) )
+  end
+
+  def test_recursive_recover
+    assert_equal( '°', @mapper.recover( '°' ) )
+    assert_equal( 'AP – Greenlake', @mapper.recover( 'AP – Greenlake' ) )
+    assert_equal( 'you’re', @mapper.recover( 'you’re' ) )
+  end
+
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification 
+name: mojibake
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: ruby
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-21 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.4"
+    type: :development
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.2
+    type: :development
+    version_requirements: *id002
+description: "Mojibake occurs in English most frequently due to misinterpreting and\n\
+  bad-transcoding between Windows-1252, ISO-8859-1, and UTF-8. This\n\
+  module provides a mojibake sequence to original character mapping\n\
+  table, and utility to recover mojibake\xE2\x80\x99d text.\n\n\
+  Testing has been with English but other Latin based languages, where\n\
+  Windows-1252 is in the wild, should also benefit."
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - mojibake
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - bin/mojibake
+  - lib/mojibake/base.rb
+  - lib/mojibake.rb
+  - lib/mojibake/mapper.rb
+  - test/test.txt
+  - test/test_mojibake.rb
+has_rdoc: true
+homepage: http://github.com/dekellum/mojibake
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: mojibake
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Mojibake occurs in English most frequently due to misinterpreting and bad-transcoding between Windows-1252, ISO-8859-1, and UTF-8
+test_files: 
+  - test/test_mojibake.rb