cmess 0.3.1 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6388ed1476587af57eb196817003d94d58ecb268
+  data.tar.gz: 41551903b954d2de73b62203467d93955ac2516f
+SHA512:
+  metadata.gz: 4ffc33d0ed9392797a97a5eb535cb0652687e0b8abcd2b6a30de85d41beb63b65ebfa434a625b6aa91bfa379ad5fe8a2f46286aa3450b930633bea9241ba30e7
+  data.tar.gz: e11d90e25d6b5fb5c0f6d81f7d2824e2c7dd662e5feef1f7978cd66588fcd0a5a18cc0d60fbe2d1ee6b3cac254198ff07a64bd985b1d7bb5f3a1f27aa7d7096a

data/ChangeLog

@@ -1,5 +1,17 @@
+# markup: rd
+
 = Revision history for cmess
 
+== 0.4.0 [2013-08-02]
+
+* Updated for Ruby 1.9.2+. <b>Ruby 1.8 no longer supported.</b>
+* guess_encoding: Added <tt>--reverse</tt> option.
+* Housekeeping.
+
+== 0.3.1 [2011-08-16]
+
+* decode_entities: Fixed regression.
+
 == 0.3.0 [2011-07-25]
 
 * Extensive refactoring

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to cmess version 0.3.1
+This documentation refers to cmess version 0.4.0
 
 
 == DESCRIPTION
@@ -22,21 +22,24 @@ there are:
                     (see CMess::BConv)
 +decode_entities+:: Decode HTML entities in a string. (see CMess::DecodeEntities)
 
-TODO: well, more of the description... ;-)
+
+== SUPPORTED PLATFORMS
+
+Requires Ruby version 1.9.2 or higher; use the latest 0.3.x release on older
+Ruby versions. CMess has been tested with ruby 2.0.0p247 on x86_64-linux.
 
 
 == LINKS
 
 <b></b>
-Documentation::     http://prometheus.rubyforge.org/cmess
-Source code::       http://github.com/blackwinter/cmess
-RubyForge project:: http://rubyforge.org/projects/prometheus
-RubyGem::           http://rubygems.org/gems/cmess
+Documentation:: http://blackwinter.github.com/cmess
+Source code::   http://github.com/blackwinter/cmess
+RubyGem::       http://rubygems.org/gems/cmess
 
 
 == AUTHORS
 
-* Jens Wille <mailto:jens.wille@uni-koeln.de>
+* Jens Wille <mailto:jens.wille@gmail.com>
 
 
 == CREDITS
@@ -48,9 +51,11 @@ RubyGem::           http://rubygems.org/gems/cmess
 
 == LICENSE AND COPYRIGHT
 
-Copyright (C) 2007-2011 University of Cologne,
+Copyright (C) 2007-2012 University of Cologne,
 Albertus-Magnus-Platz, 50923 Cologne, Germany
 
+Copyright (C) 2013 Jens Wille
+
 cmess is free software: you can redistribute it and/or modify it under the
 terms of the GNU Affero General Public License as published by the Free
 Software Foundation, either version 3 of the License, or (at your option)

data/Rakefile

@@ -6,21 +6,21 @@ begin
   require 'hen'
 
   Hen.lay! {{
-    :rubyforge => {
-      :project => %q{prometheus},
-      :package => %q{cmess}
-    },
-
-    :gem => {
-      :version      => CMess::VERSION,
-      :summary      => %Q{
-        Assist with handling messed up encodings (Currently includes the
-        following tools: #{Dir['bin/*'].map { |e| File.basename(e) }.sort.join(', ')})
-      },
-      :author       => %q{Jens Wille},
-      :email        => %q{jens.wille@uni-koeln.de},
-      :extra_files  => FileList['data/**/*'].to_a,
-      :dependencies => [['ruby-nuggets', '>= 0.3.3'], 'htmlentities']
+    gem: {
+      name:         %q{cmess},
+      version:      CMess::VERSION,
+      summary:      <<-EOT,
+Assist with handling messed up encodings (Currently includes the
+following tools: #{Dir['bin/*'].map { |e| File.basename(e) }.sort.join(', ')})
+      EOT
+      author:       %q{Jens Wille},
+      email:        %q{jens.wille@gmail.com},
+      license:      %q{AGPL},
+      homepage:     :blackwinter,
+      extra_files:  FileList['data/**/*'].to_a,
+      dependencies: [['ruby-nuggets', '>= 0.3.3'], 'htmlentities'],
+
+      required_ruby_version: '>= 1.9.2'
     }
   }}
 rescue LoadError => err

data/bin/bconv

@@ -6,12 +6,14 @@
 # bconv -- Convert between bibliographic (and other) encodings                #
 #   [A component of cmess, the encoding tool-box]                             #
 #                                                                             #
-# Copyright (C) 2008-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -33,11 +35,11 @@ require 'cmess/bconv'
 include CMess::CLI
 
 options = {
-  :input           => STDIN,
-  :output          => STDOUT,
-  :source_encoding => determine_system_encoding,
-  :target_encoding => determine_system_encoding,
-  :chartab_file    => CMess::BConv::DEFAULT_CHARTAB_FILE
+  input:           STDIN,
+  output:          STDOUT,
+  source_encoding: determine_system_encoding,
+  target_encoding: determine_system_encoding,
+  chartab_file:    CMess::BConv::DEFAULT_CHARTAB_FILE
 }
 
 parse_options { |opts|

data/bin/cinderella

@@ -6,12 +6,14 @@
 # cinderella -- Handle double encoded characters                              #
 #   [A component of cmess, the encoding tool-box]                             #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -35,14 +37,14 @@ include CMess::CLI
 progname = File.basename($0)
 
 options = {
-  :input           => STDIN,
-  :output          => STDOUT,
-  :pot             => nil,
-  :crop            => nil,
-  :source_encoding => nil,
-  :target_encoding => determine_system_encoding,
-  :csets           => [CMess::Cinderella::DEFAULT_CSETS_DIR],
-  :repair          => false
+  input:           STDIN,
+  output:          STDOUT,
+  pot:             nil,
+  crop:            nil,
+  source_encoding: nil,
+  target_encoding: determine_system_encoding,
+  csets:           [CMess::Cinderella::DEFAULT_CSETS_DIR],
+  repair:          false
 }
 
 parse_options { |opts|
@@ -169,5 +171,5 @@ cli do
 
   trailing_args_as_input(options)
 
-  CMess::Cinderella.pick(options.merge(:chars => YAML.load_file(char_file)))
+  CMess::Cinderella.pick(options.merge(chars: YAML.load_file(char_file)))
 end

data/bin/decode_entities

@@ -6,12 +6,14 @@
 # decode_entities -- Decode HTML entities                                     #
 #   [A component of cmess, the encoding tool-box]                             #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -33,11 +35,11 @@ require 'cmess/decode_entities'
 include CMess::CLI
 
 options = {
-  :input           => STDIN,
-  :output          => STDOUT,
-  :source_encoding => CMess::DecodeEntities::INTERMEDIATE_ENCODING,
-  :target_encoding => nil,
-  :flavour         => CMess::DecodeEntities::DEFAULT_FLAVOUR
+  input:           STDIN,
+  output:          STDOUT,
+  source_encoding: CMess::DecodeEntities::ENCODING,
+  target_encoding: nil,
+  flavour:         CMess::DecodeEntities::DEFAULT_FLAVOUR
 }
 
 parse_options { |opts|

data/bin/guess_encoding

@@ -6,12 +6,14 @@
 # guess_encoding -- Assist with guessing the encoding of some input at hand   #
 #   [A component of cmess, the encoding tool-box]                             #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -33,16 +35,17 @@ require 'cmess/guess_encoding'
 include CMess::CLI
 
 options = {
-  :input                => STDIN,
-  :line                 => 1,
-  :encodings            => nil,
-  :additional_encodings => [],
-  :target_encoding      => determine_system_encoding,
-  :manual               => false,
-  :chunk_size           => nil,
-  :ignore_bom           => false,
-  :charcodes            => nil,
-  :base                 => 16
+  input:                STDIN,
+  line:                 1,
+  encodings:            nil,
+  additional_encodings: [],
+  target_encoding:      determine_system_encoding,
+  manual:               false,
+  reverse:              false,
+  chunk_size:           nil,
+  ignore_bom:           false,
+  charcodes:            nil,
+  base:                 16
 }
 
 parse_options { |opts|
@@ -79,12 +82,8 @@ parse_options { |opts|
   opts.separator ''
 
   opts.on('-l', '--line LINE', Integer, "Line number of input file to use for testing [Default: #{options[:line]}]") { |line|
+    abort 'Line number must be greater than 0!' unless line > 0
     options[:line] = line
-
-    unless options[:line] > 0
-      options[:input].read # prevent 'Broken pipe' error
-      abort 'Line number must be greater then 0!'
-    end
   }
 
   opts.separator ''
@@ -111,6 +110,12 @@ parse_options { |opts|
     exit
   }
 
+  opts.separator ''
+
+  opts.on('-R', '--reverse', 'Reverse encoding direction (FROM target TO encodings)') {
+    options[:reverse] = true
+  }
+
   opts.separator ''
   opts.separator '  * Charcodes'
 
@@ -189,7 +194,7 @@ cli do
       abort "Line not found -- input has only #{$.} line#{'s' if $. != 1}" unless input.is_a?(String)
     end
 
-    CMess::GuessEncoding.manual(options.merge(:input => input))
+    CMess::GuessEncoding.manual(options.merge(input: input))
   else
     puts CMess::GuessEncoding.automatic(options[:input], options[:chunk_size], options[:ignore_bom])
   end

data/lib/cmess.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # cmess -- Assist with handling messed up encodings                           #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -27,7 +29,6 @@
 #++
 
 require 'cmess/version'
-require 'iconv'
 
 # See README for more information.
 

data/lib/cmess/bconv.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2008-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -33,9 +35,9 @@ require 'yaml'
 
 class CMess::BConv
 
-  VERSION = '0.0.3'
+  VERSION = '0.1.0'
 
-  INTERMEDIATE_ENCODING = 'utf-8'
+  ENCODING = 'utf-8'
 
   DEFAULT_CHARTAB_FILE = File.join(CMess::DATA_DIR, 'chartab.yaml')
 
@@ -67,8 +69,6 @@ class CMess::BConv
 
   end
 
-  attr_reader :input, :output, :source_encoding, :target_encoding, :chartab, :encodings
-
   def initialize(options)
     @input, @output, _ = CMess.ensure_options!(options,
       :input, :output, :source_encoding, :target_encoding
@@ -79,83 +79,64 @@ class CMess::BConv
 
     @chartab   = self.class.load_chartab(options[:chartab] || DEFAULT_CHARTAB_FILE)
     @encodings = self.class.encodings(@chartab)
-  end
 
-  def encoding?(encoding)
-    encodings.include?(encoding)
+    [:source_encoding, :target_encoding].each { |key|
+      instance_variable_set("@#{key}", encoding = options[key].upcase)
+      instance_variable_set("@have_#{key}", encodings.include?(encoding))
+    }
   end
 
+  attr_reader :input, :output, :source_encoding, :target_encoding, :chartab, :encodings
+
   def convert
-    if encoding?(source_encoding)
-      if encoding?(target_encoding)
-        @charmap = chartab.inject({}) { |hash, (code, map)|
-          hash.update(map[source_encoding] => map[target_encoding].pack('U*'))
-        }
+    source, target, out, charmap = source_encoding, target_encoding, output, {}
 
-        input.each_byte { |char|
-          output.print map(char)
+    if @have_source_encoding
+      if @have_target_encoding
+        chartab.each { |code, map|
+          charmap[map[source]] = map[target].pack('U*')
         }
-      else
-        iconv = iconv_to
 
-        @charmap = chartab.inject({}) { |hash, (code, map)|
-          hash.update(map[source_encoding] => [code.to_i(16)].pack('U*'))
+        input.each_byte { |char| out.print(map(char, charmap)) }
+      else
+        chartab.each { |code, map|
+          charmap[map[source]] = [code.to_i(16)].pack('U*')
         }
 
+        source = ENCODING
+
         input.each_byte { |char|
-          output.print iconv.iconv(map(char))
+          out.print(encode(map(char, charmap), source, target))
         }
       end
     else
-      if encoding?(target_encoding)
-        iconv = iconv_from
-
-        charmap = chartab.inject({}) { |hash, (code, map)|
-          hash.update(code.to_i(16) => map[target_encoding].pack('U*'))
+      if @have_target_encoding
+        chartab.each { |code, map|
+          charmap[code.to_i(16)] = map[target].pack('U*')
         }
 
+        target = ENCODING
+
         input.each { |line|
-          iconv.iconv(line).unpack('U*').each { |char|
-            output.print charmap[char]
+          encode(line, source, target).unpack('U*').each { |char|
+            out.print(charmap[char])
           }
         }
       else
-        iconv = iconv_from_to
-
-        input.each { |line|
-          output.puts iconv.iconv(line)
-        }
+        input.each { |line| out.print(encode(line, source, target)) }
       end
     end
   end
 
   private
 
-  def iconv_from_to(from = source_encoding, to = target_encoding)
-    iconv = begin
-      Iconv.new(to, from)
-    rescue Iconv::InvalidEncoding
-      raise ArgumentError, "invalid encoding: source encoding = #{from}, target encoding = #{to}"
-    end
-
-    def iconv.iconv(*args)
-      super
-    rescue Iconv::IllegalSequence, Iconv::InvalidCharacter => err
-      warn "ILLEGAL INPUT SEQUENCE: #{err}"; ''
-    end
-
-    iconv
-  end
-
-  def iconv_from(from = source_encoding)
-    iconv_from_to(from, INTERMEDIATE_ENCODING)
-  end
-
-  def iconv_to(to = target_encoding)
-    iconv_from_to(INTERMEDIATE_ENCODING, to)
+  def encode(string, source, target)
+    string.encode(target, source)
+  rescue Encoding::UndefinedConversionError => err
+    warn "ILLEGAL INPUT SEQUENCE: #{err.error_char}"
   end
 
-  def map(char, charmap = @charmap)
+  def map(char, charmap)
     unless map = charmap[[char]]
       unless map = charmap[[char, c = input.getc]]
         input.ungetc(c) if c

data/lib/cmess/cinderella.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -39,34 +41,33 @@ module CMess::Cinderella
 
   extend self
 
-  VERSION = '0.0.5'
+  VERSION = '0.1.0'
 
   DEFAULT_CSETS_DIR = File.join(CMess::DATA_DIR, 'csets')
 
   def pick(options)
-    CMess.ensure_options!(options,
+    input, pot, crop, source, target, chars = CMess.ensure_options!(options,
       :input, :pot, :crop, :source_encoding, :target_encoding, :chars
     )
 
     encoded = {}
-    iconv = Iconv.new(*options.values_at(:target_encoding, :source_encoding))
-
-    options[:chars].each { |char|
-      begin
-        encoded[iconv.iconv(char)] = char
-      rescue Iconv::IllegalSequence
-      end
-    }
+    chars.each { |char| encoded[encode(char, source, target)] = char }
 
     regexp = Regexp.union(*encoded.keys)
-    pot, crop, repair = options.values_at(:pot, :crop, :repair)
 
-    options[:input].each { |line|
-      if out = line =~ regexp ? crop : pot
-        line.gsub!(regexp) { |m| encoded[m] } if repair
-        out.puts(line)
-      end
+    input.each { |line|
+      out = line =~ regexp ? crop : pot or next
+
+      line.gsub!(regexp) { |m| encoded[m] } if repair
+      out.puts(line)
     }
   end
 
+  private
+
+  def encode(string, source, target)
+    string.encode(target, source)
+  rescue Encoding::UndefinedConversionError
+  end
+
 end

data/lib/cmess/cli.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -102,18 +104,12 @@ module CMess::CLI
   end
 
   def determine_system_encoding
-    ENV.user_encoding || begin
-      dummy = lambda {
-        abort <<-EOT
+    ENV.user_encoding || lambda {
+      abort <<-EOT
 Your system's encoding couldn't be determined automatically -- please specify
 it explicitly via the ENCODING environment variable or via the '-t' option.
-        EOT
-      }
-
-      def dummy.to_s; 'NOT FOUND' end
-
-      dummy
-    end
+      EOT
+    }.tap { |dummy| def dummy.to_s; 'NOT FOUND'; end }
   end
 
   def cli

data/lib/cmess/decode_entities.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # cmess is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -33,43 +35,38 @@ module CMess::DecodeEntities
 
   extend self
 
-  VERSION = '0.0.5'
+  VERSION = '0.1.0'
 
   # HTMLEntities requires UTF-8
-  INTERMEDIATE_ENCODING = 'utf-8'
-
-  ICONV_DUMMY = begin
-    dummy = Object.new
-
-    def dummy.iconv(string)
-      string
-    end
-
-    dummy
-  end
+  ENCODING = 'UTF-8'
 
   DEFAULT_FLAVOUR = 'xml-safe'
 
   def decode(options)
-    input, output, source_encoding = CMess.ensure_options!(options,
+    input, output, source = CMess.ensure_options!(options,
       :input, :output, :source_encoding
     )
 
-    target_encoding = options[:target_encoding] || source_encoding
-
-    iconv_in  = source_encoding != INTERMEDIATE_ENCODING ?
-      Iconv.new(INTERMEDIATE_ENCODING, source_encoding) : ICONV_DUMMY
+    target, entities, encoding = options[:target_encoding] || source,
+      HTMLEntities.new(options[:flavour] || DEFAULT_FLAVOUR), ENCODING
 
-    iconv_out = target_encoding != INTERMEDIATE_ENCODING ?
-      Iconv.new(target_encoding, INTERMEDIATE_ENCODING) : ICONV_DUMMY
-
-    html_entities = HTMLEntities.new(options[:flavour] || DEFAULT_FLAVOUR)
+    skip_source, skip_target = source == encoding, target == encoding
 
     input.each { |line|
-      output.puts iconv_out.iconv(html_entities.decode(iconv_in.iconv(line)))
+      line = encode(line, source, encoding) unless skip_source
+      line = entities.decode(line)
+      line = encode(line, encoding, target) unless skip_target
+
+      output.puts(line)
     }
   end
 
+  private
+
+  def encode(string, source, target)
+    string.encode(target, source)
+  end
+
 end
 
 class HTMLEntities  # :nodoc:

data/lib/cmess/guess_encoding.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # Contributors:                                                               #
 #     John Vorhauer <john@vorhauer.de> (idea and original implementation      #
@@ -38,7 +40,7 @@ require 'cmess'
 
 module CMess::GuessEncoding
 
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 
   autoload :Encoding,  'cmess/guess_encoding/encoding'
   autoload :Manual,    'cmess/guess_encoding/manual'

data/lib/cmess/guess_encoding/automatic.rb

@@ -5,12 +5,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # Contributors:                                                               #
 #     John Vorhauer <john@vorhauer.de> (idea and original implementation      #
@@ -32,8 +34,6 @@
 ###############################################################################
 #++
 
-$KCODE = 'u' if RUBY_VERSION < '1.9'
-
 require 'cmess/guess_encoding'
 
 require 'yaml'
@@ -57,9 +57,6 @@ class CMess::GuessEncoding::Automatic
 
   include CMess::GuessEncoding::Encoding
 
-  # Creates a converter for desired encoding (from UTF-8).
-  ICONV_FOR = Hash.new { |h, k| h[k] = Iconv.new(k, UTF_8) }
-
   # Single-byte encodings to test statistically by TEST_CHARS.
   TEST_ENCODINGS = [
     MACINTOSH,
@@ -87,22 +84,13 @@ class CMess::GuessEncoding::Automatic
   CHARS_TO_TEST = (
     '€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂ' <<
     'ÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'
-  ).split(//)
+  ).chars.to_a
 
   # Map TEST_ENCODINGS to respectively encoded CHARS_TO_TEST.
-  TEST_CHARS = Hash.new { |hash, encoding|
-    encoding = self[encoding]
-
-    encchars = CHARS_TO_TEST.map { |char|
-      begin
-        byte = *ICONV_FOR[encoding].iconv(char).unpack('C')
-      rescue Iconv::IllegalSequence
-      end
-    }.compact
-
-    TEST_ENCODINGS << encoding unless TEST_ENCODINGS.include?(encoding)
-
-    hash[encoding] = encchars
+  TEST_CHARS = Hash.new { |h, k|
+    e, f = self[k], UTF_8
+    TEST_ENCODINGS << e unless TEST_ENCODINGS.include?(e)
+    h[e] = CHARS_TO_TEST.flat_map { |c| c.encode(e, f).unpack('C') }
   }.update(YAML.load_file(File.join(CMess::DATA_DIR, 'test_chars.yaml')))
 
   # Relative count of TEST_CHARS must exceed this threshold to yield
@@ -134,10 +122,10 @@ class CMess::GuessEncoding::Automatic
 
     def encoding(*encodings, &block)
       encodings.flatten.each { |encoding|
-        next if @supported_encodings.include?(encoding)
-
-        @supported_encodings << encoding
-        @encoding_guessers   << block
+        unless @supported_encodings.include?(encoding)
+          @supported_encodings << encoding
+          @encoding_guessers   << block
+        end
       }
     end
 
@@ -146,10 +134,10 @@ class CMess::GuessEncoding::Automatic
     end
 
     def bom_encoding(encoding, &block)
-      return if @supported_boms.include?(encoding)
-
-      @supported_boms << encoding
-      @bom_guessers   << lambda { |*| encoding if instance_eval(&block) }
+      unless @supported_boms.include?(encoding)
+        @supported_boms << encoding
+        @bom_guessers   << lambda { |*| encoding if instance_eval(&block) }
+      end
     end
 
     def supported_bom?(encoding)
@@ -158,30 +146,30 @@ class CMess::GuessEncoding::Automatic
 
   end
 
-  attr_reader :input, :chunk_size, :byte_count, :byte_total, :first_byte
-
   def initialize(input, chunk_size = nil)
     @input = case input
       when IO     then input
       when String then StringIO.new(input)
-      else
-        raise ArgumentError, "don't know how to handle input of type #{input.class}"
+      else raise ArgumentError,
+        "don't know how to handle input of type #{input.class}"
     end
 
     @chunk_size = chunk_size
   end
 
+  attr_reader :input, :chunk_size, :byte_count, :byte_total, :first_byte
+
   def guess(ignore_bom = false)
     return bom if bom && !ignore_bom
 
     while read
       encoding_guessers.each { |block|
-        encoding = instance_eval(&block)
-        return encoding if encoding && supported_encoding?(encoding)
+        if encoding = instance_eval(&block) and supported_encoding?(encoding)
+          return encoding
+        end
       }
     end
 
-    # nothing suitable found :-(
     UNKNOWN
   end
 
@@ -206,14 +194,13 @@ class CMess::GuessEncoding::Automatic
     end
 
     bom_guessers.each { |block|
-      encoding = instance_eval(&block)
-      return encoding if encoding && supported_bom?(encoding)
-
-      # read bytes don't build a BOM, so rewind...
-      input.rewind
+      if encoding = instance_eval(&block) and supported_encoding?(encoding)
+        return encoding
+      else
+        input.rewind
+      end
     }
 
-    # nothing suitable found :-(
     nil
   end
 

data/lib/cmess/guess_encoding/encoding.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # Contributors:                                                               #
 #     John Vorhauer <john@vorhauer.de> (idea and original implementation      #
@@ -50,9 +52,7 @@ module CMess::GuessEncoding::Encoding
   private
 
   def get_all_encodings
-    %x{iconv -l}.split($/).map { |encoding|
-      get_or_set_encoding_const(encoding.sub(%r{/*\z}, ''))
-    }
+    Encoding.name_list.map { |encoding| get_or_set_encoding_const(encoding) }
   end
 
   def const_name_for(encoding)

data/lib/cmess/guess_encoding/manual.rb

@@ -3,12 +3,14 @@
 #                                                                             #
 # A component of cmess, the encoding tool-box.                                #
 #                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
+# Copyright (C) 2008-2012 University of Cologne,                              #
 #                         Albertus-Magnus-Platz,                              #
 #                         50923 Cologne, Germany                              #
 #                                                                             #
+# Copyright (C) 2013 Jens Wille                                               #
+#                                                                             #
 # Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
 #                                                                             #
 # Contributors:                                                               #
 #     John Vorhauer <john@vorhauer.de> (idea and original implementation      #
@@ -57,13 +59,13 @@ module CMess::GuessEncoding::Manual
     CP1252,
     CP850,
     CP852,
-    CP856,
     UTF_8
   ]
 
   # Likely candidates to suggest to the user
   CANDIDATES = [
     ANSI_X34,
+    CP856,
     EBCDIC_AT_DE,
     EBCDIC_US,
     EUC_JP,
@@ -95,19 +97,18 @@ module CMess::GuessEncoding::Manual
     # move target encoding to front
     encodings.in_order!(target)
 
-    max_length = encodings.max(:length)
+    max_length, reverse = encodings.max(:length), options[:reverse]
 
     encodings.each { |encoding|
+      args = [target, encoding]
+      args.reverse! if reverse
+
       converted = begin
-        Iconv.conv(target, encoding, input)
-      rescue Iconv::IllegalSequence, Iconv::InvalidCharacter => err
-        "ILLEGAL INPUT SEQUENCE: #{err}"
-      rescue Iconv::InvalidEncoding
-        if encoding == target
-          raise ArgumentError, "invalid encoding: #{encoding}"
-        else
-          'INVALID ENCODING!'
-        end
+        input.encode(*args)
+      rescue Encoding::UndefinedConversionError => err
+        "ILLEGAL INPUT SEQUENCE: #{err.error_char}"
+      rescue Encoding::ConverterNotFoundError => err
+        err.to_s
       end
 
       puts "%-#{max_length}s : %s" % [encoding, converted]

data/lib/cmess/version.rb

@@ -3,8 +3,8 @@ module CMess
   module Version
 
     MAJOR = 0
-    MINOR = 3
-    TINY  = 1
+    MINOR = 4
+    TINY  = 0
 
     class << self
 

metadata

@@ -1,153 +1,137 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: cmess
-version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 1
-  version: 0.3.1
+version: !ruby/object:Gem::Version
+  version: 0.4.0
 platform: ruby
-authors: 
+authors:
 - Jens Wille
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-08-16 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-08-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: ruby-nuggets
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 21
-        segments: 
-        - 0
-        - 3
-        - 3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
         version: 0.3.3
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: htmlentities
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.3
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-description: "\n        Assist with handling messed up encodings (Currently includes the\n        following tools: bconv, cinderella, decode_entities, guess_encoding)\n      "
-email: jens.wille@uni-koeln.de
-executables: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  Assist with handling messed up encodings (Currently includes the
+  following tools: bconv, cinderella, decode_entities, guess_encoding)
+email: jens.wille@gmail.com
+executables:
 - bconv
+- cinderella
 - decode_entities
 - guess_encoding
-- cinderella
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README
 - COPYING
 - ChangeLog
-files: 
+files:
 - lib/cmess.rb
-- lib/cmess/guess_encoding/automatic.rb
-- lib/cmess/guess_encoding/encoding.rb
-- lib/cmess/guess_encoding/manual.rb
 - lib/cmess/bconv.rb
-- lib/cmess/cli.rb
 - lib/cmess/cinderella.rb
-- lib/cmess/guess_encoding.rb
+- lib/cmess/cli.rb
 - lib/cmess/decode_entities.rb
+- lib/cmess/guess_encoding.rb
+- lib/cmess/guess_encoding/automatic.rb
+- lib/cmess/guess_encoding/encoding.rb
+- lib/cmess/guess_encoding/manual.rb
 - lib/cmess/version.rb
 - bin/bconv
+- bin/cinderella
 - bin/decode_entities
 - bin/guess_encoding
-- bin/cinderella
-- data/csets/latin1.yaml
-- data/csets/iso_8859-15.yaml
+- data/chartab.yaml
 - data/csets/iso_8859-1.yaml
-- data/csets/unicode/latin_extended_a.yaml
+- data/csets/iso_8859-15.yaml
+- data/csets/latin1.yaml
 - data/csets/unicode/basic_latin.yaml
+- data/csets/unicode/cyrillic-supplement.yaml
+- data/csets/unicode/cyrillic.yaml
+- data/csets/unicode/greek.yaml
 - data/csets/unicode/ipa_extensions.yaml
-- data/csets/unicode/latin_extended_b.yaml
+- data/csets/unicode/latin-extended-c.yaml
 - data/csets/unicode/latin-extended-d.yaml
-- data/csets/unicode/letterlike_symbols.yaml
+- data/csets/unicode/latin_1_supplement.yaml
+- data/csets/unicode/latin_extended_a.yaml
 - data/csets/unicode/latin_extended_additional.yaml
-- data/csets/unicode/greek.yaml
-- data/csets/unicode/latin-extended-c.yaml
+- data/csets/unicode/latin_extended_b.yaml
+- data/csets/unicode/letterlike_symbols.yaml
 - data/csets/unicode/spacing_modifier_letters.yaml
-- data/csets/unicode/cyrillic-supplement.yaml
-- data/csets/unicode/cyrillic.yaml
-- data/csets/unicode/latin_1_supplement.yaml
 - data/csets/utf-8.yaml
 - data/csets/utf8.yaml
 - data/test_chars.yaml
-- data/chartab.yaml
-- README
+- COPYING
 - ChangeLog
+- README
 - Rakefile
-- COPYING
-- example/guess_encoding/en.utf-8.txt
-- example/guess_encoding/de.utf-8.txt
-- example/guess_encoding/it.utf-8.txt
-- example/guess_encoding/check_results
-- example/guess_encoding/fr.utf-8.txt
-- example/cinderella/empty6-slash_repaired.txt
-- example/cinderella/empty6-slash.txt
 - example/cinderella/crop
-- example/cinderella/pot
 - example/cinderella/crop_repaired
-homepage: http://prometheus.rubyforge.org/cmess
-licenses: []
-
+- example/cinderella/empty6-slash.txt
+- example/cinderella/empty6-slash_repaired.txt
+- example/cinderella/pot
+- example/guess_encoding/check_results
+- example/guess_encoding/de.utf-8.txt
+- example/guess_encoding/en.utf-8.txt
+- example/guess_encoding/fr.utf-8.txt
+- example/guess_encoding/it.utf-8.txt
+homepage: http://github.com/blackwinter/cmess
+licenses:
+- AGPL
+metadata: {}
 post_install_message: 
-rdoc_options: 
-- --main
-- README
+rdoc_options:
 - --charset
 - UTF-8
+- --line-numbers
 - --all
 - --title
-- cmess Application documentation (v0.3.1)
-- --line-numbers
-require_paths: 
+- cmess Application documentation (v0.4.0)
+- --main
+- README
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: prometheus
-rubygems_version: 1.8.8
+rubyforge_project: 
+rubygems_version: 2.0.6
 signing_key: 
-specification_version: 3
-summary: "Assist with handling messed up encodings (Currently includes the following tools: bconv, cinderella, decode_entities, guess_encoding)"
+specification_version: 4
+summary: 'Assist with handling messed up encodings (Currently includes the following
+  tools: bconv, cinderella, decode_entities, guess_encoding)'
 test_files: []
-