blackwinter-cmess 0.2.0

data/lib/cmess/bconv.rb

@@ -0,0 +1,169 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of cmess, the encoding tool-box.                                #
+#                                                                             #
+# Copyright (C) 2008 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'yaml'
+require 'iconv'
+require 'cmess'
+
+# Convert between bibliographic (and other) encodings.
+
+class CMess::BConv
+
+  # our version ;-)
+  VERSION = '0.0.2'
+
+  INTERMEDIATE_ENCODING = 'utf-8'
+
+  DEFAULT_CHARTAB_FILE = File.join(CMess::DATA_DIR, 'chartab.yaml')
+
+  class << self
+
+    def encodings(chartab = DEFAULT_CHARTAB_FILE)
+      chartab = load_chartab(chartab)
+
+      chartab[chartab.keys.first].keys.map { |encoding|
+        encoding.upcase unless encoding =~ /\A__/
+      }.compact.sort
+    end
+
+    def convert(*args)
+      new(*args).convert
+    end
+
+    def load_chartab(chartab)
+      case chartab
+        when Hash
+          chartab
+        when String
+          raise "chartab file not found: #{chartab}" unless File.readable?(chartab)
+          YAML.load_file(chartab)
+        else
+          raise ArgumentError, "invalid chartab of type #{chartab.class}"
+      end
+    end
+
+  end
+
+  attr_reader :input, :output, :source_encoding, :target_encoding, :chartab, :encodings
+
+  def initialize(input, output, source_encoding, target_encoding, chartab = DEFAULT_CHARTAB_FILE)
+    @input, @output = input, output
+
+    @source_encoding = source_encoding.upcase
+    @target_encoding = target_encoding.upcase
+
+    @chartab = self.class.load_chartab(chartab)
+    @encodings = self.class.encodings(@chartab)
+  end
+
+  def encoding?(encoding)
+    encodings.include?(encoding)
+  end
+
+  def convert
+    if encoding?(source_encoding)
+      if encoding?(target_encoding)
+        @charmap = chartab.inject({}) { |hash, (code, map)|
+          hash.update(map[source_encoding] => map[target_encoding].pack('U*'))
+        }
+
+        input.each_byte { |char|
+          output.print map(char)
+        }
+      else
+        iconv = iconv_to
+
+        @charmap = chartab.inject({}) { |hash, (code, map)|
+          hash.update(map[source_encoding] => [code.to_i(16)].pack('U*'))
+        }
+
+        input.each_byte { |char|
+          output.print iconv.iconv(map(char))
+        }
+      end
+    else
+      if encoding?(target_encoding)
+        iconv = iconv_from
+
+        charmap = chartab.inject({}) { |hash, (code, map)|
+          hash.update(code.to_i(16) => map[target_encoding].pack('U*'))
+        }
+
+        input.each { |line|
+          iconv.iconv(line).unpack('U*').each { |char|
+            output.print charmap[char]
+          }
+        }
+      else
+        iconv = iconv_from_to
+
+        input.each { |line|
+          output.puts iconv.iconv(line)
+        }
+      end
+    end
+  end
+
+  private
+
+  def iconv_from_to(from = source_encoding, to = target_encoding)
+    iconv = begin
+      Iconv.new(to, from)
+    rescue Iconv::InvalidEncoding
+      raise ArgumentError, "invalid encoding: source encoding = #{from}, target encoding = #{to}"
+    end
+
+    def iconv.iconv(*args)
+      super
+    rescue Iconv::IllegalSequence, Iconv::InvalidCharacter => err
+      warn "ILLEGAL INPUT SEQUENCE: #{err}"; ''
+    end
+
+    iconv
+  end
+
+  def iconv_from(from = source_encoding)
+    iconv_from_to(from, INTERMEDIATE_ENCODING)
+  end
+
+  def iconv_to(to = target_encoding)
+    iconv_from_to(INTERMEDIATE_ENCODING, to)
+  end
+
+  def map(char, charmap = @charmap)
+    unless map = charmap[[char]]
+      unless map = charmap[[char, c = input.getc]]
+        input.ungetc(c) if c
+        map = ''
+      end
+    end
+
+    map
+  end
+
+end

data/lib/cmess/cinderella.rb

@@ -0,0 +1,66 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of cmess, the encoding tool-box.                                #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'iconv'
+require 'cmess'
+
+# Find (and possibly repair) doubly encoded characters. Here's how it's done:
+#
+# Treats characters encoded in target encoding as if they were encoded in
+# source encoding, converts them to target encoding and "grep"s for lines
+# containing those doubly encoded characters; if asked to repair doubly
+# encoded characters, substitutes them with their original character.
+
+module CMess::Cinderella
+
+  extend self
+
+  # our version ;-)
+  VERSION = '0.0.3'
+
+  DEFAULT_CSETS_DIR = File.join(CMess::DATA_DIR, 'csets')
+
+  def pick(input, pot, crop, source_encoding, target_encoding, chars, repair = false)
+    iconv = Iconv.new(target_encoding, source_encoding)
+
+    encoded = chars.inject({}) { |hash, char|
+      hash.update(iconv.iconv(char) => char)
+    }
+
+    regexp = Regexp.union(*encoded.keys)
+
+    input.each { |line|
+      if out = line =~ regexp ? crop : pot
+        line.gsub!(regexp) { |m| encoded[m] } if repair
+
+        out.puts(line)
+      end
+    }
+  end
+
+end

data/lib/cmess/cli.rb

@@ -0,0 +1,120 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of cmess, the encoding tool-box.                                #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'tempfile'
+
+require 'rubygems'
+require 'nuggets/env/user_encoding'
+
+module CMess::CLI
+
+  # how to split list of arguments
+  SPLIT_ARG_LIST_RE = /\s*[,\s]\s*/o
+
+  def ensure_readable(file)
+    abort "Can't find input file: #{file}" unless File.readable?(file)
+  end
+
+  def ensure_directory(dir)
+    abort "Directory not found: #{dir}" unless File.directory?(dir)
+  end
+
+  def open_file_in_place(file)
+    [open_temporary_input(file), File.open(file, 'w')]
+  end
+
+  def open_file_or_std(file, mode = 'r')
+    if file == '-'
+      case mode
+        when 'r' then STDIN
+        when 'w' then STDOUT
+        when 'a' then STDERR
+        else          raise ArgumentError, "don't know how to handle mode '#{mode}'"
+      end
+    else
+      ensure_readable(file) unless mode == 'w'
+      File.open(file, mode)
+    end
+  end
+
+  def open_temporary_input(*files)
+    temp = Tempfile.new('cmess_cli')
+
+    files.each { |file|
+      if file == '-'
+        STDIN.each { |line| temp << line }
+      else
+        ensure_readable(file)
+        File.open(file) { |f| f.each { |line| temp << line } }
+      end
+    }
+
+    # return File, instead of Tempfile
+    temp.close
+    temp.open
+  end
+
+  def trailing_args_as_input(options)
+    unless ARGV.empty? || options[:input_set]
+      options[:input] = if ARGV.size == 1
+        open_file_or_std(ARGV.first)
+      else
+        open_temporary_input(*ARGV)
+      end
+    end
+  end
+
+  def determine_system_encoding
+    ENV.user_encoding || begin
+      dummy = lambda {
+        abort <<-EOT
+Your system's encoding couldn't be determined automatically -- please specify
+it explicitly via the ENCODING environment variable or via the '-t' option.
+        EOT
+      }
+
+      def dummy.to_s; 'NOT FOUND' end
+
+      dummy
+    end
+  end
+
+  def cli
+    yield
+  rescue => err
+    if $VERBOSE
+      backtrace = err.backtrace
+      fromtrace = backtrace[1..-1].map { |i| "\n        from #{i}" }
+
+      abort "#{backtrace.first} #{err} (#{err.class})#{fromtrace}"
+    else
+      abort "#{err.to_s.capitalize} [#{err.backtrace.first}]"
+    end
+  end
+
+end

data/lib/cmess/decode_entities.rb

@@ -0,0 +1,69 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of cmess, the encoding tool-box.                                #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'iconv'
+require 'cmess'
+
+require 'rubygems'
+require 'htmlentities/string'
+
+module CMess::DecodeEntities
+
+  extend self
+
+  # our version ;-)
+  VERSION = '0.0.2'
+
+  # HTMLEntities requires UTF-8
+  INTERMEDIATE_ENCODING = 'utf-8'
+
+  ICONV_DUMMY = begin
+    dummy = Object.new
+
+    def dummy.iconv(string)
+      string
+    end
+
+    dummy
+  end
+
+  def decode(input, output, source_encoding, target_encoding = nil)
+    target_encoding ||= source_encoding
+
+    iconv_in  = source_encoding != INTERMEDIATE_ENCODING ?
+      Iconv.new(INTERMEDIATE_ENCODING, source_encoding) : ICONV_DUMMY
+
+    iconv_out = target_encoding != INTERMEDIATE_ENCODING ?
+      Iconv.new(target_encoding, INTERMEDIATE_ENCODING) : ICONV_DUMMY
+
+    input.each { |line|
+      output.puts iconv_out.iconv(iconv_in.iconv(line).decode_entities)
+    }
+  end
+
+end