cmess 0.0.4.136

data/ChangeLog

@@ -0,0 +1,6 @@
+= Revision history for cmess
+
+== 0.0.3 [2007-12-12]
+
+* Added automatic encoding detection to GuessEncoding. Idea and original
+  implementation provided by John. Thanks :-)

data/README

@@ -0,0 +1,53 @@
+= cmess - Assist with messed up encodings
+
+== VERSION
+
+This documentation refers to cmess version 0.0.3
+
+
+== DESCRIPTION
+
+CMess bundles several tools under its hood that aim at dealing with various
+problems occurring in the context of character sets and encodings. Currently,
+there are:
+
+guess_encoding::  Simple helper to identify the encoding of a given string.
+                  Includes the ability to automatically detect the encoding
+                  of an input.
+cinderella::      When characters are "double encoded", you can't easily
+                  convert them back -- this is where cinderella comes in,
+                  sorting the good ones into the pot and the (potentially)
+                  bad ones into the crop...
+decode_entities:: Decode HTML entities in a string.
+
+TODO: well, more of the description... ;-)
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@uni-koeln.de>
+
+
+== CREDITS
+
+* John Vorhauer <mailto:john@vorhauer.de> for the idea and
+  original implementation of the automatic encoding guesser
+  (see CMess::GuessEncoding::Guesser).
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007 University of Cologne,
+Albertus-Magnus-Platz, 50932 Cologne, Germany
+
+cmess is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+cmess is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+cmess. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,30 @@
+# Utilizes global rake-tasks: alias rake="rake -r rake -R /path/to/rakelibdir"
+# (Base tasks at <http://prometheus.khi.uni-koeln.de/svn/scratch/rake-tasks/>)
+
+$:.unshift('lib')
+
+require 'cmess'
+
+FILES = FileList['lib/**/*.rb'].to_a
+EXECS = FileList['bin/*'].to_a
+RDOCS = %w[README COPYING ChangeLog]
+OTHER = FileList['[A-Z]*', 'example/**/*', 'data/**/*'].to_a
+
+task(:doc_spec) {{
+  :title      => 'cmess Application documentation',
+  :rdoc_files => RDOCS + FILES
+}}
+
+task(:gem_spec) {{
+  :name             => 'cmess',
+  :version          => CMess::VERSION,
+  :summary          => "Assist with handling messed up encodings " <<
+                       "(Currently includes the following tools: " <<
+                       "#{EXECS.map { |e| File.basename(e) }.join(', ')})",
+  :files            => FILES + EXECS + OTHER,
+  :require_path     => 'lib',
+  :bindir           => 'bin',
+  :executables      => EXECS,
+  :extra_rdoc_files => RDOCS,
+  :dependencies     => %w[ruby-nuggets htmlentities]
+}}

data/bin/cinderella

@@ -0,0 +1,186 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# cinderella -- Handle double encoded characters                              #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'yaml'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess'
+require 'cmess/cinderella'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+options = {
+  :input           => STDIN,
+  :output          => STDOUT,
+  :pot             => nil,
+  :crop            => nil,
+  :source_encoding => nil,
+  :target_encoding => determine_system_encoding,
+  :csets           => [
+    File.expand_path(File.join(File.dirname(__FILE__), '..', 'data', 'csets'))
+  ],
+  :repair          => false
+}
+
+OptionParser.new(nil, 40) { |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+  }
+
+  opts.separator ''
+
+  opts.on('-p', '--pot FILE', "The good into the pot...") { |f|
+    options[:pot] = open_file_or_std(f, 'w')
+  }
+
+  opts.on('-c', '--crop FILE', "...the bad into the crop") { |f|
+    options[:crop] = open_file_or_std(f, 'w')
+  }
+
+  opts.separator ''
+
+  opts.on('-o', '--output [FILE]', "Write both good and bad lines to FILE or", "default [Default: STDOUT] (Particularly", "useful in combination with the '-r' option)") { |f|
+    options[:output] = open_file_or_std(f, 'w') if f
+
+    options[:pot]  = options[:output]
+    options[:crop] = options[:output]
+  }
+
+  opts.separator ''
+
+  opts.on('-I', '--in-place FILE', "Modify file in-place; sets '-i' and '-o'", "(Only really useful in combination with", "the '-r' option)") { |f|
+    options[:input], options[:output] = open_file_in_place(f)
+
+    options[:pot]  = options[:output]
+    options[:crop] = options[:output]
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--source-encoding ENCODING', "Source encoding (from) [REQUIRED]") { |e|
+    options[:source_encoding] = e
+  }
+
+  opts.on('-t', '--target-encoding ENCODING', "Target encoding (to); see '-l' for a list", "of available encodings [Default: #{options[:target_encoding]}]") { |e|
+    options[:target_encoding] = e
+  }
+
+  opts.separator ''
+
+  opts.on('-T', '--addtl-target-encodings DIRECTORY', "Directory providing additional char files", "for target encoding") { |d|
+    ensure_directory(d)
+
+    options[:csets] |= [File.expand_path(d)]
+  }
+
+  opts.separator ''
+
+  opts.on('-l', '--list-encodings', "Display a list of available target encodings", "and exit; see '-T' on how to add your own") {
+    csets = options[:csets].inject({}) { |hash, cset|
+      encodings = Dir[File.join(cset, '*.yaml')].sort.map { |yaml|
+        File.basename(yaml, '.yaml') unless File.symlink?(yaml)
+      }.compact
+
+      hash[cset] = encodings unless encodings.empty?
+      hash
+    }
+
+    if csets.empty?
+      puts "No target encodings available for #{PROGNAME}"
+    else
+      puts "Available target encodings for #{PROGNAME}:"
+      csets.each { |cset, encodings|
+        puts "[#{cset}]"
+        encodings.each { |encoding|
+          puts "  - #{encoding}"
+        }
+      }
+    end
+
+    exit
+  }
+
+  opts.separator ''
+
+  opts.on('-r', '--repair', "Try to repair corrupted characters") {
+    options[:repair] = true
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CMess::Cinderella::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator "If '-p' or '-c' is omitted, and '-o' is not given either, that particular output"
+  opts.separator "is ignored. When FILE is -, either STDIN or STDOUT is used (as appropriate)."
+}.parse!
+
+options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
+
+abort "No source encoding given! (Use the '-e' switch to do so; see '--help' for more information)" \
+  unless options[:source_encoding]
+
+yaml_file = "#{options[:target_encoding].downcase}.yaml"
+char_file = options[:csets].inject(nil) { |path, cset|
+  path = File.join(cset, yaml_file)
+  break path if File.readable?(path)
+}
+abort "Char file not found for target encoding: #{options[:target_encoding]}" \
+  unless char_file
+
+CMess::Cinderella.pick(
+  options[:input],
+  options[:pot],
+  options[:crop],
+  options[:source_encoding],
+  options[:target_encoding],
+  YAML.load_file(char_file),
+  options[:repair]
+)

data/bin/decode_entities

@@ -0,0 +1,101 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# decode_entities -- Decode HTML entities                                     #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess'
+require 'cmess/decode_entities'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+options = {
+  :input           => STDIN,
+  :output          => STDOUT,
+  :source_encoding => CMess::DecodeEntities::INTERMEDIATE_ENCODING,
+  :target_encoding => nil
+}
+
+OptionParser.new { |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+  }
+
+  opts.on('-o', '--output FILE', "Output file to write to [Default: STDOUT]") { |f|
+    options[:output] = open_file_or_std(f, 'w')
+  }
+
+  opts.on('-I', '--in-place FILE', "Modify file in-place; sets '-i' and '-o'") { |f|
+    options[:input], options[:output] = open_file_in_place(f)
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--source-encoding ENCODING', "Encoding of input file [Default: #{options[:source_encoding].upcase}]") { |e|
+    options[:source_encoding] = e.downcase
+  }
+
+  opts.on('-t', '--target-encoding ENCODING', "Desired encoding for output file [Default: <source_encoding>]") { |e|
+    options[:target_encoding] = e.downcase
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CMess::DecodeEntities::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator "When FILE is -, either STDIN or STDOUT is used (as appropriate)."
+}.parse!
+
+CMess::DecodeEntities.decode(
+  options[:input],
+  options[:output],
+  options[:source_encoding],
+  options[:target_encoding]
+)

data/bin/guess_encoding

@@ -0,0 +1,183 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# guess_encoding -- Assist with guessing the encoding of some input at hand   #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+
+require 'rubygems'
+require 'nuggets/string/word_wrap'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess'
+require 'cmess/guess_encoding'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+# short-cut
+CGE = CMess::GuessEncoding
+
+# how to split list of encodings
+SPLIT_ENCODING_LIST_RE = /\s*[,\s]\s*/o
+
+options = {
+  :input                => STDIN,
+  :line                 => 1,
+  :encodings            => nil,
+  :additional_encodings => [],
+  :target_encoding      => determine_system_encoding,
+  :guess                => false,
+  :chunk_size           => nil,
+  :ignore_bom           => false
+}
+
+OptionParser.new(nil, 40) { |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+  }
+
+  opts.separator ''
+  opts.separator '  * Manual guessing'
+  opts.separator ''
+
+  opts.on('-l', '--line LINE', "Line number of input file to use for testing [Default: #{options[:line]}]") { |l|
+    options[:line] = l.to_i
+
+    unless options[:line] > 0
+      options[:input].read  # prevent 'Broken pipe' error
+      abort "Line number must be greater then 0!"
+    end
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--encodings ENCODINGS...', "List of encodings to try >instead of< default (see below)") { |e|
+    options[:encodings] ||= []
+    options[:encodings] += e.split(SPLIT_ENCODING_LIST_RE)
+  }
+
+  opts.on('-a', '--additional-encodings ENCODINGS...', "List of encodings to try >in addition to< default (see below)") { |e|
+    options[:additional_encodings] += e.split(SPLIT_ENCODING_LIST_RE)
+  }
+
+  opts.separator ''
+
+  opts.on('-t', '--target-encoding ENCODING', "Target encoding of your system [Default: #{options[:target_encoding]}]") { |e|
+    options[:target_encoding] = e
+  }
+
+  opts.separator ''
+  opts.separator '  * Automatic guessing'
+  opts.separator ''
+
+  opts.on('-g', '--guess', "Actually guess the encoding of the input, automatically!", "(see below for a list of supported encodings)") {
+    options[:guess] = true
+  }
+
+  opts.on('-c', '--chunk-size SIZE', Integer, "Size of chunks input will be read in until a valid encoding", "has been found; by default the whole file will be read") { |s|
+    options[:chunk_size] = s
+  }
+
+  opts.separator ''
+
+  opts.on('-b', '--ignore-bom', "Ignore detected BOM (if any)", "(see below for a list of supported encodings)") {
+    options[:ignore_bom] = true
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CGE::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator 'Default encodings for manual guessing:'
+  CGE::Manual::ENCODINGS.join(', ').word_wrap(110, true).each { |l|
+    opts.separator l
+  }
+
+  opts.separator ''
+  opts.separator 'Likely candidates for additional testing:'
+  CGE::Manual::CANDIDATES.join(', ').word_wrap(110, true).each { |l|
+    opts.separator l
+  }
+
+  opts.separator ''
+  opts.separator 'Supported encodings for automatic guessing (will be tried in that order):'
+  CGE::Automatic.supported_encodings.join(', ').word_wrap(110, true).each { |l|
+    opts.separator l
+  }
+
+  opts.separator ''
+  opts.separator 'Supported encodings for BOM detection (will be tried in that order):'
+  CGE::Automatic.supported_boms.join(', ').word_wrap(110, true).each { |l|
+    opts.separator l
+  }
+
+  opts.separator ''
+  opts.separator "When FILE is -, STDIN is used."
+}.parse!
+
+if options[:guess]
+  puts CGE::Automatic.guess(options[:input], options[:chunk_size], options[:ignore_bom])
+else
+  options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
+
+  # reset line counter
+  $. = 0
+
+  input = options[:input].each { |line|
+    break line if $. == options[:line]
+  }
+  abort "Input was empty!" if $..zero?
+  abort "Line not found -- input has only #{$.} line#{'s' if $. != 1}" unless input.is_a?(String)
+
+  CGE::Manual.display(
+    input,
+    options[:target_encoding],
+    options[:encodings],
+    options[:additional_encodings]
+  )
+end