blackwinter-cmess 0.2.0

data/ChangeLog

@@ -0,0 +1,54 @@
+= Revision history for cmess
+
+== 0.2.0 [2009-05-08]
+
+* Ruby 1.9 compatibility (Syntax, at least)
+
+== 0.1.2 [2008-09-17]
+
+* Some refactoring; started to make tools more usable as a library
+* Make tools accept multiple files for input (via tempfile)
+* Use ENV.user_encoding from ruby-nuggets
+* Wrap command execution in a block catching any exceptions
+
+== 0.1.1 [2008-09-16]
+
+* Added bconv tool to convert between bibliographic encodings
+
+== 0.1.0 [2008-09-15]
+
+* Added ability to operate on all encodings available on a system
+
+== 0.0.9 [2008-08-15]
+
+* Reorganized file structure for guess_encoding
+* Added shortcuts GuessEncoding.manual/.automatic
+* GuessEncoding::Automatic now also takes a String
+  as input (will be converted to a StringIO)
+
+== 0.0.8 [2008-08-14]
+
+* Require 'cmess' inside libs, so the user doesn't have to
+
+== 0.0.7 [2008-05-19]
+
+* Fixed "Illegal seek" error when inside a pipe
+
+== 0.0.6 [2008-01-30]
+
+* Added ability to specify charcodes as input for manual guessing
+* Improved automatic guessing and further enhancements
+
+== 0.0.5 [2008-01-21]
+
+* Made automatic guessing the default for guess_encoding
+* Allow to specify input file as argument
+
+== 0.0.4 [2007-12-18]
+
+* Added BOM detection
+
+== 0.0.3 [2007-12-12]
+
+* Added automatic encoding detection to GuessEncoding. Idea and original
+  implementation provided by John. Thanks :-)

data/README

@@ -0,0 +1,63 @@
+= cmess - Assist with handling messed up encodings
+
+== VERSION
+
+This documentation refers to cmess version 0.2.0
+
+
+== DESCRIPTION
+
+CMess bundles several tools under its hood that aim at dealing with various
+problems occurring in the context of character sets and encodings. Currently,
+there are:
+
+guess_encoding::  Simple helper to identify the encoding of a given string.
+                  Includes the ability to automatically detect the encoding
+                  of an input.
+cinderella::      When characters are "double encoded", you can't easily
+                  convert them back -- this is where cinderella comes in,
+                  sorting the good ones into the pot and the (potentially)
+                  bad ones into the crop...
+bconv::           Convert between bibliographic (and other) encodings.
+decode_entities:: Decode HTML entities in a string.
+
+TODO: well, more of the description... ;-)
+
+
+== LINKS
+
+<b></b>
+Documentation::     <http://prometheus.rubyforge.org/cmess>
+Source code (old):: <http://prometheus.rubyforge.org/svn/scratch/cmess>
+Source code::       <http://github.com/blackwinter/cmess>
+Rubyforge project:: <http://rubyforge.org/projects/prometheus>
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@uni-koeln.de>
+
+
+== CREDITS
+
+* John Vorhauer <mailto:john@vorhauer.de> for the idea and
+  original implementation of the automatic encoding guesser
+  (see CMess::GuessEncoding::Automatic).
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007-2008 University of Cologne,
+Albertus-Magnus-Platz, 50932 Cologne, Germany
+
+cmess is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+cmess is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+cmess. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,51 @@
+$:.unshift('lib')
+require 'cmess'
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    :rubyforge => {
+      :package => 'cmess'
+    },
+
+    :gem => {
+      :version      => CMess::VERSION,
+      :summary      => %Q{
+        Assist with handling messed up encodings (Currently includes the
+        following tools: #{Dir['bin/*'].map { |e| File.basename(e) }.sort.join(', ')})
+      },
+      :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+      :extra_files  => FileList['[A-Z]*', 'example/**/*', 'data/**/*'].to_a,
+      :dependencies => [['ruby-nuggets', '>= 0.3.3'], 'htmlentities']
+    }
+  }}
+rescue LoadError
+  abort "Please install the 'hen' gem first."
+end
+
+namespace :guess_encoding do
+
+  require 'cmess/guess_encoding'
+  include CMess::GuessEncoding::Encoding
+
+  desc "Compare actual encoding and automatic guess of example files"
+  task :check_examples do
+    Dir[File.join(File.dirname(__FILE__), 'example', 'guess_encoding', '??.*.txt')].sort.each { |example|
+      language, encoding = File.basename(example, '.txt').split('.')
+      encoding.upcase!
+
+      guessed = CMess::GuessEncoding::Automatic.guess(File.open(example))
+
+      match = case guessed
+        when UNKNOWN:  '?'
+        when ASCII:    '#'
+        when encoding: '+'
+        else           '-'
+      end
+
+      puts '%s %s/%-11s => %s' % [match, language, encoding, guessed]
+    }
+  end
+
+end

data/bin/bconv

@@ -0,0 +1,130 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# bconv -- Convert between bibliographic (and other) encodings                #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2008 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+
+require 'rubygems'
+require 'nuggets/string/word_wrap'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess/bconv'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+options = {
+  :input           => STDIN,
+  :output          => STDOUT,
+  :source_encoding => determine_system_encoding,
+  :target_encoding => determine_system_encoding,
+  :chartab_file    => CMess::BConv::DEFAULT_CHARTAB_FILE
+}
+
+OptionParser.new(nil, 40) { |opts|
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+    options[:input_set] = true
+  }
+
+  opts.on('-o', '--output FILE', "Output file to write to [Default: STDOUT]") { |f|
+    options[:output] = open_file_or_std(f, 'w')
+  }
+
+  opts.on('-I', '--in-place FILE', "Modify file in-place; sets '-i' and '-o'") { |f|
+    options[:input], options[:output] = open_file_in_place(f)
+    options[:input_set] = true
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--source-encoding ENCODING', "Encoding of input file [Default: #{options[:source_encoding]}]") { |e|
+    options[:source_encoding] = e.downcase
+  }
+
+  opts.on('-t', '--target-encoding ENCODING', "Desired encoding for output file [Default: #{options[:target_encoding]}]") { |e|
+    options[:target_encoding] = e.downcase
+  }
+
+  opts.separator ''
+
+  opts.on('-c', '--chartab YAML_FILE', "File containing character mappings, in YAML format.", "[Default: #{options[:chartab_file]}]") { |c|
+    options[:chartab_file] = c
+  }
+
+  opts.on('-l', '--list-encodings', "Print a list of all available bibliographic", "encodings and exit; depends on <chartab>, see '-c'") {
+    options[:list_encodings] = true
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CMess::BConv::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator "When FILE is -, either STDIN or STDOUT is used (as appropriate)."
+}.parse!
+
+cli do
+  if options[:list_encodings]
+    puts CMess::BConv.encodings(options[:chartab_file])
+    exit
+  end
+
+  [:source_encoding, :target_encoding].each { |key|
+    options[key].call if options[key].respond_to?(:call)
+  }
+
+  trailing_args_as_input(options)
+
+  CMess::BConv.convert(
+    options[:input],
+    options[:output],
+    options[:source_encoding],
+    options[:target_encoding],
+    options[:chartab_file]
+  )
+end

data/bin/cinderella

@@ -0,0 +1,190 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# cinderella -- Handle double encoded characters                              #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'yaml'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess/cinderella'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+options = {
+  :input           => STDIN,
+  :output          => STDOUT,
+  :pot             => nil,
+  :crop            => nil,
+  :source_encoding => nil,
+  :target_encoding => determine_system_encoding,
+  :csets           => [CMess::Cinderella::DEFAULT_CSETS_DIR],
+  :repair          => false
+}
+
+OptionParser.new(nil, 40) { |opts|
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+    options[:input_set] = true
+  }
+
+  opts.separator ''
+
+  opts.on('-p', '--pot FILE', "The good into the pot...") { |f|
+    options[:pot] = open_file_or_std(f, 'w')
+  }
+
+  opts.on('-c', '--crop FILE', "...the bad into the crop") { |f|
+    options[:crop] = open_file_or_std(f, 'w')
+  }
+
+  opts.separator ''
+
+  opts.on('-o', '--output [FILE]', "Write both good and bad lines to FILE or", "default [Default: STDOUT] (Particularly", "useful in combination with the '-r' option)") { |f|
+    options[:output] = open_file_or_std(f, 'w') if f
+
+    options[:pot]  = options[:output]
+    options[:crop] = options[:output]
+  }
+
+  opts.separator ''
+
+  opts.on('-I', '--in-place FILE', "Modify file in-place; sets '-i' and '-o'", "(Only really useful in combination with", "the '-r' option)") { |f|
+    options[:input], options[:output] = open_file_in_place(f)
+    options[:input_set] = true
+
+    options[:pot]  = options[:output]
+    options[:crop] = options[:output]
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--source-encoding ENCODING', "Source encoding (from) [REQUIRED]") { |e|
+    options[:source_encoding] = e
+  }
+
+  opts.on('-t', '--target-encoding ENCODING', "Target encoding (to); see '-l' for a list", "of available encodings [Default: #{options[:target_encoding]}]") { |e|
+    options[:target_encoding] = e
+  }
+
+  opts.separator ''
+
+  opts.on('-T', '--addtl-target-encodings DIRECTORY', "Directory providing additional char files", "for target encoding") { |d|
+    ensure_directory(d)
+
+    options[:csets] |= [File.expand_path(d)]
+  }
+
+  opts.separator ''
+
+  opts.on('-l', '--list-encodings', "Display a list of available target encodings", "and exit; see '-T' on how to add your own") {
+    csets = options[:csets].inject({}) { |hash, cset|
+      encodings = Dir[File.join(cset, '*.yaml')].sort.map { |yaml|
+        File.basename(yaml, '.yaml') unless File.symlink?(yaml)
+      }.compact
+
+      hash[cset] = encodings unless encodings.empty?
+      hash
+    }
+
+    if csets.empty?
+      puts "No target encodings available for #{PROGNAME}"
+    else
+      puts "Available target encodings for #{PROGNAME}:"
+      csets.each { |cset, encodings|
+        puts "[#{cset}]"
+        encodings.each { |encoding|
+          puts "  - #{encoding}"
+        }
+      }
+    end
+
+    exit
+  }
+
+  opts.separator ''
+
+  opts.on('-r', '--repair', "Try to repair corrupted characters") {
+    options[:repair] = true
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CMess::Cinderella::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator "If '-p' or '-c' is omitted, and '-o' is not given either, that particular output"
+  opts.separator "is ignored. When FILE is -, either STDIN or STDOUT is used (as appropriate)."
+}.parse!
+
+cli do
+  options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
+
+  abort "No source encoding given! (Use the '-e' switch to do so; see '--help' for more information)" \
+    unless options[:source_encoding]
+
+  yaml_file = "#{options[:target_encoding].downcase}.yaml"
+  char_file = options[:csets].inject(nil) { |path, cset|
+    path = File.join(cset, yaml_file)
+    break path if File.readable?(path)
+  }
+
+  abort "Char file not found for target encoding: #{options[:target_encoding]}" \
+    unless char_file
+
+  trailing_args_as_input(options)
+
+  CMess::Cinderella.pick(
+    options[:input],
+    options[:pot],
+    options[:crop],
+    options[:source_encoding],
+    options[:target_encoding],
+    YAML.load_file(char_file),
+    options[:repair]
+  )
+end

data/bin/decode_entities

@@ -0,0 +1,106 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# decode_entities -- Decode HTML entities                                     #
+#   [A component of cmess, the encoding tool-box]                             #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# cmess is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# cmess is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with cmess. If not, see <http://www.gnu.org/licenses/>.                     #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cmess/decode_entities'
+require 'cmess/cli'
+
+include CMess::CLI
+
+PROGNAME = File.basename($0)
+
+options = {
+  :input           => STDIN,
+  :output          => STDOUT,
+  :source_encoding => CMess::DecodeEntities::INTERMEDIATE_ENCODING,
+  :target_encoding => nil
+}
+
+OptionParser.new { |opts|
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-i', '--input FILE', "Input file to read from [Default: STDIN]") { |f|
+    options[:input] = open_file_or_std(f)
+    options[:input_set] = true
+  }
+
+  opts.on('-o', '--output FILE', "Output file to write to [Default: STDOUT]") { |f|
+    options[:output] = open_file_or_std(f, 'w')
+  }
+
+  opts.on('-I', '--in-place FILE', "Modify file in-place; sets '-i' and '-o'") { |f|
+    options[:input], options[:output] = open_file_in_place(f)
+    options[:input_set] = true
+  }
+
+  opts.separator ''
+
+  opts.on('-e', '--source-encoding ENCODING', "Encoding of input file [Default: #{options[:source_encoding].upcase}]") { |e|
+    options[:source_encoding] = e.downcase
+  }
+
+  opts.on('-t', '--target-encoding ENCODING', "Desired encoding for output file [Default: <source_encoding>]") { |e|
+    options[:target_encoding] = e.downcase
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', "Print this help message and exit") {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', "Print program version and exit") {
+    puts "#{PROGNAME} v#{CMess::DecodeEntities::VERSION} (part of cmess v#{CMess::VERSION})"
+    exit
+  }
+
+  opts.separator ''
+  opts.separator "When FILE is -, either STDIN or STDOUT is used (as appropriate)."
+}.parse!
+
+cli do
+  trailing_args_as_input(options)
+
+  CMess::DecodeEntities.decode(
+    options[:input],
+    options[:output],
+    options[:source_encoding],
+    options[:target_encoding]
+  )
+end