cmess 0.1.1.283 → 0.1.2.288

data/ChangeLog

@@ -1,5 +1,12 @@
 = Revision history for cmess
 
+== 0.1.2 [2008-09-17]
+
+* Some refactoring; started to make tools more usable as a library
+* Make tools accept multiple files for input (via tempfile)
+* Use ENV.user_encoding from ruby-nuggets
+* Wrap command execution in a block catching any exceptions
+
 == 0.1.1 [2008-09-16]
 
 * Added bconv tool to convert between bibliographic encodings

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to cmess version 0.1.1
+This documentation refers to cmess version 0.1.2
 
 
 == DESCRIPTION

data/Rakefile

@@ -17,7 +17,7 @@ begin
       },
       :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
       :extra_files  => FileList['[A-Z]*', 'example/**/*', 'data/**/*'].to_a,
-      :dependencies => %w[ruby-nuggets htmlentities]
+      :dependencies => [['ruby-nuggets', '>= 0.3.3'], 'htmlentities']
     }
   }}
 rescue LoadError

data/bin/bconv

@@ -29,7 +29,6 @@
 ###############################################################################
 #++
 
-require 'yaml'
 require 'optparse'
 
 require 'rubygems'
@@ -49,11 +48,11 @@ options = {
   :output          => STDOUT,
   :source_encoding => determine_system_encoding,
   :target_encoding => determine_system_encoding,
-  :chartab_file    => File.join(DATA_DIR, 'chartab.yaml')
+  :chartab_file    => CMess::BConv::DEFAULT_CHARTAB_FILE
 }
 
 OptionParser.new(nil, 40) { |opts|
-  opts.banner = "Usage: #{$0} [options] [FILE]"
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
 
   opts.separator ''
   opts.separator 'Options:'
@@ -84,7 +83,7 @@ OptionParser.new(nil, 40) { |opts|
 
   opts.separator ''
 
-  opts.on('-c', '--chartab PATH', "File containing character mappings [Default: #{options[:chartab_file]}]") { |c|
+  opts.on('-c', '--chartab YAML_FILE', "File containing character mappings, in YAML format.", "[Default: #{options[:chartab_file]}]") { |c|
     options[:chartab_file] = c
   }
 
@@ -109,28 +108,23 @@ OptionParser.new(nil, 40) { |opts|
   opts.separator "When FILE is -, either STDIN or STDOUT is used (as appropriate)."
 }.parse!
 
-abort "Chartab file not found: #{options[:chartab_file]}" \
-  unless File.readable?(options[:chartab_file])
-
-chartab = YAML.load_file(options[:chartab_file])
+cli do
+  if options[:list_encodings]
+    puts CMess::BConv.encodings(options[:chartab_file])
+    exit
+  end
 
-if options[:list_encodings]
-  puts CMess::BConv.encodings(chartab)
-  exit
-end
+  [:source_encoding, :target_encoding].each { |key|
+    options[key].call if options[key].respond_to?(:call)
+  }
 
-[:source_encoding, :target_encoding].each { |key|
-  options[key].call if options[key].respond_to?(:call)
-}
+  trailing_args_as_input(options)
 
-if ARGV[0] && !options[:input_set]
-  options[:input] = open_file_or_std(ARGV[0])
+  CMess::BConv.convert(
+    options[:input],
+    options[:output],
+    options[:source_encoding],
+    options[:target_encoding],
+    options[:chartab_file]
+  )
 end
-
-CMess::BConv.convert(
-  options[:input],
-  options[:output],
-  options[:source_encoding],
-  options[:target_encoding],
-  chartab
-)

data/bin/cinderella

@@ -48,12 +48,12 @@ options = {
   :crop            => nil,
   :source_encoding => nil,
   :target_encoding => determine_system_encoding,
-  :csets           => [File.join(DATA_DIR, 'csets')],
+  :csets           => [CMess::Cinderella::DEFAULT_CSETS_DIR],
   :repair          => false
 }
 
 OptionParser.new(nil, 40) { |opts|
-  opts.banner = "Usage: #{$0} [options] [FILE]"
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
 
   opts.separator ''
   opts.separator 'Options:'
@@ -161,29 +161,30 @@ OptionParser.new(nil, 40) { |opts|
   opts.separator "is ignored. When FILE is -, either STDIN or STDOUT is used (as appropriate)."
 }.parse!
 
-options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
+cli do
+  options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
 
-abort "No source encoding given! (Use the '-e' switch to do so; see '--help' for more information)" \
-  unless options[:source_encoding]
+  abort "No source encoding given! (Use the '-e' switch to do so; see '--help' for more information)" \
+    unless options[:source_encoding]
 
-yaml_file = "#{options[:target_encoding].downcase}.yaml"
-char_file = options[:csets].inject(nil) { |path, cset|
-  path = File.join(cset, yaml_file)
-  break path if File.readable?(path)
-}
-abort "Char file not found for target encoding: #{options[:target_encoding]}" \
-  unless char_file
+  yaml_file = "#{options[:target_encoding].downcase}.yaml"
+  char_file = options[:csets].inject(nil) { |path, cset|
+    path = File.join(cset, yaml_file)
+    break path if File.readable?(path)
+  }
 
-if ARGV[0] && !options[:input_set]
-  options[:input] = open_file_or_std(ARGV[0])
-end
+  abort "Char file not found for target encoding: #{options[:target_encoding]}" \
+    unless char_file
 
-CMess::Cinderella.pick(
-  options[:input],
-  options[:pot],
-  options[:crop],
-  options[:source_encoding],
-  options[:target_encoding],
-  YAML.load_file(char_file),
-  options[:repair]
-)
+  trailing_args_as_input(options)
+
+  CMess::Cinderella.pick(
+    options[:input],
+    options[:pot],
+    options[:crop],
+    options[:source_encoding],
+    options[:target_encoding],
+    YAML.load_file(char_file),
+    options[:repair]
+  )
+end

data/bin/decode_entities

@@ -48,7 +48,7 @@ options = {
 }
 
 OptionParser.new { |opts|
-  opts.banner = "Usage: #{$0} [options] [FILE]"
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
 
   opts.separator ''
   opts.separator 'Options:'
@@ -94,13 +94,13 @@ OptionParser.new { |opts|
   opts.separator "When FILE is -, either STDIN or STDOUT is used (as appropriate)."
 }.parse!
 
-if ARGV[0] && !options[:input_set]
-  options[:input] = open_file_or_std(ARGV[0])
-end
+cli do
+  trailing_args_as_input(options)
 
-CMess::DecodeEntities.decode(
-  options[:input],
-  options[:output],
-  options[:source_encoding],
-  options[:target_encoding]
-)
+  CMess::DecodeEntities.decode(
+    options[:input],
+    options[:output],
+    options[:source_encoding],
+    options[:target_encoding]
+  )
+end

data/bin/guess_encoding

@@ -46,9 +46,6 @@ PROGNAME = File.basename($0)
 # short-cut
 CGE = CMess::GuessEncoding
 
-# how to split list of arguments
-SPLIT_ARG_LIST_RE = /\s*[,\s]\s*/o
-
 options = {
   :input                => STDIN,
   :line                 => 1,
@@ -64,7 +61,7 @@ options = {
 }
 
 OptionParser.new(nil, 40) { |opts|
-  opts.banner = "Usage: #{$0} [options] [FILE]"
+  opts.banner = "Usage: #{$0} [options] [FILE...]"
 
   opts.separator ''
   opts.separator 'Options:'
@@ -193,39 +190,34 @@ OptionParser.new(nil, 40) { |opts|
   opts.separator "When FILE is -, STDIN is used."
 }.parse!
 
-if ARGV[0] && !options[:input_set]
-  options[:input] = open_file_or_std(ARGV[0])
-end
+cli do
+  trailing_args_as_input(options)
+
+  if options[:manual] || options[:charcodes]
+    options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
+
+    if charcodes = options[:charcodes]
+      base  = options[:octal] ? 8 : options[:decimal] ? 10 : 16
+      input = charcodes.map { |c| c.to_i(base).chr }.join
+    else
+      # reset line counter
+      $. = 0
+
+      input = options[:input].each { |line|
+        break line if $. == options[:line]
+      }
+
+      abort "Input was empty!" if $..zero?
+      abort "Line not found -- input has only #{$.} line#{'s' if $. != 1}" unless input.is_a?(String)
+    end
 
-if options[:manual]
-  options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
-
-  # reset line counter
-  $. = 0
-
-  input = options[:input].each { |line|
-    break line if $. == options[:line]
-  }
-  abort "Input was empty!" if $..zero?
-  abort "Line not found -- input has only #{$.} line#{'s' if $. != 1}" unless input.is_a?(String)
-
-  CGE::Manual.display(
-    input,
-    options[:target_encoding],
-    options[:encodings],
-    options[:additional_encodings]
-  )
-elsif charcodes = options[:charcodes]
-  options[:target_encoding].call if options[:target_encoding].respond_to?(:call)
-
-  base = options[:octal] ? 8 : options[:decimal] ? 10 : 16
-
-  CGE::Manual.display(
-    charcodes.map { |c| c.to_i(base).chr }.join,
-    options[:target_encoding],
-    options[:encodings],
-    options[:additional_encodings]
-  )
-else  # automatic
-  puts CGE::Automatic.guess(options[:input], options[:chunk_size], options[:ignore_bom])
+    CGE::Manual.display(
+      input,
+      options[:target_encoding],
+      options[:encodings],
+      options[:additional_encodings]
+    )
+  else  # automatic
+    puts CGE::Automatic.guess(options[:input], options[:chunk_size], options[:ignore_bom])
+  end
 end

data/lib/cmess.rb

@@ -41,6 +41,9 @@
 # decode_entities:: Decode HTML entities in a string. (see DecodeEntities)
 
 module CMess
+
+  DATA_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'data'))
+
 end
 
 require 'cmess/version'

data/lib/cmess/bconv.rb

@@ -26,67 +26,101 @@
 ###############################################################################
 #++
 
+require 'yaml'
 require 'iconv'
 require 'cmess'
 
 # Convert between bibliographic (and other) encodings.
 
-module CMess::BConv
-
-  extend self
+class CMess::BConv
 
   # our version ;-)
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 
   INTERMEDIATE_ENCODING = 'utf-8'
 
-  def encodings(chartab)
-    chartab[chartab.keys.first].keys.map { |encoding|
-      encoding.upcase unless encoding =~ /\A__/
-    }.compact.sort
+  DEFAULT_CHARTAB_FILE = File.join(CMess::DATA_DIR, 'chartab.yaml')
+
+  class << self
+
+    def encodings(chartab = DEFAULT_CHARTAB_FILE)
+      chartab = load_chartab(chartab)
+
+      chartab[chartab.keys.first].keys.map { |encoding|
+        encoding.upcase unless encoding =~ /\A__/
+      }.compact.sort
+    end
+
+    def convert(*args)
+      new(*args).convert
+    end
+
+    def load_chartab(chartab)
+      case chartab
+        when Hash
+          chartab
+        when String
+          raise "chartab file not found: #{chartab}" unless File.readable?(chartab)
+          YAML.load_file(chartab)
+        else
+          raise ArgumentError, "invalid chartab of type #{chartab.class}"
+      end
+    end
+
   end
 
-  def convert(input, output, source_encoding, target_encoding, chartab)
-    source_encoding.upcase!
-    target_encoding.upcase!
+  attr_reader :input, :output, :source_encoding, :target_encoding, :chartab, :encodings
 
-    encodings = self.encodings(chartab)
+  def initialize(input, output, source_encoding, target_encoding, chartab = DEFAULT_CHARTAB_FILE)
+    @input, @output = input, output
 
-    if encodings.include?(source_encoding)
-      if encodings.include?(target_encoding)
-        charmap = chartab.inject({}) { |hash, (code, map)|
+    @source_encoding = source_encoding.upcase
+    @target_encoding = target_encoding.upcase
+
+    @chartab = self.class.load_chartab(chartab)
+    @encodings = self.class.encodings(@chartab)
+  end
+
+  def encoding?(encoding)
+    encodings.include?(encoding)
+  end
+
+  def convert
+    if encoding?(source_encoding)
+      if encoding?(target_encoding)
+        @charmap = chartab.inject({}) { |hash, (code, map)|
           hash.update(map[source_encoding] => map[target_encoding].pack('U*'))
         }
 
-        input.each_byte { |byte|
-          output.print charmap[[byte]] || charmap[[byte, input.getc]]
+        input.each_byte { |char|
+          output.print map(char)
         }
       else
-        iconv = Iconv.new(target_encoding, INTERMEDIATE_ENCODING)
+        iconv = iconv_to
 
-        charmap = chartab.inject({}) { |hash, (code, map)|
+        @charmap = chartab.inject({}) { |hash, (code, map)|
           hash.update(map[source_encoding] => [code.to_i(16)].pack('U*'))
         }
 
-        input.each_byte { |byte|
-          output.print iconv.iconv(charmap[[byte]] || charmap[[byte, input.getc]])
+        input.each_byte { |char|
+          output.print iconv.iconv(map(char))
         }
       end
     else
-      if encodings.include?(target_encoding)
-        iconv = Iconv.new(INTERMEDIATE_ENCODING, source_encoding)
+      if encoding?(target_encoding)
+        iconv = iconv_from
 
         charmap = chartab.inject({}) { |hash, (code, map)|
           hash.update(code.to_i(16) => map[target_encoding].pack('U*'))
         }
 
         input.each { |line|
-          iconv.iconv(line).unpack('U*').each { |byte|
-            output.print charmap[byte]
+          iconv.iconv(line).unpack('U*').each { |char|
+            output.print charmap[char]
           }
         }
       else
-        iconv = Iconv.new(target_encoding, source_encoding)
+        iconv = iconv_from_to
 
         input.each { |line|
           output.puts iconv.iconv(line)
@@ -95,4 +129,41 @@ module CMess::BConv
     end
   end
 
+  private
+
+  def iconv_from_to(from = source_encoding, to = target_encoding)
+    iconv = begin
+      Iconv.new(to, from)
+    rescue Iconv::InvalidEncoding
+      raise ArgumentError, "invalid encoding: source encoding = #{from}, target encoding = #{to}"
+    end
+
+    def iconv.iconv(*args)
+      super
+    rescue Iconv::IllegalSequence, Iconv::InvalidCharacter => err
+      warn "ILLEGAL INPUT SEQUENCE: #{err}"; ''
+    end
+
+    iconv
+  end
+
+  def iconv_from(from = source_encoding)
+    iconv_from_to(from, INTERMEDIATE_ENCODING)
+  end
+
+  def iconv_to(to = target_encoding)
+    iconv_from_to(INTERMEDIATE_ENCODING, to)
+  end
+
+  def map(char, charmap = @charmap)
+    unless map = charmap[[char]]
+      unless map = charmap[[char, c = input.getc]]
+        input.ungetc(c) if c
+        map = ''
+      end
+    end
+
+    map
+  end
+
 end

data/lib/cmess/cinderella.rb

@@ -43,6 +43,8 @@ module CMess::Cinderella
   # our version ;-)
   VERSION = '0.0.3'
 
+  DEFAULT_CSETS_DIR = File.join(CMess::DATA_DIR, 'csets')
+
   def pick(input, pot, crop, source_encoding, target_encoding, chars, repair = false)
     iconv = Iconv.new(target_encoding, source_encoding)
 

data/lib/cmess/cli.rb

@@ -26,9 +26,15 @@
 ###############################################################################
 #++
 
+require 'tempfile'
+
+require 'rubygems'
+require 'nuggets/env/user_encoding'
+
 module CMess::CLI
 
-  DATA_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'data'))
+  # how to split list of arguments
+  SPLIT_ARG_LIST_RE = /\s*[,\s]\s*/o
 
   def ensure_readable(file)
     abort "Can't find input file: #{file}" unless File.readable?(file)
@@ -39,8 +45,7 @@ module CMess::CLI
   end
 
   def open_file_in_place(file)
-    ensure_readable(file)
-    [File.readlines(file), File.open(file, 'w')]
+    [open_temporary_input(file), File.open(file, 'w')]
   end
 
   def open_file_or_std(file, mode = 'r')
@@ -57,25 +62,59 @@ module CMess::CLI
     end
   end
 
-  def determine_system_encoding
-    ENV['SYSTEM_ENCODING']   ||
-    ENV['LANG'][/\.(.*)/, 1] ||
-    system_encoding_not_found
-  end
+  def open_temporary_input(*files)
+    temp = Tempfile.new('cmess_cli')
 
-  def system_encoding_not_found
-    not_found = lambda {
-      abort <<-EOT
-Your system's encoding couldn't be determined automatically -- please specify it
-explicitly via the SYSTEM_ENCODING environment variable or via the '-t' option.
-      EOT
+    files.each { |file|
+      if file == '-'
+        STDIN.each { |line| temp << line }
+      else
+        ensure_readable(file)
+        File.open(file) { |f| f.each { |line| temp << line } }
+      end
     }
 
-    def not_found.to_s
-      'NOT FOUND'
+    # return File, instead of Tempfile
+    temp.close
+    temp.open
+  end
+
+  def trailing_args_as_input(options)
+    unless ARGV.empty? || options[:input_set]
+      options[:input] = if ARGV.size == 1
+        open_file_or_std(ARGV.first)
+      else
+        open_temporary_input(*ARGV)
+      end
+    end
+  end
+
+  def determine_system_encoding
+    ENV.user_encoding || begin
+      dummy = lambda {
+        abort <<-EOT
+Your system's encoding couldn't be determined automatically -- please specify
+it explicitly via the ENCODING environment variable or via the '-t' option.
+        EOT
+      }
+
+      def dummy.to_s; 'NOT FOUND' end
+
+      dummy
     end
+  end
+
+  def cli
+    yield
+  rescue => err
+    if $VERBOSE
+      backtrace = err.backtrace
+      fromtrace = backtrace[1..-1].map { |i| "\n        from #{i}" }
 
-    not_found
+      abort "#{backtrace.first} #{err} (#{err.class})#{fromtrace}"
+    else
+      abort "#{err.to_s.capitalize} [#{err.backtrace.first}]"
+    end
   end
 
 end

data/lib/cmess/guess_encoding/automatic.rb

@@ -273,11 +273,11 @@ class CMess::GuessEncoding::Automatic
   # UTF-8, if number of escape-bytes and following bytes
   # is matching (cf. http://en.wikipedia.org/wiki/UTF-8)
   encoding UTF_8 do
-    esc_bytes = byte_count_sum(0xc0..0xdf)     \
+    esc_bytes = byte_count_sum(0xc0..0xdf)     +
                 # => 110xxxxx 10xxxxxx
-              + byte_count_sum(0xe0..0xef) * 2 \
+                byte_count_sum(0xe0..0xef) * 2 +
                 # => 1110xxxx 10xxxxxx 10xxxxxx
-              + byte_count_sum(0xf0..0xf7) * 3
+                byte_count_sum(0xf0..0xf7) * 3
                 # => 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
     fol_bytes = byte_count_sum(0x80..0xbf)
                 # => 10xxxxxx
@@ -287,7 +287,7 @@ class CMess::GuessEncoding::Automatic
 
   # Analyse statistical appearance of German umlauts and other accented
   # letters (see TEST_CHARS)
-  encodings *TEST_ENCODINGS do
+  encodings(*TEST_ENCODINGS) do
     ratios = {}
 
     TEST_ENCODINGS.find(lambda {

data/lib/cmess/guess_encoding/manual.rb

@@ -95,7 +95,7 @@ module CMess::GuessEncoding::Manual
         "ILLEGAL INPUT SEQUENCE: #{err}"
       rescue Iconv::InvalidEncoding
         if encoding == target
-          abort "Invalid encoding: #{encoding}"
+          raise ArgumentError, "invalid encoding: #{encoding}"
         else
           "INVALID ENCODING!"
         end

data/lib/cmess/version.rb

@@ -30,7 +30,7 @@ module CMess::Version
 
   MAJOR = 0
   MINOR = 1
-  TINY  = 1
+  TINY  = 2
 
   class << self
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cmess
 version: !ruby/object:Gem::Version 
-  version: 0.1.1.283
+  version: 0.1.2.288
 platform: ruby
 authors: 
 - Jens Wille
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-09-16 00:00:00 +02:00
+date: 2008-09-17 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.3.3
     version: 
 - !ruby/object:Gem::Dependency 
   name: htmlentities
@@ -102,6 +102,7 @@ has_rdoc: true
 homepage: http://prometheus.rubyforge.org/cmess
 post_install_message: 
 rdoc_options: 
+- --line-numbers
 - --inline-source
 - --title
 - cmess Application documentation
@@ -110,7 +111,6 @@ rdoc_options:
 - --main
 - README
 - --all
-- --line-numbers
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement