sanzang 0.0.1

data/lib/sanzang/command/translate.rb

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+
+require_relative File.join("..", "translation_table")
+require_relative File.join("..", "translator")
+require_relative File.join("..", "version")
+
+module Sanzang::Command
+
+  # The Sanzang::Command::Reflow class provides a Unix-style command for
+  # text reformatting. This reformatting is typically for use prior to
+  # processing the text with the Sanzang::Command::Translate. The reason for
+  # this is to do initial text transformations to ensure (1) that terms will
+  # be translated reliably, and (2) that the final output of the translation
+  # will be readable by the user (i.e. lines not too long).
+  #
+  class Translate
+
+    # Create a new instance of the Translate class.
+    #
+    def initialize
+      @name = "sanzang-translate"
+      @encoding = nil
+      @batch_dir = nil
+      @infile = nil
+      @outfile = nil
+    end
+
+    # Run the Translate command with the given arguments. The parameter _args_
+    # would typically be an Array of Unix-style command parameters. Calling
+    # this with the "-h" or "--help" option will print full usage information
+    # necessary for running this command.
+    #
+    def run(args)
+      parser = option_parser
+      parser.parse!(args)
+
+      if args.length != 1
+        puts parser
+        return 1
+      end
+
+      set_data_encoding
+
+      translator = nil
+      File.open(args[0], "rb", encoding: @encoding) do |table_file|
+        table = Sanzang::TranslationTable.new(table_file)
+        translator = Sanzang::Translator.new(table)
+      end
+
+      if @batch_dir != nil
+        $stderr.puts "Batch mode (#{translator.processor_count} processors)"
+        if not translator.runs_parallel?
+          warn 'Gem not available: "parallel"'
+        end
+        puts translator.translate_batch($stdin.readlines, @batch_dir)
+      else
+        begin
+          fin = @infile ? File.open(@infile, "rb") : $stdin
+          fin.binmode.set_encoding(@encoding)
+          fout = @outfile ? File.open(@outfile, "wb") : $stdout
+          fout.binmode.set_encoding(@encoding)
+          translator.translate_io(fin, fout)
+        ensure
+          if defined?(fin) and fin != $stdin
+            fin.close if not fin.closed?
+          end
+          if defined?(fout) and fin != $stdout
+            fout.close if not fout.closed?
+          end
+        end
+      end
+
+      return 0
+    rescue SystemExit => err
+      return err.status
+    rescue Exception => err
+      $stderr.puts err.backtrace
+      $stderr.puts "ERROR: #{err.inspect}"
+      return 1
+    end
+
+    private
+
+    def set_data_encoding
+      if @encoding == nil
+        if Encoding.default_external == Encoding::IBM437
+          $stderr.puts "Switching to UTF-8 for text data encoding."
+          @encoding = Encoding::UTF_8
+        else
+          @encoding = Encoding.default_external
+        end
+      end
+    end
+
+    def option_parser
+      OptionParser.new do |pr|
+        pr.banner = "Usage: #{@name} [options] table\n"
+        pr.banner << "Usage: #{@name} -B output_dir table < file_list\n"
+
+        pr.banner << "\nTranslate text using simple table rules. Input text "
+        pr.banner << "is read from STDIN by\ndefault, and the output is "
+        pr.banner << "written to STDOUT by default. In batch mode, the \n"
+        pr.banner << "program reads file paths from STDIN, and writes them "
+        pr.banner << "to an output directory.\n"
+
+        pr.banner << "\nExamples:\n"
+        pr.banner << "    #{@name} -i text.txt -o text.sz.txt table.txt\n"
+        pr.banner << "    #{@name} -B table.txt output_dir < myfiles.txt\n"
+        pr.banner << "\nOptions:\n"
+
+        pr.on("-h", "--help", "show this help message and exit") do |v|
+          puts pr
+          exit 0
+        end
+        pr.on("-B", "--batch-dir=DIR", "process from a queue into DIR") do |v|
+          @batch_dir = v
+        end
+        pr.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
+          @encoding = Encoding.find(v)
+        end
+        pr.on("-L", "--list-encodings", "list possible encodings") do |v|
+          puts(Encoding.list.collect {|e| e.to_s }.sort)
+          exit 0
+        end
+        pr.on("-i", "--infile=FILE", "read input text from FILE") do |v|
+          @infile = v
+        end
+        pr.on("-o", "--outfile=FILE", "write output text to FILE") do |v|
+          @outfile = v
+        end
+        pr.on("-P", "--platform", "show platform information") do |v|
+          puts "Ruby version: #{RUBY_VERSION}"
+          puts "Ruby platform: #{RUBY_PLATFORM}"
+          puts "External encoding: #{Encoding::default_external}"
+          if Encoding::default_internal != nil
+            puts "Internal encoding: #{Encoding::default_internal}"
+          end
+          exit 0
+        end
+        pr.on("-V", "--version", "show version number and exit") do |v|
+          puts "Sanzang version: #{Sanzang::VERSION}"
+          exit 0
+        end
+      end
+    end
+
+    attr_reader :name
+
+  end
+end

data/lib/sanzang/text_formatter.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+module Sanzang
+
+  # This class handles formatting of text data especially to prepare the text
+  # for direct translation. This involves reformatting and reflowing text so
+  # that words are not divided between lines, and so the output is well suited
+  # for humans. For practical purposes of readability, lines of text to be
+  # translated should be succinct and easily comprehensible. The TextFormatter
+  # class includes methods for accomplishing this reformatting.
+  #
+  class TextFormatter
+
+    # Given a CJK string of text, reformat the string for greater compatibility
+    # with direct translation, and reflow the text based on its punctuation.
+    # The first step of this reformatting is to remove any CBETA-style margins
+    # at the beginning of each line, which are indicated by the double-bar
+    # character ("║" U+2551). An extra space is then inserted after each short
+    # line which may indicate that the line is part of a poem, and should be
+    # kept separate. Following this, all newlines are removed, and the text is
+    # then reformatted according to the remaining punctuation and spacing.
+    #
+    def reflow_cjk_text(s)
+      source_encoding = s.encoding
+      s.encode!(Encoding::UTF_8)
+
+      # Strip all CBETA-style margins
+      s.gsub!(/^.*║/, "")
+
+      # Starts with Hanzi space and short line: add Hanzi space at the end.
+      # This is used for avoiding conflicts between poetry and prose.
+      s.gsub!(/^(　)(.{1,15})$/, "\\1\\2　")
+
+      # Collapse all vertical whitespace.
+      using_crlf = s.include?("\r")
+      s.gsub!(/(\r|\n)/, "")
+
+      # Ender followed by non-ender: newline in between.
+      s.gsub!(/([：，；。？！」』.;:\?])([^：，；。？！」』.;:\?])/,
+        "\\1\n\\2")
+
+      # Non-starter, non-ender, followed by a starter: newline in between.
+      s.gsub!(/([^「『　\t：，；。？！」』.;:\?\n])([「『　\t])/,
+        "\\1\n\\2")
+
+      if s[-1] != "\n"
+        s << "\n"
+      end
+
+      s.gsub!("\n", "\r\n") if using_crlf
+      s.encode!(source_encoding)
+    end
+
+  end
+end

data/lib/sanzang/translation_table.rb

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+#
+module Sanzang
+
+  # TranslationTable encapsulates the set of rules used for translation by
+  # Sanzang::Translator. These rules may be loaded from a string passed in to
+  # the constructor, or loaded from an open IO object. The translation rules
+  # will then go through basic parsing to ensure the table data is in the
+  # correct format, and then the rules are reverse sorted by the length of the
+  # source language column. Thereafter, these rules are accessible through the
+  # ''records'' attribute, and metadata is available through other accessors
+  # and methods. It is the responsibility of Sanzang::Translator object to
+  # actually apply the rules of a TranslationTable to some text, as the table
+  # merely encapsulates a set of translation rules.
+  #
+  # The format for translation table data can be summarized as the following:
+  #
+  # * Plain text with one line per record
+  # * Records begin with "~|", end with "|~", and are delimited by "|".
+  # * The number of columns in each record must be consistent.
+  #
+  # An example of this format is the following:
+  #
+  #   ~|zh-term1|en-term1|~
+  #   ~|zh-term2|en-term2|~
+  #   ~|zh-term3|en-term3|~
+  #
+  class TranslationTable
+
+    # Create a new TranslationTable object from a string or by reading an IO
+    # object. If the table parameter is a kind of string, then attempt to parse
+    # the table data from this string. Otherwise treat the parameter as an open
+    # IO object, and attempt to read the string data from that. After loading
+    # and verifying the contents of the translation table, all the records are
+    # reverse sorted by length, since this is the order in which they will be
+    # applied.
+    #
+    def initialize(rules)
+      contents = rules.kind_of?(String) ? rules : rules.read
+      @encoding = contents.encoding
+
+      left = "~|".encode(@encoding)
+      right = "|~".encode(@encoding)
+      separator = "|".encode(@encoding)
+
+      @records = contents.gsub("\r", "").split("\n").collect do |rec|
+        rec = rec.strip.gsub(left, "").gsub(right, "").split(separator)
+      end
+
+      if @records.length > 0
+        @width = records[0].length
+        0.upto(@records.length - 1) do |i|
+          if @records[i].length != @width
+            raise "Column mismatch: Line #{i + 1}"
+          end
+        end
+      else
+        @width = 0
+      end
+
+      @records.sort! {|x,y| y.length <=> x.length }
+    end
+
+    # Retrieve a record by its numeric index. This is just shorthand for
+    # looking at the records attribute directly.
+    #
+    def [](index)
+      @records[index]
+    end
+
+    # Find the record where the source language field is equal to the given
+    # parameter.
+    #
+    def find(term)
+      @records.find {|rec| rec[0] == term }
+    end
+
+    # The number of records in the translation table (the table length).
+    #
+    def length
+      @records.length
+    end
+
+    # The number of columns in the translation table (the table width).
+    #
+    attr_reader :width
+
+    # The records for the translation table, as an Array.
+    #
+    attr_reader :records
+
+    # The text encoding used for all translation table data.
+    #
+    attr_reader :encoding
+
+  end
+end

data/lib/sanzang/translator.rb

@@ -0,0 +1,174 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+begin
+  require "parallel"
+rescue LoadError
+  nil
+end
+
+module Sanzang
+
+  # Translator is the main class for performing text translations with Sanzang.
+  # A Translator utilizes a TranslationTable, which is passed to it at the time
+  # of creation. The Translator can then apply these translation rules,
+  # generate full translation listings, and perform translations by reading and
+  # writing to IO objects. Finally, Translator supports a batch mode that can
+  # utilize multiprocessing if the _Parallel_ module is available, and if the
+  # platform supports Kernel#fork. Methods are also available for querying the
+  # status of this functionality.
+  #
+  class Translator
+
+    # Creates a new Translator object with the given TranslationTable. The
+    # TranslationTable stores rules for translation, while the Translator is
+    # the worker who applies these rules and can create translation listings.
+    #
+    def initialize(translation_table)
+      @table = translation_table
+    end
+
+    # Returns true if both the _Parallel_ module is available, and is also
+    # functioning on this particular implementation of Ruby. Currently the
+    # _mingw_ and _mswin_ ports of Ruby do not have Process#fork implemented.
+    #
+    def runs_parallel?
+      if not Process.respond_to?(:fork)
+        false
+      elsif defined?(Parallel) == "constant" and Parallel.class == Module
+        true
+      else
+        false
+      end
+    end
+
+    # Return the number of processors available on the current system. This
+    # will return the total number of logical processors, rather than physical
+    # processors.
+    #
+    def processor_count
+      runs_parallel? == true ? Parallel.processor_count : 1
+    end
+
+    # Return an Array of all translation rules used by a particular text.
+    # These records represent the vocabulary used by the text.
+    #
+    def text_vocab(source_text)
+      new_table = []
+      @table.records.each do |record|
+        if source_text.include?(record[0])
+          new_table << record
+        end
+      end
+      new_table
+    end
+
+    # Use the TranslationTable of the Translator to create translations for
+    # each destination language column of the translation table. These
+    # result is a simple Array of String objects, with each String object
+    # corresponding to a destination language column in the TranslationTable.
+    #
+    def translate(source_text)
+      text_collection = [source_text]
+      vocab_terms = text_vocab(source_text)
+      1.upto(@table.width - 1) do |column_i|
+        translation = String.new(source_text)
+        vocab_terms.each do |term|
+          translation.gsub!(term[0], term[column_i])
+        end
+        text_collection << translation
+      end
+      text_collection
+    end
+
+    # Generate a translation listing text string, in which the output of
+    # Translator#translate is collated and numbered for reference purposes.
+    # This is the normal text listing output of the Sanzang Translator.
+    #
+    def gen_listing(source_text)
+      newline = source_text.include?("\r") ? "\r\n" : "\n"
+      texts = translate(source_text).collect {|t| t = t.split(newline) }
+      listing = "".encode(source_text.encoding)
+
+      texts[0].length.times do |line_i|
+        @table.width.times do |col_i|
+          listing << "[#{line_i + 1}.#{col_i + 1}] #{texts[col_i][line_i]}" \
+                  << newline
+        end
+        listing << newline
+      end
+      listing
+    end
+
+    # Read a text from _input_ and write its translation listing to _output_.
+    # The parameters _input_ and _output_ can be either String objects or IO
+    # objects. If they are strings, then they are interpreted as being file
+    # paths. If they are not strings, then the I/O operations are performed on
+    # them directly.
+    #
+    def translate_io(input, output)
+      if input.class == String
+        input = File.open(input, "r", external_encoding: @table.encoding)
+      end
+      if output.class == String
+        output = File.open(output, "w", external_encoding: @table.encoding)
+      end
+      output.write(gen_listing(input.read))
+      input.close
+      output.close
+    end
+
+    # Translate a list of files to some output directory. If the _verbose_
+    # parameter is true, then print progress to STDERR. If the value of
+    # Translator#runs_parallel? is false, then the batch is processed
+    # sequentially, only utilizing one processor. However, if the value is
+    # true, then run the batch by utilizing the Parallel module for efficient
+    # multiprocessing.
+    #
+    def translate_batch(fpath_list, out_dir, verbose = true)
+      fpath_list.collect! {|f| f.chomp }
+
+      if not runs_parallel?
+        fpath_list.each do |in_fpath|
+          out_fpath = File.join(out_dir, File.basename(in_fpath))
+          translate_io(in_fpath, out_fpath)
+          if verbose
+            $stderr.write "[#{Process.pid}] #{File.expand_path(out_fpath)} \n"
+            $stderr.flush
+          end
+          out_fpath
+        end
+      else
+        Parallel.map(fpath_list) do |in_fpath|
+          out_fpath = File.join(out_dir, File.basename(in_fpath))
+          translate_io(in_fpath, out_fpath)
+          if verbose
+            $stderr.write "[#{Process.pid}] #{File.expand_path(out_fpath)} \n"
+            $stderr.flush
+          end
+          out_fpath
+        end
+      end
+    end
+
+    # The TranslationTable used by the Translator
+    #
+    attr_reader :table
+
+  end
+end