sanzang 0.0.3 → 1.0.0

data/lib/sanzang/command/translate.rb

@@ -24,27 +24,23 @@ require_relative File.join("..", "version")
 
 module Sanzang::Command
 
-  # The Sanzang::Command::Reflow class provides a Unix-style command for
-  # text reformatting. This reformatting is typically for use prior to
-  # processing the text with the Sanzang::Command::Translate. The reason for
-  # this is to do initial text transformations to ensure (1) that terms will
-  # be translated reliably, and (2) that the final output of the translation
-  # will be readable by the user (i.e. lines not too long).
+  # This class provides a command for simple translation of one file or text.
+  # Input and output text can be read from either stdin and stdout, or from
+  # files. For mass translation of texts, see Sanzang::Command::Batch.
   #
   class Translate
 
     # Create a new instance of the Translate class.
     #
     def initialize
-      @name = "sanzang-translate"
+      @name = "sanzang translate"
       @encoding = nil
-      @batch_dir = nil
       @infile = nil
       @outfile = nil
     end
 
-    # Run the Translate command with the given arguments. The parameter _args_
-    # would typically be an Array of Unix-style command parameters. Calling
+    # Run the translate command with the given arguments. The parameter _args_
+    # would typically be an array of command options and parameters. Calling
     # this with the "-h" or "--help" option will print full usage information
     # necessary for running this command.
     #
@@ -53,7 +49,7 @@ module Sanzang::Command
       parser.parse!(args)
 
       if args.length != 1
-        puts parser
+        $stderr.puts parser
         return 1
       end
 
@@ -61,30 +57,22 @@ module Sanzang::Command
 
       translator = nil
       File.open(args[0], "rb", encoding: @encoding) do |table_file|
-        table = Sanzang::TranslationTable.new(table_file)
+        table = Sanzang::TranslationTable.new(table_file.read)
         translator = Sanzang::Translator.new(table)
       end
 
-      if @batch_dir != nil
-        $stderr.puts "Batch mode (#{translator.processor_count} processors)"
-        if not translator.runs_parallel?
-          warn 'Gem not available: "parallel"'
+      begin
+        fin = @infile ? File.open(@infile, "rb") : $stdin
+        fin.binmode.set_encoding(@encoding)
+        fout = @outfile ? File.open(@outfile, "wb") : $stdout
+        fout.binmode.set_encoding(@encoding)
+        translator.translate_io(fin, fout)
+      ensure
+        if defined?(fin) and fin != $stdin
+          fin.close if not fin.closed?
         end
-        puts translator.translate_batch($stdin.readlines, @batch_dir)
-      else
-        begin
-          fin = @infile ? File.open(@infile, "rb") : $stdin
-          fin.binmode.set_encoding(@encoding)
-          fout = @outfile ? File.open(@outfile, "wb") : $stdout
-          fout.binmode.set_encoding(@encoding)
-          translator.translate_io(fin, fout)
-        ensure
-          if defined?(fin) and fin != $stdin
-            fin.close if not fin.closed?
-          end
-          if defined?(fout) and fin != $stdout
-            fout.close if not fout.closed?
-          end
+        if defined?(fout) and fin != $stdout
+          fout.close if not fout.closed?
         end
       end
 
@@ -93,12 +81,14 @@ module Sanzang::Command
       return err.status
     rescue Exception => err
       $stderr.puts err.backtrace
-      $stderr.puts "ERROR: #{err.inspect}"
+      $stderr.puts "\nERROR: #{err.inspect}\n\n"
       return 1
     end
 
     private
 
+    # Initialize the encoding for text data if it is not already set
+    #
     def set_data_encoding
       if @encoding == nil
         if Encoding.default_external == Encoding::IBM437
@@ -110,58 +100,45 @@ module Sanzang::Command
       end
     end
 
+    # An OptionParser for the command
+    #
     def option_parser
-      OptionParser.new do |pr|
-        pr.banner = "Usage: #{@name} [options] table\n"
-        pr.banner << "Usage: #{@name} -B output_dir table < file_list\n"
-
-        pr.banner << "\nTranslate text using simple table rules. Input text "
-        pr.banner << "is read from STDIN by\ndefault, and the output is "
-        pr.banner << "written to STDOUT by default. In batch mode, the \n"
-        pr.banner << "program reads file paths from STDIN, and writes them "
-        pr.banner << "to an output directory.\n"
-
-        pr.banner << "\nExamples:\n"
-        pr.banner << "    #{@name} -i text.txt -o text.sz.txt table.txt\n"
-        pr.banner << "    #{@name} -B output_dir table.txt < myfiles.txt\n"
-        pr.banner << "\nOptions:\n"
-
-        pr.on("-h", "--help", "show this help message and exit") do |v|
-          puts pr
+      OptionParser.new do |op|
+        op.banner = "Usage: #{@name} [options] table\n"
+
+        op.banner << "\nTranslate text using simple table rules. Input text "
+        op.banner << "is read from STDIN by\ndefault, and the output is "
+        op.banner << "written to STDOUT by default.\n"
+
+        op.banner << "\nExample:\n"
+        op.banner << "    #{@name} -i text.txt -o text.sz.txt table.txt\n"
+        op.banner << "\nOptions:\n"
+
+        op.on("-h", "--help", "show this help message and exit") do |v|
+          puts op
           exit 0
         end
-        pr.on("-B", "--batch-dir=DIR", "process from a queue into DIR") do |v|
-          @batch_dir = v
-        end
-        pr.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
+        op.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
           @encoding = Encoding.find(v)
         end
-        pr.on("-L", "--list-encodings", "list possible encodings") do |v|
-          puts(Encoding.list.collect {|e| e.to_s }.sort)
+        op.on("-L", "--list-encodings", "list possible encodings") do |v|
+          encodings = Encoding.list.sort do |x,y|
+            x.to_s.upcase <=> y.to_s.upcase
+          end
+          puts encodings
           exit 0
         end
-        pr.on("-i", "--infile=FILE", "read input text from FILE") do |v|
+        op.on("-i", "--infile=FILE", "read input text from FILE") do |v|
           @infile = v
         end
-        pr.on("-o", "--outfile=FILE", "write output text to FILE") do |v|
+        op.on("-o", "--outfile=FILE", "write output text to FILE") do |v|
           @outfile = v
         end
-        pr.on("-P", "--platform", "show platform information") do |v|
-          puts "Ruby version: #{RUBY_VERSION}"
-          puts "Ruby platform: #{RUBY_PLATFORM}"
-          puts "External encoding: #{Encoding.default_external}"
-          if Encoding.default_internal != nil
-            puts "Internal encoding: #{Encoding.default_internal}"
-          end
-          exit 0
-        end
-        pr.on("-V", "--version", "show version number and exit") do |v|
-          puts "Sanzang version: #{Sanzang::VERSION}"
-          exit 0
-        end
       end
     end
 
+    # Name of the command
+    #
     attr_reader :name
 
   end

data/lib/sanzang/text_formatter.rb

@@ -36,7 +36,7 @@ module Sanzang
     # kept separate. Following this, all newlines are removed, and the text is
     # then reformatted according to the remaining punctuation and spacing.
     #
-    def reflow_cjk_text(s)
+    def reflow_cjk(s)
       source_encoding = s.encoding
       s.encode!(Encoding::UTF_8)
 

data/lib/sanzang/translation_table.rb

@@ -15,41 +15,30 @@
 #
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
-#
+
 module Sanzang
 
-  # TranslationTable encapsulates the set of rules used for translation by
-  # Sanzang::Translator. These rules may be loaded from a string passed in to
-  # the constructor, or loaded from an open IO object. The translation rules
-  # will then go through basic parsing to ensure the table data is in the
-  # correct format, and then the rules are reverse sorted by the length of the
-  # source language column. Thereafter, these rules are accessible through the
-  # ''records'' attribute, and metadata is available through other accessors
-  # and methods. It is the responsibility of Sanzang::Translator object to
-  # actually apply the rules of a TranslationTable to some text, as the table
-  # merely encapsulates a set of translation rules.
-  #
-  # The format for translation table data can be summarized as the following:
-  #
-  # * Plain text with one line per record
-  # * Records begin with "~|", end with "|~", and are delimited by "|".
-  # * The number of columns in each record must be consistent.
-  #
-  # An example of this format is the following:
-  #
-  #   ~|zh-term1|en-term1|~
-  #   ~|zh-term2|en-term2|~
-  #   ~|zh-term3|en-term3|~
+  # A translation table encapsulates a set of rules for translating with
+  # the \Sanzang system. These are essentially read-only objects meant for
+  # storing well-defined translation table data.
   #
   class TranslationTable
 
-    # Create a new TranslationTable object from a string or by reading an IO
-    # object. If the table parameter is a kind of string, then attempt to parse
-    # the table data from this string. Otherwise treat the parameter as an open
-    # IO object, and attempt to read the string data from that. After loading
-    # and verifying the contents of the translation table, all the records are
-    # reverse sorted by length, since this is the order in which they will be
-    # applied.
+    # A table is created from a formatted string of translation rules. The
+    # string is in the format of delimited text. The text format can be
+    # summarized as follows:
+    #
+    # - Each line of text is a record for a translation rule.
+    # - Each record begins with "~|" and ends with "|~".
+    # - Fields in the record are separated by the "|" character.
+    # - The first field contains the term in the source language.
+    # - Subsequent fields are equivalent terms in destination languages.
+    # - The number of columns must be consistent for the entire table.
+    #
+    # The first element in a record is a term in the source language, and
+    # subsequent elements are are equivalent terms in destination languages.
+    # The number of "columns" in a translation table must be consistent across
+    # the entire table.
     #
     def initialize(rules)
       contents = rules.kind_of?(String) ? rules : rules.read
@@ -60,52 +49,48 @@ module Sanzang
       separator = "|".encode(@encoding)
 
       @records = contents.gsub("\r", "").split("\n").collect do |rec|
-        rec = rec.strip.gsub(left, "").gsub(right, "").split(separator)
+        rec.strip.gsub(left, "").gsub(right, "").split(separator)
       end
 
-      if @records.length > 0
-        @width = records[0].length
-        0.upto(@records.length - 1) do |i|
-          if @records[i].length != @width
-            raise "Column mismatch: Line #{i + 1}"
-          end
+      @width = records[0].length
+      0.upto(@records.length - 1) do |i|
+        if @records[i].length != @width
+          raise "Column mismatch: Line #{i + 1}"
         end
-      else
-        @width = 0
       end
 
-      @records.sort! {|x,y| y.length <=> x.length }
+      @records.sort! {|x,y| y[0].length <=> x[0].length }
     end
 
-    # Retrieve a record by its numeric index. This is just shorthand for
-    # looking at the records attribute directly.
+    # Retrieve a record by its numeric index.
     #
     def [](index)
       @records[index]
     end
 
-    # Find the record where the source language field is equal to the given
-    # parameter.
+    # Find a record by the source language term (first column).
     #
     def find(term)
       @records.find {|rec| rec[0] == term }
     end
 
-    # The number of records in the translation table (the table length).
+    # The number of records in the table
     #
     def length
       @records.length
     end
 
-    # The number of columns in the translation table (the table width).
+    # The number of columns in the table
     #
-    attr_reader :width
+    def width
+      @records[0].length
+    end
 
-    # The records for the translation table, as an Array.
+    # The records for the translation table, as an array
     #
     attr_reader :records
 
-    # The text encoding used for all translation table data.
+    # The text encoding used for all translation table data
     #
     attr_reader :encoding
 

data/lib/sanzang/translator.rb

@@ -16,22 +16,13 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-begin
-  require "parallel"
-rescue LoadError
-  nil
-end
-
 module Sanzang
 
   # Translator is the main class for performing text translations with Sanzang.
   # A Translator utilizes a TranslationTable, which is passed to it at the time
   # of creation. The Translator can then apply these translation rules,
   # generate full translation listings, and perform translations by reading and
-  # writing to IO objects. Finally, Translator supports a batch mode that can
-  # utilize multiprocessing if the _Parallel_ module is available, and if the
-  # platform supports Kernel#fork. Methods are also available for querying the
-  # status of this functionality.
+  # writing to IO objects.
   #
   class Translator
 
@@ -43,28 +34,6 @@ module Sanzang
       @table = translation_table
     end
 
-    # Returns true if both the _Parallel_ module is available, and is also
-    # functioning on this particular implementation of Ruby. Currently the
-    # _mingw_ and _mswin_ ports of Ruby do not have Process#fork implemented.
-    #
-    def runs_parallel?
-      if not Process.respond_to?(:fork)
-        false
-      elsif defined?(Parallel) == "constant" and Parallel.class == Module
-        true
-      else
-        false
-      end
-    end
-
-    # Return the number of processors available on the current system. This
-    # will return the total number of logical processors, rather than physical
-    # processors.
-    #
-    def processor_count
-      runs_parallel? == true ? Parallel.processor_count : 1
-    end
-
     # Return an Array of all translation rules used by a particular text.
     # These records represent the vocabulary used by the text.
     #
@@ -133,39 +102,6 @@ module Sanzang
       output.close
     end
 
-    # Translate a list of files to some output directory. If the _verbose_
-    # parameter is true, then print progress to STDERR. If the value of
-    # Translator#runs_parallel? is false, then the batch is processed
-    # sequentially, only utilizing one processor. However, if the value is
-    # true, then run the batch by utilizing the Parallel module for efficient
-    # multiprocessing.
-    #
-    def translate_batch(fpath_list, out_dir, verbose = true)
-      fpath_list.collect! {|f| f.chomp }
-
-      if not runs_parallel?
-        fpath_list.each do |in_fpath|
-          out_fpath = File.join(out_dir, File.basename(in_fpath))
-          translate_io(in_fpath, out_fpath)
-          if verbose
-            $stderr.write "[#{Process.pid}] #{File.expand_path(out_fpath)} \n"
-            $stderr.flush
-          end
-          out_fpath
-        end
-      else
-        Parallel.map(fpath_list) do |in_fpath|
-          out_fpath = File.join(out_dir, File.basename(in_fpath))
-          translate_io(in_fpath, out_fpath)
-          if verbose
-            $stderr.write "[#{Process.pid}] #{File.expand_path(out_fpath)} \n"
-            $stderr.flush
-          end
-          out_fpath
-        end
-      end
-    end
-
     # The TranslationTable used by the Translator
     #
     attr_reader :table

data/lib/sanzang/version.rb

@@ -18,7 +18,8 @@
 
 module Sanzang
 
-  # The current version number of Sanzang.
-  VERSION = "0.0.3"
+  # Current version number of Sanzang
+  #
+  VERSION = "1.0.0"
 
 end

data/test/tc_reflow_encodings.rb

@@ -25,7 +25,7 @@ class TestReflowEncodings < Test::Unit::TestCase
     text_s1.encode!(encoding)
     text_s2.encode!(encoding)
     formatter = Sanzang::TextFormatter.new
-    assert_equal(text_s2, formatter.reflow_cjk_text(text_s1))
+    assert_equal(text_s2, formatter.reflow_cjk(text_s1))
   end
 
   # Han characters, simplified and without double vertical bar. The margin
@@ -38,7 +38,7 @@ class TestReflowEncodings < Test::Unit::TestCase
     text_s1.encode!(encoding)
     text_s2.encode!(encoding)
     formatter = Sanzang::TextFormatter.new
-    assert_equal(text_s2, formatter.reflow_cjk_text(text_s1))
+    assert_equal(text_s2, formatter.reflow_cjk(text_s1))
   end
 
   # UTF-8 (Traditional Chinese)

data/test/tc_simple_translation.rb

@@ -5,10 +5,6 @@ require "test/unit"
 
 require_relative File.join("..", "lib", "sanzang")
 
-# assert_nothing_raised
-# assert_equal(x, y)
-# assert(stmt, "Error message")
-#
 class TestSanzang < Test::Unit::TestCase
 
   def table_string
@@ -45,7 +41,7 @@ class TestSanzang < Test::Unit::TestCase
   def test_translation_table
     table_path = File.join(File.dirname(__FILE__), "utf-8", "table.txt")
     fin = File.open(table_path, "rb", encoding: "UTF-8")
-    table = Sanzang::TranslationTable.new(fin)
+    table = Sanzang::TranslationTable.new(fin.read)
     fin.close
     assert(table.width.class == Fixnum, "Table width undefined")
     assert(table.length.class == Fixnum, "Table length undefined")
@@ -60,7 +56,7 @@ class TestSanzang < Test::Unit::TestCase
   end
 
   def test_reflow_cjk_string
-    text = Sanzang::TextFormatter.new.reflow_cjk_text(stage_1())
+    text = Sanzang::TextFormatter.new.reflow_cjk(stage_1())
     assert_equal(stage_2(), text)
   end
 
@@ -74,22 +70,22 @@ class TestSanzang < Test::Unit::TestCase
     table_path = File.join(File.dirname(__FILE__), "utf-8", "table.txt")
     s2_path = File.join(File.dirname(__FILE__), "utf-8", "stage_2.txt")
     s3_path = File.join(File.dirname(__FILE__), "utf-8", "stage_3.txt")
-    table = Sanzang::TranslationTable.new(table_path)
+    table = Sanzang::TranslationTable.new(IO.read(table_path))
     translator = Sanzang::Translator.new(table)
     translator.translate_io(s2_path, s3_path)
   end
 
   def test_translator_parallel
     table = Sanzang::TranslationTable.new(table_string())
-    translator = Sanzang::Translator.new(table)
-    translator.runs_parallel?
-    assert(translator.processor_count > 0, "Processor count less than zero")
+    bt = Sanzang::BatchTranslator.new(table)
+    bt.forking?
+    assert(bt.processor_count > 0, "Processor count less than zero")
   end
 
   def test_translate_batch
     table = Sanzang::TranslationTable.new(table_string())
-    translator = Sanzang::Translator.new(table)
-    translator.translate_batch(
+    bt = Sanzang::BatchTranslator.new(table)
+    bt.translate_to_dir(
         Dir.glob(File.join(File.dirname(__FILE__), "utf-8", "file_*.txt")),
         File.join(File.dirname(__FILE__), "utf-8", "batch"), false)
   end