sanzang 0.0.3 → 1.0.0

data/bin/{sanzang-reflow → sanzang}

@@ -18,4 +18,4 @@
 
 require_relative File.join("..", "lib", "sanzang")
 
-Kernel.exit(Sanzang::Command::Reflow.new.run(ARGV))
+Kernel.exit(Sanzang::Command::SanzangCmd.new.run(ARGV))

data/lib/sanzang.rb

@@ -1,19 +1,6 @@
 #!/usr/bin/env ruby -w
 # -*- encoding: UTF-8 -*-
-
-# == Description
-#
-# The Sanzang module contains a basic infrastructure for machine translation
-# using a simple direct translation method that does not attempt to change the
-# underlying grammar of the source text. The Sanzang module also contains
-# functionality for preparing source texts by reformatting them in a manner
-# that will facilitates both machine translation as well as the readability of
-# the final translation listing that is generated. All program source code for
-# the Sanzang system is contained within the Sanzang module, with code for the
-# Sanzang commands being located in the Sanzang::Command module.
-#
-# == Copyright
-#
+#--
 # Copyright (C) 2012 Lapis Lazuli Texts
 #
 # This program is free software: you can redistribute it and/or modify it under
@@ -28,38 +15,28 @@
 #
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
+
+# All program source code for the translation system is contained under the
+# Sanzang module, and code for the \Sanzang commands is located in the
+# Sanzang::Command module.
 #
-module Sanzang; end
+module Sanzang
+end
 
 require_relative File.join("sanzang", "text_formatter")
 require_relative File.join("sanzang", "translation_table")
 require_relative File.join("sanzang", "translator")
+require_relative File.join("sanzang", "batch_translator")
 require_relative File.join("sanzang", "version")
 
-# == Description
-#
 # The Sanzang::Command module contains Unix style commands utilizing the
 # Sanzang module. Each class is typically a different command, with usage
 # information given when running the command with the "-h" or "--help" options.
 #
-# == Copyright
-#
-# Copyright (C) 2012 Lapis Lazuli Texts
-#
-# This program is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation, either version 3 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program. If not, see <http://www.gnu.org/licenses/>.
-#
-module Sanzang::Command; end
+module Sanzang::Command
+end
 
+require_relative File.join("sanzang", "command", "batch")
 require_relative File.join("sanzang", "command", "reflow")
+require_relative File.join("sanzang", "command", "sanzang_cmd")
 require_relative File.join("sanzang", "command", "translate")

data/lib/sanzang/batch_translator.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "parallel"
+
+require_relative "translator"
+
+module Sanzang
+
+  # BatchTranslator can handle batches of files for translation, and may also
+  # be able to translate them in parallel using multiprocessing, if your Ruby
+  # virtual machine supports it. This class inherits from Translator.
+  #
+  class BatchTranslator < Translator
+
+    # Evaluates to true if this Ruby can execute the fork(2) system call.
+    #
+    def forking?
+      Process.respond_to?(:fork)
+    end
+
+    # The number of logical processors detected on the current system.
+    #
+    def processor_count
+      Parallel.processor_count
+    end
+
+    # Translate a batch of files. The main parameter is an array, each element
+    # of which should be a two-dimensional array with the first element being
+    # the input file path, and the second element being the output file path.
+    # If the _verbose_ parameter is true, then print progress to STDERR. The
+    # return value is an array containing all the output file paths.
+    #
+    def translate_batch(fpath_pairs, verbose = true, jobs = nil)
+      if not forking?
+        jobs = 0
+      end
+      Parallel.map(fpath_pairs, :in_processes => jobs) do |f1,f2|
+        translate_io(f1, f2)
+        if verbose
+          $stderr.write "[#{Process.pid}] #{File.expand_path(f2)} \n"
+          $stderr.flush
+        end
+        f2
+      end
+    end
+
+    # Translate a list of files to some output directory. The names of the
+    # files written to the output directory will be the same as those of their
+    # respective input files. If the _verbose_ parameter is true, then print
+    # progress to STDERR.
+    #
+    def translate_to_dir(in_fpaths, out_dir, verbose = true, jobs = nil)
+      pairs = []
+      in_fpaths.each do |f1|
+        pairs << [f1, File.join(out_dir, File.basename(f1))]
+      end
+      translate_batch(pairs, verbose, jobs)
+    end
+
+  end
+end

data/lib/sanzang/command/batch.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+
+require_relative File.join("..", "translation_table")
+require_relative File.join("..", "batch_translator")
+require_relative File.join("..", "version")
+
+module Sanzang::Command
+
+  # This class implements a command for batch translation of texts. The command
+  # presumes that the list of input files will be read from $stdin, while the
+  # output files will be written to a single directory. Usage information can
+  # be accessed by passing in the "-h" or "--help" options.
+  #
+  class Batch
+
+    # Create a new instance of the batch command.
+    #
+    def initialize
+      @name = "sanzang batch"
+      @encoding = nil
+      @outdir = nil
+      @jobs = nil
+    end
+
+    # Run the batch command with the given arguments. The parameter _args_
+    # would typically be an array of command options and parameters. Calling
+    # this method with the "-h" or "--help" option will print full usage
+    # information necessary for running the command. This method will return
+    # either 0 (success) or 1 (failure).
+    #
+    def run(args)
+      parser = option_parser
+      parser.parse!(args)
+
+      if args.length != 2
+        $stderr.puts parser
+        return 1
+      end
+
+      set_data_encoding
+
+      translator = nil
+      File.open(args[0], "rb", encoding: @encoding) do |table_file|
+        table = Sanzang::TranslationTable.new(table_file.read)
+        translator = Sanzang::BatchTranslator.new(table)
+      end
+
+      $stdin.binmode.set_encoding(@encoding)
+      puts translator.translate_to_dir($stdin.read.split, args[1], true, @jobs)
+      return 0
+    rescue SystemExit => err
+      return err.status
+    rescue Exception => err
+      $stderr.puts err.backtrace
+      $stderr.puts "\nERROR: #{err.inspect}\n\n"
+      return 1
+    end
+
+    private
+
+    # Set the encoding for text data if it is not already set
+    #
+    def set_data_encoding
+      if @encoding == nil
+        if Encoding.default_external == Encoding::IBM437
+          $stderr.puts "Switching to UTF-8 for text data encoding."
+          @encoding = Encoding::UTF_8
+        else
+          @encoding = Encoding.default_external
+        end
+      end
+    end
+
+    # Return an OptionParser object for this command
+    #
+    def option_parser
+      OptionParser.new do |op|
+        op.banner = "Usage: #{@name} [options] table output_dir < queue\n"
+
+        op.banner << "\nBatch translate files concurrently. A list of files "
+        op.banner << "is read from STDIN, while\nprogress information is "
+        op.banner << "printed to STDERR. The list of output files written is\n"
+        op.banner << "printed to STDOUT at the end of the batch. The "
+        op.banner << "output directory is specified as\na parameter.\n"
+
+        op.banner << "\nOptions:\n"
+
+        op.on("-h", "--help", "show this help message and exit") do |v|
+          puts op
+          exit 0
+        end
+        op.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
+          @encoding = Encoding.find(v)
+        end
+        op.on("-L", "--list-encodings", "list possible encodings") do |v|
+          encodings = Encoding.list.sort do |x,y|
+            x.to_s.upcase <=> y.to_s.upcase
+          end
+          puts encodings
+          exit 0
+        end
+        op.on("-j", "--jobs=N", "allow N concurrent processes") do |v|
+          @jobs = v.to_i
+        end
+      end
+    end
+
+    # Name of the command
+    #
+    attr_reader :name
+
+  end
+end

data/lib/sanzang/command/reflow.rb

@@ -23,26 +23,26 @@ require_relative File.join("..", "version")
 
 module Sanzang::Command
 
-  # The Sanzang::Command::Reflow class provides a Unix-style command for
-  # text reformatting. This reformatting is typically for use prior to
-  # processing the text with the Sanzang::Command::Translate. The reason for
-  # this is to do initial text transformations to ensure (1) that terms will
-  # be translated reliably, and (2) that the final output of the translation
-  # will be readable by the user (i.e. lines not too long).
+  # This class provides a command for text reformatting for CJK languages. This
+  # reformatting is typically for use prior to processing the text with the
+  # translation commands. The reason for doing this is so that initial text
+  # transformations will be done to ensure (1) that terms will be translated
+  # reliably, and (2) that the final output of the translation will be readable
+  # by the user (i.e. lines not too long).
   #
   class Reflow
 
-    # Create a new instance of the Reflow class.
+    # Create a new instance of the reflow command
     #
     def initialize
-      @name = "sanzang-reflow"
+      @name = "sanzang reflow"
       @encoding = Encoding.default_external
       @infile = nil
       @outfile = nil
     end
 
-    # Run the Reflow command with the given arguments. The parameter _args_
-    # would typically be an Array of Unix-style command parameters. Calling
+    # Run the reflow command with the given arguments. The parameter _args_
+    # would typically be an array of command options and parameters. Calling
     # this with the "-h" or "--help" option will print full usage information
     # necessary for running this command.
     #
@@ -51,7 +51,7 @@ module Sanzang::Command
       parser.parse!(args)
 
       if args.length != 0
-        puts(parser)
+        $stderr.puts(parser)
         return 1
       end
 
@@ -62,7 +62,7 @@ module Sanzang::Command
         fin.binmode.set_encoding(@encoding)
         fout = @outfile ? File.open(@outfile, "w") : $stdout
         fout.binmode.set_encoding(@encoding)
-        fout.write(Sanzang::TextFormatter.new.reflow_cjk_text(fin.read))
+        fout.write(Sanzang::TextFormatter.new.reflow_cjk(fin.read))
       ensure
         if defined?(fin) and fin != $stdin
           fin.close if not fin.closed?
@@ -77,12 +77,14 @@ module Sanzang::Command
       return err.status
     rescue Exception => err
       $stderr.puts err.backtrace
-      $stderr.puts "ERROR: #{err.inspect}"
+      $stderr.puts "\nERROR: #{err.inspect}\n\n"
       return 1
     end
 
     private
 
+    # Initialize the encoding for text data if it is not already set
+    #
     def set_data_encoding
       if @encoding == nil
         if Encoding.default_external == Encoding::IBM437
@@ -94,41 +96,42 @@ module Sanzang::Command
       end
     end
 
+    # An OptionParser for the command
+    #
     def option_parser
-      OptionParser.new do |pr|
-        pr.banner = "Usage: #{@name} [options]\n"
+      OptionParser.new do |op|
+        op.banner = "Usage: #{@name} [options]\n"
 
-        pr.banner << "\nReformat text file contents into lines based on "
-        pr.banner << "spacing, punctuation, etc.\n"
-        pr.banner << "\nExamples:\n"
-        pr.banner << "    #{@name} -i in/mytext.txt -o out/mytext.txt\n"
-        pr.banner << "\nOptions:\n"
+        op.banner << "\nReformat text file contents into lines based on "
+        op.banner << "spacing, punctuation, etc.\n"
+        op.banner << "\nExamples:\n"
+        op.banner << "    #{@name} -i in/mytext.txt -o out/mytext.txt\n"
+        op.banner << "\nOptions:\n"
 
-        pr.on("-h", "--help", "show this help message and exit") do |v|
-          puts pr
+        op.on("-h", "--help", "show this help message and exit") do |v|
+          puts op
           exit 0
         end
-        pr.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
+        op.on("-E", "--encoding=ENC", "set data encoding to ENC") do |v|
           @encoding = Encoding.find(v)
         end
-        pr.on("-L", "--list-encodings", "list possible encodings") do |v|
-          puts(Encoding.list.collect {|e| e.to_s }.sort)
+        op.on("-L", "--list-encodings", "list possible encodings") do |v|
+          encodings = Encoding.list.sort do |x,y|
+            x.to_s.upcase <=> y.to_s.upcase
+          end
+          puts encodings
           exit 0
         end
-        pr.on("-i", "--infile=FILE", "read input text from FILE") do |v|
+        op.on("-i", "--infile=FILE", "read input text from FILE") do |v|
           @infile = v
         end
-        pr.on("-o", "--outfile=FILE", "write output text to FILE") do |v|
+        op.on("-o", "--outfile=FILE", "write output text to FILE") do |v|
           @outfile = v
         end
-        pr.on("-V", "--version", "show version number and exit") do |v|
-          puts "Sanzang version: #{Sanzang::VERSION}"
-          exit 0
-        end
       end
     end
 
-    # The standard name for the command.
+    # The name of the command
     #
     attr_reader :name
 

data/lib/sanzang/command/sanzang_cmd.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+require "parallel"
+
+require_relative "reflow"
+require_relative "translate"
+require_relative "batch"
+
+require_relative File.join("..", "version")
+
+module Sanzang::Command
+
+  # This class provides a frontend for all Sanzang operations and subcommands.
+  #
+  class SanzangCmd
+
+    # Create a new instance of the sanzang command
+    #
+    def initialize
+      @name = "sanzang"
+      @commands = [
+        ["batch", Sanzang::Command::Batch],
+        ["reflow", Sanzang::Command::Reflow],
+        ["translate", Sanzang::Command::Translate]
+      ]
+    end
+
+    # Run the sanzang command with the given arguments. If the first argument
+    # is the name of a sanzang subcommand or the beginning of a subcommand,
+    # then that subcommand is executed. The sanzang command also accepts
+    # several options such as showing usage and platform information.
+    #
+    def run(args)
+      parser = option_parser
+
+      if args.length < 1
+        $stderr.puts parser
+        return 1
+      end
+
+      @commands.each do |key,cmd|
+        if key.start_with?(args[0])
+          return cmd.new.run(args[1..-1])
+        end
+      end
+
+      parser.parse!(args)
+
+      $stderr.puts parser
+      return 1
+    rescue SystemExit => err
+      return err.status
+    rescue Exception => err
+      $stderr.puts err.backtrace
+      $stderr.puts "ERROR: #{err.inspect}"
+      return 1
+    end
+
+    # A string giving a listing of platform information
+    #
+    def platform_info
+      info = "Ruby platform: #{RUBY_PLATFORM}\n"
+      info << "Ruby version: #{RUBY_VERSION}\n"
+      info << "External encoding: #{Encoding.default_external}\n"
+      info << "Internal encoding: #{Encoding.default_internal or 'none'}\n"
+      info << "Fork implemented: #{Process.respond_to?(:fork)}\n"
+      info << "Parallel version: #{Parallel::VERSION}\n"
+      info << "Processors found: #{Parallel.processor_count}\n"
+      info << "Sanzang version: #{Sanzang::VERSION}\n"
+    end
+
+    # This is a string giving a brief one-line summary of version information
+    #
+    def version_info
+      "sanzang #{Sanzang::VERSION} [ruby_#{RUBY_VERSION}] [#{RUBY_PLATFORM}]"
+    end
+
+    private
+
+    # An OptionParser object for parsing command options and parameters
+    #
+    def option_parser
+      OptionParser.new do |op|
+        op.banner = "Usage: #{@name} [options]\n"
+        op.banner << "Usage: #{@name} <command> [options] [args]\n\n"
+
+        op.banner << "Use \"--help\" with commands for usage information.\n"
+
+        op.banner << "\nSanzang commands:\n"
+        op.banner << "    batch       translate many files in parallel\n"
+        op.banner << "    reflow      format CJK text for translation\n"
+        op.banner << "    translate   standard single text translation\n"
+        op.banner << "\nOptions:\n"
+
+        op.on("-h", "--help", "show this help message and exit") do |v|
+          puts op
+          exit 0
+        end
+        op.on("-P", "--platform", "show platform information and exit") do |v|
+          puts platform_info
+          exit 0
+        end
+        op.on("-V", "--version", "show version number and exit") do |v|
+          puts version_info
+          exit 0
+        end
+      end
+    end
+
+    # Name of the command
+    #
+    attr_reader :name
+
+  end
+end