sanzang 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed9732f3291f8bebe17d1abc1cceb68765aed70e
-  data.tar.gz: c3560faf30781e7e30b76ef8b6c41234725606e9
+  metadata.gz: a5bd5aafaee153f9753cdeedf952c8581eaf770b
+  data.tar.gz: 111fc689cb0f19ca0dd20356f50d7483e6a51420
 SHA512:
-  metadata.gz: 64942f119d7618ba75ed87e0e47911b0c8179445d4bdd2425b6b85665e6c5967a32a57492adba2ad3c5a9a45423dbec5540f74b3440bbce182c43408758d3b07
-  data.tar.gz: 85624b92a30b276e0148b3d7a1449fe6b987be2fb70b1ffc1eebb1f59b0e2d1d200c07731efdb38168304e1c2d494ba2d7d54c254b8b2a866bda36ad924983d1
+  metadata.gz: 17a663b61ef1523e24d76d7d37a65c56ff8d73c2bbb43e3aa226437dbc4b224a96b7ebaa02370bd8054490c2d57885d24f6be5844f6767fa59565705c3c37fa4
+  data.tar.gz: a453e356504a8c334cefcf42455769bcb79bba82af4a48138780a3bbd3a086c474c3fa37659c3a7c33ca38f30c78e5f8237a9caf2f8cc316d95374b71bf31920

data/NEWS.rdoc

@@ -2,6 +2,12 @@
 
 == Release History
 
+=== v1.2.0
+* Updated vocab building code for faster term matching
+* Refactored the TextFormatter class into a Formatting module
+* Added methods for merging translation tables into one another
+* Consolidated Sanzang module "requires" into a central location
+
 === v1.1.2
 * Cleaned up table loading code to be faster and simpler
 * Added RDoc option to set documentation encoding to UTF-8 (RDoc 3.x)

data/README.rdoc

@@ -40,6 +40,6 @@ command to verify your installation and print version information.
 
 This command should show a summary of your \Sanzang version and environment.
 
-    sanzang 1.1.1 (UTF-8) ruby-2.1.0p0 x86_64-linux
+    sanzang 1.2.0 ruby-2.1.1 x86_64-linux (UTF-8)
 
 You now have \Sanzang installed on your computer.

data/lib/sanzang.rb

@@ -1,6 +1,6 @@
 # coding: UTF-8
 #--
-# Copyright (C) 2012-2013 Lapis Lazuli Texts
+# Copyright (C) 2012-2014 Lapis Lazuli Texts
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -22,11 +22,11 @@
 module Sanzang
 end
 
-require_relative File.join("sanzang", "batch_translator")
 require_relative File.join("sanzang", "platform")
-require_relative File.join("sanzang", "text_formatter")
+require_relative File.join("sanzang", "formatting")
 require_relative File.join("sanzang", "translation_table")
 require_relative File.join("sanzang", "translator")
+require_relative File.join("sanzang", "batch_translator")
 require_relative File.join("sanzang", "version")
 
 # The Sanzang::Command module contains Unix style commands utilizing the

data/lib/sanzang/batch_translator.rb

@@ -15,11 +15,6 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-require "parallel"
-
-require_relative "platform"
-require_relative "translator"
-
 module Sanzang
 
   # BatchTranslator can handle batches of files for translation, and may also

data/lib/sanzang/command/batch.rb

@@ -15,12 +15,8 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-require "optparse"
-
-require_relative File.join("..", "platform")
-require_relative File.join("..", "translation_table")
-require_relative File.join("..", "batch_translator")
-require_relative File.join("..", "version")
+require 'optparse'
+require 'parallel'
 
 module Sanzang::Command
 

data/lib/sanzang/command/reflow.rb

@@ -15,11 +15,7 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-require "optparse"
-
-require_relative File.join("..", "platform")
-require_relative File.join("..", "text_formatter")
-require_relative File.join("..", "version")
+require 'optparse'
 
 module Sanzang::Command
 
@@ -61,7 +57,7 @@ module Sanzang::Command
         fin.binmode.set_encoding(@encoding)
         fout = @outfile ? File.open(@outfile, "w") : $stdout
         fout.binmode.set_encoding(@encoding)
-        fout.write(Sanzang::TextFormatter.new.reflow_cjk(fin.read))
+        fout.write(Sanzang::Formatting.reflow_cjk(fin.read))
       ensure
         if defined?(fin) and fin.class == File
           fin.close if not fin.closed?

data/lib/sanzang/command/sanzang_cmd.rb

@@ -15,15 +15,7 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-require "optparse"
-require "parallel"
-
-require_relative "reflow"
-require_relative "translate"
-require_relative "batch"
-
-require_relative File.join("..", "platform")
-require_relative File.join("..", "version")
+require 'optparse'
 
 module Sanzang::Command
 
@@ -80,6 +72,8 @@ module Sanzang::Command
     # A string giving a listing of platform information
     #
     def platform_info
+      require 'parallel'
+
       info = "host_arch = #{Sanzang::Platform.machine_arch}\n"
       info << "host_os = #{Sanzang::Platform.os_name}\n"
       info << "host_processors = #{Sanzang::Platform.processor_count}\n"
@@ -96,8 +90,9 @@ module Sanzang::Command
     # This is a string giving a brief one-line summary of version information
     #
     def version_info
-      "sanzang #{Sanzang::VERSION} (#{Sanzang::Platform.data_encoding})" \
-        + " ruby-#{RUBY_VERSION}p#{RUBY_PATCHLEVEL} #{RUBY_PLATFORM}"
+      "sanzang #{Sanzang::VERSION} ruby-#{RUBY_VERSION} #{RUBY_PLATFORM} " \
+        + "(#{Sanzang::Platform.data_encoding})"
+#     "sanzang #{Sanzang::VERSION} #{Sanzang::Platform.data_encoding}"
     end
 
     # Name of the command

data/lib/sanzang/command/translate.rb

@@ -15,12 +15,7 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-require "optparse"
-
-require_relative File.join("..", "platform")
-require_relative File.join("..", "translation_table")
-require_relative File.join("..", "translator")
-require_relative File.join("..", "version")
+require 'optparse'
 
 module Sanzang::Command
 

data/lib/sanzang/{text_formatter.rb → formatting.rb}

@@ -1,6 +1,6 @@
 # coding: UTF-8
 #--
-# Copyright (C) 2012-2013 Lapis Lazuli Texts
+# Copyright (C) 2012-2014 Lapis Lazuli Texts
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -15,16 +15,14 @@
 # You should have received a copy of the GNU General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-module Sanzang
-
-  # This class handles formatting of text data especially to prepare the text
-  # for direct translation. This involves reformatting and reflowing text so
-  # that words are not divided between lines, and so the output is well suited
-  # for humans. For practical purposes of readability, lines of text to be
-  # translated should be succinct and easily comprehensible. The TextFormatter
-  # class includes methods for accomplishing this reformatting.
-  #
-  class TextFormatter
+# This module handles formatting of text data especially to prepare the text
+# for direct translation. This involves reformatting and reflowing text so
+# that words are not divided between lines, and so the output is well suited
+# for humans. For practical purposes of readability, lines of text to be
+# translated should be succinct and easily comprehensible.
+#
+module Sanzang::Formatting
+  class << self
 
     # Given a CJK string of text, reformat the string for greater compatibility
     # with direct translation, and reflow the text based on its punctuation.

data/lib/sanzang/translation_table.rb

@@ -17,9 +17,8 @@
 
 module Sanzang
 
-  # A translation table encapsulates a set of rules for translating with
-  # the \Sanzang system. These are essentially read-only objects meant for
-  # storing well-defined translation table data.
+  # A translation table encapsulates a set of rules for translating with the
+  # \Sanzang system.
   #
   class TranslationTable
 
@@ -39,6 +38,8 @@ module Sanzang
     # - Subsequent fields are equivalent terms in destination languages.
     # - The number of columns must be consistent for the entire table.
     #
+    # The rules passed in here may either be a file descriptor or a string.
+    #
     def initialize(rules)
       contents = rules.kind_of?(String) ? rules : rules.read
       @source_encoding = contents.encoding
@@ -51,8 +52,9 @@ module Sanzang
       end
 
       @records = contents.strip.split("\n").collect {|r| r.strip.split("|") }
+      @sorted = false
       check_dims
-      sort!
+     #sort!
     end
 
     # Retrieve a record by its numeric index.
@@ -76,10 +78,18 @@ module Sanzang
       end
     end
 
+    # Check if the table records are sorted
+    #
+    def sorted?
+      @sorted
+    end
+
     # Reverse sort all records by length
     #
     def sort!
       @records.sort! {|x,y| y[0].size <=> x[0].size }
+      @sorted = true
+      nil
     end
 
     # The text encoding used internally for all translation table data
@@ -94,6 +104,46 @@ module Sanzang
       @records.find {|rec| rec[0] == term }
     end
 
+    # Convert to a hash. The original records are the values.
+    #
+    # For example: "A" => ["A", "B", "C"]
+    #
+    def to_h
+      h = Hash.new
+      @records.each {|rec| h[rec[0]] = rec if not h[rec[0]] }
+      h
+    end
+
+    # Only include unique source values. The resulting table is unsorted.
+    #
+    def uniq!
+      @records = to_h.values
+      @sorted = false
+      nil
+    end
+
+    # Merge another table into this one. If the same source term exists in
+    # both tables, then the record from the other table will be used instead.
+    # Note: after a merge, the resulting table is unsorted.
+    #
+    def merge!(tab2)
+      if tab2.width != width
+        raise "Table widths must match when merging tables"
+      end
+      h1 = to_h
+      tab2.records.each do |rec|
+        h1[rec[0]] = rec
+      end
+      @records = h1.values
+      @sorted = false
+    end
+
+    # Return a CSV formatted string
+    #
+    def to_csv
+      @records.map {|r| r.join("|") }.join("\n")
+    end
+
     # The number of records in the table
     #
     def length

data/lib/sanzang/translator.rb

@@ -37,13 +37,10 @@ module Sanzang
     # These records represent the vocabulary used by the text.
     #
     def text_vocab(source_text)
-      new_table = []
-      @table.records.each do |record|
-        if source_text.include?(record[0])
-          new_table << record
-        end
+      text_copy = String.new(source_text)
+      @table.records.select do |r|
+        text_copy.include?(r[0]) ? text_copy.gsub!(r[0], "\x1F") : false
       end
-      new_table
     end
 
     # Use the TranslationTable of the Translator to create translations for
@@ -80,7 +77,7 @@ module Sanzang
 
       listing = ""
       texts[0].length.times do |line_i|
-        @table.width.times do |col_i|
+        texts.length.times do |col_i|
           listing << "[#{pos + line_i}.#{col_i + 1}] #{texts[col_i][line_i]}" \
                   << newline
         end

data/lib/sanzang/version.rb

@@ -19,6 +19,6 @@ module Sanzang
 
   # Current version number of Sanzang
   #
-  VERSION = "1.1.2"
+  VERSION = "1.2.0"
 
 end

data/test/tc_reflow_encodings.rb

@@ -23,8 +23,7 @@ class TestReflowEncodings < Test::Unit::TestCase
     text_s2 = "　　　　大唐三藏法師玄奘奉\n　詔譯\n　\n"
     text_s1.encode!(encoding)
     text_s2.encode!(encoding)
-    formatter = Sanzang::TextFormatter.new
-    assert_equal(text_s2, formatter.reflow_cjk(text_s1))
+    assert_equal(text_s2, Sanzang::Formatting.reflow_cjk(text_s1))
   end
 
   # Han characters, simplified and without double vertical bar. The margin
@@ -36,8 +35,7 @@ class TestReflowEncodings < Test::Unit::TestCase
     text_s2 = "　　　　大唐三藏法师玄奘奉\n　诏译\n　\n"
     text_s1.encode!(encoding)
     text_s2.encode!(encoding)
-    formatter = Sanzang::TextFormatter.new
-    assert_equal(text_s2, formatter.reflow_cjk(text_s1))
+    assert_equal(text_s2, Sanzang::Formatting.reflow_cjk(text_s1))
   end
 
   # UTF-8 (Traditional Chinese)

data/test/tc_simple_translation.rb

@@ -55,7 +55,7 @@ class TestSanzang < Test::Unit::TestCase
   end
 
   def test_reflow_cjk_string
-    text = Sanzang::TextFormatter.new.reflow_cjk(stage_1())
+    text = Sanzang::Formatting.reflow_cjk(stage_1())
     assert_equal(stage_2(), text)
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sanzang
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Lapis Lazuli Texts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-13 00:00:00.000000000 Z
+date: 2014-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -52,8 +52,8 @@ files:
 - lib/sanzang/command/reflow.rb
 - lib/sanzang/command/sanzang_cmd.rb
 - lib/sanzang/command/translate.rb
+- lib/sanzang/formatting.rb
 - lib/sanzang/platform.rb
-- lib/sanzang/text_formatter.rb
 - lib/sanzang/translation_table.rb
 - lib/sanzang/translator.rb
 - lib/sanzang/version.rb
@@ -90,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - parallel ~> 0.8
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Machine translation from CJK languages