sanzang 0.0.1

data/lib/sanzang/version.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+#--
+# Copyright (C) 2012 Lapis Lazuli Texts
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+module Sanzang
+
+  # The current version number of Sanzang.
+  VERSION = "0.0.1"
+
+end

data/test/tc_commands.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+
+require "test/unit"
+
+require_relative File.join("..", "lib", "sanzang")
+
+class TestCommands < Test::Unit::TestCase
+
+  def run_reflow(args)
+  end
+
+  def run_translate(args)
+  end
+
+
+end

data/test/tc_reflow_encodings.rb

@@ -0,0 +1,98 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+
+require "test/unit"
+
+require_relative File.join("..", "lib", "sanzang")
+
+# Test "reflow" operation with all major encodings for conversion and accuracy.
+#
+# Most encodings deemed as "important" here are Unicode encodings and those
+# commonly used for Chinese. Some encodings do not function due to converters
+# for these encodings being unimplemented in Ruby 1.9. Such encodings include
+# the following:
+#
+# * EUC-TW (Traditional Chinese)
+#
+class TestReflowEncodings < Test::Unit::TestCase
+
+  # Han characters, traditional, including a CBETA-style margin, which should
+  # be automatically stripped out by the text formatter.
+  #
+  def reflow_zh_hant(encoding)
+    text_s1 = "T31n1586_p0060a19(00)║　　　　大唐三藏法師玄奘奉　詔譯"
+    text_s2 = "　　　　大唐三藏法師玄奘奉\n　詔譯\n　\n"
+    text_s1.encode!(encoding)
+    text_s2.encode!(encoding)
+    formatter = Sanzang::TextFormatter.new
+    assert_equal(text_s2, formatter.reflow_cjk_text(text_s1))
+  end
+
+  # Han characters, simplified and without double vertical bar. The margin
+  # was dropped from the text due to GB2312 not supporting the "double bar"
+  # (U+2551) character.
+  #
+  def reflow_zh_hans(encoding)
+    text_s1 = "　　　　大唐三藏法师玄奘奉　诏译"
+    text_s2 = "　　　　大唐三藏法师玄奘奉\n　诏译\n　\n"
+    text_s1.encode!(encoding)
+    text_s2.encode!(encoding)
+    formatter = Sanzang::TextFormatter.new
+    assert_equal(text_s2, formatter.reflow_cjk_text(text_s1))
+  end
+
+  # UTF-8 (Traditional Chinese)
+  #
+  def test_reflow_hanzi_utf_8
+    reflow_zh_hant("UTF-8")
+  end
+
+  # UTF-16LE (Traditional Chinese)
+  #
+  def test_reflow_hanzi_utf_16le
+    reflow_zh_hant("UTF-16LE")
+  end
+
+  # UTF-16BE (Traditional Chinese)
+  #
+  def test_reflow_hanzi_utf_16be
+    reflow_zh_hant("UTF-16BE")
+  end
+
+  # UTF-32LE (Traditional Chinese)
+  #
+  def test_reflow_hanzi_utf_32le
+    reflow_zh_hant("UTF-32LE")
+  end
+
+  # UTF-32BE (Traditional Chinese)
+  #
+  def test_reflow_hanzi_utf_32be
+    reflow_zh_hant("UTF-32BE")
+  end
+
+  # Big5 (Traditional Chinese)
+  #
+  def test_reflow_hanzi_big5
+    reflow_zh_hant("Big5")
+  end
+
+  # GB2312 (Simplified Chinese)
+  # Double vertical bar glyph (U+2551) is not present in GB2312
+  #
+  def test_reflow_hanzi_gb2312
+    reflow_zh_hans("GB2312")
+  end
+
+  # GBK (Traditional Chinese)
+  #
+  def test_reflow_hanzi_gbk
+    reflow_zh_hant("GBK")
+  end
+
+  # GB18030 (Traditional Chinese)
+  #
+  def test_reflow_hanzi_gb18030
+    reflow_zh_hant("GB18030")
+  end
+end

data/test/tc_simple_translation.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+
+require "test/unit"
+
+require_relative File.join("..", "lib", "sanzang")
+
+# assert_nothing_raised
+# assert_equal(x, y)
+# assert(stmt, "Error message")
+#
+class TestSanzang < Test::Unit::TestCase
+
+  def table_string
+    "~|三藏| sānzàng| tripiṭaka|~
+     ~|法師| fǎshī| dharma-master|~
+     ~|玄奘| xuánzàng| xuanzang|~
+     ~|奉| fèng| reverently|~
+     ~|唐| táng| tang|~
+     ~|大| dà| great|~
+     ~|詔| zhào| imperial-order|~
+     ~|譯| yì| translate/interpret|~"
+  end
+
+  def stage_1
+    "T31n1586_p0060a19(00)║　　　　大唐三藏法師玄奘奉　詔譯\r\n"
+  end
+
+  def stage_2
+    "　　　　大唐三藏法師玄奘奉\r\n　詔譯\r\n"
+  end
+
+  def stage_3
+    "[1.1] 　　　　大唐三藏法師玄奘奉\r\n" \
+    << "[1.2] 　　　　 dà táng sānzàng fǎshī xuánzàng fèng\r\n" \
+    << "[1.3] 　　　　 great tang tripiṭaka dharma-master xuanzang " \
+    << "reverently\r\n" \
+    << "\r\n" \
+    << "[2.1] 　詔譯\r\n" \
+    << "[2.2] 　 zhào yì\r\n" \
+    << "[2.3] 　 imperial-order translate/interpret\r\n" \
+    << "\r\n"
+  end
+
+  def test_translation_table
+    table_path = File.join(File.dirname(__FILE__), "utf-8", "table.txt")
+    fin = File.open(table_path, "rb", encoding: "UTF-8")
+    table = Sanzang::TranslationTable.new(fin)
+    fin.close
+    assert(table.width.class == Fixnum, "Table width undefined")
+    assert(table.length.class == Fixnum, "Table length undefined")
+    assert(table.records.class == Array, "Table contents not an array")
+    rec0_length = table.records[0].length
+    table.records.each do |rec|
+      assert(rec.class == Array, "Malformed table records")
+      assert(rec.length == rec0_length, "Inconsistent table records")
+    end
+    assert(table.width > 0, "Zero-width table")
+    assert(table.length > 0, "Zero-length table")
+  end
+
+  def test_reflow_cjk_string
+    text = Sanzang::TextFormatter.new.reflow_cjk_text(stage_1())
+    assert_equal(stage_2(), text)
+  end
+
+  def test_translate_string
+    table = Sanzang::TranslationTable.new(table_string())
+    text = Sanzang::Translator.new(table).gen_listing(stage_2())
+    assert_equal(stage_3(), text)
+  end
+
+  def test_translate_file
+    table_path = File.join(File.dirname(__FILE__), "utf-8", "table.txt")
+    s2_path = File.join(File.dirname(__FILE__), "utf-8", "stage_2.txt")
+    s3_path = File.join(File.dirname(__FILE__), "utf-8", "stage_3.txt")
+    table = Sanzang::TranslationTable.new(table_path)
+    translator = Sanzang::Translator.new(table)
+    translator.translate_io(s2_path, s3_path)
+  end
+
+  def test_translator_parallel
+    table = Sanzang::TranslationTable.new(table_string())
+    translator = Sanzang::Translator.new(table)
+    translator.runs_parallel?
+    assert(translator.processor_count > 0, "Processor count less than zero")
+  end
+
+  def test_translate_batch
+    table = Sanzang::TranslationTable.new(table_string())
+    translator = Sanzang::Translator.new(table)
+    translator.translate_batch(
+        Dir.glob(File.join(File.dirname(__FILE__), "utf-8", "file_*.txt")),
+        File.join(File.dirname(__FILE__), "utf-8", "batch"), false)
+  end
+
+end

data/test/utf-8/batch/file_1.txt

@@ -0,0 +1,8 @@
+[1.1] 　　　　大唐三藏法師玄奘奉
+[1.2] 　　　　 dà táng sānzàng fǎshī xuánzàng fèng
+[1.3] 　　　　 great tang tripiṭaka dharma-master xuanzang reverently
+
+[2.1] 　詔譯
+[2.2] 　 zhào yì
+[2.3] 　 imperial-order translate/interpret
+

data/test/utf-8/batch/file_2.txt

@@ -0,0 +1,8 @@
+[1.1] 　　　　大唐三藏法師玄奘奉
+[1.2] 　　　　 dà táng sānzàng fǎshī xuánzàng fèng
+[1.3] 　　　　 great tang tripiṭaka dharma-master xuanzang reverently
+
+[2.1] 　詔譯
+[2.2] 　 zhào yì
+[2.3] 　 imperial-order translate/interpret
+

data/test/utf-8/batch/file_3.txt

@@ -0,0 +1,8 @@
+[1.1] 　　　　大唐三藏法師玄奘奉
+[1.2] 　　　　 dà táng sānzàng fǎshī xuánzàng fèng
+[1.3] 　　　　 great tang tripiṭaka dharma-master xuanzang reverently
+
+[2.1] 　詔譯
+[2.2] 　 zhào yì
+[2.3] 　 imperial-order translate/interpret
+

data/test/utf-8/batch/file_4.txt

@@ -0,0 +1,8 @@
+[1.1] 　　　　大唐三藏法師玄奘奉
+[1.2] 　　　　 dà táng sānzàng fǎshī xuánzàng fèng
+[1.3] 　　　　 great tang tripiṭaka dharma-master xuanzang reverently
+
+[2.1] 　詔譯
+[2.2] 　 zhào yì
+[2.3] 　 imperial-order translate/interpret
+

data/test/utf-8/file_1.txt

@@ -0,0 +1,2 @@
+　　　　大唐三藏法師玄奘奉
+　詔譯

data/test/utf-8/file_2.txt

@@ -0,0 +1,2 @@
+　　　　大唐三藏法師玄奘奉
+　詔譯

data/test/utf-8/file_3.txt

@@ -0,0 +1,2 @@
+　　　　大唐三藏法師玄奘奉
+　詔譯

data/test/utf-8/file_4.txt

@@ -0,0 +1,2 @@
+　　　　大唐三藏法師玄奘奉
+　詔譯

data/test/utf-8/stage_1.txt

@@ -0,0 +1 @@
+T31n1586_p0060a19(00)║　　　　大唐三藏法師玄奘奉　詔譯

data/test/utf-8/stage_2.txt

@@ -0,0 +1,2 @@
+　　　　大唐三藏法師玄奘奉
+　詔譯

data/test/utf-8/stage_3.txt

@@ -0,0 +1,4 @@
+[1.1] 　　　　大唐三藏法師玄奘奉
+
+[2.1] 　詔譯
+

data/test/utf-8/table.txt

@@ -0,0 +1,8 @@
+三藏| sānzàng| tripiṭaka
+法師| fǎshī| dharma-master
+玄奘| xuánzàng| xuanzang
+奉| fèng| reverently
+唐| táng| tang
+大| dà| great
+詔| zhào| imperial-order
+譯| yì| translate/interpret

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: sanzang
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Lapis Lazuli Texts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: &81673770 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.18
+  type: :runtime
+  prerelease: false
+  version_requirements: *81673770
+description: Sanzang is an application built for direct machine translation of natural
+  languages. This application is particularly suitable as a translation aid for for
+  ancient Chinese texts. Sanzang uses simple direct translation rules organized into
+  translation tables, which are stored in a straightforward text format. Batch translations
+  utilize multiprocessing to translate files in parallel, naturally scaling to the
+  number of processors available. Sanzang is available under the GNU General Public
+  License, version 3.
+email:
+- lapislazulitexts@gmail.com
+executables:
+- sanzang-reflow
+- sanzang-translate
+extensions: []
+extra_rdoc_files:
+- HACKING
+- LICENSE
+- README
+files:
+- bin/sanzang-reflow
+- bin/sanzang-translate
+- test/tc_commands.rb
+- test/utf-8/batch/file_4.txt
+- test/utf-8/batch/file_3.txt
+- test/utf-8/batch/file_1.txt
+- test/utf-8/batch/file_2.txt
+- test/utf-8/stage_3.txt
+- test/utf-8/file_4.txt
+- test/utf-8/file_3.txt
+- test/utf-8/file_1.txt
+- test/utf-8/stage_1.txt
+- test/utf-8/table.txt
+- test/utf-8/file_2.txt
+- test/utf-8/stage_2.txt
+- test/tc_reflow_encodings.rb
+- test/tc_simple_translation.rb
+- lib/sanzang.rb
+- lib/sanzang/text_formatter.rb
+- lib/sanzang/translation_table.rb
+- lib/sanzang/version.rb
+- lib/sanzang/command/reflow.rb
+- lib/sanzang/command/translate.rb
+- lib/sanzang/translator.rb
+- HACKING
+- LICENSE
+- README
+homepage: http://www.lapislazulitexts.com
+licenses:
+- GPL-3
+post_install_message: 
+rdoc_options:
+- --main
+- README
+- --title
+- Sanzang
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Sanzang
+test_files:
+- test/tc_commands.rb
+- test/tc_reflow_encodings.rb
+- test/tc_simple_translation.rb