konjak 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea6b25de9fe1bb666b02b8db7a3253619fe6cc5b
-  data.tar.gz: 4bf4b8cdc708b5904c16e8d3b656c37b15c26f4e
+  metadata.gz: 0150635f9f3db7b2d052aee30b1db085162eceaa
+  data.tar.gz: f2a5b91c9dc7a9a99d84c9ac979a67893959ab56
 SHA512:
-  metadata.gz: 94607b5df832840753b9a863dec409c98a283d9be758f40be2af1522cdd0815fe7615390f868fd2afa89133046a9418de25b1c568146cc5f3ceb491854c423c6
-  data.tar.gz: bef5ef91e252bfadde9f6b8f6cb8fd6674efd4ae4e2d4c28a4f65e88c9a0330e365faf7fe4ced678a56a876f185dd8dac0d87f3877ef4d13f7d12be4939a94f1
+  metadata.gz: add182fac41dada9dce73a25f1a7b90ca1663fb4e52c0ae836e190cbd0bc3795356917b7f88918af1a61c8d2f7dc73ca09c6a268398f5c6f3a78d1c62a18d365
+  data.tar.gz: 21a97c4060b425a7ebfc9b3df5b6edd4de26d9c2ec874df619155257067f844b70a7efc71707cb1bb89a53b9a64015ff0aee739256b4df9acbf1a5062f85d873

data/{LICENSE.txt → LICENSE.md}

@@ -1,6 +1,6 @@
-Copyright (c) 2014 Seiei Higa
+Copyright © 2014-2015 [YassLab](http://yasslab.jp)
 
-MIT License
+The MIT License
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -31,3 +31,9 @@ Or install it yourself as:
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## License
+
+Copyright © 2015 [YassLab](http://yasslab.jp)
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/lib/konjak/html_segmentor.rb

@@ -0,0 +1,27 @@
+module Konjak
+  class HtmlSegmentor < Segmentor
+    def segments
+      segments = [content.dup]
+
+      begin
+        size = segments.size
+
+        segments_patterns = [
+          %r{<(?<start>p|h1|h2|h3|h4|h5|h6|li|title|td)>(.*?)</\k<start>>}m,
+          %r{<(?<start>p|h1|h2|h3|h4|h5|h6|li|title|td) [^>]*?>(.*?)</\k<start>>}m,
+          %r{<div>(.*?)</div>}m,
+          %r{<div [^>]*?>(.*?)</div>}m
+        ]
+        segments_patterns.each do |pattern|
+          segments.map! do |s|
+            s.partition(pattern)
+          end
+          segments.flatten!
+          segments.reject!(&:empty?)
+        end
+      end while segments.size != size
+
+      segments
+    end
+  end
+end

data/lib/konjak/parser.rb

@@ -2,7 +2,13 @@ require 'nokogiri'
 
 module Konjak
   class Parser
-    def parse(xml, gtt: false)
+    attr_accessor :gtt
+
+    def initialize(gtt: false)
+      @gtt = gtt
+    end
+
+    def parse(xml)
       if gtt
         # FIXME
         xml = xml.gsub(/&(#\d+|#x[0-9a-fA-F]+|[0-9a-zA-Z]+);/) { "&#{$1};" }

data/lib/konjak/polytex_segmentor.rb

@@ -0,0 +1,40 @@
+module Konjak
+  class PolytexSegmentor < Segmentor
+    def segments
+      segments = [content.dup]
+
+      begin
+        size = segments.size
+
+        segments_patterns = [
+          /\\begin\{(?<start>[^\}]+)\}([\n.]*?)\\end\{\k<start>\}/m,
+          /(?<=\\chapter\{)[^\}]+(?=\})/,
+          /(?<=\\section\{)[^\}]+(?=\})/,
+          /(?<=\\subsection\{)[^\}]+(?=\})/,
+          /\\footnote\{(?<gr>\\(?!footnote)[^\{]+\{[^\}]+\}(?:\{[^\}]+\})?\g<gr>|[^{])+\}/m,
+          /(?<=\\footnote\{)(?<gr>\\(?!footnote)[^\{]+\{[^\}]+\}(?:\{[^\}]+\})?\g<gr>|[^{])+(?=\})/m,
+          /(?<=\\codecaption\{).+(?= \\|\}$)/,
+          /(?<=\\caption\{).+(?=\\label\{.*\}\}$)/,
+          /(?<=\n)^.*$(?=\n)/m,
+          /# .*$/,
+          /(?<=^).+?[\.\?\!](?= |\n|\t)/,
+          /(?<=\()[^\.\n]+[\.\?\!](?=\))/,
+          /^ (?=[\w\\]+)/,
+          /^\s+% .*$/,
+          /^$/,
+          /\\noindent /,
+          /\\item /,
+        ]
+        segments_patterns.each do |pattern|
+          segments.map! do |s|
+            s.partition(pattern)
+          end
+          segments.flatten!
+          segments.reject!(&:empty?)
+        end
+      end while segments.size != size
+
+      segments
+    end
+  end
+end

data/lib/konjak/segment/gtt.rb

@@ -0,0 +1,39 @@
+module Konjak
+  class Segment < StructuralElement
+    module GTT
+      Tag = Struct.new(:gtt, :html)
+
+      def compile_gtt_html_pattern
+        regexp = Regexp.escape(text)
+        gtt_tag_ns.each do |n|
+          regexp = regexp.sub(/\\\{#{n}\\\}/)    { "(?<n#{n}><(?<_#{n}>\\w+)[^>]*>)" }
+          regexp = regexp.gsub(/\\\{#{n}\\\}/)   { "\\k<n#{n}>" }
+          regexp = regexp.gsub(/\\\{\/#{n}\\\}/) { "</\\k<_#{n}>>" }
+        end
+        Regexp.compile(regexp)
+      end
+
+      def extract_gtt_tags_from(text)
+        m = text.match(compile_gtt_html_pattern)
+        gtt_tag_ns.each_with_object([]) do |n, tags|
+          tags << Tag.new("{#{n}}", m["n#{n}"])
+          tags << Tag.new("{/#{n}}", "</#{m["_#{n}"]}>")
+        end
+      end
+
+      def interpolate_gtt_tags(tags)
+        new_text = self.text.dup
+        tags.each do |tag|
+          new_text = new_text.gsub(tag[:gtt], tag[:html])
+        end
+        new_text
+      end
+
+      private
+
+      def gtt_tag_ns
+        text.scan(/\{(\d+)\}/).flatten.uniq
+      end
+    end
+  end
+end

data/lib/konjak/segment.rb

@@ -1,6 +1,10 @@
+require 'konjak/segment/gtt'
+
 module Konjak
   # container
   class Segment < StructuralElement
+    include GTT
+
     # children
     def text
       Text.new(super)
@@ -10,5 +14,13 @@ module Konjak
     def can_contain?(element)
       [Text, BeginPairedTag, EndPairedTag, IsolatedTag, Placeholder, Highlight].any? {|c| c === element }
     end
+
+    def translation_unit
+      TranslationUnit.new(translation_unit_variant.parent)
+    end
+
+    def translation_unit_variant
+      TranslationUnitVariant.new(parent)
+    end
   end
 end

data/lib/konjak/segmentor.rb

@@ -0,0 +1,10 @@
+module Konjak
+  class Segmentor
+    attr_accessor :content, :options
+
+    def initialize(content, options)
+      @content = content
+      @options = options
+    end
+  end
+end

data/lib/konjak/tmx_segmentor/gtt_html_strategy.rb

@@ -0,0 +1,28 @@
+require 'konjak/tmx_segmentor/segment_string'
+require 'konjak/tmx_segmentor/strategy'
+
+module Konjak
+  class TmxSegmentor < Segmentor
+    class GttHtmlStrategy < Strategy
+
+      private
+
+      def split(translation_unit, text)
+        segment = translation_unit.variant(@lang).segment
+        pattern = segment.compile_gtt_html_pattern
+
+        texts = []
+        while true
+          head, match, tail = text.partition(pattern)
+          break if match.empty?
+          texts << head unless head.empty?
+
+          texts << SegmentString.new(match, segment)
+
+          text = tail
+        end
+        texts << text
+      end
+    end
+  end
+end

data/lib/konjak/tmx_segmentor/segment_string.rb

@@ -0,0 +1,14 @@
+require 'konjak/segmentor'
+
+module Konjak
+  class TmxSegmentor < Segmentor
+    class SegmentString < String
+      attr_accessor :segment
+
+      def initialize(str, segment)
+        super(str)
+        @segment = segment
+      end
+    end
+  end
+end

data/lib/konjak/tmx_segmentor/strategy.rb

@@ -0,0 +1,37 @@
+require 'mem'
+require 'konjak/segmentor'
+require 'konjak/tmx_segmentor/segment_string'
+
+module Konjak
+  class TmxSegmentor < Segmentor
+    class Strategy
+      include Mem
+
+      def initialize(tmx, lang)
+        @tmx = tmx
+        @lang = lang
+      end
+
+      def segmentize(text)
+        segments = [text]
+        translation_units.each do |translation_unit|
+          segments.map! {|text|
+            next text if text.is_a?(SegmentString)
+
+            split(translation_unit, text)
+          }.flatten!
+        end
+        segments
+      end
+
+      private
+
+      def translation_units
+        @tmx.body.translation_units.sort_by {|tu|
+          -tu.variant(@lang).segment.text.length
+        }
+      end
+      memoize :translation_units
+    end
+  end
+end

data/lib/konjak/tmx_segmentor/text_strategy.rb

@@ -0,0 +1,28 @@
+require 'konjak/tmx_segmentor/segment_string'
+require 'konjak/tmx_segmentor/strategy'
+
+module Konjak
+  class TmxSegmentor < Segmentor
+    class TextStrategy < Strategy
+
+      private
+
+      def split(translation_unit, text)
+        segment = translation_unit.variant(@lang).segment
+        segment_text = segment.text
+
+        texts = []
+        while true
+          head, match, tail = text.partition(segment_text)
+          break if match.empty?
+          texts << head unless head.empty?
+
+          texts << SegmentString.new(match, segment)
+
+          text = tail
+        end
+        texts << text
+      end
+    end
+  end
+end

data/lib/konjak/tmx_segmentor.rb

@@ -0,0 +1,40 @@
+require 'konjak/segmentor'
+require 'konjak/tmx_segmentor/gtt_html_strategy'
+require 'konjak/tmx_segmentor/text_strategy'
+
+module Konjak
+  class TmxSegmentor < Segmentor
+
+    STRATEGIES = {
+      text:     TextStrategy,
+      gtt_html: GttHtmlStrategy,
+    }
+
+    def segments
+      strategy.segmentize(content)
+    end
+
+    private
+
+    def tmx
+      @options[:tmx] or raise 'tmx option is not set'
+    end
+
+    def lang
+      @options[:lang] or raise 'lang option is not set'
+    end
+
+    def format
+      if STRATEGIES.has_key?(options[:format])
+        options[:format]
+      else
+        :text
+      end
+    end
+
+    def strategy
+      STRATEGIES[format].new(tmx, lang)
+    end
+
+  end
+end

data/lib/konjak/translator.rb

@@ -1,7 +1,5 @@
 require 'mem'
-require 'konjak/translator/gtt_html_translate'
-require 'konjak/translator/text_translate'
-require 'konjak/translator/translated_string'
+require 'konjak/tmx_segmentor'
 
 module Konjak
   class Translator
@@ -17,49 +15,24 @@ module Konjak
       @options     = options
     end
 
-    def translate(doc)
-      translated_docs = [doc.dup]
-      translation_units.each do |tu|
-        translated_docs.map! { |text|
-          next text if text.is_a?(TranslatedString)
-
-          env = translate_env.dup
-          env.local_variable_set(:tu, tu)
-          env.local_variable_set(:src_lang, src_lang)
-          env.local_variable_set(:target_lang, target_lang)
-          env.local_variable_set(:text, text)
-          eval('tu.translate(src_lang, target_lang, text)', env)
-        }.flatten!
+    def translate(content)
+      segmentor(content).segments.map do |text|
+        next text unless text.is_a?(TmxSegmentor::SegmentString)
+        source_segment = text.segment
+        target_segment = source_segment.translation_unit.variant(target_lang).segment
+        target_segment.interpolate_gtt_tags(source_segment.extract_gtt_tags_from(text))
       end
-      translated_docs
     end
 
     private
 
-    TRANSLATE_ENVS= {
-      text:     Class.new { using TextTranslate;    break binding },
-      gtt_html: Class.new { using GttHtmlTranslate; break binding }
-    }
-
-    def format
-      if TRANSLATE_ENVS.has_key?(options[:format])
-        options[:format]
-      else
-        :text
-      end
-    end
-
-    def translate_env
-      TRANSLATE_ENVS[format]
-    end
-
-    def translation_units
-      tmx.body.translation_units.select { |tu|
-        tu.has_translation?(src_lang, target_lang)
-      }.sort_by {|tu|
-        -tu.variant(src_lang).segment.text.length
-      }
+    def segmentor(content)
+      TmxSegmentor.new(
+        content,
+        tmx: tmx,
+        lang: src_lang,
+        format: options[:format]
+      )
     end
-    memoize :translation_units
   end
 end

data/lib/konjak/version.rb

@@ -1,3 +1,3 @@
 module Konjak
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/konjak.rb

@@ -34,14 +34,20 @@ require 'konjak/unknown_tag'
 # translator
 require 'konjak/translator'
 
+# segmentor
+require 'konjak/segmentor'
+require 'konjak/html_segmentor'
+require 'konjak/polytex_segmentor'
+require 'konjak/tmx_segmentor'
+
 module Konjak
   class << self
     def parse(xml, **options)
-      Parser.new.parse(xml, **options)
+      Parser.new(**options).parse(xml)
     end
 
     def translate(doc, xml_or_tmx, src_lang, target_lang, **options)
-      tmx = xml_or_tmx.kind_of?(Tmx) ? xml_or_tmx : parse(xml_or_tmx)
+      tmx = xml_or_tmx.kind_of?(Tmx) ? xml_or_tmx : parse(xml_or_tmx, **options)
       Translator.new(tmx, src_lang, target_lang, **options).translate(doc)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: konjak
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Seiei Higa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-05 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mem
@@ -134,7 +134,7 @@ files:
 - ".rspec"
 - ".travis.yml"
 - Gemfile
-- LICENSE.txt
+- LICENSE.md
 - README.md
 - Rakefile
 - bin/konjak
@@ -147,24 +147,30 @@ files:
 - lib/konjak/end_paired_tag.rb
 - lib/konjak/header.rb
 - lib/konjak/highlight.rb
+- lib/konjak/html_segmentor.rb
 - lib/konjak/inline_element.rb
 - lib/konjak/isolated_tag.rb
 - lib/konjak/map.rb
 - lib/konjak/note.rb
 - lib/konjak/parser.rb
 - lib/konjak/placeholder.rb
+- lib/konjak/polytex_segmentor.rb
 - lib/konjak/property.rb
 - lib/konjak/segment.rb
+- lib/konjak/segment/gtt.rb
+- lib/konjak/segmentor.rb
 - lib/konjak/structural_element.rb
 - lib/konjak/sub_flow.rb
 - lib/konjak/text.rb
 - lib/konjak/tmx.rb
+- lib/konjak/tmx_segmentor.rb
+- lib/konjak/tmx_segmentor/gtt_html_strategy.rb
+- lib/konjak/tmx_segmentor/segment_string.rb
+- lib/konjak/tmx_segmentor/strategy.rb
+- lib/konjak/tmx_segmentor/text_strategy.rb
 - lib/konjak/translation_unit.rb
 - lib/konjak/translation_unit_variant.rb
 - lib/konjak/translator.rb
-- lib/konjak/translator/gtt_html_translate.rb
-- lib/konjak/translator/text_translate.rb
-- lib/konjak/translator/translated_string.rb
 - lib/konjak/unknown_tag.rb
 - lib/konjak/user_defined_encoding.rb
 - lib/konjak/version.rb

data/lib/konjak/translator/gtt_html_translate.rb

@@ -1,47 +0,0 @@
-module Konjak
-  class Translator
-    module GttHtmlTranslate
-      refine(Text) do
-        def gtt_tag_ns
-          scan(/\{(\d+)\}/).flatten.uniq
-        end
-
-        def compile_gtt_html_pattern
-          regexp = Regexp.escape(self)
-          gtt_tag_ns.each do |n|
-            regexp = regexp.sub(/\\\{#{n}\\\}/)    { "(?<n#{n}><(?<_#{n}>\\w+)[^>]*>)" }
-            regexp = regexp.gsub(/\\\{#{n}\\\}/)   { "\\k<n#{n}>" }
-            regexp = regexp.gsub(/\\\{\/#{n}\\\}/) { "</\\k<_#{n}>>" }
-          end
-          Regexp.compile(regexp)
-        end
-
-        def interpolate_gtt_html_pattern(match_data)
-          new_text = dup
-          gtt_tag_ns.each do |n|
-            new_text = new_text.gsub("{#{n}}", match_data["n#{n}"])
-            new_text = new_text.gsub("{/#{n}}", "</#{match_data["_#{n}"]}>")
-          end
-          new_text
-        end
-      end
-
-      refine(TranslationUnit) do
-        def translate(src_lang, target_lang, text)
-          pattern     = variant(src_lang).segment.text.compile_gtt_html_pattern
-          target_text = variant(target_lang).segment.text
-
-          texts = []
-          while true
-            head, match, tail = text.partition(pattern)
-            break if match.empty?
-            texts << head unless head.empty?
-            texts << TranslatedString.new(target_text.interpolate_gtt_html_pattern($~))
-            text = tail
-          end
-          texts << text
-        end
-      end
-    end
-  end
-end

data/lib/konjak/translator/text_translate.rb

@@ -1,22 +0,0 @@
-module Konjak
-  class Translator
-    module TextTranslate
-      refine(TranslationUnit) do
-        def translate(src_lang, target_lang, text)
-          s = variant(src_lang).segment.text
-          t = variant(target_lang).segment.text
-
-          texts = []
-          while true
-            head, match, tail = text.partition(s)
-            break if match.empty?
-            texts << head unless head.empty?
-            texts << TranslatedString.new(t)
-            text = tail
-          end
-          texts << text
-        end
-      end
-    end
-  end
-end

data/lib/konjak/translator/translated_string.rb

@@ -1,6 +0,0 @@
-module Konjak
-  class Translator
-    class TranslatedString < String
-    end
-  end
-end