konjak 0.0.18 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4ffc2d0548951d62d07f5612bd4399e8a8f6519
-  data.tar.gz: 77a83feb0b806e2e6a87b56246c7062fa175f364
+  metadata.gz: 768f8768ce6cf9b74a7efdbc099fe192d8315263
+  data.tar.gz: 6ea6d582ac4df35043317bc274bf9391716fe5f3
 SHA512:
-  metadata.gz: c92f4f5d4328b6fe29e5a92451b7e648796c37ec7c044a58ecfbb70e7620cf67494fce6a8052bbce6901caa85a529a5d35001ccafe236c7dd0b97efcd7e399d3
-  data.tar.gz: 995318581a91f8fbc41ddf97258777ba3942e6e7c69ab1b303208be8d7f1aecca44c2b0561c1a10adc70f5055e583223f97ad8fcca7787a8165f85ff37f8a8e8
+  metadata.gz: 90a225fc3be21ca3a273ff7304c5c803d72ed372449a53143090909b48e4272ba8cf9f47bc0522dc502e50582ebcd0c76c3dffce4ef605c7322e3448c9a0ab48
+  data.tar.gz: cdf1f709324975a459b808e7c855efcfa981cee5f96839905ab3744cd52fa80a7247f199957617059b1f509211582aa32558c1a8e666b827ae303a503f29f6b2

data/lib/konjak/segment.rb

@@ -23,8 +23,8 @@ module Konjak
 
     def compile_pattern
       regexp = Regexp.escape(text)
-      regexp.gsub!(/(?<!^)\\\s/)          { WHITE_SPACE_PATTERN_TEXT }
-      regexp.gsub!(/(?<!^)(?:\\s)+(?!$)/) {|s| s + POSSESSIVE_QUALIFIER }
+      regexp.gsub!(/(?<!^)\\\s/)     { WHITE_SPACE_PATTERN_TEXT }
+      regexp.gsub!(/(?<!^)(?:\\s)+/) {|s| s + POSSESSIVE_QUALIFIER }
       Regexp.compile(regexp)
     end
 

data/lib/konjak/tmx_segmentor/strategy.rb

@@ -16,22 +16,81 @@ module Konjak
       end
 
       def segmentize(text)
-        segments = [text]
-        translation_units(text).each do |translation_unit|
-          segment = translation_unit.variant(@lang).segment
+        range_segment_pairs = []
 
-          pat = compile_pattern(segment)
+        translation_units.each {|tu|
+          segment = tu.variant(@lang).segment
+          text.scan(compile_pattern(segment)) {
+            range_segment_pairs << [($~.begin(0)...$~.end(0)), segment]
+          }
+        }
+
+        # Can't split text
+        return [text] if range_segment_pairs.empty?
+
+        range_segment_pairs.uniq! {|rsp| [rsp[0], rsp[1].text] }
+        range_segment_pairs.sort_by! {|(m, s)|
+          [m.begin, -s.text.size]
+        }
 
-          segments.map! {|text|
-            next text if text.length < min_segment_length
-            next text if text.is_a?(SegmentString)
+        max_weight_range_segments = max_weight_range_segments(range_segment_pairs)
 
-            split(pat, segment, text)
-          }.flatten!
+        segments = []
+        prev_text_index = 0
+        max_weight_range_segments.each do |(range, segment)|
+          prev_text = text[prev_text_index...range.begin]
+
+          segments << prev_text unless prev_text.empty?
+
+          segments << SegmentString.new(text[range.begin, range.size], segment)
+
+          prev_text_index = range.end
         end
+        after_text = text[prev_text_index..-1]
+        segments << after_text unless after_text.empty?
         segments
       end
 
+
+      def max_weight_range_segments(range_segment_pairs)
+        edges      = []
+        prev_nodes = Array.new(range_segment_pairs.size, -1)
+        weights    = range_segment_pairs.map {|rsp| rsp[0].size }
+
+        range_segment_pairs.each_with_index do |rsp, rsp_i|
+          ((rsp_i + 1)...range_segment_pairs.size).each do |rsp2_i|
+            rsp2 = range_segment_pairs[rsp2_i]
+
+            next if rsp2[0].begin < rsp[0].end
+
+            edges << [rsp_i, rsp2_i]
+          end
+        end
+
+        edges.each do |(rsp_i, rsp2_i)|
+          new_rsp2_weight = weights[rsp_i] + range_segment_pairs[rsp2_i][0].size
+
+          if weights[rsp2_i] < new_rsp2_weight
+            weights[rsp2_i] = new_rsp2_weight
+            prev_nodes[rsp2_i] = rsp_i
+          end
+        end
+
+        node_index = weights.index(weights.max)
+
+        max_weight_range_segment_indexes = Enumerator.new {|y|
+          loop do
+            break if node_index == -1
+            y << node_index
+            node_index = prev_nodes[node_index]
+          end
+        }.to_a.reverse
+
+        max_weight_range_segment_indexes.map {|i|
+          range_segment_pairs[i]
+        }
+      end
+
       private
 
       def default_options
@@ -49,73 +108,16 @@ module Konjak
         @options[:max_segment_length]
       end
 
-      def split(pat, segment, text)
-        texts = []
-        while true
-          break if text.length < min_segment_length
-
-          break unless text =~ pat
-
-          head  = $`
-          match = $&
-          tail  = $'
-
-          texts << head unless head.empty?
-
-          texts << SegmentString.new(match, segment)
-
-          text = tail
-        end
-        texts << text
-      end
-
-      def translation_units(text)
-        tus = @tmx.body.translation_units
-
-        tus.select! {|tu|
+      def translation_units
+        @translation_units ||= @tmx.body.translation_units.select {|tu|
           segment = tu.variant(@lang).segment
           segment_length = segment.text.length
 
           next false if segment_length < min_segment_length
           next false if max_segment_length && max_segment_length < segment_length
 
-          text =~ compile_pattern(tu.variant(@lang).segment)
+          true
         }
-
-        simular_translation_units_map = {}
-
-        tus.sort_by! {|tu|
-          tu_segment = tu.variant(@lang).segment
-          segment_text = tu_segment.text
-
-          unless simular_translation_units_map[segment_text]
-            simular_translation_units = tus.select {|tu2|
-              tu2.variant(@lang).segment.text.include?(segment_text)
-            }.sort_by! {|tu2| tu2.variant(@lang).segment.text.size }
-
-            simular_translation_units.each do |tu2|
-              simular_translation_units_map[tu2.variant(@lang).segment.text] = simular_translation_units
-            end
-          end
-
-          rank = simular_translation_units_map[segment_text].index {|tu2|
-            tu2.variant(@lang).segment.text == segment_text
-          }
-
-          # GTTの場合
-          translation_timestamp = nil
-          if tm_entry = tu.at('entry_metadata').try(:at, 'tm_entry')
-            source_info = tm_entry.at('source_info')
-            if source_info.try(:at, 'source_lang').try(:text) == @lang && source_info.try(:at, 'source').try(:text) == segment_text
-              translation_timestamp = tm_entry.at('translation').try(:attr, 'translation_timestamp').to_i
-            end
-          end
-          translation_timestamp ||= 0
-
-          [-rank, -translation_timestamp, -segment_text.length]
-        }
-
-        tus
       end
     end
   end

data/lib/konjak/version.rb

@@ -1,3 +1,3 @@
 module Konjak
-  VERSION = "0.0.18"
+  VERSION = "0.0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: konjak
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.0.19
 platform: ruby
 authors:
 - Seiei Higa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-18 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport