text_alignment 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dba8334b3fbdc77976ae32ed2e5844a716f954c850bf24c0937462c1cd2b220
-  data.tar.gz: 806ac498264b81111ef1055dbb8592fa5ec7fd4755f0b2106c2851c45c6eb498
+  metadata.gz: 8c1c45ed630cdd60291606b59e1944f0b854a689cfa0d281ae8b8879bf01e806
+  data.tar.gz: 9b33688f3a08f9110f556b4357fbe598d42d1147bc06933323cae4df7187341f
 SHA512:
-  metadata.gz: 4732107de89daff9e8bbe89254e0d138db517396958e75b56f7f0697e3ff9d38e6b64082bbc84482bf12a91da7a03cfacdb3691729aaebd3eeac6aa836bf07c5
-  data.tar.gz: e101fd3c1f5b8a5d9604f4998218a816d0eeecd5e7afbed23bf7504403d97de5e282f0dcad37abb7a46804f1f0749f6b996430bc47e93ae0b4947a916c25f40d
+  metadata.gz: 856a8fca63f80be4cea7f6beff85dcf475d9237d68bc96728bcfcc030397f414637d2a8e32b139a6fdbccfd8603d327738ff7f3f59d80dc9e61e55a11a04bf20
+  data.tar.gz: 6055f50827354461f194a50da74259e8d97473be9b085634df270964bb815d96d9f916bbad2ef3961ddd845291a80e13b1680f1c42bafeab156b0d59fa3ba952

data/lib/text_alignment/approximate_fit.rb

@@ -8,7 +8,7 @@ module TextAlignment
   SIGNATURE_NGRAM = 5
   MIN_LENGTH_FOR_APPROXIMATION = 50
   BUFFER_RATE = 0.1
-  TEXT_SIMILARITY_TRESHOLD = 0.8
+  TEXT_SIMILARITY_TRESHOLD = 0.7
 end
 
 class << TextAlignment
@@ -28,6 +28,7 @@ class << TextAlignment
     signature_ngrams = ngram_shared.select{|g| ngram2.count(g) == 1}
     return nil, nil if signature_ngrams.empty? #raise "no signature ngram"
 
+    cache = {}
     fit_begin, fit_end = nil, nil
     signature_ngrams.each do |signature_ngram|
       loc_signature_ngram_in_str1 = str1.index(signature_ngram)
@@ -42,13 +43,15 @@ class << TextAlignment
       fit_end = loc_signature_ngram_in_str2 + offset_end + (offset_end * TextAlignment::BUFFER_RATE).to_i
       fit_end = str2.length if fit_end > str2.length
 
+      next if cache.has_key?("#{fit_begin}-#{fit_end}")
       text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
+      cache["#{fit_begin}-#{fit_end}"] = text_similarity
+
       break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
       fit_begin, fit_end = nil, nil
     end
-
-    return nil, nil if fit_begin >= fit_end
-    return fit_begin, fit_end
+    return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end
+    return nil, nil
   end
 
   private

data/lib/text_alignment/find_divisions.rb

@@ -8,7 +8,7 @@ module TextAlignment; end unless defined? TextAlignment
 # to assume that there is no bag representation to this method
 
 module TextAlignment
-  TextAlignment::SIMILARITY_THRESHOLD = 0.8
+  TextAlignment::SIMILARITY_THRESHOLD = 0.7
 end
 
 class << TextAlignment
@@ -28,12 +28,102 @@ class << TextAlignment
     target.tr!(characters_from, characters_to)
     sources.each{|source| source[:text].tr!(characters_from, characters_to)}
 
+    # to process smaller ones first
     sources.sort!{|s1, s2| s1[:text].size <=> s2[:text].size}
 
     TextAlignment._find_divisions(target, sources)
   end
 
-  def _find_divisions(target, sources)
+  def _find_divisions(_target, _sources)
+    indice = []
+    history = []
+    cache = {}
+    target = _target.dup
+    sources = _sources.dup
+    until target.strip.empty? || sources.empty?
+      mode, cmp = nil, nil
+      candidates = []
+      sources.each_with_index do |source, i|
+        if target.size < source[:text].size
+          mode = :t_in_s
+          str1 = target
+          str2 = source[:text]
+        else
+          mode = :s_in_t
+          str1 = source[:text]
+          str2 = target
+        end
+
+        len1 = str1.length
+        len2 = str2.length
+
+        offset_begin, offset_end = if (len2 - len1) > len1 * (1 - TextAlignment::SIMILARITY_THRESHOLD)
+          approximate_fit(str1, str2)
+        else
+          # the whole target
+          [0, -1]
+        end
+
+        unless offset_begin.nil?
+          key = str1 + ' _:_ ' + str2[offset_begin .. offset_end]
+          cmp = if cache.has_key? key
+            cache[key]
+          else
+            cmp = TextAlignment::LCSComparison.new(str1, str2[offset_begin .. offset_end])
+          end
+          cache[key] = cmp
+
+          if (cmp.similarity > TextAlignment::SIMILARITY_THRESHOLD) && ((len1 - (cmp.str1_match_final - cmp.str1_match_initial + 1)) < len1 * (1 - TextAlignment::SIMILARITY_THRESHOLD))
+            candidates << {idx:i, offset:offset_begin, mode:mode, cmp:cmp}
+          end
+        end
+      end
+
+      # return remaining target and sources if m.nil?
+      break if candidates.empty?
+
+      choice = candidates.max{|a, b| a[:cmp].similarity <=> a[:cmp].similarity}
+      m = choice[:idx]
+      mode = choice[:mode]
+
+      index = if mode == :t_in_s
+        {divid:sources[m][:divid], region:[0, target.size]}
+      else # :s_in_t
+        cmp = choice[:cmp]
+        offset = choice[:offset]
+        {divid:sources[m][:divid], region:[cmp.str2_match_initial + offset, cmp.str2_match_final + offset + 1]}
+      end
+
+      target = target[0 ... index[:region][0]] + target[index[:region][1] .. -1]
+      history << index[:region].dup
+
+      before_begin = index[:region][0]
+      before_end = index[:region][1]
+
+      rhistory = history.reverse
+      rhistory.shift
+      rhistory.each do |h|
+        gap = h[1] - h[0]
+        index[:region][0] += gap if index[:region][0] >= h[0]
+        index[:region][1] += gap if index[:region][1] >  h[0]
+      end
+
+      indice << index
+
+      sources.delete_at(m)
+    end
+
+    unless target.strip.empty? && sources.empty?
+      index = {divid:nil}
+      index[:remaining_target] = target unless target.strip.empty?
+      index[:remaining_sources] = sources.collect{|s| s[:divid]} unless sources.empty?
+      indice << index
+    end
+
+    indice
+  end
+
+  def _find_divisions_old(target, sources)
     mode, m, c, offset_begin = nil, nil, nil, nil
 
     sources.each_with_index do |source, i|
@@ -88,6 +178,7 @@ class << TextAlignment
       return [index] + more_index
     end
   end
+
 end
 
 if __FILE__ == $0
@@ -98,20 +189,29 @@ if __FILE__ == $0
 
     sources = JSON.parse File.read(ARGV[1]), :symbolize_names => true
     div_index = TextAlignment::find_divisions(target_text, sources)
+    pp div_index
 
     # str1 = File.read(ARGV[0]).strip
     # str2 = File.read(ARGV[1]).strip
     # div_index = TextAlignment::find_divisions(str1, [str2])
 
-    puts "target length: #{target_text.length}"
-    div_index.each do |i|
-      if i[0] >= 0
-        puts "[Div: #{i[0]}] (#{i[1][0]}, #{i[1][1]})"
-        puts target_text[i[1][0] ... i[1][1]]
-        puts "=========="
-      else
-        p i
-      end
-    end
+    # puts "target length: #{target_text.length}"
+    # div_index.each do |i|
+    #   unless i[:divid].nil?
+    #     puts "[Div: #{i[:divid]}] (#{i[:region][0]}, #{i[:region][1]})"
+    #     puts target_text[i[:region][0] ... i[:region][1]]
+    #     puts "=========="
+    #   else
+    #     p i
+    #   end
+
+    #   # if i[0] >= 0
+    #   #   puts "[Div: #{i[0]}] (#{i[1][0]}, #{i[1][1]})"
+    #   #   puts target_text[i[1][0] ... i[1][1]]
+    #   #   puts "=========="
+    #   # else
+    #   #   p i
+    #   # end
+    # end
   end
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-  VERSION = '0.2.8'
+  VERSION = '0.2.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-04 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary