text_alignment 0.2.4 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c61a711930d19f5a72bd4a4128f5f36038a185eeb203b9ca68afce143694dbd3
-  data.tar.gz: 173fa1ed0277f0384f0804e7a1ea02c3bd1a1eda20a5d3f57523028f815e69ac
+  metadata.gz: 8c1c45ed630cdd60291606b59e1944f0b854a689cfa0d281ae8b8879bf01e806
+  data.tar.gz: 9b33688f3a08f9110f556b4357fbe598d42d1147bc06933323cae4df7187341f
 SHA512:
-  metadata.gz: 8f2a018145a07ec1d9a5c2277fd4d1e4bdb694726cf0936c92e8f25addb325e485545b7a1373a265e3b60fe2da0e39cdccbad586968aef093ed2154070242173
-  data.tar.gz: 488390234e0e3b9d7d67389f303aed6e59bb0e16b1ad94908c8020be64b9c0bac9a9f404e42944fd37ece188377c930fd43596678a80ee24f3b00174c9e33aa4
+  metadata.gz: 856a8fca63f80be4cea7f6beff85dcf475d9237d68bc96728bcfcc030397f414637d2a8e32b139a6fdbccfd8603d327738ff7f3f59d80dc9e61e55a11a04bf20
+  data.tar.gz: 6055f50827354461f194a50da74259e8d97473be9b085634df270964bb815d96d9f916bbad2ef3961ddd845291a80e13b1680f1c42bafeab156b0d59fa3ba952

data/Gemfile

@@ -1,8 +1,9 @@
 source 'https://rubygems.org'
-ruby '2.3.4'
+ruby '2.5.5'
 
 gem 'diff-lcs', '~> 1.3'
 gem 'ruby-dictionary', '~>1.1', '>=1.1.1'
+gem 'string-similarity', '~> 2.1'
 
 group :test do
 	gem 'rspec', '~>3.0'

data/Gemfile.lock

@@ -15,6 +15,7 @@ GEM
       rspec-support (~> 3.0.0)
     rspec-support (3.0.4)
     ruby-dictionary (1.1.1)
+    string-similarity (2.1.0)
 
 PLATFORMS
   ruby
@@ -23,9 +24,10 @@ DEPENDENCIES
   diff-lcs (~> 1.3)
   rspec (~> 3.0)
   ruby-dictionary (~> 1.1, >= 1.1.1)
+  string-similarity (~> 2.1)
 
 RUBY VERSION
-   ruby 2.3.4p301
+   ruby 2.5.5p157
 
 BUNDLED WITH
    1.17.3

data/bin/align_annotations

@@ -17,14 +17,35 @@ str2 = anns2[:text]
 
 denotations = anns1[:denotations]
 
+puts "[Alignment1]====="
 align = TextAlignment::TextAlignment.new(str1, str2, TextAlignment::MAPPINGS)
 puts TextAlignment::sdiff2cdiff(align.sdiff)
-puts "\n=====\n\n"
-
+puts
+puts "[Similarity]\n#{align.similarity}"
+puts
+puts '[Denotations original]'
+pp denotations
+puts
+puts '[Denotations transformed]'
+new_denotations = align.transform_hdenotations(denotations)
+pp new_denotations
+puts
+puts "[Alignment2 (downcased)]====="
+align = TextAlignment::TextAlignment.new(str1.downcase, str2.downcase, TextAlignment::MAPPINGS)
+puts TextAlignment::sdiff2cdiff(align.sdiff)
+puts
+puts "[Similarity]\n#{align.similarity}"
+puts
+puts '[Denotations original]'
 pp denotations
-puts "-----"
+puts
+puts '[Denotations transformed]'
 new_denotations = align.transform_hdenotations(denotations)
 pp new_denotations
+puts
+puts '[Annotations transformed]'
+anns2[:denotations] = new_denotations
+puts anns2.to_json
 
 # p align.common_elements
 # puts "---------------"

data/lib/text_alignment/approximate_fit.rb

@@ -1,11 +1,14 @@
 #!/usr/bin/env ruby
+require 'string-similarity'
+
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
 module TextAlignment
   SIGNATURE_NGRAM = 5
   MIN_LENGTH_FOR_APPROXIMATION = 50
-  BUFFER_RATE = 0.2
+  BUFFER_RATE = 0.1
+  TEXT_SIMILARITY_TRESHOLD = 0.7
 end
 
 class << TextAlignment
@@ -22,29 +25,43 @@ class << TextAlignment
     # If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
     return nil, nil if ngram_shared.empty?
 
-    # approximate the beginning of the fit
-    signature_ngram = ngram_shared.detect{|g| ngram2.count(g) == 1}
+    signature_ngrams = ngram_shared.select{|g| ngram2.count(g) == 1}
+    return nil, nil if signature_ngrams.empty? #raise "no signature ngram"
+
+    cache = {}
+    fit_begin, fit_end = nil, nil
+    signature_ngrams.each do |signature_ngram|
+      loc_signature_ngram_in_str1 = str1.index(signature_ngram)
+      loc_signature_ngram_in_str2 = str2.index(signature_ngram)
+
+      # approximate the beginning of the fit
+      fit_begin = loc_signature_ngram_in_str2 - loc_signature_ngram_in_str1 - (loc_signature_ngram_in_str1 * TextAlignment::BUFFER_RATE).to_i
+      fit_begin = 0 if fit_begin < 0
 
-    return nil, nil if signature_ngram.nil? #raise "no signature ngram"
-    offset = str1.index(signature_ngram)
-    fit_begin = str2.index(signature_ngram) - offset - (offset * TextAlignment::BUFFER_RATE).to_i
-    fit_begin = 0 if fit_begin < 0    
+      # approximate the end of the fit
+      offset_end = str1.length - loc_signature_ngram_in_str1
+      fit_end = loc_signature_ngram_in_str2 + offset_end + (offset_end * TextAlignment::BUFFER_RATE).to_i
+      fit_end = str2.length if fit_end > str2.length
 
-    # to change the order according to ngram2
-    ngram_shared = ngram2 & ngram1
+      next if cache.has_key?("#{fit_begin}-#{fit_end}")
+      text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
+      cache["#{fit_begin}-#{fit_end}"] = text_similarity
 
-    # approximate the end of the fit
-    ngram_shared_reverse = ngram_shared.reverse
-    ngram2_reverse = ngram2.reverse
-    signature_ngram = ngram_shared_reverse.detect{|g| ngram2_reverse.count(g) == 1}
-    return nil, nil if signature_ngram.nil? # raise "no signature ngram" 
-    offset = str1.length - str1.rindex(signature_ngram)
-    fit_end = str2.rindex(signature_ngram) + offset + (offset * TextAlignment::BUFFER_RATE).to_i
-    fit_end = str2.length if fit_end > str2.length
+      break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+      fit_begin, fit_end = nil, nil
+    end
+    return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end
+    return nil, nil
+  end
+
+  private
 
-    return nil, nil if fit_begin >= fit_end
-    return fit_begin, fit_end
+  def text_similarity(str1, str2, ngram_order = 3)
+    _str1 = str1.delete(" \t\r\n")
+    _str2 = str2.delete(" \t\r\n")
+    String::Similarity.cosine(_str1, _str2, ngram:2)
   end
+
 end
 
 if __FILE__ == $0

data/lib/text_alignment/find_divisions.rb

@@ -8,7 +8,7 @@ module TextAlignment; end unless defined? TextAlignment
 # to assume that there is no bag representation to this method
 
 module TextAlignment
-  TextAlignment::SIMILARITY_THRESHOLD = 0.8
+  TextAlignment::SIMILARITY_THRESHOLD = 0.7
 end
 
 class << TextAlignment
@@ -28,12 +28,102 @@ class << TextAlignment
     target.tr!(characters_from, characters_to)
     sources.each{|source| source[:text].tr!(characters_from, characters_to)}
 
+    # to process smaller ones first
     sources.sort!{|s1, s2| s1[:text].size <=> s2[:text].size}
 
     TextAlignment._find_divisions(target, sources)
   end
 
-  def _find_divisions(target, sources)
+  def _find_divisions(_target, _sources)
+    indice = []
+    history = []
+    cache = {}
+    target = _target.dup
+    sources = _sources.dup
+    until target.strip.empty? || sources.empty?
+      mode, cmp = nil, nil
+      candidates = []
+      sources.each_with_index do |source, i|
+        if target.size < source[:text].size
+          mode = :t_in_s
+          str1 = target
+          str2 = source[:text]
+        else
+          mode = :s_in_t
+          str1 = source[:text]
+          str2 = target
+        end
+
+        len1 = str1.length
+        len2 = str2.length
+
+        offset_begin, offset_end = if (len2 - len1) > len1 * (1 - TextAlignment::SIMILARITY_THRESHOLD)
+          approximate_fit(str1, str2)
+        else
+          # the whole target
+          [0, -1]
+        end
+
+        unless offset_begin.nil?
+          key = str1 + ' _:_ ' + str2[offset_begin .. offset_end]
+          cmp = if cache.has_key? key
+            cache[key]
+          else
+            cmp = TextAlignment::LCSComparison.new(str1, str2[offset_begin .. offset_end])
+          end
+          cache[key] = cmp
+
+          if (cmp.similarity > TextAlignment::SIMILARITY_THRESHOLD) && ((len1 - (cmp.str1_match_final - cmp.str1_match_initial + 1)) < len1 * (1 - TextAlignment::SIMILARITY_THRESHOLD))
+            candidates << {idx:i, offset:offset_begin, mode:mode, cmp:cmp}
+          end
+        end
+      end
+
+      # return remaining target and sources if m.nil?
+      break if candidates.empty?
+
+      choice = candidates.max{|a, b| a[:cmp].similarity <=> a[:cmp].similarity}
+      m = choice[:idx]
+      mode = choice[:mode]
+
+      index = if mode == :t_in_s
+        {divid:sources[m][:divid], region:[0, target.size]}
+      else # :s_in_t
+        cmp = choice[:cmp]
+        offset = choice[:offset]
+        {divid:sources[m][:divid], region:[cmp.str2_match_initial + offset, cmp.str2_match_final + offset + 1]}
+      end
+
+      target = target[0 ... index[:region][0]] + target[index[:region][1] .. -1]
+      history << index[:region].dup
+
+      before_begin = index[:region][0]
+      before_end = index[:region][1]
+
+      rhistory = history.reverse
+      rhistory.shift
+      rhistory.each do |h|
+        gap = h[1] - h[0]
+        index[:region][0] += gap if index[:region][0] >= h[0]
+        index[:region][1] += gap if index[:region][1] >  h[0]
+      end
+
+      indice << index
+
+      sources.delete_at(m)
+    end
+
+    unless target.strip.empty? && sources.empty?
+      index = {divid:nil}
+      index[:remaining_target] = target unless target.strip.empty?
+      index[:remaining_sources] = sources.collect{|s| s[:divid]} unless sources.empty?
+      indice << index
+    end
+
+    indice
+  end
+
+  def _find_divisions_old(target, sources)
     mode, m, c, offset_begin = nil, nil, nil, nil
 
     sources.each_with_index do |source, i|
@@ -88,6 +178,7 @@ class << TextAlignment
       return [index] + more_index
     end
   end
+
 end
 
 if __FILE__ == $0
@@ -98,20 +189,29 @@ if __FILE__ == $0
 
     sources = JSON.parse File.read(ARGV[1]), :symbolize_names => true
     div_index = TextAlignment::find_divisions(target_text, sources)
+    pp div_index
 
     # str1 = File.read(ARGV[0]).strip
     # str2 = File.read(ARGV[1]).strip
     # div_index = TextAlignment::find_divisions(str1, [str2])
 
-    puts "target length: #{target_text.length}"
-    div_index.each do |i|
-      if i[0] >= 0
-        puts "[Div: #{i[0]}] (#{i[1][0]}, #{i[1][1]})"
-        puts target_text[i[1][0] ... i[1][1]]
-        puts "=========="
-      else
-        p i
-      end
-    end
+    # puts "target length: #{target_text.length}"
+    # div_index.each do |i|
+    #   unless i[:divid].nil?
+    #     puts "[Div: #{i[:divid]}] (#{i[:region][0]}, #{i[:region][1]})"
+    #     puts target_text[i[:region][0] ... i[:region][1]]
+    #     puts "=========="
+    #   else
+    #     p i
+    #   end
+
+    #   # if i[0] >= 0
+    #   #   puts "[Div: #{i[0]}] (#{i[1][0]}, #{i[1][1]})"
+    #   #   puts target_text[i[1][0] ... i[1][1]]
+    #   #   puts "=========="
+    #   # else
+    #   #   p i
+    #   # end
+    # end
   end
 end

data/lib/text_alignment/lcs_cdiff.rb

@@ -37,7 +37,7 @@ class << TextAlignment
       end
     end
 
-    cdiff_str1.gsub(/\n/, ' ') + "\n" + cdiff_str2.gsub(/\n/, ' ')
+    cdiff_str1.gsub(/\n/, ' ') + "\n>>>>><<<<<\n" + cdiff_str2.gsub(/\n/, ' ')
   end
 
 end

data/lib/text_alignment/text_alignment.rb

@@ -79,7 +79,7 @@ class TextAlignment::TextAlignment
   end
 
   def transform_a_span(span)
-    {:begin=>@position_map_begin[span[:begin]], :end=>@position_map_end[span[:end]]}
+    {begin: @position_map_begin[span[:begin]], end: @position_map_end[span[:end]]}
   end
 
   def transform_spans(spans)
@@ -91,11 +91,8 @@ class TextAlignment::TextAlignment
   end
 
   def transform_hdenotations(hdenotations)
-    unless hdenotations.nil?
-      hdenotations_new = Array.new(hdenotations)
-      (0...hdenotations.length).each {|i| hdenotations_new[i][:span] = transform_a_span(hdenotations[i][:span])}
-      hdenotations_new
-    end
+    return nil if hdenotations.nil?
+    hdenotations.collect{|d| d.dup.merge({span:transform_a_span(d[:span])})}
   end
 
   private

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-  VERSION = '0.2.4'
+  VERSION = '0.2.9'
 end

data/text_alignment.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_development_dependency 'ruby-dictionary', '~>1.1', '>=1.1.1'
+  gem.add_runtime_dependency 'ruby-dictionary', '~>1.1', '>=1.1.1'
+  gem.add_runtime_dependency 'string-similarity', '~> 2.1'
   gem.add_development_dependency 'rspec', '~>3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.9
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-11 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -20,7 +20,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.1
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -30,6 +30,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.1
+- !ruby/object:Gem::Dependency
+  name: string-similarity
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Ruby class for aligning two character strings