text_alignment 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e9a7bcce440fe7a86655b4c8edceb92607c810a4cdf6d05baee3ffe55199db6
-  data.tar.gz: 739bed915b416c59c908c396f312f6b04df79cbad4075cbcbfab3a82db610fd5
+  metadata.gz: 45bf8e55a66daff4d79765aec0e4d7482151aa7b86749e054e5d93030ffffabe
+  data.tar.gz: 6251835e35077e2ae828666026a663ae63be7f8d7b7d785b4898763021f546e2
 SHA512:
-  metadata.gz: f2fe7ba9a6bee8150572f7813f9ce680c8fd3fe1252292f351082d95293392969d4094e492e04f844c0bc956cbb8934213937e11b75789f4b6432f31c1e7fad0
-  data.tar.gz: aed5b749142693d86f26c8bdd0d68ba89727f5b8dbab916f0f476d56d253ccf63a0ea908230192d4d5e1511124b515c013fde90ffa6873a28b3124702c926339
+  metadata.gz: 4750b2507b4eb172123ca9a82320c2456e66d3f06ca3e3bdd52cf31055e9cb46af53013a484dbb87b1bb3236701cc8a69e1c4c64b576ee41a3a4957c3431be65
+  data.tar.gz: 1819b94e47bdf064d53712bb667f15d74f8e37bd49df7976075c3cbdc0a992d25bb318f76bfc0522cea0b594d002bcab0ebcef6112bf9e26128c245a7fb67bcf

data/Gemfile

@@ -1,8 +1,9 @@
 source 'https://rubygems.org'
-ruby '2.3.4'
+ruby '2.5.5'
 
 gem 'diff-lcs', '~> 1.3'
 gem 'ruby-dictionary', '~>1.1', '>=1.1.1'
+gem 'string-similarity', '~> 2.1'
 
 group :test do
 	gem 'rspec', '~>3.0'

data/Gemfile.lock

@@ -15,6 +15,7 @@ GEM
       rspec-support (~> 3.0.0)
     rspec-support (3.0.4)
     ruby-dictionary (1.1.1)
+    string-similarity (2.1.0)
 
 PLATFORMS
   ruby
@@ -23,9 +24,10 @@ DEPENDENCIES
   diff-lcs (~> 1.3)
   rspec (~> 3.0)
   ruby-dictionary (~> 1.1, >= 1.1.1)
+  string-similarity (~> 2.1)
 
 RUBY VERSION
-   ruby 2.3.4p301
+   ruby 2.5.5p157
 
 BUNDLED WITH
    1.17.3

data/bin/align_annotations

@@ -42,6 +42,10 @@ puts
 puts '[Denotations transformed]'
 new_denotations = align.transform_hdenotations(denotations)
 pp new_denotations
+puts
+puts '[Annotations transformed]'
+anns2[:denotations] = new_denotations
+puts anns2.to_json
 
 # p align.common_elements
 # puts "---------------"

data/lib/text_alignment/approximate_fit.rb

@@ -1,11 +1,14 @@
 #!/usr/bin/env ruby
+require 'string-similarity'
+
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
 module TextAlignment
   SIGNATURE_NGRAM = 5
   MIN_LENGTH_FOR_APPROXIMATION = 50
-  BUFFER_RATE = 0.2
+  BUFFER_RATE = 0.1
+  TEXT_SIMILARITY_TRESHOLD = 0.8
 end
 
 class << TextAlignment
@@ -22,29 +25,40 @@ class << TextAlignment
     # If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
     return nil, nil if ngram_shared.empty?
 
-    # approximate the beginning of the fit
-    signature_ngram = ngram_shared.detect{|g| ngram2.count(g) == 1}
+    signature_ngrams = ngram_shared.select{|g| ngram2.count(g) == 1}
+    return nil, nil if signature_ngrams.empty? #raise "no signature ngram"
+
+    fit_begin, fit_end = nil, nil
+    signature_ngrams.each do |signature_ngram|
+      loc_signature_ngram_in_str1 = str1.index(signature_ngram)
+      loc_signature_ngram_in_str2 = str2.index(signature_ngram)
 
-    return nil, nil if signature_ngram.nil? #raise "no signature ngram"
-    offset = str1.index(signature_ngram)
-    fit_begin = str2.index(signature_ngram) - offset - (offset * TextAlignment::BUFFER_RATE).to_i
-    fit_begin = 0 if fit_begin < 0    
+      # approximate the beginning of the fit
+      fit_begin = loc_signature_ngram_in_str2 - loc_signature_ngram_in_str1 - (loc_signature_ngram_in_str1 * TextAlignment::BUFFER_RATE).to_i
+      fit_begin = 0 if fit_begin < 0
 
-    # to change the order according to ngram2
-    ngram_shared = ngram2 & ngram1
+      # approximate the end of the fit
+      offset_end = str1.length - loc_signature_ngram_in_str1
+      fit_end = loc_signature_ngram_in_str2 + offset_end + (offset_end * TextAlignment::BUFFER_RATE).to_i
+      fit_end = str2.length if fit_end > str2.length
 
-    # approximate the end of the fit
-    ngram_shared_reverse = ngram_shared.reverse
-    ngram2_reverse = ngram2.reverse
-    signature_ngram = ngram_shared_reverse.detect{|g| ngram2_reverse.count(g) == 1}
-    return nil, nil if signature_ngram.nil? # raise "no signature ngram" 
-    offset = str1.length - str1.rindex(signature_ngram)
-    fit_end = str2.rindex(signature_ngram) + offset + (offset * TextAlignment::BUFFER_RATE).to_i
-    fit_end = str2.length if fit_end > str2.length
+      text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
+      break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+      fit_begin, fit_end = nil, nil
+    end
 
     return nil, nil if fit_begin >= fit_end
     return fit_begin, fit_end
   end
+
+  private
+
+  def text_similarity(str1, str2, ngram_order = 3)
+    _str1 = str1.delete(" \t\r\n")
+    _str2 = str2.delete(" \t\r\n")
+    String::Similarity.cosine(_str1, _str2, ngram:2)
+  end
+
 end
 
 if __FILE__ == $0

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-  VERSION = '0.2.6'
+  VERSION = '0.2.7'
 end

data/text_alignment.gemspec

@@ -18,5 +18,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_development_dependency 'ruby-dictionary', '~>1.1', '>=1.1.1'
+  gem.add_development_dependency 'string-similarity', '~> 2.1'
   gem.add_development_dependency 'rspec', '~>3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-12 00:00:00.000000000 Z
+date: 2020-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -30,6 +30,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.1
+- !ruby/object:Gem::Dependency
+  name: string-similarity
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Ruby class for aligning two character strings