text_alignment 0.3.22 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa961ae295a43e3912878dab34fac18a8cbca395eea999a781a538ce17c61bf3
-  data.tar.gz: 38ae815d192104b9bff4664c5749910675ae4e5d9c895fa63c26fdc24f6d7b91
+  metadata.gz: 52bc948955e2df858e397b14eabb4411f73b3ff1e4d879ff4b7015d3b5e03308
+  data.tar.gz: fd20caec51c95bdc475e0698a52bb7fdebc9e22c43bb47267a883bcc75862268
 SHA512:
-  metadata.gz: 185bdcd0298932a09795723085734d79aaddf638a45c871ff1d29e6c89e02041ddd12f848e3350244b27ac050adc7c52f7e1dedfe6664159d781c25d6dd55f1b
-  data.tar.gz: 0b88cd8aff0f529b5cad32a360aba476e90d525bc77f89c5c5c59dcef5a6d9077c849acdf9f289f61fea2a50ebe927fd9a53c1eb0bdca9c7661900afdaa8fb94
+  metadata.gz: dbcb7ab70a64d4a398a5c5761cc5b2f5de6835ccc0e2d0854556f03ef91d0c0294986cc2ff1273788e6b7b0c73dfdf86fd16ee1ef8ce35ecc11d61f8eaab9521
+  data.tar.gz: 01d21cdcc0ab81d61e08ff1f52360ba35973756fd5060ce866391ff622d4cf87da945dba43a622a0581d63ee96c8723e1cb28991bfa02f4e1e803896bdc64d7f

data/lib/text_alignment/anchor_finder.rb

@@ -1,17 +1,15 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
-
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil)
+	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
 		@size_window = _size_window || TextAlignment::SIZE_WINDOW
+		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
 
 		@reverse = (target_str.length < source_str.length)
 
@@ -43,10 +41,10 @@ class TextAlignment::AnchorFinder
 				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
 
 				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
 
 				search_position = @beg_s2 + 1
 			end
@@ -87,7 +85,8 @@ class TextAlignment::AnchorFinder
 	private
 
 	def get_left_windows
-		return if @beg_s1 < @size_window || @beg_s2 < @size_window
+		# commend below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
 
 		window_s1 = ''
 		loc = @beg_s1 - 1
@@ -115,7 +114,8 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# commend below with the assumption that the end of a document gives a significant locational
+		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram

data/lib/text_alignment/approximate_fit.rb

@@ -1,13 +1,11 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::MIN_LENGTH_FOR_APPROXIMATION = 50 unless defined? TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class << TextAlignment
 
@@ -16,8 +14,8 @@ class << TextAlignment
 		raise ArgumentError, 'nil string' if str1.nil? || str2.nil?
 		return 0, str2.length if str2.length < TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
 
-		ngram1 = (0 .. str1.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str1[i, TextAlignment::SIGNATURE_NGRAM]}
-		ngram2 = (0 .. str2.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str2[i, TextAlignment::SIGNATURE_NGRAM]}
+		ngram1 = (0 .. str1.length - TextAlignment::SIZE_NGRAM).collect{|i| str1[i, TextAlignment::SIZE_NGRAM]}
+		ngram2 = (0 .. str2.length - TextAlignment::SIZE_NGRAM).collect{|i| str2[i, TextAlignment::SIZE_NGRAM]}
 		ngram_shared = ngram1 & ngram2
 
 		# If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
@@ -45,7 +43,7 @@ class << TextAlignment
 			text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
 			cache["#{fit_begin}-#{fit_end}"] = text_similarity
 
-			break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break if text_similarity > TextAlignment::TEXT_SIMILARITY_THRESHOLD
 			fit_begin, fit_end = nil, nil
 		end
 		return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end

data/lib/text_alignment/constants.rb

@@ -0,0 +1,7 @@
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 60 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
+TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.9 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/glcs_alignment_fast.rb

@@ -9,8 +9,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 5 unless defined? TextAlignment::SIGNATURE_NGRAM
-
 class TextAlignment::GLCSTextAlignment
 	attr_reader :position_map_begin, :position_map_end
 	attr_reader :common_elements, :mapped_elements

data/lib/text_alignment/mixed_alignment.rb

@@ -10,8 +10,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::NOMATCH_CHARS = "@^|#$%&_" unless defined? TextAlignment::NOMATCH_CHARS
-
 class TextAlignment::MixedAlignment
 	attr_reader :sdiff
 	attr_reader :position_map_begin, :position_map_end
@@ -21,58 +19,7 @@ class TextAlignment::MixedAlignment
 
 	def initialize(str1, str2, mappings = [])
 		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
-		raise ArgumentError, "nil mappings" if mappings.nil?
-
-		## preprocessing
-		str1 = str1.dup
-		str2 = str2.dup
-		mappings = mappings.dup
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if str2.index(c).nil?
-				@nomatch_char1 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char1.nil?
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if c != @nomatch_char1 && str1.index(c).nil?
-				@nomatch_char2 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char2.nil?
-
-		# single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@nomatch_char1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@nomatch_char2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+		mappings ||= []
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end

data/lib/text_alignment/text_alignment.rb

@@ -1,23 +1,21 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'text_alignment/anchor_finder'
 require 'text_alignment/mixed_alignment'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
-
+TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, mappings = nil)
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
+		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
 
-		mappings ||= TextAlignment::MAPPINGS
+		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
@@ -26,7 +24,7 @@ class TextAlignment::TextAlignment
 			return @block_alignments
 		end
 
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
 
 		# To collect matched blocks
 		mblocks = []
@@ -232,4 +230,60 @@ class TextAlignment::TextAlignment
 		r
 	end
 
+	private
+
+
+	def string_preprocessing(_str1, _str2)
+		str1 = _str1.dup
+		str2 = _str2.dup
+		mappings = TextAlignment::MAPPINGS.dup
+
+		## single character mappings
+		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+		characters_from = character_mappings.collect{|m| m[0]}.join
+		characters_to   = character_mappings.collect{|m| m[1]}.join
+		characters_to.gsub!(/-/, '\-')
+
+		str1.tr!(characters_from, characters_to)
+		str2.tr!(characters_from, characters_to)
+
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+		## long to one character mappings
+		pletters = TextAlignment::PADDING_LETTERS
+
+		# find the padding letter for str1
+		padding_letter1 = begin
+			i = pletters.index{|l| str2.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# find the padding letter for str2
+		padding_letter2 = begin
+			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# ASCII foldings
+		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+		ascii_foldings.each do |f|
+			from = f[1]
+
+			if str2.index(f[0])
+				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				str1.gsub!(from, to)
+			end
+
+			if str1.index(f[0])
+				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				str2.gsub!(from, to)
+			end
+		end
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+		[str1, str2, mappings]
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.22'
+	VERSION = '0.4.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.22
+  version: 0.4.3
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-28 00:00:00.000000000 Z
+date: 2020-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -77,6 +77,7 @@ files:
 - lib/text_alignment.rb
 - lib/text_alignment/anchor_finder.rb
 - lib/text_alignment/approximate_fit.rb
+- lib/text_alignment/constants.rb
 - lib/text_alignment/find_divisions.rb
 - lib/text_alignment/glcs_alignment.rb
 - lib/text_alignment/glcs_alignment_fast.rb