text_alignment 0.3.11 → 0.3.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01525b6ca5f7e0ae1ebb9dfce2083006e095a66b3ab468ccef0a584bc3005556
-  data.tar.gz: bc1137052a12b8db97635b183f299518b0158f3ccecf36d42fa45d746b4f3792
+  metadata.gz: 5fc0292470086de4cc05275d8d19dcd534b1bfe66c5560d2043451530a657c5e
+  data.tar.gz: c6eee42e0b2b2111f063abd4bd37cbd83a6eff7c236eb032832a23fe77bb7ed5
 SHA512:
-  metadata.gz: 9596bea2616c3b4d939c8314d026941a6e627f4380183409051df62722a0ee5e3b35302da3066b0d32e8322582c999877b05a09c54749d878a284a062247342e
-  data.tar.gz: 361e3e7a23697167b41e037e7d272bbd286ac397416defb125821ebbcb17bfa386341c75fecbb94fe7f9976cfbc2a8b5f7f9be7c150d23daf6d8c16410509b5d
+  metadata.gz: 22d172f3fdaa4549edd4f9e4541c03b947c6ae8bab61e46f55c0378120b632639c70d7a30361b321065a978be6930844aedbce6c5e7591521c016c429f4eeb14
+  data.tar.gz: 3babca5408aa2dd7ce23e40dbb7909cb4b60f492583c48cb9a6fe085f84e7cf1d4f9b07ea3942455ca984c98dc3d91fce425b8dc116c98a5b8d0ff753fc232c6

data/bin/align_annotations

@@ -103,6 +103,34 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
+	pp alignment
+
+	# verification
+	source_text = source_annotations[:text]
+	puts "=====BEGIN"
+	(0 ... source_text.length).each do |p|
+		t = alignment.transform_begin_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	puts "=====BEGIN"
+	(0 .. source_text.length).each do |p|
+		t = alignment.transform_end_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
 	# alignment.block_alignments.each do |a|
 	# 	if a[:alignment].nil? || a[:alignment] == :empty
 	# 		# p [a[:source], a[:target]]
@@ -119,7 +147,22 @@ else
 	# end
 	# exit
 
+	# verification of source denotations
+	puts "[Invalid source denotations]"
+	source_annotations[:denotations] do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	end
+	puts "====="
+	puts
+
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
+
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 
 	source_annotations.merge({text:target_text, denotations:denotations})
@@ -159,7 +202,7 @@ if lost_annotations
 	warn "#{lost_annotations.length}"
 end
 
-puts target_annotations.to_json
+#puts target_annotations.to_json
 
 # denotations = anns1[:denotations]
 

data/lib/text_alignment/anchor_finder.rb

@@ -3,9 +3,9 @@ require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 5 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 10 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
+TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class TextAlignment::AnchorFinder
 
@@ -23,6 +23,8 @@ class TextAlignment::AnchorFinder
 
 		# current position in s1
 		@beg_s1 = 0
+		@end_s1_prev = 0
+		@end_s2_prev = 0
 	end
 
 	def get_next_anchor
@@ -31,15 +33,16 @@ class TextAlignment::AnchorFinder
 			anchor = @s1[@beg_s1, @size_ngram]
 
 			search_position = 0
+			# search_position = @end_s2_prev
 			while @beg_s2 = @s2.index(anchor, search_position)
 				# if both the begining points are sufficiantly close to the end points of the last match
-				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 - @end_s2_prev < 5)
+				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1  && text_similarity(left_window_s1, left_window_s2)  > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
 				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && text_similarity(right_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
 				search_position = @beg_s2 + 1
 			end
@@ -54,7 +57,7 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 > -1 && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
 		b1 += 1; b2 += 1
@@ -107,7 +110,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram < (@s1.length - @size_window)) || (@beg_s2 + @size_ngram < (@s2.length - @size_window))
+		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram
@@ -137,6 +140,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def text_similarity(str1, str2, ngram_order = 2)
+		return 0 if str1.nil? || str2.nil?
 		String::Similarity.cosine(str1, str2, ngram:ngram_order)
 	end
 

data/lib/text_alignment/text_alignment.rb

@@ -6,6 +6,8 @@ module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
+
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
@@ -70,7 +72,7 @@ class TextAlignment::TextAlignment
 						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
-						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
@@ -135,9 +137,11 @@ class TextAlignment::TextAlignment
 					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
-					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 					e1 = _str1.length < len_buffer ? str1.length : b1 + len_buffer
-					e2 = _str2.length < len_buffer ? str1.length : b2 + len_buffer
+					e2 = _str2.length < len_buffer ? str2.length : b2 + len_buffer
+					_str1 = str1[b1 ... e1]
+					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
@@ -166,10 +170,12 @@ class TextAlignment::TextAlignment
 			if begin_position == block_alignment[:source][:begin]
 				block_alignment[:target][:begin]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 
@@ -183,10 +189,12 @@ class TextAlignment::TextAlignment
 			if end_position == block_alignment[:source][:end]
 				block_alignment[:target][:end]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.11'
+	VERSION = '0.3.17'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.17
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-10 00:00:00.000000000 Z
+date: 2020-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary