text_alignment 0.3.13 → 0.3.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19a2dfcf8dfffa752dfc0c3363d2d3e1cb3ef7498f79023cdd16e38aa8c46afd
-  data.tar.gz: 94d925dfc71d24b05fd6861a4f7f7344428b68785db84eeae8f430563b4e3318
+  metadata.gz: 5b6821076dd780721d97403f3ca49568a66100f34f7fffe32acbf28572c36429
+  data.tar.gz: 0d29d5b64897b4030de7479eaa8fc3990e13f2135298315a18b42b06ee3aa300
 SHA512:
-  metadata.gz: 72e61cf30c98df2c3d5ac19717c813c936b55daad22cb8c6e8b44bdb45321dab98c69d5f90820e9993d86263b04bdad2e96e8010afc8a57eee916126b673c8cc
-  data.tar.gz: d92c04294d58845f4a88cb8d9e3db42e9a18e0dd02d0398e3a95bf94662f64a33752754dd637163ef9bc4af77dc602c12fe683d49e9ac0ebb61a2469e5e08216
+  metadata.gz: efb442cee5dfa76e7428d516b5e8ba768cbf80229580f396d02fcd5aa99f5573f88b2af662d82159669fc2c58f84ad77f6bde72696162f3c0a7fb74a75d4f7e0
+  data.tar.gz: d6fe000e32862fea2c511fb7937c85ca65e75dd53fe33214f0a637cd7b081a947f268919dd050e425d4fa2fca4268125dcffcb1a54fb933c6eb0bd3c4971b058

data/bin/align_annotations

@@ -105,6 +105,32 @@ else
 
 	pp alignment
 
+	# verification
+	source_text = source_annotations[:text]
+	puts "=====BEGIN"
+	(0 ... source_text.length).each do |p|
+		t = alignment.transform_begin_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	puts "=====BEGIN"
+	(0 .. source_text.length).each do |p|
+		t = alignment.transform_end_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
 	# alignment.block_alignments.each do |a|
 	# 	if a[:alignment].nil? || a[:alignment] == :empty
 	# 		# p [a[:source], a[:target]]
@@ -121,7 +147,22 @@ else
 	# end
 	# exit
 
+	# verification of source denotations
+	puts "[Invalid source denotations]"
+	source_annotations[:denotations] do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	end
+	puts "====="
+	puts
+
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
+
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 
 	source_annotations.merge({text:target_text, denotations:denotations})

data/lib/text_alignment/anchor_finder.rb

@@ -4,7 +4,7 @@ require 'string-similarity'
 module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 20 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
 TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class TextAlignment::AnchorFinder
@@ -23,6 +23,7 @@ class TextAlignment::AnchorFinder
 
 		# current position in s1
 		@beg_s1 = 0
+		@end_s1_prev = 0
 		@end_s2_prev = 0
 	end
 
@@ -31,8 +32,8 @@ class TextAlignment::AnchorFinder
 		while @beg_s1 < (@s1.length - @size_ngram)
 			anchor = @s1[@beg_s1, @size_ngram]
 
-			search_position = 0
-			# search_position = @end_s2_prev
+			# search_position = 0
+			search_position = @end_s2_prev
 			while @beg_s2 = @s2.index(anchor, search_position)
 				# if both the begining points are sufficiantly close to the end points of the last match
 				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
@@ -56,7 +57,7 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 > -1 && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
 		b1 += 1; b2 += 1

data/lib/text_alignment/text_alignment.rb

@@ -6,6 +6,8 @@ module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
+
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
@@ -70,7 +72,7 @@ class TextAlignment::TextAlignment
 						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
-						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
@@ -135,9 +137,11 @@ class TextAlignment::TextAlignment
 					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
-					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 					e1 = _str1.length < len_buffer ? str1.length : b1 + len_buffer
-					e2 = _str2.length < len_buffer ? str1.length : b2 + len_buffer
+					e2 = _str2.length < len_buffer ? str2.length : b2 + len_buffer
+					_str1 = str1[b1 ... e1]
+					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
@@ -166,10 +170,12 @@ class TextAlignment::TextAlignment
 			if begin_position == block_alignment[:source][:begin]
 				block_alignment[:target][:begin]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 
@@ -183,10 +189,12 @@ class TextAlignment::TextAlignment
 			if end_position == block_alignment[:source][:end]
 				block_alignment[:target][:end]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.13'
+	VERSION = '0.3.18'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.18
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-12 00:00:00.000000000 Z
+date: 2020-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary