text_alignment 0.3.9 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4a8ed8b4cff6f5b04b10c91704939936530ad0dc820a126a514a58cce7a4df6
-  data.tar.gz: '087412fa8b9779073c67fa1d9a0afc05e32d7f1baad4518c375fcf804a45ecd4'
+  metadata.gz: 8252929e7b74251db493ad991332d0da8a92f35441e2a442d05b6fb29139d657
+  data.tar.gz: c9e41127fb231e4db2843696c1029b35711449c5bf5104158da301d03b817131
 SHA512:
-  metadata.gz: e5f56b58d35a614c6b9a72ccb8282b775d19c2fb576d68420153b96703d954e47471cfbcd9b384bd244f19110cc436d28e409d5014f1ade66c23390a928111fc
-  data.tar.gz: 4dbdd214b0e2aab9b32751305517160016cfd16b60f5380282b2b1ba2e6946e3097789f4fd234b577cf8ab00e06ea8d3497f59843d600eac2b12b8d60c32441c
+  metadata.gz: 1fc8da7324d71cf25edbec9765ab512928323079472736ea4e294abb12dfafc87f55d71cb49c371470811775bd489d3c91cce4a787b99faa305f2f326dc80c77
+  data.tar.gz: f694c99216b59dd693a6acdfffc727fe74b5c189b4b9583b31fb7e6394319a3176de76237a142dac3770bcd1fbbc467d5d1e97a7225ed993a21246a66de8b2ec

data/bin/align_annotations

@@ -103,6 +103,35 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
+	# verification
+	source_text = source_annotations[:text]
+	puts "=====BEGIN"
+	(0 ... source_text.length).each do |p|
+		t = alignment.transform_begin_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	puts "=====BEGIN"
+	(0 .. source_text.length).each do |p|
+		t = alignment.transform_end_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	pp alignment
+
+	exit
 	# alignment.block_alignments.each do |a|
 	# 	if a[:alignment].nil? || a[:alignment] == :empty
 	# 		# p [a[:source], a[:target]]
@@ -119,7 +148,22 @@ else
 	# end
 	# exit
 
+	# verification of source denotations
+	puts "[Invalid source denotations]"
+	source_annotations[:denotations] do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	end
+	puts "====="
+	puts
+
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
+
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 
 	source_annotations.merge({text:target_text, denotations:denotations})
@@ -159,7 +203,7 @@ if lost_annotations
 	warn "#{lost_annotations.length}"
 end
 
-puts target_annotations.to_json
+#puts target_annotations.to_json
 
 # denotations = anns1[:denotations]
 

data/lib/text_alignment/anchor_finder.rb

@@ -3,9 +3,9 @@ require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 5 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 10 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
+TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 30 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class TextAlignment::AnchorFinder
 
@@ -23,39 +23,35 @@ class TextAlignment::AnchorFinder
 
 		# current position in s1
 		@beg_s1 = 0
+		@end_s2_prev = 0
 	end
 
 	def get_next_anchor
 		# find the position of an anchor ngram in s1 and s2
-		@beg_s2 = nil
 		while @beg_s1 < (@s1.length - @size_ngram)
-			while @beg_s1 < (@s1.length - @size_ngram)
-				anchor = @s1[@beg_s1, @size_ngram]
-				@beg_s2 = if defined? @end_s2_prev
-					@s2.index(anchor, @end_s2_prev)
-				else
-					@s2.index(anchor)
-				end
-				break unless @beg_s2.nil?
-				@beg_s1 += 1
-			end
+			anchor = @s1[@beg_s1, @size_ngram]
+
+			search_position = 0
+			# search_position = @end_s2_prev
+			while @beg_s2 = @s2.index(anchor, search_position)
+				# if both the begining points are sufficiantly close to the end points of the last match
+				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
-			# The loop above is terminated with beg_s2 == nil, which means no more anchor
-			break if @beg_s2.nil?
+				left_window_s1, left_window_s2 = get_left_windows
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
-			# if both the begining points are sufficiantly close to the end points of the last match
-			break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 - @end_s2_prev < 5)
+				right_window_s1, right_window_s2 = get_right_windows
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
-			left_window_s1, left_window_s2 = get_left_windows
-			break if left_window_s1  && text_similarity(left_window_s1, left_window_s2)  > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+				search_position = @beg_s2 + 1
+			end
 
-			right_window_s1, right_window_s2 = get_right_windows
-			break if right_window_s2 && text_similarity(right_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break unless @beg_s2.nil?
 
 			@beg_s1 += 1
 		end
 
-		return nil if @beg_s2.nil?
+		return nil if @beg_s1 >= (@s1.length - @size_ngram)
 
 		# extend the block
 		b1 = @beg_s1
@@ -67,7 +63,7 @@ class TextAlignment::AnchorFinder
 
 		e1 = @beg_s1 + @size_ngram
 		e2 = @beg_s2 + @size_ngram
-		while @s1[e1] == @s2[e2]
+		while @s1[e1] && @s1[e1] == @s2[e2]
 			e1 += 1; e2 += 1
 		end
 
@@ -113,7 +109,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram < (@s1.length - @size_window)) || (@beg_s2 + @size_ngram < (@s2.length - @size_window))
+		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram
@@ -143,6 +139,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def text_similarity(str1, str2, ngram_order = 2)
+		return 0 if str1.nil? || str2.nil?
 		String::Similarity.cosine(str1, str2, ngram:ngram_order)
 	end
 

data/lib/text_alignment/text_alignment.rb

@@ -6,6 +6,8 @@ module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
+
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
@@ -70,7 +72,7 @@ class TextAlignment::TextAlignment
 						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
-						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
@@ -135,9 +137,11 @@ class TextAlignment::TextAlignment
 					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
-					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 					e1 = _str1.length < len_buffer ? str1.length : b1 + len_buffer
-					e2 = _str2.length < len_buffer ? str1.length : b2 + len_buffer
+					e2 = _str2.length < len_buffer ? str2.length : b2 + len_buffer
+					_str1 = str1[b1 ... e1]
+					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
@@ -166,7 +170,8 @@ class TextAlignment::TextAlignment
 			if begin_position == block_alignment[:source][:begin]
 				block_alignment[:target][:begin]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
 			block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
@@ -183,7 +188,8 @@ class TextAlignment::TextAlignment
 			if end_position == block_alignment[:source][:end]
 				block_alignment[:target][:end]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
 			block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
@@ -199,13 +205,9 @@ class TextAlignment::TextAlignment
 	end
 
 	def transform_denotations!(denotations)
-		puts "hereherehere========"
 		return nil if denotations.nil?
 		@lost_annotations = []
 
-		pp denotations
-		puts "-----"
-
 		denotations.each do |d|
 			begin
 				d.begin = transform_begin_position(d.begin);
@@ -217,7 +219,7 @@ class TextAlignment::TextAlignment
 			end
 		end
 
-		pp denotations
+		@lost_annotations
 	end
 
 	def transform_hdenotations(hdenotations)

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.9'
+	VERSION = '0.3.15'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.15
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary