text_alignment 0.3.10 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76e19ea61fda3543526ef26d582ad4ec9cfb3eec33ad03b77a998529af762eb6
-  data.tar.gz: e386cd97317e6c32c0d4c9ec347b664c2640fcad17bb7dc906b6b5b9c5784b54
+  metadata.gz: 61f98e83ee9c1d220dd228be6bb708b79f36d1c691f04dcb14d4af55f398b6da
+  data.tar.gz: f692e98a27a555baab2797ebe37407ad7133916db172a977e95415b9004e471c
 SHA512:
-  metadata.gz: dbb72e3c50713edc4fbf3328bdcc5f78723d1fe03cad4f451f2142bd05075da0ca9c3ea9b7bcdaeda009bf94cb8e26f4ede32ab96f819479fd3e2f9a988f41e9
-  data.tar.gz: 31443cca978d00e03bd5cf4339575ce4b070b06ad690938277b19f12045de2c33e04395c16799cfe4e336ac1b14264dccaa76b1c4fc48f71dbe9558f4efc6221
+  metadata.gz: 2dd8f865c245601c362e335df4e26413501fa682a97010b8aebd3ebc01864ae4772f6e716725331f9c6bc8f688818d665ef7a21384906211efc0e630b46f2313
+  data.tar.gz: 4c43199f474b94c825d8ec8ca2085b06107ec34df3d5ee988294f7423caef317c893bbba9879637d197fb2e7ae426c9e43c67cf042b7339959b638d5e5f60d01

data/bin/align_annotations

@@ -103,6 +103,35 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
+	# verification
+	source_text = source_annotations[:text]
+	puts "=====BEGIN"
+	(0 ... source_text.length).each do |p|
+		t = alignment.transform_begin_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	puts "=====BEGIN"
+	(0 .. source_text.length).each do |p|
+		t = alignment.transform_end_position(p)
+		if t.nil?
+			print source_text[p]
+		else
+			print '.'
+		end
+	end
+	puts
+	puts "=====END"
+
+	pp alignment
+
+	exit
 	# alignment.block_alignments.each do |a|
 	# 	if a[:alignment].nil? || a[:alignment] == :empty
 	# 		# p [a[:source], a[:target]]
@@ -119,7 +148,22 @@ else
 	# end
 	# exit
 
+	# verification of source denotations
+	puts "[Invalid source denotations]"
+	source_annotations[:denotations] do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	end
+	puts "====="
+	puts
+
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
+
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 
 	source_annotations.merge({text:target_text, denotations:denotations})
@@ -159,7 +203,7 @@ if lost_annotations
 	warn "#{lost_annotations.length}"
 end
 
-puts target_annotations.to_json
+#puts target_annotations.to_json
 
 # denotations = anns1[:denotations]
 

data/lib/text_alignment/anchor_finder.rb

@@ -3,9 +3,9 @@ require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 5 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 10 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
+TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class TextAlignment::AnchorFinder
 
@@ -23,44 +23,41 @@ class TextAlignment::AnchorFinder
 
 		# current position in s1
 		@beg_s1 = 0
+		@end_s1_prev = 0
+		@end_s2_prev = 0
 	end
 
 	def get_next_anchor
 		# find the position of an anchor ngram in s1 and s2
-		@beg_s2 = nil
 		while @beg_s1 < (@s1.length - @size_ngram)
-			while @beg_s1 < (@s1.length - @size_ngram)
-				anchor = @s1[@beg_s1, @size_ngram]
-				@beg_s2 = if defined? @end_s2_prev
-					@s2.index(anchor, @end_s2_prev)
-				else
-					@s2.index(anchor)
-				end
-				break unless @beg_s2.nil?
-				@beg_s1 += 1
-			end
+			anchor = @s1[@beg_s1, @size_ngram]
+
+			search_position = 0
+			# search_position = @end_s2_prev
+			while @beg_s2 = @s2.index(anchor, search_position)
+				# if both the begining points are sufficiantly close to the end points of the last match
+				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
-			# The loop above is terminated with beg_s2 == nil, which means no more anchor
-			break if @beg_s2.nil?
+				left_window_s1, left_window_s2 = get_left_windows
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
-			# if both the begining points are sufficiantly close to the end points of the last match
-			break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 - @end_s2_prev < 5)
+				right_window_s1, right_window_s2 = get_right_windows
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
 
-			left_window_s1, left_window_s2 = get_left_windows
-			break if left_window_s1  && text_similarity(left_window_s1, left_window_s2)  > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+				search_position = @beg_s2 + 1
+			end
 
-			right_window_s1, right_window_s2 = get_right_windows
-			break if right_window_s2 && text_similarity(right_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break unless @beg_s2.nil?
 
 			@beg_s1 += 1
 		end
 
-		return nil if @beg_s2.nil?
+		return nil if @beg_s1 >= (@s1.length - @size_ngram)
 
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 > -1 && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
 		b1 += 1; b2 += 1
@@ -113,7 +110,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram < (@s1.length - @size_window)) || (@beg_s2 + @size_ngram < (@s2.length - @size_window))
+		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram
@@ -143,6 +140,7 @@ class TextAlignment::AnchorFinder
 	end
 
 	def text_similarity(str1, str2, ngram_order = 2)
+		return 0 if str1.nil? || str2.nil?
 		String::Similarity.cosine(str1, str2, ngram:ngram_order)
 	end
 

data/lib/text_alignment/text_alignment.rb

@@ -6,6 +6,8 @@ module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
+
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
@@ -38,17 +40,17 @@ class TextAlignment::TextAlignment
 			end
 		end
 
-		# mblocks.each do |b|
-		# 	p [b[:source], b[:target]]
-		# 	puts "---"
-		# 	puts str1[b[:source][:begin] ... b[:source][:end]]
-		# 	puts "---"
-		# 	puts str2[b[:target][:begin] ... b[:target][:end]]
-		# 	puts "====="
-		# 	puts
-		# end
-		# puts "-=-=-=-=-"
-		# puts
+		mblocks.each do |b|
+			p [b[:source], b[:target]]
+			puts "---"
+			puts str1[b[:source][:begin] ... b[:source][:end]]
+			puts "---"
+			puts str2[b[:target][:begin] ... b[:target][:end]]
+			puts "====="
+			puts
+		end
+		puts "-=-=-=-=-"
+		puts
 
 		## To find block alignments
 		@block_alignments = []
@@ -70,7 +72,7 @@ class TextAlignment::TextAlignment
 						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
-						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
@@ -135,9 +137,11 @@ class TextAlignment::TextAlignment
 					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
-					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i
+					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 					e1 = _str1.length < len_buffer ? str1.length : b1 + len_buffer
-					e2 = _str2.length < len_buffer ? str1.length : b2 + len_buffer
+					e2 = _str2.length < len_buffer ? str2.length : b2 + len_buffer
+					_str1 = str1[b1 ... e1]
+					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
@@ -166,9 +170,12 @@ class TextAlignment::TextAlignment
 			if begin_position == block_alignment[:source][:begin]
 				block_alignment[:target][:begin]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
+			p begin_position
+			puts "-----"
 			block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
 		end
 	end
@@ -183,7 +190,8 @@ class TextAlignment::TextAlignment
 			if end_position == block_alignment[:source][:end]
 				block_alignment[:target][:end]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
 			block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.10'
+	VERSION = '0.3.16'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.16
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-29 00:00:00.000000000 Z
+date: 2020-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary