text_alignment 0.3.14 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6f98465bb47a2b241dda72c8532530f5c7fdf4de49a403366bd08c256b7ff0e
-  data.tar.gz: 44a6c920f8f05ab3ee29a0b9fe4de38e2f6fac2386838625b77d99486189ebf0
+  metadata.gz: ea57d01970fdb56a95a7929803949a553965692fb3f4748eec72fe026f9a79cf
+  data.tar.gz: 96397baa91646b3eb05a346ff699930b6dacf7d38075273b64ce7916f32d6275
 SHA512:
-  metadata.gz: 17f038d6d7366b8223cdd66b5ef9f3d79c8ecc39f432ac15dbfd0f3311e1197bc9c40c5cbd38a69d5778278405dcd100bc18187870ee563a7e5999246845b049
-  data.tar.gz: f0ded392d47821bc99c640700955686f14cc9550a13b3b8141af2af7f88f79400a3de6632f2bc3223c9e0dc82311d461de84a5ffa16aff443394b3c76540a74c
+  metadata.gz: 1d1e7650c35d9bae35a7f1dc2948dbf97fa8f71a86f1f83c5dda9cb64b7179e96ae219db88951e65c2988f078900d960784c4c74489a074654ed893c408be97f
+  data.tar.gz: 49afaec2b6332dc4038c1f0d0e930bf20ce61a6d4933ff3758bbbfea3e05cf347277aefc341d9c3b3d1f8a4692cec24b901528c2875875bacecc1caa7cf9159c

data/bin/align_annotations

@@ -103,6 +103,8 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
+	pp alignment
+
 	# verification
 	source_text = source_annotations[:text]
 	puts "=====BEGIN"
@@ -129,8 +131,6 @@ else
 	puts
 	puts "=====END"
 
-	# pp alignment
-
 	# alignment.block_alignments.each do |a|
 	# 	if a[:alignment].nil? || a[:alignment] == :empty
 	# 		# p [a[:source], a[:target]]
@@ -153,8 +153,16 @@ else
 		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
 	end
 	puts "====="
+	puts
 
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
+
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 
 	source_annotations.merge({text:target_text, denotations:denotations})

data/lib/text_alignment/anchor_finder.rb

@@ -3,8 +3,8 @@ require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 20 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
 TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class TextAlignment::AnchorFinder
@@ -23,6 +23,7 @@ class TextAlignment::AnchorFinder
 
 		# current position in s1
 		@beg_s1 = 0
+		@end_s1_prev = 0
 		@end_s2_prev = 0
 	end
 
@@ -31,8 +32,8 @@ class TextAlignment::AnchorFinder
 		while @beg_s1 < (@s1.length - @size_ngram)
 			anchor = @s1[@beg_s1, @size_ngram]
 
-			search_position = 0
-			# search_position = @end_s2_prev
+			# search_position = 0
+			search_position = @end_s2_prev
 			while @beg_s2 = @s2.index(anchor, search_position)
 				# if both the begining points are sufficiantly close to the end points of the last match
 				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
@@ -56,7 +57,7 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 > -1 && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
 		b1 += 1; b2 += 1

data/lib/text_alignment/text_alignment.rb

@@ -40,6 +40,10 @@ class TextAlignment::TextAlignment
 			end
 		end
 
+		pp mblocks
+		puts "-----"
+		puts
+
 		# mblocks.each do |b|
 		# 	p [b[:source], b[:target]]
 		# 	puts "---"
@@ -170,10 +174,12 @@ class TextAlignment::TextAlignment
 			if begin_position == block_alignment[:source][:begin]
 				block_alignment[:target][:begin]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 
@@ -187,10 +193,12 @@ class TextAlignment::TextAlignment
 			if end_position == block_alignment[:source][:end]
 				block_alignment[:target][:end]
 			else
-				raise "lost annotation"
+				# raise "lost annotation"
+				nil
 			end
 		else
-			block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin]) + block_alignment[:target][:begin]
+			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
+			r.nil? ? nil : r + block_alignment[:target][:begin]
 		end
 	end
 

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.14'
+	VERSION = '0.3.19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.19
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-13 00:00:00.000000000 Z
+date: 2020-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary