text_alignment 0.3.18 → 0.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b6821076dd780721d97403f3ca49568a66100f34f7fffe32acbf28572c36429
-  data.tar.gz: 0d29d5b64897b4030de7479eaa8fc3990e13f2135298315a18b42b06ee3aa300
+  metadata.gz: 7678cbfd749833e03a17dcf0b06699def0215e7c39b981d2ff8ddfb3a79b2ea4
+  data.tar.gz: 0251b99cbd984dcd8e23ea38e6190a449a1f83f98119e68251f1f17e5854ac73
 SHA512:
-  metadata.gz: efb442cee5dfa76e7428d516b5e8ba768cbf80229580f396d02fcd5aa99f5573f88b2af662d82159669fc2c58f84ad77f6bde72696162f3c0a7fb74a75d4f7e0
-  data.tar.gz: d6fe000e32862fea2c511fb7937c85ca65e75dd53fe33214f0a637cd7b081a947f268919dd050e425d4fa2fca4268125dcffcb1a54fb933c6eb0bd3c4971b058
+  metadata.gz: '08055cb7659458e4a5834a193442c7592f9eecab09e206a8e5f20aab658c26cb24cda2949804db3569c160f03c6e71e9a28bdcdb352f8d207f1eb31aa427d963'
+  data.tar.gz: 43f0c0111662212c049a8b4baf2eeb7b2cc15a01d9f02d7cfbc501a9b016fa1344516685b8052c862227c72f8f9866b3232425e06d862899003860ad4700fc70

data/bin/align_annotations

@@ -108,7 +108,7 @@ else
 	# verification
 	source_text = source_annotations[:text]
 	puts "=====BEGIN"
-	(0 ... source_text.length).each do |p|
+	(0 ... source_text.rstrip.length).each do |p|
 		t = alignment.transform_begin_position(p)
 		if t.nil?
 			print source_text[p]
@@ -120,7 +120,7 @@ else
 	puts "=====END"
 
 	puts "=====BEGIN"
-	(0 .. source_text.length).each do |p|
+	(0 .. source_text.rstrip.length).each do |p|
 		t = alignment.transform_end_position(p)
 		if t.nil?
 			print source_text[p]

data/lib/text_alignment/anchor_finder.rb

@@ -3,7 +3,7 @@ require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 10 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
 TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
 TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
@@ -30,13 +30,17 @@ class TextAlignment::AnchorFinder
 	def get_next_anchor
 		# find the position of an anchor ngram in s1 and s2
 		while @beg_s1 < (@s1.length - @size_ngram)
+			if [' ', "\n", "\t"].include? @s1[@beg_s1]
+				@beg_s1 += 1
+				next
+			end
 			anchor = @s1[@beg_s1, @size_ngram]
 
 			# search_position = 0
 			search_position = @end_s2_prev
 			while @beg_s2 = @s2.index(anchor, search_position)
 				# if both the begining points are sufficiantly close to the end points of the last match
-				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
+				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
 				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
@@ -57,9 +61,10 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 >= @end_s2_prev && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
+
 		b1 += 1; b2 += 1
 
 		e1 = @beg_s1 + @size_ngram
@@ -82,7 +87,8 @@ class TextAlignment::AnchorFinder
 	private
 
 	def get_left_windows
-		return if @beg_s1 < @size_window || @beg_s2 < @size_window
+		# commend below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
 
 		window_s1 = ''
 		loc = @beg_s1 - 1
@@ -110,7 +116,8 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# commend below with the assumption that the end of a document gives a significant locational
+		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram

data/lib/text_alignment/text_alignment.rb

@@ -40,6 +40,9 @@ class TextAlignment::TextAlignment
 			end
 		end
 
+		# pp mblocks
+		# puts "-----"
+		# puts
 		# mblocks.each do |b|
 		# 	p [b[:source], b[:target]]
 		# 	puts "---"
@@ -78,6 +81,8 @@ class TextAlignment::TextAlignment
 
 						@block_alignments << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
 
+						_str1 = str1[b1 ... e1]
+						_str2 = str2[b2 ... e2]
 						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 						if alignment.similarity < 0.6
 							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
@@ -113,24 +118,11 @@ class TextAlignment::TextAlignment
 		end
 
 		# Final step
-		if mblocks[-1][:source][:end] < str1.length
-			b1 = mblocks[-1][:source][:end]
-			b2 = mblocks[-1][:target][:end]
-
-			if mblocks[-1][:target][:end] < str2.length
-
-			else
-				e1 = str1.length
-				e2 = str2.length
-				@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
-			end
-		end
-
 		if  mblocks[-1][:source][:end] < str1.length && mblocks[-1][:target][:end] < str2.length
 			b1 = mblocks[-1][:source][:end]
 			b2 = mblocks[-1][:target][:end]
-			_str1 = str1[b1 ... -1]
-			_str2 = str2[b2 ... -1]
+			_str1 = str1[b1 ... str1.length]
+			_str2 = str2[b2 ... str2.length]
 
 			unless _str1.strip.empty?
 				if _str2.strip.empty?

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.18'
+	VERSION = '0.3.23'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.3.23
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-18 00:00:00.000000000 Z
+date: 2020-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary