text_alignment 0.3.19 → 0.3.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea57d01970fdb56a95a7929803949a553965692fb3f4748eec72fe026f9a79cf
-  data.tar.gz: 96397baa91646b3eb05a346ff699930b6dacf7d38075273b64ce7916f32d6275
+  metadata.gz: fd59fb1ad977d3286f358d8c08315824e86d99cc6f9d72814adb760f2e680107
+  data.tar.gz: baafd1b76f6c6447a5763ff731b77ad07e449fc87ef94abed74a978376f6334e
 SHA512:
-  metadata.gz: 1d1e7650c35d9bae35a7f1dc2948dbf97fa8f71a86f1f83c5dda9cb64b7179e96ae219db88951e65c2988f078900d960784c4c74489a074654ed893c408be97f
-  data.tar.gz: 49afaec2b6332dc4038c1f0d0e930bf20ce61a6d4933ff3758bbbfea3e05cf347277aefc341d9c3b3d1f8a4692cec24b901528c2875875bacecc1caa7cf9159c
+  metadata.gz: d417878396803e1169a24fae67a9a4b0d4e84948d0c6bc678626641d6f1b6ac1fe16c94e9f07ee747b98f83e4305fa553220a607475363378f32fac4d43a65c7
+  data.tar.gz: efaf3640a67be46dddcf7dda9d819cbcb47828a3966d305887b6024eee6ba517a597e9303790f584264b8995f69671d90bdc0c7883fc9244f3b1746695960bf8

data/bin/align_annotations

@@ -108,7 +108,7 @@ else
 	# verification
 	source_text = source_annotations[:text]
 	puts "=====BEGIN"
-	(0 ... source_text.length).each do |p|
+	(0 ... source_text.rstrip.length).each do |p|
 		t = alignment.transform_begin_position(p)
 		if t.nil?
 			print source_text[p]
@@ -120,7 +120,7 @@ else
 	puts "=====END"
 
 	puts "=====BEGIN"
-	(0 .. source_text.length).each do |p|
+	(0 .. source_text.rstrip.length).each do |p|
 		t = alignment.transform_end_position(p)
 		if t.nil?
 			print source_text[p]

data/lib/text_alignment/anchor_finder.rb

@@ -30,13 +30,17 @@ class TextAlignment::AnchorFinder
 	def get_next_anchor
 		# find the position of an anchor ngram in s1 and s2
 		while @beg_s1 < (@s1.length - @size_ngram)
+			if [' ', "\n", "\t"].include? @s1[@beg_s1]
+				@beg_s1 += 1
+				next
+			end
 			anchor = @s1[@beg_s1, @size_ngram]
 
 			# search_position = 0
 			search_position = @end_s2_prev
 			while @beg_s2 = @s2.index(anchor, search_position)
 				# if both the begining points are sufficiantly close to the end points of the last match
-				break if @end_s1_prev && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
+				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
 				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
@@ -57,9 +61,10 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 >= @end_s2_prev && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
+
 		b1 += 1; b2 += 1
 
 		e1 = @beg_s1 + @size_ngram
@@ -82,7 +87,8 @@ class TextAlignment::AnchorFinder
 	private
 
 	def get_left_windows
-		return if @beg_s1 < @size_window || @beg_s2 < @size_window
+		# commend below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
 
 		window_s1 = ''
 		loc = @beg_s1 - 1
@@ -110,7 +116,8 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# commend below with the assumption that the end of a document gives a significant locational
+		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram

data/lib/text_alignment/text_alignment.rb

@@ -6,7 +6,7 @@ module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
+TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
 
 
 class TextAlignment::TextAlignment
@@ -40,10 +40,9 @@ class TextAlignment::TextAlignment
 			end
 		end
 
-		pp mblocks
-		puts "-----"
-		puts
-
+		# pp mblocks
+		# puts "-----"
+		# puts
 		# mblocks.each do |b|
 		# 	p [b[:source], b[:target]]
 		# 	puts "---"
@@ -82,6 +81,8 @@ class TextAlignment::TextAlignment
 
 						@block_alignments << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
 
+						_str1 = str1[b1 ... e1]
+						_str2 = str2[b2 ... e2]
 						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 						if alignment.similarity < 0.6
 							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
@@ -117,24 +118,11 @@ class TextAlignment::TextAlignment
 		end
 
 		# Final step
-		if mblocks[-1][:source][:end] < str1.length
-			b1 = mblocks[-1][:source][:end]
-			b2 = mblocks[-1][:target][:end]
-
-			if mblocks[-1][:target][:end] < str2.length
-
-			else
-				e1 = str1.length
-				e2 = str2.length
-				@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
-			end
-		end
-
 		if  mblocks[-1][:source][:end] < str1.length && mblocks[-1][:target][:end] < str2.length
 			b1 = mblocks[-1][:source][:end]
 			b2 = mblocks[-1][:target][:end]
-			_str1 = str1[b1 ... -1]
-			_str2 = str2[b2 ... -1]
+			_str1 = str1[b1 ... str1.length]
+			_str2 = str2[b2 ... str2.length]
 
 			unless _str1.strip.empty?
 				if _str2.strip.empty?

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.19'
+	VERSION = '0.3.24'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.19
+  version: 0.3.24
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-21 00:00:00.000000000 Z
+date: 2020-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary