text_alignment 0.4.3 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52bc948955e2df858e397b14eabb4411f73b3ff1e4d879ff4b7015d3b5e03308
-  data.tar.gz: fd20caec51c95bdc475e0698a52bb7fdebc9e22c43bb47267a883bcc75862268
+  metadata.gz: fb5dd06236d0b1a8a9c8c5fcb92807a62bdd30e0648bcbd636b95b2a8a45b9b4
+  data.tar.gz: 9266b852993bfee999daa92e3f38ec93e2aec77171fee27c1fea6ac2a17e4d23
 SHA512:
-  metadata.gz: dbcb7ab70a64d4a398a5c5761cc5b2f5de6835ccc0e2d0854556f03ef91d0c0294986cc2ff1273788e6b7b0c73dfdf86fd16ee1ef8ce35ecc11d61f8eaab9521
-  data.tar.gz: 01d21cdcc0ab81d61e08ff1f52360ba35973756fd5060ce866391ff622d4cf87da945dba43a622a0581d63ee96c8723e1cb28991bfa02f4e1e803896bdc64d7f
+  metadata.gz: 7ee2a590fb31bcc27121a4a227d7fcefe2e8e80646bea3898bb86729ca3ca299e0aebcf23bea30e2391687e6ec0d6573c04a4605f728562482c7edbd0c0285e0
+  data.tar.gz: 73612c185fe533b0daa22d44e7776ed610025cb1bd874f05d95761079f95d1e8a06ead68c88b84bab4d33e8a676edff1e98880912254d9a7ecb5c4ead5eb01fb

data/bin/align_annotations

@@ -35,6 +35,10 @@ def align_mdoc(source_annotations, target_annotations)
 	source_annotations.each do |annotations|
 		alignment = TextAlignment::TextAlignment.new(annotations[:text], target_annotations[:text])
 
+		puts alignment.alignment_show
+		puts "-----"
+		puts
+
 		# alignment.block_alignments.each do |a|
 			# p {source:a[:source], target:a[:target]}
 			# puts "--"
@@ -103,48 +107,39 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
-	pp alignment
+	# pp alignment
 
 	# verification
-	source_text = source_annotations[:text]
-	puts "=====BEGIN"
-	(0 ... source_text.rstrip.length).each do |p|
-		t = alignment.transform_begin_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	puts "=====BEGIN"
-	(0 .. source_text.rstrip.length).each do |p|
-		t = alignment.transform_end_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	# alignment.block_alignments.each do |a|
-	# 	if a[:alignment].nil? || a[:alignment] == :empty
-	# 		# p [a[:source], a[:target]]
-	# 		# p a[:alignment]
+	# source_text = source_annotations[:text]
+	# puts "=====BEGIN"
+	# (0 ... source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_begin_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
+	# 	else
+	# 		print '.'
+	# 	end
+	# end
+	# puts
+	# puts "=====END"
+
+	# puts "=====BEGIN"
+	# (0 .. source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_end_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
 	# 	else
-	# 		p [a[:source], a[:target]]
-	# 		p a[:alignment].similarity
-	# 		puts "--"
-	# 		puts source_annotations[:text][a[:source][:begin] ... a[:source][:end]]
-	# 		puts "--"
-	# 		puts target_text[a[:target][:begin] ... a[:target][:end]]
-	# 		puts "======"
+	# 		print '.'
 	# 	end
 	# end
+	# puts
+	# puts "=====END"
+
+	source_text = source_annotations[:text]
+
+	puts "[block alignment]"
+	puts alignment.alignment_show
+	puts "====="
 	# exit
 
 	# verification of source denotations

data/lib/text_alignment/lcs_comparison.rb

@@ -33,7 +33,9 @@ class TextAlignment::LCSComparison
 			@str2_match_initial = sdiff[match_initial].new_position
 			@str1_match_final   = sdiff[match_final].old_position
 			@str2_match_final   = sdiff[match_final].new_position
-			@similarity  = 2 * lcs / ((@str1_match_final - @str1_match_initial + 1) + (@str2_match_final - @str2_match_initial + 1)).to_f
+			mlcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+			@similarity  = 2 * mlcs / (str1[@str1_match_initial .. @str1_match_final].scan(/\S/).count + str2[@str2_match_initial .. @str2_match_final].scan(/\S/).count).to_f
+			# @similarity  = 2 * lcs / (str1[@str1_match_initial .. @str1_match_final].length + str2[@str2_match_initial .. @str2_match_final].length).to_f
 		else
 			@str1_match_initial = 0
 			@str2_match_initial = 0

data/lib/text_alignment/text_alignment.rb

@@ -8,20 +8,29 @@ module TextAlignment; end unless defined? TextAlignment
 TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
 
 class TextAlignment::TextAlignment
-	attr_reader :block_alignments
+	attr_reader :block_alignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
 	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
 
+		@block_alignment = {source_text:_str1, target_text:_str2}
+
 		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
 		unless block_begin.nil?
-			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
-			return @block_alignments
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
+		end
+
+		# try exact match
+		block_begin = str2.downcase.index(str1.downcase)
+		unless block_begin.nil?
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
 		end
 
 		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
@@ -54,7 +63,7 @@ class TextAlignment::TextAlignment
 		# puts
 
 		## To find block alignments
-		@block_alignments = []
+		@block_alignment[:blocks] = []
 		return if mblocks.empty?
 
 		# Initial step
@@ -63,35 +72,36 @@ class TextAlignment::TextAlignment
 			e2 = mblocks[0][:target][:begin]
 
 			if mblocks[0][:target][:begin] == 0
-				@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
+				@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
 			else
 				_str1 = str1[0 ... e1]
 				_str2 = str2[0 ... e2]
 
 				unless _str1.strip.empty?
 					if _str2.strip.empty?
-						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+						@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
 						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
-						@block_alignments << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
+						@block_alignment[:blocks] << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
 
 						_str1 = str1[b1 ... e1]
 						_str2 = str2[b2 ... e2]
 						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-						if alignment.similarity < 0.6
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+						similarity = alignment_similarity(_str1, _str2, alignment)
+						if similarity < 0.6
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty, similarity: similarity}
 						else
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
 						end
 					end
 				end
 			end
 		end
-		@block_alignments << mblocks[0]
+		@block_alignment[:blocks] << mblocks[0].merge(alignment: :block)
 
 		(1 ... mblocks.length).each do |i|
 			b1 = mblocks[i - 1][:source][:end]
@@ -102,17 +112,18 @@ class TextAlignment::TextAlignment
 			_str2 = str2[b2 ... e2]
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 				else
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					similarity = alignment_similarity(_str1, _str2, alignment)
+					if similarity < 0.6
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 				end
 			end
-			@block_alignments << mblocks[i]
+			@block_alignment[:blocks] << mblocks[i].merge(alignment: :block)
 		end
 
 		# Final step
@@ -124,7 +135,7 @@ class TextAlignment::TextAlignment
 
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
 					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
@@ -134,57 +145,58 @@ class TextAlignment::TextAlignment
 					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					similarity = alignment_similarity(_str1, _str2, alignment)
+					if similarity < 0.6
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 
-					@block_alignments << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
+					@block_alignment[:blocks] << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
 				end
 			end
 		end
 
-		@block_alignments.each do |a|
+		@block_alignment[:blocks].each do |a|
 			a[:delta] = a[:target][:begin] - a[:source][:begin]
 		end
 	end
 
 	def transform_begin_position(begin_position)
-		i = @block_alignments.index{|b| b[:source][:end] > begin_position}
-		block_alignment = @block_alignments[i]
-
-		b = if block_alignment[:alignment].nil?
-			begin_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if begin_position == block_alignment[:source][:begin]
-				block_alignment[:target][:begin]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
+		block = @block_alignment[:blocks][i]
+
+		b = if block[:alignment] == :block
+			begin_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if begin_position == block[:source][:begin]
+				block[:target][:begin]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_begin_position(begin_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
 	def transform_end_position(end_position)
-		i = @block_alignments.index{|b| b[:source][:end] >= end_position}
-		block_alignment = @block_alignments[i]
-
-		e = if block_alignment[:alignment].nil?
-			end_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if end_position == block_alignment[:source][:end]
-				block_alignment[:target][:end]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
+		block = @block_alignment[:blocks][i]
+
+		e = if block[:alignment] == :block
+			end_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if end_position == block[:source][:end]
+				block[:target][:end]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_end_position(end_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
@@ -230,8 +242,63 @@ class TextAlignment::TextAlignment
 		r
 	end
 
-	private
+	def alignment_show
+		stext = @block_alignment[:source_text]
+		ttext = @block_alignment[:target_text]
+
+		show = ''
+		@block_alignment[:blocks].each do |a|
+			show += case a[:alignment]
+			when :block
+				"===== common =====\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :empty
+				"xxxxx disparate texts (similarity: #{a[:similarity]})\n" +
+				"<<<<< string 1\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
+				">>>>> string 2\n" +
+				ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
+			else
+				astr1 = ''
+				astr2 = ''
+
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						stext[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						stext[c.old_position + base]
+					when '!'
+						stext[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						ttext[c.new_position + base]
+					when '+'
+						ttext[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + ttext[c.new_position + base]
+					end
+				end.join('')
 
+				"***** local mismatch\n" +
+				"[#{astr1}]\n" +
+				"[#{astr2}]\n\n"
+			end
+		end
+		show
+	end
+
+	private
 
 	def string_preprocessing(_str1, _str2)
 		str1 = _str1.dup
@@ -253,15 +320,15 @@ class TextAlignment::TextAlignment
 		pletters = TextAlignment::PADDING_LETTERS
 
 		# find the padding letter for str1
-		padding_letter1 = begin
+		@padding_letter1 = begin
 			i = pletters.index{|l| str2.index(l).nil?}
 			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
 			TextAlignment::PADDING_LETTERS[i]
 		end
 
 		# find the padding letter for str2
-		padding_letter2 = begin
-			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+		@padding_letter2 = begin
+			i = pletters.index{|l| l != @padding_letter1 && str1.index(l).nil?}
 			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
 			TextAlignment::PADDING_LETTERS[i]
 		end
@@ -272,12 +339,12 @@ class TextAlignment::TextAlignment
 			from = f[1]
 
 			if str2.index(f[0])
-				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				to   = f[0] + (@padding_letter1 * (f[1].length - 1))
 				str1.gsub!(from, to)
 			end
 
 			if str1.index(f[0])
-				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				to   = f[0] + (@padding_letter2 * (f[1].length - 1))
 				str2.gsub!(from, to)
 			end
 		end
@@ -286,4 +353,13 @@ class TextAlignment::TextAlignment
 		[str1, str2, mappings]
 	end
 
+	def alignment_similarity(_s1, _s2, alignment)
+		# compute the lcs only with non-whitespace letters
+		lcs = alignment.sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+
+		s1 = _s1.tr(@padding_letter1, ' ')
+		s2 = _s2.tr(@padding_letter2, ' ')
+		similarity  = 2 * lcs / (s1.scan(/\S/).count + s2.scan(/\S/).count).to_f
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.4.3'
+	VERSION = '0.6.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.6.1
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-02 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary