text_alignment 0.5 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f5f7f27c8628123530d51d0a68060aa6fb850bcef8c7089c8bf990f7257a80b
-  data.tar.gz: 45f768df4e7d89c931295985adb31df9e725156e5a85d7f78a5b7cd26d00be4d
+  metadata.gz: c4b2cdf0c257b74c6bec90b93d1907787f3c102108046731c2755684a1b156e9
+  data.tar.gz: 85334dad09a046432503183e3d3ad83841612299038f2f2dac1f9d5d208e1939
 SHA512:
-  metadata.gz: 4004293fa10eb247110764d16a24900a10e714072227dd6e5626d2123deca4bcec00b7255e1affaba4d7dda75d7e42049aabf6b1e6c51f4c005124443b5f9ffc
-  data.tar.gz: eb5c8a2c89c8973242bb77457e0f8d9922486d9743e7c39ee12ea4025ece6d888be5308eaddec076e5a21c84149dcb31bc0ed6fcd68881fb6c8c4063e49fb64d
+  metadata.gz: 9272bdd6c56717b53d39b3f2009259accb608ea86b99758b6a7ee9cee1e7b275330db55af4e0eba1eba80ee69275a21a3179243394d24139b3018996f659abe1
+  data.tar.gz: a6a9d97d2bf81ac0c2972fd6e9d5202116156d8ff2e5e81a9bf0306e313dbc601522f887bcbcebff8b9d888cc06826a8ce69ba908dce29fa8decad85d53008af

data/bin/align_annotations

@@ -35,6 +35,10 @@ def align_mdoc(source_annotations, target_annotations)
 	source_annotations.each do |annotations|
 		alignment = TextAlignment::TextAlignment.new(annotations[:text], target_annotations[:text])
 
+		puts alignment.alignment_show
+		puts "-----"
+		puts
+
 		# alignment.block_alignments.each do |a|
 			# p {source:a[:source], target:a[:target]}
 			# puts "--"
@@ -133,26 +137,26 @@ else
 
 	source_text = source_annotations[:text]
 
-	# puts "[block alignment]"
-	puts alignment.alignment_table
-	# puts "====="
+	puts "[block alignment]"
+	puts alignment.alignment_show
+	puts "====="
 	# exit
 
 	# verification of source denotations
-	# puts "[Invalid source denotations]"
-	# source_annotations[:denotations] do |d|
-	# 	p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
-	# end
-	# puts "====="
-	# puts
+	puts "[Invalid source denotations]"
+	source_annotations[:denotations] do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	end
+	puts "====="
+	puts
 
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
-	# puts "[Invalid transformation]"
-	# denotations.each do |d|
-	# 	p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
-	# end
-	# puts "====="
-	# puts
+	puts "[Invalid transformation]"
+	denotations.each do |d|
+		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	end
+	puts "====="
+	puts
 
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 

data/lib/text_alignment/lcs_comparison.rb

@@ -33,7 +33,9 @@ class TextAlignment::LCSComparison
 			@str2_match_initial = sdiff[match_initial].new_position
 			@str1_match_final   = sdiff[match_final].old_position
 			@str2_match_final   = sdiff[match_final].new_position
-			@similarity  = 2 * lcs / ((@str1_match_final - @str1_match_initial + 1) + (@str2_match_final - @str2_match_initial + 1)).to_f
+			mlcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+			@similarity  = 2 * mlcs / (str1[@str1_match_initial .. @str1_match_final].scan(/\S/).count + str2[@str2_match_initial .. @str2_match_final].scan(/\S/).count).to_f
+			# @similarity  = 2 * lcs / (str1[@str1_match_initial .. @str1_match_final].length + str2[@str2_match_initial .. @str2_match_final].length).to_f
 		else
 			@str1_match_initial = 0
 			@str2_match_initial = 0

data/lib/text_alignment/text_alignment.rb

@@ -8,30 +8,29 @@ module TextAlignment; end unless defined? TextAlignment
 TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
 
 class TextAlignment::TextAlignment
-	attr_reader :block_alignments
+	attr_reader :block_alignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
 	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
 
-		@ostr1 = _str1
-		@ostr2 = _str2
+		@block_alignment = {source_text:_str1, target_text:_str2}
 
 		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
 		unless block_begin.nil?
-			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
-			return @block_alignments
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
 		end
 
 		# try exact match
 		block_begin = str2.downcase.index(str1.downcase)
 		unless block_begin.nil?
-			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
-			return @block_alignments
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
 		end
 
 		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
@@ -64,7 +63,7 @@ class TextAlignment::TextAlignment
 		# puts
 
 		## To find block alignments
-		@block_alignments = []
+		@block_alignment[:blocks] = []
 		return if mblocks.empty?
 
 		# Initial step
@@ -73,35 +72,36 @@ class TextAlignment::TextAlignment
 			e2 = mblocks[0][:target][:begin]
 
 			if mblocks[0][:target][:begin] == 0
-				@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
+				@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
 			else
 				_str1 = str1[0 ... e1]
 				_str2 = str2[0 ... e2]
 
 				unless _str1.strip.empty?
 					if _str2.strip.empty?
-						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+						@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
 						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
-						@block_alignments << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
+						@block_alignment[:blocks] << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
 
 						_str1 = str1[b1 ... e1]
 						_str2 = str2[b2 ... e2]
 						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-						if alignment.similarity < 0.6
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+						similarity = alignment_similarity(_str1, _str2, alignment)
+						if similarity < 0.6
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty, similarity: similarity}
 						else
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
 						end
 					end
 				end
 			end
 		end
-		@block_alignments << mblocks[0].merge(alignment: :block)
+		@block_alignment[:blocks] << mblocks[0].merge(alignment: :block)
 
 		(1 ... mblocks.length).each do |i|
 			b1 = mblocks[i - 1][:source][:end]
@@ -112,17 +112,18 @@ class TextAlignment::TextAlignment
 			_str2 = str2[b2 ... e2]
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 				else
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					similarity = alignment_similarity(_str1, _str2, alignment)
+					if similarity < 0.6
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 				end
 			end
-			@block_alignments << mblocks[i].merge(alignment: :block)
+			@block_alignment[:blocks] << mblocks[i].merge(alignment: :block)
 		end
 
 		# Final step
@@ -134,7 +135,7 @@ class TextAlignment::TextAlignment
 
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
 					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
@@ -144,57 +145,58 @@ class TextAlignment::TextAlignment
 					_str2 = str2[b2 ... e2]
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					similarity = alignment_similarity(_str1, _str2, alignment)
+					if similarity < 0.6
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 
-					@block_alignments << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
+					@block_alignment[:blocks] << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
 				end
 			end
 		end
 
-		@block_alignments.each do |a|
+		@block_alignment[:blocks].each do |a|
 			a[:delta] = a[:target][:begin] - a[:source][:begin]
 		end
 	end
 
 	def transform_begin_position(begin_position)
-		i = @block_alignments.index{|b| b[:source][:end] > begin_position}
-		block_alignment = @block_alignments[i]
-
-		b = if block_alignment[:alignment] == :block
-			begin_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if begin_position == block_alignment[:source][:begin]
-				block_alignment[:target][:begin]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
+		block = @block_alignment[:blocks][i]
+
+		b = if block[:alignment] == :block
+			begin_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if begin_position == block[:source][:begin]
+				block[:target][:begin]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_begin_position(begin_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
 	def transform_end_position(end_position)
-		i = @block_alignments.index{|b| b[:source][:end] >= end_position}
-		block_alignment = @block_alignments[i]
-
-		e = if block_alignment[:alignment] == :block
-			end_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if end_position == block_alignment[:source][:end]
-				block_alignment[:target][:end]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
+		block = @block_alignment[:blocks][i]
+
+		e = if block[:alignment] == :block
+			end_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if end_position == block[:source][:end]
+				block[:target][:end]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_end_position(end_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
@@ -240,87 +242,23 @@ class TextAlignment::TextAlignment
 		r
 	end
 
-	def alignment_table
-		table = <<-TABLE
-			<table class='text_alignment_table'>
-				<thead>
-					<tr>
-						<th class='text_alignment_left' style='width:50%'>Text 1</th>
-						<th class='text_alignment_rigt'>Text 2</th>
-					</tr>
-				</thead>
-				<tbody>
-		TABLE
-
-		@block_alignments.each do |a|
-			table += alignment_table_th(a)
-			table += "<tr>\n" + case a[:alignment]
-			when :block
-				"<td colspan='2' class='text_alignment_common'>" +
-				@ostr1[a[:source][:begin] ... a[:source][:end]] +
-				"</td>\n"
-			when :empty
-				"<td class='text_alignment_left'>"  + @ostr1[a[:source][:begin] ... a[:source][:end]] + "</td>\n" +
-				"<td class='text_alignment_right'>" + @ostr2[a[:target][:begin] ... a[:target][:end]] + "</td>\n"
-			else
-				base = a[:source][:begin]
-				astr1 = a[:alignment].sdiff.map do |c|
-					case c.action
-					when '='
-						@ostr1[c.old_position + base]
-					when '+'
-						'_'
-					when '-'
-						@ostr1[c.old_position + base]
-					when '!'
-						@ostr1[c.old_position + base] + '_'
-					end
-				end.join('')
-
-				base = a[:target][:begin]
-				astr2 = a[:alignment].sdiff.map do |c|
-					case c.action
-					when '='
-						@ostr2[c.new_position + base]
-					when '+'
-						@ostr2[c.new_position + base]
-					when '-'
-						'_'
-					when '!'
-						'_' + @ostr2[c.new_position + base]
-					end
-				end.join('')
-
-				"<td class='text_alignment_left'>"  + astr1 + "</td>\n" +
-				"<td class='text_alignment_right'>" + astr2 + "</td>\n"
-			end + "</tr>\n"
-		end
-		table += '</tbody></table>'
-	end
-
-	def alignment_table_th(a)
-		"<tr>" +
-		"<th class='text_alignment_left'>#{a[:source][:begin]} - #{a[:source][:end]}</th>" +
-		"<th class='text_alignment_right'>#{a[:target][:begin]} - #{a[:target][:end]}</th>" +
-		"</tr>"
-	end
-
 	def alignment_show
+		stext = @block_alignment[:source_text]
+		ttext = @block_alignment[:target_text]
+
 		show = ''
-		@block_alignments.each do |a|
+		@block_alignment[:blocks].each do |a|
 			show += case a[:alignment]
 			when :block
 				"===== common =====\n" +
-				@ostr1[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
 			when :empty
-				puts "<<<<< string 1"
-				p @ostr1[a[:source][:begin] ... a[:source][:end]]
-				puts
-				puts ">>>>> string 2"
-				p @ostr2[a[:target][:begin] ... a[:target][:end]]
-				puts
+				"xxxxx disparate texts (similarity: #{a[:similarity]})\n" +
+				"<<<<< string 1\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
+				">>>>> string 2\n" +
+				ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
 			else
-				puts "***** local mismatch"
 				astr1 = ''
 				astr2 = ''
 
@@ -328,13 +266,13 @@ class TextAlignment::TextAlignment
 				astr1 = a[:alignment].sdiff.map do |c|
 					case c.action
 					when '='
-						@ostr1[c.old_position + base]
+						stext[c.old_position + base]
 					when '+'
 						'_'
 					when '-'
-						@ostr1[c.old_position + base]
+						stext[c.old_position + base]
 					when '!'
-						@ostr1[c.old_position + base] + '_'
+						stext[c.old_position + base] + '_'
 					end
 				end.join('')
 
@@ -342,21 +280,22 @@ class TextAlignment::TextAlignment
 				astr2 = a[:alignment].sdiff.map do |c|
 					case c.action
 					when '='
-						@ostr2[c.new_position + base]
+						ttext[c.new_position + base]
 					when '+'
-						@ostr2[c.new_position + base]
+						ttext[c.new_position + base]
 					when '-'
 						'_'
 					when '!'
-						'_' + @ostr2[c.new_position + base]
+						'_' + ttext[c.new_position + base]
 					end
 				end.join('')
 
-				puts '[' + astr1 + ']'
-				puts '[' + astr2 + ']'
-				puts
+				"***** local mismatch\n" +
+				"[#{astr1}]\n" +
+				"[#{astr2}]\n\n"
 			end
 		end
+		show
 	end
 
 	private
@@ -381,15 +320,15 @@ class TextAlignment::TextAlignment
 		pletters = TextAlignment::PADDING_LETTERS
 
 		# find the padding letter for str1
-		padding_letter1 = begin
+		@padding_letter1 = begin
 			i = pletters.index{|l| str2.index(l).nil?}
 			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
 			TextAlignment::PADDING_LETTERS[i]
 		end
 
 		# find the padding letter for str2
-		padding_letter2 = begin
-			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+		@padding_letter2 = begin
+			i = pletters.index{|l| l != @padding_letter1 && str1.index(l).nil?}
 			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
 			TextAlignment::PADDING_LETTERS[i]
 		end
@@ -400,12 +339,12 @@ class TextAlignment::TextAlignment
 			from = f[1]
 
 			if str2.index(f[0])
-				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				to   = f[0] + (@padding_letter1 * (f[1].length - 1))
 				str1.gsub!(from, to)
 			end
 
 			if str1.index(f[0])
-				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				to   = f[0] + (@padding_letter2 * (f[1].length - 1))
 				str2.gsub!(from, to)
 			end
 		end
@@ -414,4 +353,16 @@ class TextAlignment::TextAlignment
 		[str1, str2, mappings]
 	end
 
+	def alignment_similarity(_s1, _s2, alignment)
+		return 0 if alignment.sdiff.nil?
+
+		# compute the lcs only with non-whitespace letters
+		lcs = alignment.sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+
+		s1 = _s1.tr(@padding_letter1, ' ')
+		s2 = _s2.tr(@padding_letter2, ' ')
+
+		similarity = 2 * lcs / (s1.scan(/\S/).count + s2.scan(/\S/).count).to_f
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.5'
+	VERSION = '0.6.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: 0.6.2
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-04 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary