text_alignment 0.4.2 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d56b44ee3cc3451b78ddb87134fefd6c043b4730f1170b07069a417bb162bad1
-  data.tar.gz: 67318d70bee5bb82127c98e79e0b45e5e337832a2b8d23469474215168579f20
+  metadata.gz: dc820991f5f694f154b94c369158909ccba3760829e0d881c7fd2e6ef7ddd149
+  data.tar.gz: 40ae6f2e388405426a77682bd1a3fb7a3c853076eced9b7301b632081dfd0a57
 SHA512:
-  metadata.gz: 60ae2292556d15d860e0c7377be9c1fd5aec6741b26f562fee8c0c2a9a123d0b8c656b2f7f34b8f74f83ea9ac4878b53f17454e3b25d27d00cd98ecf8175a387
-  data.tar.gz: fe8fe545101cddb638c00ffb6f6785f8b2f5ddb109b4010739e399a2c50fe41164d791fe4a5599a76fca5553c494f78fd49630368d48e406a8211fc0516916f5
+  metadata.gz: 5802241b4a8394d3c570c1d4b8f5e1d7706c72852e2d6e6fb23bda2f6e2972fa09f7001695db026667144e2af982eeb91ed0b700bd8151af6df794c98e3c069b
+  data.tar.gz: 8d7c93acbef6ab12bb2a0291444a7bcc73b0236bb5b0d06d274e95aa30c9ffc829965653b58270686147a9ac30ccf570518b3ad266120b320dfb20cd1620f5f9

data/bin/align_annotations

@@ -35,6 +35,10 @@ def align_mdoc(source_annotations, target_annotations)
 	source_annotations.each do |annotations|
 		alignment = TextAlignment::TextAlignment.new(annotations[:text], target_annotations[:text])
 
+		puts alignment.alignment_show
+		puts "-----"
+		puts
+
 		# alignment.block_alignments.each do |a|
 			# p {source:a[:source], target:a[:target]}
 			# puts "--"
@@ -103,48 +107,39 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
-	pp alignment
+	# pp alignment
 
 	# verification
-	source_text = source_annotations[:text]
-	puts "=====BEGIN"
-	(0 ... source_text.rstrip.length).each do |p|
-		t = alignment.transform_begin_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	puts "=====BEGIN"
-	(0 .. source_text.rstrip.length).each do |p|
-		t = alignment.transform_end_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	# alignment.block_alignments.each do |a|
-	# 	if a[:alignment].nil? || a[:alignment] == :empty
-	# 		# p [a[:source], a[:target]]
-	# 		# p a[:alignment]
+	# source_text = source_annotations[:text]
+	# puts "=====BEGIN"
+	# (0 ... source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_begin_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
+	# 	else
+	# 		print '.'
+	# 	end
+	# end
+	# puts
+	# puts "=====END"
+
+	# puts "=====BEGIN"
+	# (0 .. source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_end_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
 	# 	else
-	# 		p [a[:source], a[:target]]
-	# 		p a[:alignment].similarity
-	# 		puts "--"
-	# 		puts source_annotations[:text][a[:source][:begin] ... a[:source][:end]]
-	# 		puts "--"
-	# 		puts target_text[a[:target][:begin] ... a[:target][:end]]
-	# 		puts "======"
+	# 		print '.'
 	# 	end
 	# end
+	# puts
+	# puts "=====END"
+
+	source_text = source_annotations[:text]
+
+	puts "[block alignment]"
+	puts alignment.alignment_show
+	puts "====="
 	# exit
 
 	# verification of source denotations

data/lib/text_alignment/constants.rb

@@ -4,4 +4,4 @@ TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
 TextAlignment::SIZE_WINDOW = 60 unless defined? TextAlignment::SIZE_WINDOW
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
 TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
-TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.87 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD
+TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.9 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/lcs_comparison.rb

@@ -33,7 +33,9 @@ class TextAlignment::LCSComparison
 			@str2_match_initial = sdiff[match_initial].new_position
 			@str1_match_final   = sdiff[match_final].old_position
 			@str2_match_final   = sdiff[match_final].new_position
-			@similarity  = 2 * lcs / ((@str1_match_final - @str1_match_initial + 1) + (@str2_match_final - @str2_match_initial + 1)).to_f
+			mlcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+			@similarity  = 2 * mlcs / (str1[@str1_match_initial .. @str1_match_final].scan(/\S/).count + str2[@str2_match_initial .. @str2_match_final].scan(/\S/).count).to_f
+			# @similarity  = 2 * lcs / (str1[@str1_match_initial .. @str1_match_final].length + str2[@str2_match_initial .. @str2_match_final].length).to_f
 		else
 			@str1_match_initial = 0
 			@str2_match_initial = 0

data/lib/text_alignment/mixed_alignment.rb

@@ -10,8 +10,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::NOMATCH_CHARS = "@^|#$%&_" unless defined? TextAlignment::NOMATCH_CHARS
-
 class TextAlignment::MixedAlignment
 	attr_reader :sdiff
 	attr_reader :position_map_begin, :position_map_end
@@ -21,58 +19,7 @@ class TextAlignment::MixedAlignment
 
 	def initialize(str1, str2, mappings = [])
 		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
-		raise ArgumentError, "nil mappings" if mappings.nil?
-
-		## preprocessing
-		str1 = str1.dup
-		str2 = str2.dup
-		mappings = mappings.dup
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if str2.index(c).nil?
-				@nomatch_char1 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char1.nil?
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if c != @nomatch_char1 && str1.index(c).nil?
-				@nomatch_char2 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char2.nil?
-
-		# single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@nomatch_char1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@nomatch_char2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+		mappings ||= []
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end

data/lib/text_alignment/text_alignment.rb

@@ -5,28 +5,35 @@ require 'text_alignment/mixed_alignment'
 
 module TextAlignment; end unless defined? TextAlignment
 
+TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
+
 class TextAlignment::TextAlignment
-	attr_reader :block_alignments
+	attr_reader :block_alignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
+		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
 
-		size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
-		size_window = _size_window || TextAlignment::SIZE_WINDOW
-		sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
+		@block_alignment = {source_text:_str1, target_text:_str2}
 
-		mappings ||= TextAlignment::MAPPINGS
+		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
 		unless block_begin.nil?
-			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
-			return @block_alignments
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
+		end
+
+		# try exact match
+		block_begin = str2.downcase.index(str1.downcase)
+		unless block_begin.nil?
+			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignment
 		end
 
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, size_ngram, size_window, sim_threshold)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
 
 		# To collect matched blocks
 		mblocks = []
@@ -56,7 +63,7 @@ class TextAlignment::TextAlignment
 		# puts
 
 		## To find block alignments
-		@block_alignments = []
+		@block_alignment[:blocks] = []
 		return if mblocks.empty?
 
 		# Initial step
@@ -65,35 +72,35 @@ class TextAlignment::TextAlignment
 			e2 = mblocks[0][:target][:begin]
 
 			if mblocks[0][:target][:begin] == 0
-				@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
+				@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:0}, alignment: :empty}
 			else
 				_str1 = str1[0 ... e1]
 				_str2 = str2[0 ... e2]
 
 				unless _str1.strip.empty?
 					if _str2.strip.empty?
-						@block_alignments << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+						@block_alignment[:blocks] << {source:{begin:0, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 					else
 						len_min = [_str1.length, _str2.length].min
 						len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b1 = _str1.length < len_buffer ? 0 : e1 - len_buffer
 						b2 = _str2.length < len_buffer ? 0 : e2 - len_buffer
 
-						@block_alignments << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
+						@block_alignment[:blocks] << {source:{begin:0, end:b1}, target:{begin:0, end:b2}, alignment: :empty} if b1 > 0
 
 						_str1 = str1[b1 ... e1]
 						_str2 = str2[b2 ... e2]
 						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 						if alignment.similarity < 0.6
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty}
 						else
-							@block_alignments << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
 						end
 					end
 				end
 			end
 		end
-		@block_alignments << mblocks[0]
+		@block_alignment[:blocks] << mblocks[0].merge(alignment: :block)
 
 		(1 ... mblocks.length).each do |i|
 			b1 = mblocks[i - 1][:source][:end]
@@ -104,17 +111,17 @@ class TextAlignment::TextAlignment
 			_str2 = str2[b2 ... e2]
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 				else
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 				end
 			end
-			@block_alignments << mblocks[i]
+			@block_alignment[:blocks] << mblocks[i].merge(alignment: :block)
 		end
 
 		# Final step
@@ -126,7 +133,7 @@ class TextAlignment::TextAlignment
 
 			unless _str1.strip.empty?
 				if _str2.strip.empty?
-					@block_alignments << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
+					@block_alignment[:blocks] << {source:{begin:b1, end:str1.length}, target:{begin:b2, end:str2.length}, alignment: :empty}
 				else
 					len_min = [_str1.length, _str2.length].min
 					len_buffer = (len_min * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
@@ -137,56 +144,56 @@ class TextAlignment::TextAlignment
 
 					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
 					if alignment.similarity < 0.6
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 					else
-						@block_alignments << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
 					end
 
-					@block_alignments << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
+					@block_alignment[:blocks] << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
 				end
 			end
 		end
 
-		@block_alignments.each do |a|
+		@block_alignment[:blocks].each do |a|
 			a[:delta] = a[:target][:begin] - a[:source][:begin]
 		end
 	end
 
 	def transform_begin_position(begin_position)
-		i = @block_alignments.index{|b| b[:source][:end] > begin_position}
-		block_alignment = @block_alignments[i]
-
-		b = if block_alignment[:alignment].nil?
-			begin_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if begin_position == block_alignment[:source][:begin]
-				block_alignment[:target][:begin]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
+		block = @block_alignment[:blocks][i]
+
+		b = if block[:alignment] == :block
+			begin_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if begin_position == block[:source][:begin]
+				block[:target][:begin]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_begin_position(begin_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_begin_position(begin_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
 	def transform_end_position(end_position)
-		i = @block_alignments.index{|b| b[:source][:end] >= end_position}
-		block_alignment = @block_alignments[i]
-
-		e = if block_alignment[:alignment].nil?
-			end_position + block_alignment[:delta]
-		elsif block_alignment[:alignment] == :empty
-			if end_position == block_alignment[:source][:end]
-				block_alignment[:target][:end]
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
+		block = @block_alignment[:blocks][i]
+
+		e = if block[:alignment] == :block
+			end_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if end_position == block[:source][:end]
+				block[:target][:end]
 			else
 				# raise "lost annotation"
 				nil
 			end
 		else
-			r = block_alignment[:alignment].transform_end_position(end_position - block_alignment[:source][:begin])
-			r.nil? ? nil : r + block_alignment[:target][:begin]
+			r = block[:alignment].transform_end_position(end_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
 		end
 	end
 
@@ -232,4 +239,114 @@ class TextAlignment::TextAlignment
 		r
 	end
 
+	def alignment_show
+		stext = @block_alignment[:source_text]
+		ttext = @block_alignment[:target_text]
+
+		show = ''
+		@block_alignment[:blocks].each do |a|
+			show += case a[:alignment]
+			when :block
+				"===== common =====\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :empty
+				"<<<<< string 1\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
+				">>>>> string 2\n" +
+				ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
+			else
+				astr1 = ''
+				astr2 = ''
+
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						stext[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						stext[c.old_position + base]
+					when '!'
+						stext[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						ttext[c.new_position + base]
+					when '+'
+						ttext[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + ttext[c.new_position + base]
+					end
+				end.join('')
+
+				"***** local mismatch\n" +
+				"[#{astr1}]\n" +
+				"[#{astr2}]\n\n"
+			end
+		end
+		show
+	end
+
+	private
+
+	def string_preprocessing(_str1, _str2)
+		str1 = _str1.dup
+		str2 = _str2.dup
+		mappings = TextAlignment::MAPPINGS.dup
+
+		## single character mappings
+		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+		characters_from = character_mappings.collect{|m| m[0]}.join
+		characters_to   = character_mappings.collect{|m| m[1]}.join
+		characters_to.gsub!(/-/, '\-')
+
+		str1.tr!(characters_from, characters_to)
+		str2.tr!(characters_from, characters_to)
+
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+		## long to one character mappings
+		pletters = TextAlignment::PADDING_LETTERS
+
+		# find the padding letter for str1
+		padding_letter1 = begin
+			i = pletters.index{|l| str2.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# find the padding letter for str2
+		padding_letter2 = begin
+			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# ASCII foldings
+		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+		ascii_foldings.each do |f|
+			from = f[1]
+
+			if str2.index(f[0])
+				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				str1.gsub!(from, to)
+			end
+
+			if str1.index(f[0])
+				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				str2.gsub!(from, to)
+			end
+		end
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+		[str1, str2, mappings]
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.4.2'
+	VERSION = '0.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: '0.6'
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-02 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary