text_alignment 0.4 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35f83f76cd3e9ca59604327710d2eac4684b8abca9dbd167b2b391d98ea561f9
-  data.tar.gz: 77007a1bfcf72d0681ac9d21b35221e7d519a291383f3d0d8fa0574d631b65ea
+  metadata.gz: 07e02285cce988b857421b5eae20c6b39394dbd0c904de2f416344cad69b725f
+  data.tar.gz: 6ccdf9930bc97fc5bc6fc2a2d92f732867744342a6c828eaa3ac8029339f33c3
 SHA512:
-  metadata.gz: 2f936569e072e6693f279b5d3a349965b286192d8f694fffbbf41110f66afbd5d80f865a9232c2bef60ec94920e0a814142c8756f0ba3ff29a109cbbe1f7abec
-  data.tar.gz: 955b85aa639f733361f354cc68ac3d8b14a17a14784f1e6ec24f240934ef0d160c6b8cf01232e1e4243c2eecffcaf1665999cc3e455bfb0e61c0dd905d2cb8b4
+  metadata.gz: 477e77a9349857cd9ab69b5da4d048dd2b36b1d51a7af742a9d05cb1ee20eba8c6d67cd4882c380f5e2737be1ca0d6fa13c0a81b2c3b257a7f249ccb1f4e589c
+  data.tar.gz: 904fe737512c8774a03c23ae7229df55dd0727fd1a41d3a73031b96e6458920c78c02c66990577624c830525734918c54a8821d1e700eb55d7c8b871ecd6edf3

data/bin/align_annotations

@@ -35,6 +35,10 @@ def align_mdoc(source_annotations, target_annotations)
 	source_annotations.each do |annotations|
 		alignment = TextAlignment::TextAlignment.new(annotations[:text], target_annotations[:text])
 
+		puts alignment.alignment_show
+		puts "-----"
+		puts
+
 		# alignment.block_alignments.each do |a|
 			# p {source:a[:source], target:a[:target]}
 			# puts "--"
@@ -103,48 +107,39 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
-	pp alignment
+	# pp alignment
 
 	# verification
-	source_text = source_annotations[:text]
-	puts "=====BEGIN"
-	(0 ... source_text.rstrip.length).each do |p|
-		t = alignment.transform_begin_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	puts "=====BEGIN"
-	(0 .. source_text.rstrip.length).each do |p|
-		t = alignment.transform_end_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	# alignment.block_alignments.each do |a|
-	# 	if a[:alignment].nil? || a[:alignment] == :empty
-	# 		# p [a[:source], a[:target]]
-	# 		# p a[:alignment]
+	# source_text = source_annotations[:text]
+	# puts "=====BEGIN"
+	# (0 ... source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_begin_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
+	# 	else
+	# 		print '.'
+	# 	end
+	# end
+	# puts
+	# puts "=====END"
+
+	# puts "=====BEGIN"
+	# (0 .. source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_end_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
 	# 	else
-	# 		p [a[:source], a[:target]]
-	# 		p a[:alignment].similarity
-	# 		puts "--"
-	# 		puts source_annotations[:text][a[:source][:begin] ... a[:source][:end]]
-	# 		puts "--"
-	# 		puts target_text[a[:target][:begin] ... a[:target][:end]]
-	# 		puts "======"
+	# 		print '.'
 	# 	end
 	# end
+	# puts
+	# puts "=====END"
+
+	source_text = source_annotations[:text]
+
+	puts "[block alignment]"
+	puts alignment.alignment_show
+	puts "====="
 	# exit
 
 	# verification of source denotations

data/lib/text_alignment/anchor_finder.rb

@@ -1,18 +1,15 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
-
 class TextAlignment::AnchorFinder
 
 	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
 		@size_window = _size_window || TextAlignment::SIZE_WINDOW
-		@sim_threshold = _size_window || TextAlignment::TEXT_SIMILARITY_TRESHOLD
+		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
 
 		@reverse = (target_str.length < source_str.length)
 

data/lib/text_alignment/approximate_fit.rb

@@ -1,13 +1,11 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::MIN_LENGTH_FOR_APPROXIMATION = 50 unless defined? TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class << TextAlignment
 
@@ -16,8 +14,8 @@ class << TextAlignment
 		raise ArgumentError, 'nil string' if str1.nil? || str2.nil?
 		return 0, str2.length if str2.length < TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
 
-		ngram1 = (0 .. str1.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str1[i, TextAlignment::SIGNATURE_NGRAM]}
-		ngram2 = (0 .. str2.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str2[i, TextAlignment::SIGNATURE_NGRAM]}
+		ngram1 = (0 .. str1.length - TextAlignment::SIZE_NGRAM).collect{|i| str1[i, TextAlignment::SIZE_NGRAM]}
+		ngram2 = (0 .. str2.length - TextAlignment::SIZE_NGRAM).collect{|i| str2[i, TextAlignment::SIZE_NGRAM]}
 		ngram_shared = ngram1 & ngram2
 
 		# If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
@@ -45,7 +43,7 @@ class << TextAlignment
 			text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
 			cache["#{fit_begin}-#{fit_end}"] = text_similarity
 
-			break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break if text_similarity > TextAlignment::TEXT_SIMILARITY_THRESHOLD
 			fit_begin, fit_end = nil, nil
 		end
 		return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end

data/lib/text_alignment/constants.rb

@@ -0,0 +1,7 @@
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 60 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
+TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.9 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/glcs_alignment_fast.rb

@@ -9,8 +9,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 5 unless defined? TextAlignment::SIGNATURE_NGRAM
-
 class TextAlignment::GLCSTextAlignment
 	attr_reader :position_map_begin, :position_map_end
 	attr_reader :common_elements, :mapped_elements

data/lib/text_alignment/lcs_comparison.rb

@@ -33,7 +33,8 @@ class TextAlignment::LCSComparison
 			@str2_match_initial = sdiff[match_initial].new_position
 			@str1_match_final   = sdiff[match_final].old_position
 			@str2_match_final   = sdiff[match_final].new_position
-			@similarity  = 2 * lcs / ((@str1_match_final - @str1_match_initial + 1) + (@str2_match_final - @str2_match_initial + 1)).to_f
+			mlcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+			@similarity  = 2 * mlcs / (str1[@str1_match_initial .. @str1_match_final].scan(/\S/).count + str2[@str2_match_initial .. @str2_match_final].scan(/\S/).count).to_f
 		else
 			@str1_match_initial = 0
 			@str2_match_initial = 0

data/lib/text_alignment/mixed_alignment.rb

@@ -10,8 +10,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::NOMATCH_CHARS = "@^|#$%&_" unless defined? TextAlignment::NOMATCH_CHARS
-
 class TextAlignment::MixedAlignment
 	attr_reader :sdiff
 	attr_reader :position_map_begin, :position_map_end
@@ -21,58 +19,7 @@ class TextAlignment::MixedAlignment
 
 	def initialize(str1, str2, mappings = [])
 		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
-		raise ArgumentError, "nil mappings" if mappings.nil?
-
-		## preprocessing
-		str1 = str1.dup
-		str2 = str2.dup
-		mappings = mappings.dup
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if str2.index(c).nil?
-				@nomatch_char1 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char1.nil?
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if c != @nomatch_char1 && str1.index(c).nil?
-				@nomatch_char2 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char2.nil?
-
-		# single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@nomatch_char1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@nomatch_char2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+		mappings ||= []
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end

data/lib/text_alignment/text_alignment.rb

@@ -1,32 +1,40 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'text_alignment/anchor_finder'
 require 'text_alignment/mixed_alignment'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
-
+TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, mappings = nil)
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
+		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
+
+		@ostr1 = _str1
+		@ostr2 = _str2
 
-		mappings ||= TextAlignment::MAPPINGS
+		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
 		unless block_begin.nil?
-			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
+			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+			return @block_alignments
+		end
+
+		# try exact match
+		block_begin = str2.downcase.index(str1.downcase)
+		unless block_begin.nil?
+			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
 			return @block_alignments
 		end
 
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
 
 		# To collect matched blocks
 		mblocks = []
@@ -93,7 +101,7 @@ class TextAlignment::TextAlignment
 				end
 			end
 		end
-		@block_alignments << mblocks[0]
+		@block_alignments << mblocks[0].merge(alignment: :block)
 
 		(1 ... mblocks.length).each do |i|
 			b1 = mblocks[i - 1][:source][:end]
@@ -114,7 +122,7 @@ class TextAlignment::TextAlignment
 					end
 				end
 			end
-			@block_alignments << mblocks[i]
+			@block_alignments << mblocks[i].merge(alignment: :block)
 		end
 
 		# Final step
@@ -156,7 +164,7 @@ class TextAlignment::TextAlignment
 		i = @block_alignments.index{|b| b[:source][:end] > begin_position}
 		block_alignment = @block_alignments[i]
 
-		b = if block_alignment[:alignment].nil?
+		b = if block_alignment[:alignment] == :block
 			begin_position + block_alignment[:delta]
 		elsif block_alignment[:alignment] == :empty
 			if begin_position == block_alignment[:source][:begin]
@@ -175,7 +183,7 @@ class TextAlignment::TextAlignment
 		i = @block_alignments.index{|b| b[:source][:end] >= end_position}
 		block_alignment = @block_alignments[i]
 
-		e = if block_alignment[:alignment].nil?
+		e = if block_alignment[:alignment] == :block
 			end_position + block_alignment[:delta]
 		elsif block_alignment[:alignment] == :empty
 			if end_position == block_alignment[:source][:end]
@@ -232,4 +240,176 @@ class TextAlignment::TextAlignment
 		r
 	end
 
+	def alignment_table
+		table = <<-TABLE
+			<table class='text_alignment_table'>
+				<thead>
+					<tr>
+						<th class='text_alignment_left' style='width:50%'>Text 1</th>
+						<th class='text_alignment_rigt'>Text 2</th>
+					</tr>
+				</thead>
+				<tbody>
+		TABLE
+
+		@block_alignments.each do |a|
+			table += alignment_table_th(a)
+			table += "<tr>\n" + case a[:alignment]
+			when :block
+				"<td colspan='2' class='text_alignment_common'>" +
+				@ostr1[a[:source][:begin] ... a[:source][:end]] +
+				"</td>\n"
+			when :empty
+				"<td class='text_alignment_left'>"  + @ostr1[a[:source][:begin] ... a[:source][:end]] + "</td>\n" +
+				"<td class='text_alignment_right'>" + @ostr2[a[:target][:begin] ... a[:target][:end]] + "</td>\n"
+			else
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr1[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						@ostr1[c.old_position + base]
+					when '!'
+						@ostr1[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr2[c.new_position + base]
+					when '+'
+						@ostr2[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + @ostr2[c.new_position + base]
+					end
+				end.join('')
+
+				"<td class='text_alignment_left'>"  + astr1 + "</td>\n" +
+				"<td class='text_alignment_right'>" + astr2 + "</td>\n"
+			end + "</tr>\n"
+		end
+		table += '</tbody></table>'
+	end
+
+	def alignment_table_th(a)
+		"<tr>" +
+		"<th class='text_alignment_left'>#{a[:source][:begin]} - #{a[:source][:end]}</th>" +
+		"<th class='text_alignment_right'>#{a[:target][:begin]} - #{a[:target][:end]}</th>" +
+		"</tr>"
+	end
+
+	def alignment_show
+		show = ''
+		@block_alignments.each do |a|
+			show += case a[:alignment]
+			when :block
+				"===== common =====\n" +
+				@ostr1[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :empty
+				"<<<<< string 1\n" +
+				@ostr1[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
+				">>>>> string 2\n" +
+				@ostr2[a[:target][:begin] ... a[:target][:end]] + "\n\n"
+			else
+				astr1 = ''
+				astr2 = ''
+
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr1[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						@ostr1[c.old_position + base]
+					when '!'
+						@ostr1[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr2[c.new_position + base]
+					when '+'
+						@ostr2[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + @ostr2[c.new_position + base]
+					end
+				end.join('')
+
+				"***** local mismatch\n" +
+				"[#{astr1}]\n" +
+				"[#{astr2}]\n\n"
+			end
+		end
+		show
+	end
+
+	private
+
+	def string_preprocessing(_str1, _str2)
+		str1 = _str1.dup
+		str2 = _str2.dup
+		mappings = TextAlignment::MAPPINGS.dup
+
+		## single character mappings
+		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+		characters_from = character_mappings.collect{|m| m[0]}.join
+		characters_to   = character_mappings.collect{|m| m[1]}.join
+		characters_to.gsub!(/-/, '\-')
+
+		str1.tr!(characters_from, characters_to)
+		str2.tr!(characters_from, characters_to)
+
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+		## long to one character mappings
+		pletters = TextAlignment::PADDING_LETTERS
+
+		# find the padding letter for str1
+		padding_letter1 = begin
+			i = pletters.index{|l| str2.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# find the padding letter for str2
+		padding_letter2 = begin
+			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# ASCII foldings
+		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+		ascii_foldings.each do |f|
+			from = f[1]
+
+			if str2.index(f[0])
+				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				str1.gsub!(from, to)
+			end
+
+			if str1.index(f[0])
+				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				str2.gsub!(from, to)
+			end
+		end
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+		[str1, str2, mappings]
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.4'
+	VERSION = '0.5.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: 0.5.2
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-29 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -77,6 +77,7 @@ files:
 - lib/text_alignment.rb
 - lib/text_alignment/anchor_finder.rb
 - lib/text_alignment/approximate_fit.rb
+- lib/text_alignment/constants.rb
 - lib/text_alignment/find_divisions.rb
 - lib/text_alignment/glcs_alignment.rb
 - lib/text_alignment/glcs_alignment_fast.rb