text_alignment 0.3.23 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7678cbfd749833e03a17dcf0b06699def0215e7c39b981d2ff8ddfb3a79b2ea4
-  data.tar.gz: 0251b99cbd984dcd8e23ea38e6190a449a1f83f98119e68251f1f17e5854ac73
+  metadata.gz: 9f5f7f27c8628123530d51d0a68060aa6fb850bcef8c7089c8bf990f7257a80b
+  data.tar.gz: 45f768df4e7d89c931295985adb31df9e725156e5a85d7f78a5b7cd26d00be4d
 SHA512:
-  metadata.gz: '08055cb7659458e4a5834a193442c7592f9eecab09e206a8e5f20aab658c26cb24cda2949804db3569c160f03c6e71e9a28bdcdb352f8d207f1eb31aa427d963'
-  data.tar.gz: 43f0c0111662212c049a8b4baf2eeb7b2cc15a01d9f02d7cfbc501a9b016fa1344516685b8052c862227c72f8f9866b3232425e06d862899003860ad4700fc70
+  metadata.gz: 4004293fa10eb247110764d16a24900a10e714072227dd6e5626d2123deca4bcec00b7255e1affaba4d7dda75d7e42049aabf6b1e6c51f4c005124443b5f9ffc
+  data.tar.gz: eb5c8a2c89c8973242bb77457e0f8d9922486d9743e7c39ee12ea4025ece6d888be5308eaddec076e5a21c84149dcb31bc0ed6fcd68881fb6c8c4063e49fb64d

data/bin/align_annotations

@@ -103,65 +103,56 @@ target_annotations = if source_annotations.class == Array
 else
 	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text)
 
-	pp alignment
+	# pp alignment
 
 	# verification
-	source_text = source_annotations[:text]
-	puts "=====BEGIN"
-	(0 ... source_text.rstrip.length).each do |p|
-		t = alignment.transform_begin_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	puts "=====BEGIN"
-	(0 .. source_text.rstrip.length).each do |p|
-		t = alignment.transform_end_position(p)
-		if t.nil?
-			print source_text[p]
-		else
-			print '.'
-		end
-	end
-	puts
-	puts "=====END"
-
-	# alignment.block_alignments.each do |a|
-	# 	if a[:alignment].nil? || a[:alignment] == :empty
-	# 		# p [a[:source], a[:target]]
-	# 		# p a[:alignment]
+	# source_text = source_annotations[:text]
+	# puts "=====BEGIN"
+	# (0 ... source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_begin_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
+	# 	else
+	# 		print '.'
+	# 	end
+	# end
+	# puts
+	# puts "=====END"
+
+	# puts "=====BEGIN"
+	# (0 .. source_text.rstrip.length).each do |p|
+	# 	t = alignment.transform_end_position(p)
+	# 	if t.nil?
+	# 		print source_text[p]
 	# 	else
-	# 		p [a[:source], a[:target]]
-	# 		p a[:alignment].similarity
-	# 		puts "--"
-	# 		puts source_annotations[:text][a[:source][:begin] ... a[:source][:end]]
-	# 		puts "--"
-	# 		puts target_text[a[:target][:begin] ... a[:target][:end]]
-	# 		puts "======"
+	# 		print '.'
 	# 	end
 	# end
+	# puts
+	# puts "=====END"
+
+	source_text = source_annotations[:text]
+
+	# puts "[block alignment]"
+	puts alignment.alignment_table
+	# puts "====="
 	# exit
 
 	# verification of source denotations
-	puts "[Invalid source denotations]"
-	source_annotations[:denotations] do |d|
-		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
-	end
-	puts "====="
-	puts
+	# puts "[Invalid source denotations]"
+	# source_annotations[:denotations] do |d|
+	# 	p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < source_text.length
+	# end
+	# puts "====="
+	# puts
 
 	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
-	puts "[Invalid transformation]"
-	denotations.each do |d|
-		p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
-	end
-	puts "====="
-	puts
+	# puts "[Invalid transformation]"
+	# denotations.each do |d|
+	# 	p d unless d[:span][:begin] && d[:span][:end] && d[:span][:begin] < d[:span][:end] && d[:span][:begin] >= 0 && d[:span][:end] < target_text.length
+	# end
+	# puts "====="
+	# puts
 
 	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
 

data/lib/text_alignment/anchor_finder.rb

@@ -1,17 +1,15 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
-
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil)
+	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
 		@size_window = _size_window || TextAlignment::SIZE_WINDOW
+		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
 
 		@reverse = (target_str.length < source_str.length)
 
@@ -43,10 +41,10 @@ class TextAlignment::AnchorFinder
 				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
 
 				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
 
 				search_position = @beg_s2 + 1
 			end

data/lib/text_alignment/approximate_fit.rb

@@ -1,13 +1,11 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::MIN_LENGTH_FOR_APPROXIMATION = 50 unless defined? TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class << TextAlignment
 
@@ -16,8 +14,8 @@ class << TextAlignment
 		raise ArgumentError, 'nil string' if str1.nil? || str2.nil?
 		return 0, str2.length if str2.length < TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
 
-		ngram1 = (0 .. str1.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str1[i, TextAlignment::SIGNATURE_NGRAM]}
-		ngram2 = (0 .. str2.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str2[i, TextAlignment::SIGNATURE_NGRAM]}
+		ngram1 = (0 .. str1.length - TextAlignment::SIZE_NGRAM).collect{|i| str1[i, TextAlignment::SIZE_NGRAM]}
+		ngram2 = (0 .. str2.length - TextAlignment::SIZE_NGRAM).collect{|i| str2[i, TextAlignment::SIZE_NGRAM]}
 		ngram_shared = ngram1 & ngram2
 
 		# If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
@@ -45,7 +43,7 @@ class << TextAlignment
 			text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
 			cache["#{fit_begin}-#{fit_end}"] = text_similarity
 
-			break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break if text_similarity > TextAlignment::TEXT_SIMILARITY_THRESHOLD
 			fit_begin, fit_end = nil, nil
 		end
 		return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end

data/lib/text_alignment/constants.rb

@@ -0,0 +1,7 @@
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 60 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
+TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.9 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/glcs_alignment_fast.rb

@@ -9,8 +9,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 5 unless defined? TextAlignment::SIGNATURE_NGRAM
-
 class TextAlignment::GLCSTextAlignment
 	attr_reader :position_map_begin, :position_map_end
 	attr_reader :common_elements, :mapped_elements

data/lib/text_alignment/mixed_alignment.rb

@@ -10,8 +10,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::NOMATCH_CHARS = "@^|#$%&_" unless defined? TextAlignment::NOMATCH_CHARS
-
 class TextAlignment::MixedAlignment
 	attr_reader :sdiff
 	attr_reader :position_map_begin, :position_map_end
@@ -21,58 +19,7 @@ class TextAlignment::MixedAlignment
 
 	def initialize(str1, str2, mappings = [])
 		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
-		raise ArgumentError, "nil mappings" if mappings.nil?
-
-		## preprocessing
-		str1 = str1.dup
-		str2 = str2.dup
-		mappings = mappings.dup
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if str2.index(c).nil?
-				@nomatch_char1 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char1.nil?
-
-		## find the first nomatch character
-		TextAlignment::NOMATCH_CHARS.each_char do |c|
-			if c != @nomatch_char1 && str1.index(c).nil?
-				@nomatch_char2 = c
-				break
-			end
-		end
-		raise RuntimeError, "Cannot find nomatch character" if @nomatch_char2.nil?
-
-		# single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@nomatch_char1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@nomatch_char2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+		mappings ||= []
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end

data/lib/text_alignment/text_alignment.rb

@@ -1,23 +1,24 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'text_alignment/anchor_finder'
 require 'text_alignment/mixed_alignment'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
-
+TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
 
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, mappings = nil)
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
+		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
+
+		@ostr1 = _str1
+		@ostr2 = _str2
 
-		mappings ||= TextAlignment::MAPPINGS
+		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		# try exact match
 		block_begin = str2.index(str1)
@@ -26,7 +27,14 @@ class TextAlignment::TextAlignment
 			return @block_alignments
 		end
 
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2)
+		# try exact match
+		block_begin = str2.downcase.index(str1.downcase)
+		unless block_begin.nil?
+			@block_alignments = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin}]
+			return @block_alignments
+		end
+
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
 
 		# To collect matched blocks
 		mblocks = []
@@ -93,7 +101,7 @@ class TextAlignment::TextAlignment
 				end
 			end
 		end
-		@block_alignments << mblocks[0]
+		@block_alignments << mblocks[0].merge(alignment: :block)
 
 		(1 ... mblocks.length).each do |i|
 			b1 = mblocks[i - 1][:source][:end]
@@ -114,7 +122,7 @@ class TextAlignment::TextAlignment
 					end
 				end
 			end
-			@block_alignments << mblocks[i]
+			@block_alignments << mblocks[i].merge(alignment: :block)
 		end
 
 		# Final step
@@ -156,7 +164,7 @@ class TextAlignment::TextAlignment
 		i = @block_alignments.index{|b| b[:source][:end] > begin_position}
 		block_alignment = @block_alignments[i]
 
-		b = if block_alignment[:alignment].nil?
+		b = if block_alignment[:alignment] == :block
 			begin_position + block_alignment[:delta]
 		elsif block_alignment[:alignment] == :empty
 			if begin_position == block_alignment[:source][:begin]
@@ -175,7 +183,7 @@ class TextAlignment::TextAlignment
 		i = @block_alignments.index{|b| b[:source][:end] >= end_position}
 		block_alignment = @block_alignments[i]
 
-		e = if block_alignment[:alignment].nil?
+		e = if block_alignment[:alignment] == :block
 			end_position + block_alignment[:delta]
 		elsif block_alignment[:alignment] == :empty
 			if end_position == block_alignment[:source][:end]
@@ -232,4 +240,178 @@ class TextAlignment::TextAlignment
 		r
 	end
 
+	def alignment_table
+		table = <<-TABLE
+			<table class='text_alignment_table'>
+				<thead>
+					<tr>
+						<th class='text_alignment_left' style='width:50%'>Text 1</th>
+						<th class='text_alignment_rigt'>Text 2</th>
+					</tr>
+				</thead>
+				<tbody>
+		TABLE
+
+		@block_alignments.each do |a|
+			table += alignment_table_th(a)
+			table += "<tr>\n" + case a[:alignment]
+			when :block
+				"<td colspan='2' class='text_alignment_common'>" +
+				@ostr1[a[:source][:begin] ... a[:source][:end]] +
+				"</td>\n"
+			when :empty
+				"<td class='text_alignment_left'>"  + @ostr1[a[:source][:begin] ... a[:source][:end]] + "</td>\n" +
+				"<td class='text_alignment_right'>" + @ostr2[a[:target][:begin] ... a[:target][:end]] + "</td>\n"
+			else
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr1[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						@ostr1[c.old_position + base]
+					when '!'
+						@ostr1[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr2[c.new_position + base]
+					when '+'
+						@ostr2[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + @ostr2[c.new_position + base]
+					end
+				end.join('')
+
+				"<td class='text_alignment_left'>"  + astr1 + "</td>\n" +
+				"<td class='text_alignment_right'>" + astr2 + "</td>\n"
+			end + "</tr>\n"
+		end
+		table += '</tbody></table>'
+	end
+
+	def alignment_table_th(a)
+		"<tr>" +
+		"<th class='text_alignment_left'>#{a[:source][:begin]} - #{a[:source][:end]}</th>" +
+		"<th class='text_alignment_right'>#{a[:target][:begin]} - #{a[:target][:end]}</th>" +
+		"</tr>"
+	end
+
+	def alignment_show
+		show = ''
+		@block_alignments.each do |a|
+			show += case a[:alignment]
+			when :block
+				"===== common =====\n" +
+				@ostr1[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :empty
+				puts "<<<<< string 1"
+				p @ostr1[a[:source][:begin] ... a[:source][:end]]
+				puts
+				puts ">>>>> string 2"
+				p @ostr2[a[:target][:begin] ... a[:target][:end]]
+				puts
+			else
+				puts "***** local mismatch"
+				astr1 = ''
+				astr2 = ''
+
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr1[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						@ostr1[c.old_position + base]
+					when '!'
+						@ostr1[c.old_position + base] + '_'
+					end
+				end.join('')
+
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						@ostr2[c.new_position + base]
+					when '+'
+						@ostr2[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + @ostr2[c.new_position + base]
+					end
+				end.join('')
+
+				puts '[' + astr1 + ']'
+				puts '[' + astr2 + ']'
+				puts
+			end
+		end
+	end
+
+	private
+
+	def string_preprocessing(_str1, _str2)
+		str1 = _str1.dup
+		str2 = _str2.dup
+		mappings = TextAlignment::MAPPINGS.dup
+
+		## single character mappings
+		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+		characters_from = character_mappings.collect{|m| m[0]}.join
+		characters_to   = character_mappings.collect{|m| m[1]}.join
+		characters_to.gsub!(/-/, '\-')
+
+		str1.tr!(characters_from, characters_to)
+		str2.tr!(characters_from, characters_to)
+
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+		## long to one character mappings
+		pletters = TextAlignment::PADDING_LETTERS
+
+		# find the padding letter for str1
+		padding_letter1 = begin
+			i = pletters.index{|l| str2.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# find the padding letter for str2
+		padding_letter2 = begin
+			i = pletters.index{|l| l != padding_letter1 && str1.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# ASCII foldings
+		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+		ascii_foldings.each do |f|
+			from = f[1]
+
+			if str2.index(f[0])
+				to   = f[0] + (padding_letter1 * (f[1].length - 1))
+				str1.gsub!(from, to)
+			end
+
+			if str1.index(f[0])
+				to   = f[0] + (padding_letter2 * (f[1].length - 1))
+				str2.gsub!(from, to)
+			end
+		end
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+		[str1, str2, mappings]
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.23'
+	VERSION = '0.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.23
+  version: '0.5'
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-29 00:00:00.000000000 Z
+date: 2020-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -77,6 +77,7 @@ files:
 - lib/text_alignment.rb
 - lib/text_alignment/anchor_finder.rb
 - lib/text_alignment/approximate_fit.rb
+- lib/text_alignment/constants.rb
 - lib/text_alignment/find_divisions.rb
 - lib/text_alignment/glcs_alignment.rb
 - lib/text_alignment/glcs_alignment_fast.rb