text_alignment 0.10.1 → 0.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 710fa3dfe07e268c62de77c67abb52e1893a36a9c0fbdfbf7400328ab7f8358a
-  data.tar.gz: 31a086666978c4776a60b0ce19fde48beeb8a5bce599b7572f20089cfb5c7500
+  metadata.gz: abf4387fb9ea598d924eb0a7369ca58ec0be7cdd35ac89abc57647d665c00207
+  data.tar.gz: 30ab0ccea5c04cae8132a70872cf5de4704ce7fd4e2104f7fddc6527794424c5
 SHA512:
-  metadata.gz: d0a752d2203f65e48b4f10ff29eb29a77504e7cef4da1dd3602e2ea5107a99b7edff4c3609c79c7dcba0f9cc458780dc4788ff8123a781a4d77ae89236170bf5
-  data.tar.gz: '094ff5f662ccdc2fc14f81549126362be49fd64610f994c6ba5c4457fbbbececc1931257589ebde49211c2bce744ca033bfe3673c8af5c8a01af7679bfb626cc'
+  metadata.gz: 2033123f2289c991b2a53826921ca19489f7f39c53918df5f962e321b1f3ff0996760e9df302872d07a7e638e6c56ad86428d1ce773d6e14c54180ad2360b1be
+  data.tar.gz: a902ca48bba502af787deb3abf6c5a2b4c9a121394452ef25e95c61212ee0cfa4e991f83315bc6e03a52d45ead909acedeaf0abd9cdc278ccc58b7cb315ab556

data/bin/align_annotations

@@ -26,9 +26,8 @@ def read_text(filename)
 	end
 end
 
-def align_denotations(denotations, source_text, target_text, debug = false, cm = nil)
-	alignment = TextAlignment::TextAlignment.new(source_text, target_text, denotations, cm)
-	cm = alignment.cultivation_map
+def align_denotations(denotations, source_text, alignment, debug = false)
+	alignment.align(source_text, denotations)
 	new_denotations = alignment.transform_hdenotations(denotations)
 
 	if debug
@@ -38,7 +37,7 @@ def align_denotations(denotations, source_text, target_text, debug = false, cm =
 	end
 
 	lost_annotations = alignment.lost_annotations
-	unless lost_annotations.empty?
+	unless lost_annotations.nil? || lost_annotations.empty?
 		warn "\n[lost annotations] #{lost_annotations.length}"
 		lost_annotations.each do |a|
 			warn "#{a}"
@@ -48,23 +47,22 @@ def align_denotations(denotations, source_text, target_text, debug = false, cm =
 	warn
 
 	# return target annotations
-	[new_denotations, cm]
+	new_denotations
 end
 
-def align_mannotations(source_annotations, target_text, debug = false)
-	target_annotations = {text:target_text}
+def align_mannotations(source_annotations, reference_text, alignment, debug = false)
+	target_annotations = {text:reference_text}
 
 	idnum_denotations = 0
 	idnum_relations = 0
 	idnum_attributes = 0
 	idnum_modifications = 0
 
-	cm = nil
 	source_annotations.each_with_index do |annotations, i|
 		if annotations.has_key?(:denotations) && !annotations[:denotations].empty?
 			ididx = {}
 			warn "[#{i}]-=-=-=-=-"
-			denotations, cm = align_denotations(annotations[:denotations], annotations[:text], target_text, debug, cm)
+			denotations = align_denotations(annotations[:denotations], annotations[:text], alignment, debug)
 
 			denotations.each do |d|
 				reid = 'T' + (idnum_denotations += 1).to_s
@@ -79,7 +77,9 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:relations].each do |r|
 					reid = 'R' + (idnum_relations += 1).to_s
 					ididx[r[:id]] = reid
-					target_annotations[:relations] << r.dup.merge({id:reid, subj:ididx[r[:subj]], obj:ididx[r[:obj]]})
+					sid = ididx[r[:subj]]
+					oid = ididx[r[:obj]]
+					target_annotations[:relations] << r.dup.merge({id:reid, subj:sid, obj:oid}) unless sid.nil? || oid.nil?
 				end
 			end
 
@@ -88,7 +88,8 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:attributes].each do |a|
 					reid = 'A' + (idnum_attributes += 1).to_s
 					ididx[a[:id]] = reid
-					target_annotations[:attributes] << a.dup.merge({id:reid, subj:ididx[a[:subj]]})
+					sid = ididx[a[:subj]]
+					target_annotations[:attributes] << a.dup.merge({id:reid, subj:sid}) unless sid.nil?
 				end
 			end
 
@@ -97,7 +98,8 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:modifications].each do |m|
 					reid = 'M' + (idnum_modifications += 1).to_s
 					ididx[m[:id]] = reid
-					target_annotations[:modifications] << m.dup.merge({id:reid, obj:ididx[m[:obj]]})
+					oid = ididx[m[:obj]]
+					target_annotations[:modifications] << m.dup.merge({id:reid, obj:oid}) unless oid.nil?
 				end
 			end
 		end
@@ -112,14 +114,18 @@ unless ARGV.length == 2
 end
 
 source_annotations = read_annotations(ARGV[0])
-target_text = read_text(ARGV[1])
+reference_text = read_text(ARGV[1])
+
+alignment = TextAlignment::TextAlignment.new(reference_text, true)
 
 target_annotations = if source_annotations.class == Array
-	align_mannotations(source_annotations, target_text, false)
+	# align_mannotations(source_annotations, reference_text, alignment, true)
+	align_mannotations(source_annotations, reference_text, alignment, false)
 else
-	denotations, cm = align_denotations(source_annotations[:denotations], source_annotations[:text], target_text, false)
-	# denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], target_text, true)
-	source_annotations.merge({text:target_text, denotations:denotations})
+	# denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment)
+	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, false)
+	source_annotations.merge({text:reference_text, denotations:denotations})
 end
 
-puts target_annotations.to_json
+# pp alignment.block_alignment
+# puts target_annotations.to_json

data/lib/text_alignment/anchor_finder.rb

@@ -7,11 +7,8 @@ module TextAlignment; end unless defined? TextAlignment
 class TextAlignment::AnchorFinder
 
 	def initialize(source_str, target_str, cultivation_map)
-		@s1, @s2 = if reverse?(source_str, target_str)
-			[target_str.downcase, source_str.downcase]
-		else
-			[source_str.downcase, target_str.downcase]
-		end
+		@s1 = source_str.downcase
+		@s2 = target_str.downcase
 
 		@cultivation_map = cultivation_map
 
@@ -19,19 +16,13 @@ class TextAlignment::AnchorFinder
 		@size_window = TextAlignment::SIZE_WINDOW
 		@sim_threshold = TextAlignment::TEXT_SIMILARITY_THRESHOLD
 		@pos_s1_final_possible_begin = @s1.length - @size_ngram - 1
+		@pos_s2_final_possible_end = @s2.length
 
 		# positions of last match
 		@pos_s1_last_match = 0
 		@pos_s2_last_match = 0
 	end
 
-	def reverse?(source_str = nil, target_str = nil)
-		unless source_str.nil?
-			@reverse_p = target_str.length < source_str.length
-		end
-		@reverse_p
-	end
-
 	def get_next_anchor
 		# To find the beginning positions of an anchor ngram in s1 and s2, beginning from the last positions matched
 		beg_s2 = for beg_s1 in @pos_s1_last_match .. @pos_s1_final_possible_begin
@@ -49,26 +40,23 @@ class TextAlignment::AnchorFinder
 		# To extend the block to the left
 		b1 = beg_s1
 		b2 = beg_s2
-		while b1 >= @pos_s1_last_match && b2 >= @pos_s2_last_match && @s1[b1] == @s2[b2]
+		left_boundary_b2 = [@pos_s2_last_match, (@cultivation_map.last_cultivated_position(b2) || 0)].max
+		while b1 > @pos_s1_last_match && b2 > left_boundary_b2 && @s1[b1 - 1] == @s2[b2 - 1]
 			b1 -= 1; b2 -= 1
 		end
-		b1 += 1; b2 += 1
 
 		# To extend the block to the right
 		e1 = beg_s1 + @size_ngram
 		e2 = beg_s2 + @size_ngram
-		while @s1[e1] && @s1[e1] == @s2[e2]
+		right_boundary_b2 = @cultivation_map.next_cultivated_position(e2) || @pos_s2_final_possible_end
+		while @s1[e1] && e2 < right_boundary_b2 && @s1[e1] == @s2[e2]
 			e1 += 1; e2 += 1
 		end
 
 		@pos_s1_last_match = e1
 		@pos_s2_last_match = e2
 
-		if reverse?
-			{source:{begin:b2 , end:e2}, target:{begin:b1, end:e1}}
-		else
-			{source:{begin:b1 , end:e1}, target:{begin:b2, end:e2}}
-		end
+		{source:{begin:b1 , end:e1}, target:{begin:b2, end:e2}}
 	end
 
 	private
@@ -91,13 +79,7 @@ class TextAlignment::AnchorFinder
 	# return nil if the anchor is too much frequent
 	def find_beg_s2_candidates(anchor, search_position)
 		candidates = []
-		while _beg_s2 = @s2.index(anchor, search_position)
-			search_again_position = @cultivation_map.search_again_position(_beg_s2)
-			unless search_again_position.nil?
-				search_position = search_again_position
-				next
-			end
-
+		while _beg_s2 = @cultivation_map.index(anchor, @s2, search_position)
 			candidates << _beg_s2
 
 			# for speed, skip anchor of high frequency
@@ -126,14 +108,14 @@ class TextAlignment::AnchorFinder
 					next
 				end
 
-				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2)
+				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2, size_window)
 				if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
 					break unless valid_beg_s2.nil?
 					valid_beg_s2 = beg_s2
 					next
 				end
 
-				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2)
+				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2, size_window)
 				if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
 					break unless valid_beg_s2.nil?
 					valid_beg_s2 = beg_s2
@@ -143,7 +125,11 @@ class TextAlignment::AnchorFinder
 
 			# r == nil means that the inner loop was broken (multiple candidates had passed the tests)
 			# r != nil means that the inner loop was completed (with or w/o a valid beg_s2 found)
-			break unless r.nil?
+			if r.nil?
+				valid_beg_s2 = nil
+			else
+				break
+			end
 		end
 
 		valid_beg_s2

data/lib/text_alignment/char_mapping.rb

@@ -61,6 +61,7 @@ TextAlignment::CHAR_MAPPING = [
 	["•", "*"],				#U+2022 (bullet)
 	[" ", " "],				#U+2009 (thin space)
 	[" ", " "],				#U+200A (hair space)
+	[" ", " "],				#U+202F (narrow no-break space)
 	[" ", " "],				#U+00A0 (Non-Breaking space)
 	["　", " "],				#U+3000 (ideographic space)
 	["‐", "-"],				#U+2010 (Hyphen)
@@ -77,11 +78,11 @@ TextAlignment::CHAR_MAPPING = [
 
 
 class TextAlignment::CharMapping
-	attr_reader :str
+	attr_reader :mapped_text
 
-	def initialize(_str, char_mapping = nil)
+	def initialize(_text, char_mapping = nil)
 		char_mapping ||= TextAlignment::CHAR_MAPPING
-		@str, offset_mapping = enmap_str(_str, char_mapping)
+		@mapped_text, offset_mapping = enmap_text(_text, char_mapping)
 		@index_enmap = offset_mapping.to_h
 		@index_demap = offset_mapping.map{|m| m.reverse}.to_h
 	end
@@ -94,20 +95,22 @@ class TextAlignment::CharMapping
 		@index_demap[position]
 	end
 
-	def enmap_denotations(_denotations)
-		denotations = _denotations.map do |d|
+	def enmap_denotations(denotations)
+		return nil if denotations.nil?
+
+		denotations.map do |d|
 			d.dup.merge(span:{begin:enmap_position(d[:span][:begin]), end:enmap_position(d[:span][:end])})
 		end
 	end
 
 	private
 
-	def enmap_str(_str, char_mapping)
-		str = _str.dup
+	def enmap_text(_text, char_mapping)
+		text = _text.dup
 
 		# To execute the single letter mapping
 		char_mapping.each do |one, long|
-			str.gsub!(one, long) if long.length == 1
+			text.gsub!(one, long) if long.length == 1
 		end
 
 		# To get the (location, length) index for replacements
@@ -116,18 +119,18 @@ class TextAlignment::CharMapping
 			next if long.length == 1
 
 			init_next = 0
-			while loc = str.index(long, init_next)
+			while loc = text.index(long, init_next)
 				loc_len << [loc, long.length]
 				init_next = loc + long.length
 			end
 
 			# a workaround to avoid messing-up due to embedding
-			str.gsub!(long, one * long.length)
+			text.gsub!(long, one * long.length)
 		end
 
 		# To get the (location, length) index for consecutive whitespace sequences
 		init_next = 0
-		while loc = str.index(/\s{2,}/, init_next)
+		while loc = text.index(/\s{2,}/, init_next)
 			len = $~[0].length
 			loc_len << [loc, len]
 			init_next = loc + len
@@ -148,20 +151,20 @@ class TextAlignment::CharMapping
 			init_next = loc + len
 		end
 
-		offset_mapping += (init_next .. str.length).map do |i|
+		offset_mapping += (init_next .. text.length).map do |i|
 			j += 1
 			[i, j - 1]
 		end
 
 		# To execute the long letter mapping
 		char_mapping.each do |one, long|
-			str.gsub!(one * long.length, one) if long.length > 1
+			text.gsub!(one * long.length, one) if long.length > 1
 		end
 
 		# To replace multi whitespace sequences to a space
-		str.gsub!(/\s{2,}/, ' ')
+		text.gsub!(/\s{2,}/, ' ')
 
-		[str, offset_mapping]
+		[text, offset_mapping]
 	end
 end
 
@@ -173,15 +176,15 @@ if __FILE__ == $0
 		exit
 	end
 	annotations = JSON.parse File.read(ARGV[0]).strip, symbolize_names: true
-	denotations = annotations[:denotations]
+	denotations = annotations[:denotations] || []
 	if denotations.nil? && annotations[:tracks]
 		denotations = annotations[:tracks].first[:denotations]
 	end
 
-	str_mapping = TextAlignment::CharMapping.new(annotations[:text])
-	str_mapped = str_mapping.str
-	denotations_mapped = str_mapping.enmap_denotations(denotations)
-	new_annotations = {text:str_mapped, denotations:denotations_mapped}
+	text_mapping = TextAlignment::CharMapping.new(annotations[:text])
+	text_mapped = text_mapping.mapped_text
+	denotations_mapped = text_mapping.enmap_denotations(denotations)
+	new_annotations = {text:text_mapped, denotations:denotations_mapped}
 
 	puts new_annotations.to_json
 end

data/lib/text_alignment/cultivation_map.rb

@@ -4,16 +4,91 @@ class TextAlignment::CultivationMap
 	attr_reader :map
 
 	def initialize
-		@map = {}
+		@map = []
 	end
 
 	def cultivate(regions)
-		regions.each do |b, e|
-			(b ... e).each{|p| @map[p] = e}
+		@map += regions
+		@map.sort!{|a, b| a[0] <=> b[0]}
+		new_map = []
+		@map.each do |region|
+			if new_map.empty?
+				new_map << region
+			elsif new_map.last[1] > region[0]
+				raise "Overlapping regions: #{new_map.last} : #{region}"
+			elsif new_map.last[1] == region[0]
+				new_map.last[1] == region[1]
+			else
+				new_map << region
+			end
 		end
+		@map = new_map
 	end
 
-	def search_again_position(position)
-		@map[position]
+	def search_again_position(position, end_position = nil)
+		end_position ||= position
+		region = @map.bsearch{|r| end_position < r[1]}
+		if region.nil? || region[0] > position
+			nil
+		else
+			region[1]
+		end
+	end
+
+	def last_cultivated_position(position)
+		ridx = @map.rindex{|r| r[1] <= position}
+		ridx.nil? ? nil : @map[ridx][1]
+	end
+
+	def next_cultivated_position(position)
+		region = @map.bsearch{|r| position <= r[0]}
+		region.nil? ? nil : region[0]
+	end
+
+	def in_regions(region)
+		@map.select{|r| (r[1] > region[0] && r[1] <= region[1]) || (r[0] < region[1] && r[0] >= region[0])}
+	end
+
+	def region_state(region)
+		closed_parts = in_regions(region)
+		if closed_parts.empty?
+			[:open, region]
+		else
+			if front_open?(region, closed_parts)
+				if rear_open?(region, closed_parts)
+					[:middle_closed, [closed_parts.first[0], closed_parts.last[1]]]
+				else
+					[:front_open, [region[0], closed_parts.first[0]]]
+				end
+			else
+				if rear_open?(region, closed_parts)
+					[:rear_open, [closed_parts.last[1], region[1]]]
+				else
+					[:closed, nil]
+				end
+			end
+		end
+	end
+
+	def index(target, string, position = 0)
+		length = target.length
+		loop do
+			_begin = string.index(target, position)
+			break if _begin.nil?
+			position = search_again_position(_begin)
+			next unless position.nil?
+			break _begin if region_state([_begin, _begin + length])[0] == :open
+			position = _begin + 1
+		end
+	end
+
+	private
+
+	def front_open?(region, closed_parts)
+		closed_parts.first[0] > region[0]
+	end
+
+	def rear_open?(region, closed_parts)
+		closed_parts.last[1] < region[1]
 	end
 end

data/lib/text_alignment/mixed_alignment.rb

@@ -144,11 +144,17 @@ class TextAlignment::MixedAlignment
 	def compute_similarity(s1, s2, sdiff)
 		return 0 if sdiff.nil?
 
-		# compute the lcs only with non-whitespace letters
-		lcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
-		return 0 if lcs == 0
+		# recoverbility
+		count_nws =	sdiff.count{|d| d.old_element =~ /\S/}
+		count_nws_match =	sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/}
 
-		similarity = lcs.to_f / [s1.scan(/\S/).count, s2.scan(/\S/).count].min
-	end
+		coverage = count_nws_match.to_f / count_nws
+
+		# fragmentation rate
+		count_ofrag = sdiff.count{|d| d.old_element =~ /\s/} + 1
+		count_frag = sdiff.collect{|d| (d.action == '=') && (d.old_element =~/\s/) ? ' ' : d.action}.join.scan(/=+/).count
+		rate_frag = count_ofrag.to_f / count_frag
 
+		similarity = coverage * rate_frag
+	end
 end

data/lib/text_alignment/text_alignment.rb

@@ -10,56 +10,49 @@ class TextAlignment::TextAlignment
 	attr_reader :block_alignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
-	attr_reader :cultivation_map
 
-	def initialize(_str1, _str2, _denotations = nil, _cultivation_map = nil)
-		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
+	# Initialize with a reference text, again which texts will be aligned
+	def initialize(reference_text, to_prevent_overlap = false)
+		raise ArgumentError, "nil text" if reference_text.nil?
 
-		@block_alignment = {source_text: _str1, target_text: _str2, denotations: _denotations}
-		@original_str1 = _str1
-		@original_str2 = _str2
+		@original_reference_text = reference_text
+		@rtext_mapping = TextAlignment::CharMapping.new(reference_text)
+		@mapped_reference_text = @rtext_mapping.mapped_text
+		@to_prevent_overlap = to_prevent_overlap
 
-		@str1_mapping = TextAlignment::CharMapping.new(_str1)
-		@str2_mapping = TextAlignment::CharMapping.new(_str2)
+		@original_text = nil
+		@blocks = nil
+		@cultivation_map = TextAlignment::CultivationMap.new
+	end
 
-		str1 = @str1_mapping.str
-		denotations = @str1_mapping.enmap_denotations(_denotations)
+	def align(text, denotations = nil)
+		# To maintain the cultivation map
+		update_cultivation_map if @to_prevent_overlap
 
-		str2 = @str2_mapping.str
+		# In case the input text is the same as the previous one, reuse the previous text mapping
+		unless @original_text && @original_text == text
+			@original_text = text
+			@text_mapping = TextAlignment::CharMapping.new(text)
+		end
 
-		@cultivation_map = _cultivation_map || TextAlignment::CultivationMap.new
+		@mapped_text = @text_mapping.mapped_text
+		denotations_mapped = @text_mapping.enmap_denotations(denotations)
 
-		@block_alignment[:blocks] = if r = whole_block_alignment(str1, str2, @cultivation_map)
-			# whole block alignment
+		## To generate the block_alignment of the input text against the reference text
+		@blocks = if r = whole_block_alignment(@mapped_text, @mapped_reference_text, @cultivation_map)
 			r
 		else
-			find_block_alignment(str1, str2, denotations, @cultivation_map)
+			find_block_alignment(@mapped_text, @mapped_reference_text, denotations_mapped, @cultivation_map)
 		end
 
-		newly_cultivated_regions = @block_alignment[:blocks].collect do |b|
-			if b[:alignment] == :block || b[:alignment] == :term
-				[b[:target][:begin], b[:target][:end]]
-			else
-				nil
-			end
-		end.compact
-		newly_cultivated_regions_condensed = newly_cultivated_regions.inject([]) do |condensed, region|
-			if condensed.empty? || (condensed.last.last + 1 < region.first)
-				condensed.push region
-			else
-				condensed.last[1] = region.last
-			end
-			condensed
-		end
-
-		@cultivation_map.cultivate(newly_cultivated_regions_condensed)
+		@block_alignment = {text: @original_text, reference_text: @original_reference_text, denotations: denotations, blocks: demap_blocks(@blocks)}
 	end
 
 	def transform_begin_position(_begin_position)
-		begin_position = @str1_mapping.enmap_position(_begin_position)
+		begin_position = @text_mapping.enmap_position(_begin_position)
 
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
-		block = @block_alignment[:blocks][i]
+		i = @blocks.index{|b| b[:source][:end] > begin_position}
+		block = @blocks[i]
 
 		b = if block[:alignment] == :block || block[:alignment] == :term
 			begin_position + block[:delta]
@@ -74,14 +67,14 @@ class TextAlignment::TextAlignment
 			r.nil? ? nil : r + block[:target][:begin]
 		end
 
-		@str2_mapping.demap_position(b)
+		@rtext_mapping.demap_position(b)
 	end
 
 	def transform_end_position(_end_position)
-		end_position = @str1_mapping.enmap_position(_end_position)
+		end_position = @text_mapping.enmap_position(_end_position)
 
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
-		block = @block_alignment[:blocks][i]
+		i = @blocks.index{|b| b[:source][:end] >= end_position}
+		block = @blocks[i]
 
 		e = if block[:alignment] == :block || block[:alignment] == :term
 			end_position + block[:delta]
@@ -96,7 +89,7 @@ class TextAlignment::TextAlignment
 			r.nil? ? nil : r + block[:target][:begin]
 		end
 
-		@str2_mapping.demap_position(e)
+		@rtext_mapping.demap_position(e)
 	end
 
 	def transform_a_span(span)
@@ -115,7 +108,7 @@ class TextAlignment::TextAlignment
 			source = {begin:d.begin, end:d.end}
 			d.begin = transform_begin_position(d.begin);
 			d.end = transform_end_position(d.end);
-			raise "invalid transform" unless !d.begin.nil? && !d.end.nil? && d.begin >= 0 && d.end > d.begin && d.end <= @original_str2.length
+			raise "invalid transform" unless !d.begin.nil? && !d.end.nil? && d.begin >= 0 && d.end > d.begin && d.end <= @original_reference_text.length
 		rescue
 			@lost_annotations << {source: source, target:{begin:d.begin, end:d.end}}
 			d.begin = nil
@@ -131,7 +124,7 @@ class TextAlignment::TextAlignment
 
 		r = hdenotations.collect do |d|
 			t = transform_a_span(d[:span])
-			raise "invalid transform" unless !t[:begin].nil? && !t[:end].nil? && t[:begin] >= 0 && t[:end] > t[:begin] && t[:end] <= @original_str2.length
+			raise "invalid transform" unless !t[:begin].nil? && !t[:end].nil? && t[:begin] >= 0 && t[:end] > t[:begin] && t[:end] <= @original_reference_text.length
 			new_d = d.dup.merge({span:t})
 		rescue
 			@lost_annotations << {source: d[:span], target:t}
@@ -142,8 +135,8 @@ class TextAlignment::TextAlignment
 	end
 
 	def alignment_show
-		stext = @block_alignment[:source_text]
-		ttext = @block_alignment[:target_text]
+		stext = @block_alignment[:text]
+		ttext = @block_alignment[:reference_text]
 
 		show = ''
 		@block_alignment[:blocks].each do |a|
@@ -199,7 +192,7 @@ class TextAlignment::TextAlignment
 
 				"***** local mismatch [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}] (similarity: #{a[:similarity]})\n" +
 				"[#{astr1}]\n" +
-				"[#{astr2}]\n\n"
+				"[#{astr2.gsub("\n", " ")}]\n\n"
 			end
 		end
 		show
@@ -238,142 +231,125 @@ class TextAlignment::TextAlignment
 		# puts "-=-=-=-=-"
 		# puts
 
-		## to fill the gaps
-		last_block = nil
-		blocks2 = blocks.inject([]) do |sum, block|
-			b1 = last_block ? last_block[:source][:end] : 0
-			e1 = block[:source][:begin]
+		## To fill the gaps
+		## lblock: last block, cblock: current block
+		lblock = nil
+		blocks2 = (blocks + [nil]).inject([]) do |sum, cblock|
+			b1 = lblock.nil? ? 0 : lblock[:source][:end]
+			e1 = cblock.nil? ? str1.length : cblock[:source][:begin]
 
-			sum += if b1 == e1
-				[block]
-			else
-				b2 = last_block ? last_block[:target][:end] : 0
-				e2 = block[:target][:begin]
-
-				if b2 == e2
-					[
-						{source:{begin:b1, end:e1}, alignment: :empty},
-						block
-					]
+			if b1 < e1
+				b2 = lblock.nil? ? 0 : lblock[:target][:end]
+				e2 = cblock.nil? ? str2.length : cblock[:target][:begin]
+				_str1 = str1[b1 ... e1]
+				_str2 = str2[b2 ... e2]
+
+				sum += if _str1.strip.empty? || _str2.strip.empty?
+					[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
 				else
 					len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
-
-					if b1 == 0 && b2 == 0
-						b2 = e2 - len_buffer if e2 > len_buffer
-					end
-
-					_str1 = str1[b1 ... e1]
-					_str2 = str2[b2 ... e2]
-
-					if _str1.strip.empty? || _str2.strip.empty?
-						[
-							{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty},
-							block
-						]
-					elsif ((e2 - b2) - (e1 - b1)) > len_buffer
-						la_block1 = local_alignment_blocks(str1, b1, e1, str2, b2, b2 + len_buffer, denotations)
-						la_block2 = local_alignment_blocks(str1, b1, e1, str2, e2 - len_buffer, e2, denotations)
-						[la_block2, la_block2].max{|a, b| a.first[:similarity] <=> b.first[:similarity]} << block
-					else
-						local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations) << block
+					region_state, state_region = cultivation_map.region_state([b2, e2])
+					case region_state
+					when :closed
+						[{source:{begin:b1, end:e1}, alignment: :empty}]
+					when :front_open
+						if sum.empty? # when there is no preceding matched block
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							oe2 = state_region[1]
+							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+						end
+					when :rear_open
+						if cblock.nil? # when there is no following matched block
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							ob2 = state_region[0]
+							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
+							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
+						end
+					when :middle_closed
+						attempt1 = if sum.empty?
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							oe2 = state_region[0]
+							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+						end
+						if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
+							ob2 = state_region[1]
+							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
+							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
+						else
+							attempt1
+						end
+					else # :open
+						if (e2 - b2) > len_buffer
+							attempt1 = if sum.empty?
+								[{source:{begin:b1, end:e1}, alignment: :empty}]
+							else
+								local_alignment(str1, b1, e1, str2, b2, b2 + len_buffer, denotations, cultivation_map)
+							end
+							if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
+								local_alignment(str1, b1, e1, str2, e2 - len_buffer, e2, denotations, cultivation_map)
+							else
+								attempt1
+							end
+						else
+							local_alignment(str1, b1, e1, str2, b2, e2, denotations, cultivation_map)
+						end
 					end
 				end
 			end
 
-			last_block = block
-			sum
+			lblock = cblock
+			cblock.nil? ? sum : sum << cblock
 		end
 
-		# the last step
-		blocks2 += if last_block.nil?
-			local_alignment_blocks(str1, 0, str1.length, str2, 0, str2.length, denotations)
-		else
-			b1 = last_block[:source][:end]
-			if b1 < str1.length
-				e1 = str1.length
-				b2 = last_block[:target][:end]
-
-				_str1 = str1[b1 ... e1]
-				if _str1.strip.empty?
-					[{source:{begin:b1, end:e1}, alignment: :empty}]
-				else
-					if b2 < str2.length
-						len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
-						e2 = (str2.length - b2) > len_buffer ? b2 + len_buffer : str2.length
-
-						local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations)
-					else
-						[{source:{begin:b1, end:e1}, alignment: :empty}]
-					end
-				end
-			else
-				[]
-			end
-		end
 	end
 
 	def whole_block_alignment(str1, str2, cultivation_map)
-		## Block exact match
-		search_position = 0
-
-		block_begin = begin
-			_block_begin = str2.index(str1, search_position)
-			break if _block_begin.nil?
-			search_position = cultivation_map.search_again_position(_block_begin)
-			_block_begin
-		end until search_position.nil?
-
-		unless block_begin.nil?
-			return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
-		end
+		block_begin = cultivation_map.index(str1, str2)
+		return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}] unless block_begin.nil?
 
-		search_position = 0
+		block_begin = cultivation_map.index(str1.downcase, str2.downcase)
+		return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}] unless block_begin.nil?
 
-		dstr1 = str1.downcase
-		dstr2 = str2.downcase
-		block_begin = begin
-			_block_begin = dstr2.index(dstr1, search_position)
-			break if _block_begin.nil?
-			search_position = cultivation_map.search_again_position(_block_begin)
-			_block_begin
-		end until search_position.nil?
+		nil
+	end
 
-		unless block_begin.nil?
-			return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+	def local_alignment(str1, b1, e1, str2, b2, e2, denotations = nil, cultivation_map)
+		tblocks = term_based_alignment(str1, b1, e1, str2, b2, e2, denotations, cultivation_map)
+		if tblocks.empty? || tblocks.first[:alignment] == :empty
+			lcs_alignment(str1, b1, e1, str2, b2, e2, cultivation_map)
+		else
+			tblocks
 		end
-
-		nil
 	end
 
-	def local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations = nil)
-		block2 = str2[b2 ... e2]
+	def term_based_alignment(str1, b1, e1, str2, b2, e2, denotations = nil, cultivation_map)
+		str2_block = str2[0 ... e2]
 
 		## term-based alignment
 		tblocks = if denotations
-			ds_in_scope = denotations.select{|d| d[:span][:begin] >= b1 && d[:span][:end] <= e1}.
+			denotations_in_scope = denotations.select{|d| d[:span][:begin] >= b1 && d[:span][:end] <= e1}.
 							sort{|d1, d2| d1[:span][:begin] <=> d2[:span][:begin] || d2[:span][:end] <=> d1[:span][:end] }.
 							map{|d| d.merge(lex:str1[d[:span][:begin] ... d[:span][:end]])}
 
-			position = 0
-			_tblocks = ds_in_scope.map do |term|
-				lex = term[:lex]
-				r = block2.index(lex, position)
-				if r.nil?
-					position = nil
-					break
-				end
-				position = r + lex.length
-				{source:term[:span], target:{begin:r + b2, end:r + b2 + lex.length}, alignment: :term, similarity: 0.9, delta: r + b2 - term[:span][:begin]}
+			search_position = b2
+			_tblocks = denotations_in_scope.map do |denotation|
+				lex = denotation[:lex]
+				term_begin = cultivation_map.index(lex, str2_block, search_position)
+				break [] if term_begin.nil? # break the loop if a missing term is found
+				search_position = term_begin + lex.length
+				{source:denotation[:span], target:{begin:term_begin, end:term_begin + lex.length}, alignment: :term, similarity: 0.9, delta: term_begin - denotation[:span][:begin]}
 			end
 
-			# missing term found
-			_tblocks = [] if position.nil?
-
 			# redundant matching found
-			unless position.nil?
-				ds_in_scope.each do |term|
-					lex = term[:lex]
-					look_forward = block2.index(lex, position)
+			unless _tblocks.empty?
+				search_position = _tblocks.last[:target][:end]
+				denotations_in_scope.each do |term|
+					look_forward = cultivation_map.index(term[:lex], str2_block, search_position)
 					unless look_forward.nil?
 						_tblocks = []
 						break
@@ -386,73 +362,72 @@ class TextAlignment::TextAlignment
 			[]
 		end
 
-		if tblocks.empty?
-			if b1 == 0 && e1 == str1.length
-				if (e1 > 2000) || (e2 > 2000)
-					[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
-				else
-					block1 = str1[b1 ... e1]
-					block2 = str2[b2 ... e2]
-
-					## character-based alignment
-					alignment = TextAlignment::MixedAlignment.new(block1.downcase, block2.downcase)
-					if alignment.sdiff.nil?
-						[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
-					else
-						[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: alignment, similarity: alignment.similarity}]
-					end
-				end
-			else
-				block1 = str1[b1 ... e1]
-				block2 = str2[b2 ... e2]
+		ltblock = nil
+		tblocks2 = (tblocks + [nil]).inject([]) do |sum, ctblock|
+			tb1 = ltblock.nil? ? b1 : ltblock[:source][:end]
+			te1 = ctblock.nil? ? e1 : ctblock[:source][:begin]
 
-				## character-based alignment
-				alignment = TextAlignment::MixedAlignment.new(block1.downcase, block2.downcase)
-				if alignment.sdiff.nil?
-					[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
-				else
-					[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: alignment, similarity: alignment.similarity}]
-				end
+			if te1 > tb1
+				tb2 = ltblock.nil? ? b2 : ltblock[:target][:end]
+				te2 = ctblock.nil? ? e2 : ctblock[:target][:begin]
+				sum << {source:{begin:tb1, end:te1}, target:{begin:tb2, end:te2}, alignment: :empty}
 			end
-		else
-			last_tblock = nil
-			lblocks = tblocks.inject([]) do |sum, tblock|
-				tb1 = last_tblock ? last_tblock[:source][:end] : b1
-				te1 = tblock[:source][:begin]
 
-				sum += if te1 == tb1
-					[tblock]
-				else
-					tb2 = last_tblock ? last_tblock[:target][:end] : b2
-					te2 = tblock[:target][:begin]
-
-					if b2 == e2
-						[
-							{source:{begin:tb1, end:te1}, alignment: :empty},
-							tblock
-						]
-					else
-						[
-							{source:{begin:tb1, end:te1}, target:{begin:tb2, end:te2}, alignment: :empty},
-							tblock
-						]
-					end
-				end
+			ltblock = ctblock
+			ctblock.nil? ? sum : sum << ctblock
+		end
 
-				last_tblock = tblock
-				sum
+		tblocks2
+	end
+
+	def lcs_alignment(str1, b1, e1, str2, b2, e2, cultivation_map)
+		source = {begin:b1, end:e1}
+		target = {begin:b2, end:e2}
+
+		if (e1 - b1) > 2000
+			[{source:source, target:target, alignment: :empty}]
+		else
+			alignment = TextAlignment::MixedAlignment.new(str1[b1 ... e1].downcase, str2[b2 ... e2].downcase)
+			if alignment.similarity < 0.5
+				[{source:source, target:target, alignment: :empty}]
+			else
+				[{source:source, target:target, alignment: alignment, similarity: alignment.similarity}]
 			end
+		end
+	end
 
-			if last_tblock[:source][:end] < e1
-				if last_tblock[:target][:end] < e2
-					lblocks << {source:{begin:last_tblock[:source][:end], end:e1}, target:{begin:last_tblock[:target][:end], end:e2}, alignment: :empty}
-				else
-					lblocks << {source:{begin:last_tblock[:source][:end], end:e1}, alignment: :empty}
-				end
+	def update_cultivation_map
+		return if @blocks.nil?
+
+		## To update the cultivation map
+		newly_cultivated_regions = @blocks.collect do |b|
+			if b[:alignment] == :block || b[:alignment] == :term
+				[b[:target][:begin], b[:target][:end]]
+			else
+				nil
+			end
+		end.compact.inject([]) do |condensed, region|
+			if condensed.empty? || (condensed.last.last + 1 < region.first)
+				condensed.push region
+			else
+				condensed.last[1] = region.last
 			end
+			condensed
+		end
+
+		@cultivation_map.cultivate(newly_cultivated_regions)
+	end
 
-			lblocks
+	def demap_blocks(_blocks)
+		return nil if _blocks.nil?
+
+		blocks = _blocks.map{|b| b.dup}
+		blocks.each do |b|
+			b[:source] = {begin:@text_mapping.demap_position(b[:source][:begin]), end:@text_mapping.demap_position(b[:source][:end])} if b[:source]
+			b[:target] = {begin:@rtext_mapping.demap_position(b[:target][:begin]), end:@rtext_mapping.demap_position(b[:target][:end])} if b[:target]
 		end
+
+		blocks
 	end
 
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.10.1'
+	VERSION = '0.11.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.5
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-03 00:00:00.000000000 Z
+date: 2021-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary