text_alignment 0.7.3 → 0.11.0

data/lib/text_alignment/char_mapping.rb

@@ -0,0 +1,189 @@
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::CHAR_MAPPING = [
+	["©", "(c)"],			#U+00A9 (Copyright Sign)
+
+	["α", "alpha"],		#U+03B1 (greek small letter alpha)
+	["β", "beta"],		#U+03B2 (greek small letter beta)
+	["γ", "gamma"],		#U+03B3 (greek small letter gamma)
+	["δ", "delta"],		#U+03B4 (greek small letter delta)
+	["ε", "epsilon"],	#U+03B5 (greek small letter epsilon)
+	["ζ", "zeta"],		#U+03B6 (greek small letter zeta)
+	["η", "eta"],			#U+03B7 (greek small letter eta)
+	["θ", "theta"],		#U+03B7 (greek small letter eta)
+	["ι", "iota"],		#U+03B7 (greek small letter eta)
+	["κ", "kappa"],		#U+03BA (greek small letter kappa)
+	["λ", "lambda"],	#U+03BB (greek small letter lambda)
+	["λ", "lamda"],		#U+03BB (greek small letter lambda)
+	["μ", "mu"],			#U+03BC (greek small letter mu)
+	["ν", "nu"],			#U+03BD (greek small letter nu)
+	["ξ", "xi"],			#U+03BE (greek small letter xi)
+	["ο", "omicron"],	#U+03BF (greek small letter omicron)
+	["π", "pi"],			#U+03C0 (greek small letter pi)
+	["ρ", "rho"],			#U+03C1 (greek small letter rho)
+	["σ", "sigma"],		#U+03C3 (greek small letter sigma)
+	["τ", "tau"],			#U+03C4 (greek small letter tau)
+	["υ", "upsilon"],	#U+03C5 (greek small letter upsilon)
+	["φ", "phi"],			#U+03C6 (greek small letter phi)
+	["χ", "chi"],			#U+03C7 (greek small letter chi)
+	["ψ", "psi"],			#U+03C8 (greek small letter psi)
+	["ω", "omega"],		#U+03C9 (greek small letter omega)
+
+	["Α", "Alpha"],		#U+0391 (greek capital letter alpha)
+	["Β", "Beta"],		#U+0392 (greek capital letter beta)
+	["Γ", "Gamma"],		#U+0393 (greek capital letter gamma)
+	["Δ", "Delta"],		#U+0394 (greek capital letter delta)
+	["Ε", "Epsilon"],	#U+0395 (greek capital letter epsilon)
+	["Ζ", "Zeta"],		#U+0396 (greek capital letter zeta)
+	["Η", "Eta"],			#U+0397 (greek capital letter eta)
+	["Θ", "Theta"],		#U+0398 (greek capital letter theta)
+	["Ι", "Iota"],		#U+0399 (greek capital letter iota)
+	["Κ", "Kappa"],		#U+039A (greek capital letter kappa)
+	["Λ", "Lambda"],	#U+039B (greek capital letter lambda)
+	["Λ", "Lamda"],		#U+039B (greek capital letter lambda)
+	["Μ", "Mu"],			#U+039C (greek capital letter mu)
+	["Ν", "Nu"],			#U+039D (greek capital letter nu)
+	["Ξ", "Xi"],			#U+039E (greek capital letter xi)
+	["Ο", "Omicron"],	#U+039F (greek capital letter omicron)
+	["Π", "Pi"],			#U+03A0 (greek capital letter pi)
+	["Ρ", "Rho"],			#U+03A1 (greek capital letter rho)
+	["Σ", "Sigma"],		#U+03A3 (greek capital letter sigma)
+	["Τ", "Tau"],			#U+03A4 (greek capital letter tau)
+	["Υ", "Upsilon"],	#U+03A5 (greek capital letter upsilon)
+	["Φ", "Phi"],			#U+03A6 (greek capital letter phi)
+	["Χ", "Chi"],			#U+03A7 (greek capital letter chi)
+	["Ψ", "Psi"],			#U+03A8 (greek capital letter Psi)
+	["Ω", "Omega"],		#U+03A9 (greek capital letter omega)
+
+	["ϕ", "phi"],			#U+03D5 (greek phi symbol)
+
+	["×", "x"],				#U+00D7 (multiplication sign)
+	["•", "*"],				#U+2022 (bullet)
+	[" ", " "],				#U+2009 (thin space)
+	[" ", " "],				#U+200A (hair space)
+	[" ", " "],				#U+00A0 (Non-Breaking space)
+	["　", " "],				#U+3000 (ideographic space)
+	["‐", "-"],				#U+2010 (Hyphen)
+	["‑", "-"],				#U+2011 (Non-Breaking Hyphen)
+	["−", "-"],				#U+2212 (minus sign)
+	["–", "-"],				#U+2013 (en dash)
+	["′", "'"],				#U+2032 (prime)
+	["‘", "'"],				#U+2018 (left single quotation mark)
+	["’", "'"],				#U+2019 (right single quotation mark)
+	["“", '"'],				#U+201C (left double quotation mark)
+	["”", '"'],				#U+201D (right double quotation mark)
+	['"', "''"]
+]
+
+
+class TextAlignment::CharMapping
+	attr_reader :mapped_text
+
+	def initialize(_text, char_mapping = nil)
+		char_mapping ||= TextAlignment::CHAR_MAPPING
+		@mapped_text, offset_mapping = enmap_text(_text, char_mapping)
+		@index_enmap = offset_mapping.to_h
+		@index_demap = offset_mapping.map{|m| m.reverse}.to_h
+	end
+
+	def enmap_position(position)
+		@index_enmap[position]
+	end
+
+	def demap_position(position)
+		@index_demap[position]
+	end
+
+	def enmap_denotations(_denotations)
+		return nil if _denotations.nil?
+
+		denotations = _denotations.map do |d|
+			d.dup.merge(span:{begin:enmap_position(d[:span][:begin]), end:enmap_position(d[:span][:end])})
+		end
+	end
+
+	private
+
+	def enmap_text(_text, char_mapping)
+		text = _text.dup
+
+		# To execute the single letter mapping
+		char_mapping.each do |one, long|
+			text.gsub!(one, long) if long.length == 1
+		end
+
+		# To get the (location, length) index for replacements
+		loc_len = []
+		char_mapping.each do |one, long|
+			next if long.length == 1
+
+			init_next = 0
+			while loc = text.index(long, init_next)
+				loc_len << [loc, long.length]
+				init_next = loc + long.length
+			end
+
+			# a workaround to avoid messing-up due to embedding
+			text.gsub!(long, one * long.length)
+		end
+
+		# To get the (location, length) index for consecutive whitespace sequences
+		init_next = 0
+		while loc = text.index(/\s{2,}/, init_next)
+			len = $~[0].length
+			loc_len << [loc, len]
+			init_next = loc + len
+		end
+
+		loc_len.sort!{|a, b| a[0] <=> b[0]}
+
+		# To get the offset_mapping before and after replacement
+		offset_mapping = []
+		init_next = 0
+		j = 0
+
+		loc_len.each do |loc, len|
+			offset_mapping += (init_next .. loc).map do |i|
+				j += 1
+				[i, j - 1]
+			end
+			init_next = loc + len
+		end
+
+		offset_mapping += (init_next .. text.length).map do |i|
+			j += 1
+			[i, j - 1]
+		end
+
+		# To execute the long letter mapping
+		char_mapping.each do |one, long|
+			text.gsub!(one * long.length, one) if long.length > 1
+		end
+
+		# To replace multi whitespace sequences to a space
+		text.gsub!(/\s{2,}/, ' ')
+
+		[text, offset_mapping]
+	end
+end
+
+if __FILE__ == $0
+	require 'json'
+
+	unless ARGV.length == 1
+		warn "#{$0} an_annotation_json_file.json" 
+		exit
+	end
+	annotations = JSON.parse File.read(ARGV[0]).strip, symbolize_names: true
+	denotations = annotations[:denotations]
+	if denotations.nil? && annotations[:tracks]
+		denotations = annotations[:tracks].first[:denotations]
+	end
+
+	text_mapping = TextAlignment::CharMapping.new(annotations[:text])
+	text_mapped = text_mapping.mapped_text
+	denotations_mapped = text_mapping.enmap_denotations(denotations)
+	new_annotations = {text:text_mapped, denotations:denotations_mapped}
+
+	puts new_annotations.to_json
+end

data/lib/text_alignment/constants.rb

@@ -1,7 +1,7 @@
 module TextAlignment; end unless defined? TextAlignment
 
 TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 30 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::SIZE_WINDOW = 10 unless defined? TextAlignment::SIZE_WINDOW
 TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
 TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
 TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.9 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/cultivation_map.rb

@@ -0,0 +1,19 @@
+module TextAlignment; end unless defined? TextAlignment
+
+class TextAlignment::CultivationMap
+	attr_reader :map
+
+	def initialize
+		@map = {}
+	end
+
+	def cultivate(regions)
+		regions.each do |b, e|
+			(b ... e).each{|p| @map[p] = e}
+		end
+	end
+
+	def search_again_position(position)
+		@map[position]
+	end
+end

data/lib/text_alignment/glcs_alignment_fast.rb

@@ -5,7 +5,7 @@ require 'text_alignment/find_divisions'
 require 'text_alignment/lcs_comparison'
 require 'text_alignment/lcs_alignment'
 require 'text_alignment/glcs_alignment'
-require 'text_alignment/mappings'
+require 'text_alignment/char_mapping'
 
 module TextAlignment; end unless defined? TextAlignment
 
@@ -106,7 +106,7 @@ if __FILE__ == $0
 
 	dictionary = [["β", "beta"]]
 	# align = TextAlignment::TextAlignment.new(str1, str2)
-	align = TextAlignment::TextAlignment.new(str1, str2, TextAlignment::MAPPINGS)
+	align = TextAlignment::TextAlignment.new(str1, str2, TextAlignment::CHAR_MAPPING)
 	p align.common_elements
 	p align.mapped_elements
 end

data/lib/text_alignment/mixed_alignment.rb

@@ -6,7 +6,7 @@ require 'text_alignment/lcs_comparison'
 require 'text_alignment/lcs_alignment'
 require 'text_alignment/lcs_cdiff'
 require 'text_alignment/glcs_alignment'
-require 'text_alignment/mappings'
+require 'text_alignment/char_mapping'
 
 module TextAlignment; end unless defined? TextAlignment
 
@@ -17,10 +17,12 @@ class TextAlignment::MixedAlignment
 	attr_reader :similarity
 	attr_reader :str1_match_initial, :str1_match_final, :str2_match_initial, :str2_match_final
 
-	def initialize(_str1, _str2)
+	def initialize(_str1, _str2, _mappings = nil)
 		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
 
-		str1, str2, mappings = string_preprocessing(_str1, _str2)
+		mappings ||= TextAlignment::CHAR_MAPPING
+		str1 = _str1.dup
+		str2 = _str2.dup
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end
@@ -139,72 +141,14 @@ class TextAlignment::MixedAlignment
 		@position_map_end = posmap_end.sort.to_h
 	end
 
-	private
-
-	def string_preprocessing(_str1, _str2)
-		str1 = _str1.dup
-		str2 = _str2.dup
-		mappings = TextAlignment::MAPPINGS.dup
-
-		## single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		## long to one character mappings
-		pletters = TextAlignment::PADDING_LETTERS
-
-		# find the padding letter for str1
-		@padding_letter1 = begin
-			i = pletters.index{|l| str2.index(l).nil?}
-			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
-			TextAlignment::PADDING_LETTERS[i]
-		end
-
-		# find the padding letter for str2
-		@padding_letter2 = begin
-			i = pletters.index{|l| l != @padding_letter1 && str1.index(l).nil?}
-			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
-			TextAlignment::PADDING_LETTERS[i]
-		end
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@padding_letter1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@padding_letter2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
-
-		[str1, str2, mappings]
-	end
-
-	def compute_similarity(_s1, _s2, sdiff)
+	def compute_similarity(s1, s2, sdiff)
 		return 0 if sdiff.nil?
 
 		# compute the lcs only with non-whitespace letters
 		lcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
 		return 0 if lcs == 0
 
-		s1 = _s1.tr(@padding_letter1, ' ')
-		s2 = _s2.tr(@padding_letter2, ' ')
-
-		similarity = lcs / [s1.scan(/\S/).count, s2.scan(/\S/).count].min.to_f
+		similarity = lcs.to_f / [s1.scan(/\S/).count, s2.scan(/\S/).count].min
 	end
 
 end

data/lib/text_alignment/text_alignment.rb

@@ -2,39 +2,233 @@
 require 'text_alignment/constants'
 require 'text_alignment/anchor_finder'
 require 'text_alignment/mixed_alignment'
+require 'text_alignment/cultivation_map'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
-
 class TextAlignment::TextAlignment
 	attr_reader :block_alignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, denotations = nil, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+	# Initialize with a reference text, again which texts will be aligned
+	def initialize(reference_text, to_prevent_overlap = false)
+		raise ArgumentError, "nil text" if reference_text.nil?
 
-		@block_alignment = {source_text:str1, target_text:str2}
-		@str1 = str1
-		@str2 = str2
+		@original_rtext = reference_text
+		@rtext_mapping = TextAlignment::CharMapping.new(reference_text)
+		@to_prevent_overlap = to_prevent_overlap
 
-		## Block exact match
-		block_begin = str2.index(str1)
-		unless block_begin.nil?
-			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
-			return
+		@original_text = nil
+		@block_alignment = nil
+		@cultivation_map = TextAlignment::CultivationMap.new
+	end
+
+	def align(text, denotations = nil)
+		# To maintain the cultivation map
+		update_cultivation_map if @to_prevent_overlap
+
+		# In case the input text is the same as the previous one, reuse the previous text mapping
+		unless @original_text && @original_text == text
+			@original_text = text
+			@text_mapping = TextAlignment::CharMapping.new(text)
 		end
 
-		block_begin = str2.downcase.index(str1.downcase)
-		unless block_begin.nil?
-			@block_alignment[:blocks] = [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
-			return
+		text_mapped = @text_mapping.mapped_text
+		denotations_mapped = @text_mapping.enmap_denotations(denotations)
+
+		rtext_mapped = @rtext_mapping.mapped_text
+
+		## To generate the block_alignment of the input text against the reference text
+
+		# Initialization
+		@block_alignment = {text: @original_text, reference_text: @original_rtext, denotations: denotations}
+
+		# Generation
+		@block_alignment[:blocks] = if r = whole_block_alignment(text_mapped, rtext_mapped, @cultivation_map)
+			r
+		else
+			find_block_alignment(text_mapped, rtext_mapped, denotations_mapped, @cultivation_map)
+		end
+	end
+
+	def update_cultivation_map
+		return if @block_alignment.nil? || @block_alignment[:blocks].nil?
+
+		## To update the cultivation map
+		newly_cultivated_regions = @block_alignment[:blocks].collect do |b|
+			if b[:alignment] == :block || b[:alignment] == :term
+				[b[:target][:begin], b[:target][:end]]
+			else
+				nil
+			end
+		end.compact.inject([]) do |condensed, region|
+			if condensed.empty? || (condensed.last.last + 1 < region.first)
+				condensed.push region
+			else
+				condensed.last[1] = region.last
+			end
+			condensed
+		end
+
+		@cultivation_map.cultivate(newly_cultivated_regions)
+	end
+
+	def transform_begin_position(_begin_position)
+		begin_position = @text_mapping.enmap_position(_begin_position)
+
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
+		block = @block_alignment[:blocks][i]
+
+		b = if block[:alignment] == :block || block[:alignment] == :term
+			begin_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if begin_position == block[:source][:begin]
+				block[:target][:begin]
+			else
+				nil
+			end
+		else
+			r = block[:alignment].transform_begin_position(begin_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
+		end
+
+		@rtext_mapping.demap_position(b)
+	end
+
+	def transform_end_position(_end_position)
+		end_position = @text_mapping.enmap_position(_end_position)
+
+		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
+		block = @block_alignment[:blocks][i]
+
+		e = if block[:alignment] == :block || block[:alignment] == :term
+			end_position + block[:delta]
+		elsif block[:alignment] == :empty
+			if end_position == block[:source][:end]
+				block[:target][:end]
+			else
+				nil
+			end
+		else
+			r = block[:alignment].transform_end_position(end_position - block[:source][:begin])
+			r.nil? ? nil : r + block[:target][:begin]
+		end
+
+		@rtext_mapping.demap_position(e)
+	end
+
+	def transform_a_span(span)
+		{begin: transform_begin_position(span[:begin]), end: transform_end_position(span[:end])}
+	end
+
+	def transform_spans(spans)
+		spans.map{|span| transform_a_span(span)}
+	end
+
+	def transform_denotations!(denotations)
+		return nil if denotations.nil?
+		@lost_annotations = []
+
+		denotations.each do |d|
+			source = {begin:d.begin, end:d.end}
+			d.begin = transform_begin_position(d.begin);
+			d.end = transform_end_position(d.end);
+			raise "invalid transform" unless !d.begin.nil? && !d.end.nil? && d.begin >= 0 && d.end > d.begin && d.end <= @original_rtext.length
+		rescue
+			@lost_annotations << {source: source, target:{begin:d.begin, end:d.end}}
+			d.begin = nil
+			d.end = nil
 		end
 
+		@lost_annotations
+	end
+
+	def transform_hdenotations(hdenotations)
+		return nil if hdenotations.nil?
+		@lost_annotations = []
+
+		r = hdenotations.collect do |d|
+			t = transform_a_span(d[:span])
+			raise "invalid transform" unless !t[:begin].nil? && !t[:end].nil? && t[:begin] >= 0 && t[:end] > t[:begin] && t[:end] <= @original_rtext.length
+			new_d = d.dup.merge({span:t})
+		rescue
+			@lost_annotations << {source: d[:span], target:t}
+			nil
+		end.compact
+
+		r
+	end
+
+	def alignment_show
+		stext = @block_alignment[:text]
+		ttext = @block_alignment[:reference_text]
+
+		show = ''
+		@block_alignment[:blocks].each do |a|
+			show += case a[:alignment]
+			when :block
+				"===== common (block) ===== [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :term
+				"===== common (term) ===== [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
+			when :empty
+				"xxxxx disparate texts (similarity: #{a[:similarity]})\n" +
+				"<<<<< string 1 [#{a[:source][:begin]} - #{a[:source][:end]}]\n" +
+				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
+				">>>>> string 2 " +
+				if a[:target]
+					"[#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
+					ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
+				else
+					"[-]\n\n"
+				end
+			else
+				astr1 = ''
+				astr2 = ''
+
+				base = a[:source][:begin]
+				astr1 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						stext[c.old_position + base]
+					when '+'
+						'_'
+					when '-'
+						stext[c.old_position + base]
+					when '!'
+						stext[c.old_position + base] + '_'
+					end
+				end.join('')
 
+				base = a[:target][:begin]
+				astr2 = a[:alignment].sdiff.map do |c|
+					case c.action
+					when '='
+						ttext[c.new_position + base]
+					when '+'
+						ttext[c.new_position + base]
+					when '-'
+						'_'
+					when '!'
+						'_' + ttext[c.new_position + base]
+					end
+				end.join('')
+
+				"***** local mismatch [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}] (similarity: #{a[:similarity]})\n" +
+				"[#{astr1}]\n" +
+				"[#{astr2}]\n\n"
+			end
+		end
+		show
+	end
+
+	private
+
+	def find_block_alignment(str1, str2, denotations, cultivation_map)
 		## to find block alignments
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, _size_ngram, _size_window, _text_similiarity_threshold)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, cultivation_map)
 
 		blocks = []
 		while block = anchor_finder.get_next_anchor
@@ -77,12 +271,13 @@ class TextAlignment::TextAlignment
 
 				if b2 == e2
 					[
-						{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty},
+						{source:{begin:b1, end:e1}, alignment: :empty},
 						block
 					]
 				else
+					len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
+
 					if b1 == 0 && b2 == 0
-						len_buffer = (e1 * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
 						b2 = e2 - len_buffer if e2 > len_buffer
 					end
 
@@ -94,6 +289,10 @@ class TextAlignment::TextAlignment
 							{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty},
 							block
 						]
+					elsif ((e2 - b2) - (e1 - b1)) > len_buffer
+						la_block1 = local_alignment_blocks(str1, b1, e1, str2, b2, b2 + len_buffer, denotations)
+						la_block2 = local_alignment_blocks(str1, b1, e1, str2, e2 - len_buffer, e2, denotations)
+						[la_block2, la_block2].max{|a, b| a.first[:similarity] <=> b.first[:similarity]} << block
 					else
 						local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations) << block
 					end
@@ -111,21 +310,58 @@ class TextAlignment::TextAlignment
 			b1 = last_block[:source][:end]
 			if b1 < str1.length
 				e1 = str1.length
-
 				b2 = last_block[:target][:end]
-				if b2 < str2.length
-					len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
-					e2 = (str2.length - b2) > len_buffer ? b2 + len_buffer : str2.length
-					local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations)
+
+				_str1 = str1[b1 ... e1]
+				if _str1.strip.empty?
+					[{source:{begin:b1, end:e1}, alignment: :empty}]
 				else
-					[{source:{begin:last_block[:source][:end], end:str1.length}, alignment: :empty}]
+					if b2 < str2.length
+						len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
+						e2 = (str2.length - b2) > len_buffer ? b2 + len_buffer : str2.length
+
+						local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations)
+					else
+						[{source:{begin:b1, end:e1}, alignment: :empty}]
+					end
 				end
 			else
 				[]
 			end
 		end
+	end
+
+	def whole_block_alignment(str1, str2, cultivation_map)
+		## Block exact match
+		search_position = 0
+
+		block_begin = begin
+			_block_begin = str2.index(str1, search_position)
+			break if _block_begin.nil?
+			search_position = cultivation_map.search_again_position(_block_begin)
+			_block_begin
+		end until search_position.nil?
+
+		unless block_begin.nil?
+			return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+		end
+
+		search_position = 0
 
-		@block_alignment[:blocks] = blocks2
+		dstr1 = str1.downcase
+		dstr2 = str2.downcase
+		block_begin = begin
+			_block_begin = dstr2.index(dstr1, search_position)
+			break if _block_begin.nil?
+			search_position = cultivation_map.search_again_position(_block_begin)
+			_block_begin
+		end until search_position.nil?
+
+		unless block_begin.nil?
+			return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}]
+		end
+
+		nil
 	end
 
 	def local_alignment_blocks(str1, b1, e1, str2, b2, e2, denotations = nil)
@@ -138,7 +374,7 @@ class TextAlignment::TextAlignment
 							map{|d| d.merge(lex:str1[d[:span][:begin] ... d[:span][:end]])}
 
 			position = 0
-			tblocks = ds_in_scope.map do |term|
+			_tblocks = ds_in_scope.map do |term|
 				lex = term[:lex]
 				r = block2.index(lex, position)
 				if r.nil?
@@ -146,11 +382,11 @@ class TextAlignment::TextAlignment
 					break
 				end
 				position = r + lex.length
-				{source:term[:span], target:{begin:r + b2, end:r + b2 + lex.length}, alignment: :term, delta: r - term[:span][:begin]}
+				{source:term[:span], target:{begin:r + b2, end:r + b2 + lex.length}, alignment: :term, similarity: 0.9, delta: r + b2 - term[:span][:begin]}
 			end
 
 			# missing term found
-			tblocks = [] if position.nil?
+			_tblocks = [] if position.nil?
 
 			# redundant matching found
 			unless position.nil?
@@ -158,14 +394,15 @@ class TextAlignment::TextAlignment
 					lex = term[:lex]
 					look_forward = block2.index(lex, position)
 					unless look_forward.nil?
-						puts lex
-						tblocks = []
+						_tblocks = []
 						break
 					end
 				end
 			end
 
-			tblocks
+			_tblocks
+		else
+			[]
 		end
 
 		if tblocks.empty?
@@ -237,153 +474,4 @@ class TextAlignment::TextAlignment
 		end
 	end
 
-
-	def indices(str, target)
-	  position = 0
-	  len = target.len
-	  Enumerator.new do |yielder|
-	    while idx = str.index(target, position)
-	      yielder << idx
-	      position = idx + len
-	    end
-	  end
-	end
-
-	def transform_begin_position(begin_position)
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
-		block = @block_alignment[:blocks][i]
-
-		b = if block[:alignment] == :block || block[:alignment] == :term
-			begin_position + block[:delta]
-		elsif block[:alignment] == :empty
-			if begin_position == block[:source][:begin]
-				block[:target][:begin]
-			else
-				nil
-			end
-		else
-			r = block[:alignment].transform_begin_position(begin_position - block[:source][:begin])
-			r.nil? ? nil : r + block[:target][:begin]
-		end
-	end
-
-	def transform_end_position(end_position)
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
-		block = @block_alignment[:blocks][i]
-
-		e = if block[:alignment] == :block || block[:alignment] == :term
-			end_position + block[:delta]
-		elsif block[:alignment] == :empty
-			if end_position == block[:source][:end]
-				block[:target][:end]
-			else
-				nil
-			end
-		else
-			r = block[:alignment].transform_end_position(end_position - block[:source][:begin])
-			r.nil? ? nil : r + block[:target][:begin]
-		end
-	end
-
-	def transform_a_span(span)
-		{begin: transform_begin_position(span[:begin]), end: transform_end_position(span[:end])}
-	end
-
-	def transform_spans(spans)
-		spans.map{|span| transform_a_span(span)}
-	end
-
-	def transform_denotations!(denotations)
-		return nil if denotations.nil?
-		@lost_annotations = []
-
-		denotations.each do |d|
-			source = {begin:d.begin, end:d.end}
-			d.begin = transform_begin_position(d.begin);
-			d.end = transform_end_position(d.end);
-			raise "invalid transform" unless !d.begin.nil? && !d.end.nil? && d.begin >= 0 && d.end > d.begin && d.end <= @str2.length
-		rescue
-			@lost_annotations << {source: source, target:{begin:d.begin, end:d.end}}
-			d.begin = nil
-			d.end = nil
-		end
-
-		@lost_annotations
-	end
-
-	def transform_hdenotations(hdenotations)
-		return nil if hdenotations.nil?
-		@lost_annotations = []
-
-		r = hdenotations.collect do |d|
-			t = transform_a_span(d[:span])
-			raise "invalid transform" unless !t[:begin].nil? && !t[:end].nil? && t[:begin] >= 0 && t[:end] > t[:begin] && t[:end] <= @str2.length
-			new_d = d.dup.merge({span:t})
-		rescue
-			@lost_annotations << {source: d[:span], target:t}
-			nil
-		end.compact
-
-		r
-	end
-
-	def alignment_show
-		stext = @block_alignment[:source_text]
-		ttext = @block_alignment[:target_text]
-
-		show = ''
-		@block_alignment[:blocks].each do |a|
-			show += case a[:alignment]
-			when :block
-				"===== common (block) ===== [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
-				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
-			when :term
-				"===== common (term) ===== [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
-				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
-			when :empty
-				"xxxxx disparate texts (similarity: #{a[:similarity]})\n" +
-				"<<<<< string 1 [#{a[:source][:begin]} - #{a[:source][:end]}]\n" +
-				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
-				">>>>> string 2 [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
-				ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
-			else
-				astr1 = ''
-				astr2 = ''
-
-				base = a[:source][:begin]
-				astr1 = a[:alignment].sdiff.map do |c|
-					case c.action
-					when '='
-						stext[c.old_position + base]
-					when '+'
-						'_'
-					when '-'
-						stext[c.old_position + base]
-					when '!'
-						stext[c.old_position + base] + '_'
-					end
-				end.join('')
-
-				base = a[:target][:begin]
-				astr2 = a[:alignment].sdiff.map do |c|
-					case c.action
-					when '='
-						ttext[c.new_position + base]
-					when '+'
-						ttext[c.new_position + base]
-					when '-'
-						'_'
-					when '!'
-						'_' + ttext[c.new_position + base]
-					end
-				end.join('')
-
-				"***** local mismatch [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}] (similarity: #{a[:similarity]})\n" +
-				"[#{astr1}]\n" +
-				"[#{astr2}]\n\n"
-			end
-		end
-		show
-	end
-
 end