text_alignment 0.9 → 0.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c44cc3036273c8c34800d8f78a79316c53efb80fe45ad81092a6172da3b03c6
-  data.tar.gz: 50ab44cc66b50bf732e99f900c10584025c6ed498603ccf3afd75de90cac4b79
+  metadata.gz: 81994ec9a8c7c08d2aad32b351b5942fef1748e4035ba762af546d4f3fe7cee8
+  data.tar.gz: bfa75451d33b9d21c2baa1a52280f03486bf10a41b5ac2a97469f0ca3a4f7379
 SHA512:
-  metadata.gz: 98645c1ba4566c822d1e6ba6488e4ecdfe100c30923cc7effe7d2a4390ebb6901707e8c9f6a12145e2f98515bc6792afef4f9bfa5fcd683c77d3a5cf599094c7
-  data.tar.gz: 11657abdb8acb64c8edfd5271bbf78d2a75024753180988030c5ce6722b4da2781760e583ca6e33ed469cca85e4a2f8e28af6ef4dc62029ada5bd8a184200dfb
+  metadata.gz: 73d0ff212a89d6ad33751f87f14a9b292b45ce177c61efd0ede5f852eb3834d1bd3940d202c4d87cfb0422c4dc566dea30c560f8500a220dcd3e1dd492e29eac
+  data.tar.gz: 6c763a564e339267624bec5c809b334b0adf9951d25409eac21ae4b4582beae4a08739119dfa369022c44c19d4faacfd0e06766c6223bd219a43c3961cfab08c

data/bin/align_annotations

@@ -26,8 +26,8 @@ def read_text(filename)
 	end
 end
 
-def align_denotations(denotations, source_text, target_text, debug = false)
-	alignment = TextAlignment::TextAlignment.new(source_text, target_text, denotations)
+def align_denotations(denotations, source_text, alignment, debug = false)
+	alignment.align(source_text, denotations)
 	new_denotations = alignment.transform_hdenotations(denotations)
 
 	if debug
@@ -37,7 +37,7 @@ def align_denotations(denotations, source_text, target_text, debug = false)
 	end
 
 	lost_annotations = alignment.lost_annotations
-	unless lost_annotations.empty?
+	unless lost_annotations.nil? || lost_annotations.empty?
 		warn "\n[lost annotations] #{lost_annotations.length}"
 		lost_annotations.each do |a|
 			warn "#{a}"
@@ -50,8 +50,8 @@ def align_denotations(denotations, source_text, target_text, debug = false)
 	new_denotations
 end
 
-def align_mannotations(source_annotations, target_text, debug = false)
-	target_annotations = {text:target_text}
+def align_mannotations(source_annotations, reference_text, alignment, debug = false)
+	target_annotations = {text:reference_text}
 
 	idnum_denotations = 0
 	idnum_relations = 0
@@ -62,7 +62,8 @@ def align_mannotations(source_annotations, target_text, debug = false)
 		if annotations.has_key?(:denotations) && !annotations[:denotations].empty?
 			ididx = {}
 			warn "[#{i}]-=-=-=-=-"
-			denotations = align_denotations(annotations[:denotations], annotations[:text], target_text, debug)
+			denotations = align_denotations(annotations[:denotations], annotations[:text], alignment, debug)
+
 			denotations.each do |d|
 				reid = 'T' + (idnum_denotations += 1).to_s
 				ididx[d[:id]] = reid
@@ -76,7 +77,9 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:relations].each do |r|
 					reid = 'R' + (idnum_relations += 1).to_s
 					ididx[r[:id]] = reid
-					target_annotations[:relations] << r.dup.merge({id:reid, subj:ididx[r[:subj]], obj:ididx[r[:obj]]})
+					sid = ididx[r[:subj]]
+					oid = ididx[r[:obj]]
+					target_annotations[:relations] << r.dup.merge({id:reid, subj:sid, obj:oid}) unless sid.nil? || oid.nil?
 				end
 			end
 
@@ -85,7 +88,8 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:attributes].each do |a|
 					reid = 'A' + (idnum_attributes += 1).to_s
 					ididx[a[:id]] = reid
-					target_annotations[:attributes] << a.dup.merge({id:reid, subj:ididx[a[:subj]]})
+					sid = ididx[a[:subj]]
+					target_annotations[:attributes] << a.dup.merge({id:reid, subj:sid}) unless sid.nil?
 				end
 			end
 
@@ -94,7 +98,8 @@ def align_mannotations(source_annotations, target_text, debug = false)
 				annotations[:modifications].each do |m|
 					reid = 'M' + (idnum_modifications += 1).to_s
 					ididx[m[:id]] = reid
-					target_annotations[:modifications] << m.dup.merge({id:reid, obj:ididx[m[:obj]]})
+					oid = ididx[m[:obj]]
+					target_annotations[:modifications] << m.dup.merge({id:reid, obj:oid}) unless oid.nil?
 				end
 			end
 		end
@@ -109,13 +114,18 @@ unless ARGV.length == 2
 end
 
 source_annotations = read_annotations(ARGV[0])
-target_text = read_text(ARGV[1])
+reference_text = read_text(ARGV[1])
+
+alignment = TextAlignment::TextAlignment.new(reference_text, true)
 
 target_annotations = if source_annotations.class == Array
-	align_mannotations(source_annotations, target_text, false)
+	# align_mannotations(source_annotations, reference_text, alignment, true)
+	align_mannotations(source_annotations, reference_text, alignment, false)
 else
-	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], target_text, false)
-	source_annotations.merge({text:target_text, denotations:denotations})
+	# denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment)
+	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, false)
+	source_annotations.merge({text:reference_text, denotations:denotations})
 end
 
-# puts target_annotations.to_json
+# pp alignment.block_alignment
+# puts target_annotations.to_json

data/lib/text_alignment/anchor_finder.rb

@@ -6,92 +6,141 @@ module TextAlignment; end unless defined? TextAlignment
 
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
-		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
-		@size_window = _size_window || TextAlignment::SIZE_WINDOW
-		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
+	def initialize(source_str, target_str, cultivation_map)
+		@s1 = source_str.downcase
+		@s2 = target_str.downcase
 
-		@reverse = (target_str.length < source_str.length)
+		@cultivation_map = cultivation_map
 
-		@s1, @s2 = if @reverse
-			[target_str.downcase, source_str.downcase]
-		else
-			[source_str.downcase, target_str.downcase]
-		end
+		@size_ngram  = TextAlignment::SIZE_NGRAM
+		@size_window = TextAlignment::SIZE_WINDOW
+		@sim_threshold = TextAlignment::TEXT_SIMILARITY_THRESHOLD
+		@pos_s1_final_possible_begin = @s1.length - @size_ngram - 1
+		@pos_s2_final_possible_end = @s2.length
 
-		# current position in s1
-		@beg_s1 = 0
-		@end_s1_prev = 0
-		@end_s2_prev = 0
+		# positions of last match
+		@pos_s1_last_match = 0
+		@pos_s2_last_match = 0
 	end
 
 	def get_next_anchor
-		# find the position of an anchor ngram in s1 and s2
-		while @beg_s1 < (@s1.length - @size_ngram)
-			if [' ', "\n", "\t"].include? @s1[@beg_s1]
-				@beg_s1 += 1
-				next
-			end
-			anchor = @s1[@beg_s1, @size_ngram]
-
-			# search_position = 0
-			search_position = @end_s2_prev
-			while @beg_s2 = @s2.index(anchor, search_position)
-				# if both the begining points are sufficiantly close to the end points of the last match
-				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
+		# To find the beginning positions of an anchor ngram in s1 and s2, beginning from the last positions matched
+		beg_s2 = for beg_s1 in @pos_s1_last_match .. @pos_s1_final_possible_begin
 
-				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
+			# To skip whitespace letters
+			next if [' ', "\n", "\t"].include? @s1[beg_s1]
 
-				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
+			_beg_s2 = get_beg_s2(beg_s1)
+			break _beg_s2 unless _beg_s2.nil?
+		end
 
-				search_position = @beg_s2 + 1
-			end
+		# To return nil when it fails to find an anchor
+		return nil if beg_s2.class == Range
 
-			break unless @beg_s2.nil?
+		# To extend the block to the left
+		b1 = beg_s1
+		b2 = beg_s2
+		left_boundary_b2 = [@pos_s2_last_match, (@cultivation_map.last_cultivated_position(b2) || 0)].max
+		while b1 > @pos_s1_last_match && b2 > left_boundary_b2 && @s1[b1 - 1] == @s2[b2 - 1]
+			b1 -= 1; b2 -= 1
+		end
 
-			@beg_s1 += 1
+		# To extend the block to the right
+		e1 = beg_s1 + @size_ngram
+		e2 = beg_s2 + @size_ngram
+		right_boundary_b2 = @cultivation_map.next_cultivated_position(e2) || @pos_s2_final_possible_end
+		while @s1[e1] && e2 < right_boundary_b2 && @s1[e1] == @s2[e2]
+			e1 += 1; e2 += 1
 		end
 
-		return nil if @beg_s1 >= (@s1.length - @size_ngram)
+		@pos_s1_last_match = e1
+		@pos_s2_last_match = e2
 
-		# extend the block
-		b1 = @beg_s1
-		b2 = @beg_s2
-		while b1 >= @end_s1_prev && b2 >= @end_s2_prev && @s1[b1] == @s2[b2]
-			b1 -= 1; b2 -= 1
-		end
+		{source:{begin:b1 , end:e1}, target:{begin:b2, end:e2}}
+	end
 
-		b1 += 1; b2 += 1
+	private
 
-		e1 = @beg_s1 + @size_ngram
-		e2 = @beg_s2 + @size_ngram
-		while @s1[e1] && @s1[e1] == @s2[e2]
-			e1 += 1; e2 += 1
+	def get_beg_s2(beg_s1)
+		# to get the anchor to search for in s2
+		anchor = @s1[beg_s1, @size_ngram]
+
+		# comment out below with the assumption that texts are in the same order
+		# search_position = 0
+		search_position = @pos_s2_last_match
+
+		beg_s2_candidates = find_beg_s2_candidates(anchor, search_position)
+		return nil if beg_s2_candidates.empty?
+
+		find_valid_beg_s2(beg_s1, beg_s2_candidates)
+	end
+
+	# To find beg_s2 which match to the anchor
+	# return nil if the anchor is too much frequent
+	def find_beg_s2_candidates(anchor, search_position)
+		candidates = []
+		while _beg_s2 = @cultivation_map.index(anchor, @s2, search_position)
+			candidates << _beg_s2
+
+			# for speed, skip anchor of high frequency
+			if candidates.length > 5
+				candidates.clear
+				break
+			end
+
+			search_position = _beg_s2 + 1
 		end
+		candidates
+	end
+
+	def find_valid_beg_s2(beg_s1, beg_s2_candidates)
+		valid_beg_s2 = nil
 
-		@end_s1_prev = e1
-		@end_s2_prev = e2
-		@beg_s1 = e1
+		(10 .. 30).step(10).each do |size_window|
+			valid_beg_s2 = nil
 
-		if @reverse
-			{source:{begin:b2 , end:e2}, target:{begin:b1, end:e1}}
-		else
-			{source:{begin:b1 , end:e1}, target:{begin:b2, end:e2}}
+			r = beg_s2_candidates.each do |beg_s2|
+				# if both the begining points are sufficiantly close to the end points of the last match
+				# break if beg_s1 > 0 && beg_s2 > 0 && (beg_s1 - @pos_s1_last_match < 5) && (beg_s2 >= @pos_s2_last_match) && (beg_s2 - @pos_s2_last_match < 5)
+				if beg_s1 > 0 && beg_s2 > 0 && (beg_s1 - @pos_s1_last_match < 5) && (beg_s2 - @pos_s2_last_match < 5)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+
+				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2)
+				if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+
+				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2)
+				if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+			end
+
+			# r == nil means that the inner loop was broken (multiple candidates had passed the tests)
+			# r != nil means that the inner loop was completed (with or w/o a valid beg_s2 found)
+			break unless r.nil?
 		end
+
+		valid_beg_s2
 	end
 
-	private
+	def get_left_windows(beg_s1, beg_s2, size_window = nil)
+		size_window ||= @size_window
 
-	def get_left_windows
-		# commend below with the assumption that the beginning of a document gives a significant locational information
-		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
+		# comment out below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < size_window || @beg_s2 < size_window
 
 		window_s1 = ''
-		loc = @beg_s1 - 1
+		loc = beg_s1 - 1
 		count = 0
-		while count < @size_window && loc >= 0
+		while count < size_window && loc >= 0
 			if @s1[loc] =~ /[0-9a-zA-Z]/
 				window_s1 += @s1[loc]
 				count += 1
@@ -100,9 +149,9 @@ class TextAlignment::AnchorFinder
 		end
 
 		window_s2 = ''
-		loc = @beg_s2 - 1
+		loc = beg_s2 - 1
 		count = 0
-		while count < @size_window && loc >= 0
+		while count < size_window && loc >= 0
 			if @s2[loc] =~ /[0-9a-zA-Z]/
 				window_s2 += @s2[loc]
 				count += 1
@@ -113,15 +162,17 @@ class TextAlignment::AnchorFinder
 		[window_s1, window_s2]
 	end
 
-	def get_right_windows
+	def get_right_windows(beg_s1, beg_s2, size_window = nil)
+		size_window ||= @size_window
+
 		# commend below with the assumption that the end of a document gives a significant locational
-		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# return if (@beg_s1 + @size_ngram > (@s1.length - size_window)) || (@beg_s2 + @size_ngram > (@s2.length - size_window))
 
 		window_s1 = ''
-		loc = @beg_s1 + @size_ngram
+		loc = beg_s1 + @size_ngram
 		len_s1 = @s1.length
 		count = 0
-		while count < @size_window && loc < len_s1
+		while count < size_window && loc < len_s1
 			if @s1[loc] =~ /[0-9a-zA-Z]/
 				window_s1 += @s1[loc]
 				count += 1
@@ -130,10 +181,10 @@ class TextAlignment::AnchorFinder
 		end
 
 		window_s2 = ''
-		loc = @beg_s2 + @size_ngram
+		loc = beg_s2 + @size_ngram
 		len_s2 = @s2.length
 		count = 0
-		while count < @size_window && loc < len_s2
+		while count < size_window && loc < len_s2
 			if @s2[loc] =~ /[0-9a-zA-Z]/
 				window_s2 += @s2[loc]
 				count += 1
@@ -148,5 +199,4 @@ class TextAlignment::AnchorFinder
 		return 0 if str1.nil? || str2.nil?
 		String::Similarity.cosine(str1, str2, ngram:ngram_order)
 	end
-
-end
+end

data/lib/text_alignment/{mappings.rb → char_mapping.rb}

@@ -1,6 +1,6 @@
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::MAPPINGS = [
+TextAlignment::CHAR_MAPPING = [
 	["©", "(c)"],			#U+00A9 (Copyright Sign)
 
 	["α", "alpha"],		#U+03B1 (greek small letter alpha)
@@ -61,9 +61,11 @@ TextAlignment::MAPPINGS = [
 	["•", "*"],				#U+2022 (bullet)
 	[" ", " "],				#U+2009 (thin space)
 	[" ", " "],				#U+200A (hair space)
-	[" ", " "],				#U+00A0 (no-break space)
+	[" ", " "],				#U+202F (narrow no-break space)
+	[" ", " "],				#U+00A0 (Non-Breaking space)
 	["　", " "],				#U+3000 (ideographic space)
-	["‑", "-"],				#U+2211 (Non-Breaking Hyphen)
+	["‐", "-"],				#U+2010 (Hyphen)
+	["‑", "-"],				#U+2011 (Non-Breaking Hyphen)
 	["−", "-"],				#U+2212 (minus sign)
 	["–", "-"],				#U+2013 (en dash)
 	["′", "'"],				#U+2032 (prime)
@@ -75,98 +77,114 @@ TextAlignment::MAPPINGS = [
 ]
 
 
-TextAlignment::PADDING_LETTERS = ['@', '^', '|', '#', '$', '%', '&', '_'] unless defined? TextAlignment::PADDING_LETTERS
+class TextAlignment::CharMapping
+	attr_reader :mapped_text
 
+	def initialize(_text, char_mapping = nil)
+		char_mapping ||= TextAlignment::CHAR_MAPPING
+		@mapped_text, offset_mapping = enmap_text(_text, char_mapping)
+		@index_enmap = offset_mapping.to_h
+		@index_demap = offset_mapping.map{|m| m.reverse}.to_h
+	end
 
-class << TextAlignment
-	def single_character_mapping_preprocessing(_str1, _str2, _mappings = nil)
-		_mappings ||= TextAlignment::MAPPINGS
-
-		character_mappings = _mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		if character_mappings.empty?
-			[_str1, _str2, _mappings]
-		else
-			characters_from = character_mappings.collect{|m| m[0]}.join
-			characters_to   = character_mappings.collect{|m| m[1]}.join
-			characters_to.gsub!(/-/, '\-')
+	def enmap_position(position)
+		@index_enmap[position]
+	end
 
-			str1 = _str1.tr(characters_from, characters_to)
-			str2 = _str2.tr(characters_from, characters_to)
+	def demap_position(position)
+		@index_demap[position]
+	end
 
-			mappings = _mappings.select{|m| m[0].length > 1 || m[1].length > 1}
+	def enmap_denotations(denotations)
+		return nil if denotations.nil?
 
-			[str1, str2, mappings]
+		denotations.map do |d|
+			d.dup.merge(span:{begin:enmap_position(d[:span][:begin]), end:enmap_position(d[:span][:end])})
 		end
 	end
 
-	def long_to_one_mapping_preprocessing(_str1, _str2, _mappings = nil)
-		_mappings ||= TextAlignment::MAPPINGS
-
-		long_to_one_mappings = _mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		if long_to_one_mappings.empty?
-			[_str1, _str2, _mappings]
-		else
-			## long to one character mappings
-			pletters = TextAlignment::PADDING_LETTERS
-
-			# find the padding letter for str1
-			@padding_letter1 = begin
-				i = pletters.index{|l| _str2.index(l).nil?}
-				raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
-				TextAlignment::PADDING_LETTERS[i]
-			end
+	private
 
-			# find the padding letter for str2
-			@padding_letter2 = begin
-				i = pletters.index{|l| l != @padding_letter1 && _str1.index(l).nil?}
-				raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
-				TextAlignment::PADDING_LETTERS[i]
-			end
+	def enmap_text(_text, char_mapping)
+		text = _text.dup
 
-			str1 = str2 = nil
-			long_to_one_mappings.each do |f|
-				from = f[1]
-
-				str1 = if _str2.index(f[0])
-					to = f[0] + (@padding_letter1 * (f[1].length - 1))
-					_str1.gsub(from, to)
-				else
-					_str1
-				end
-
-				str2 = if _str1.index(f[0])
-					to = f[0] + (@padding_letter2 * (f[1].length - 1))
-					_str2.gsub(from, to)
-				else
-					_str2
-				end
+		# To execute the single letter mapping
+		char_mapping.each do |one, long|
+			text.gsub!(one, long) if long.length == 1
+		end
+
+		# To get the (location, length) index for replacements
+		loc_len = []
+		char_mapping.each do |one, long|
+			next if long.length == 1
+
+			init_next = 0
+			while loc = text.index(long, init_next)
+				loc_len << [loc, long.length]
+				init_next = loc + long.length
 			end
-			mappings = _mappings.select{|m| m[0].length > 1 || m[1].length == 1}
 
-			[str1, str2, mappings]
+			# a workaround to avoid messing-up due to embedding
+			text.gsub!(long, one * long.length)
 		end
-	end
 
-	def compute_similarity(_s1, _s2, sdiff)
-		return 0 if sdiff.nil?
+		# To get the (location, length) index for consecutive whitespace sequences
+		init_next = 0
+		while loc = text.index(/\s{2,}/, init_next)
+			len = $~[0].length
+			loc_len << [loc, len]
+			init_next = loc + len
+		end
+
+		loc_len.sort!{|a, b| a[0] <=> b[0]}
+
+		# To get the offset_mapping before and after replacement
+		offset_mapping = []
+		init_next = 0
+		j = 0
 
-		# compute the lcs only with non-whitespace letters
-		lcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
-		return 0 if lcs == 0
+		loc_len.each do |loc, len|
+			offset_mapping += (init_next .. loc).map do |i|
+				j += 1
+				[i, j - 1]
+			end
+			init_next = loc + len
+		end
 
-		s1 = if @padding_letter1
-			_s1.tr(@padding_letter1, ' ')
-		else
-			_s1
+		offset_mapping += (init_next .. text.length).map do |i|
+			j += 1
+			[i, j - 1]
 		end
 
-		s2 = if @padding_letter2
-			_s2.tr(@padding_letter2, ' ')
-		else
-			_s2
+		# To execute the long letter mapping
+		char_mapping.each do |one, long|
+			text.gsub!(one * long.length, one) if long.length > 1
 		end
 
-		similarity = lcs.to_f / [s1.scan(/\S/).count, s2.scan(/\S/).count].min
+		# To replace multi whitespace sequences to a space
+		text.gsub!(/\s{2,}/, ' ')
+
+		[text, offset_mapping]
 	end
+end
+
+if __FILE__ == $0
+	require 'json'
+
+	unless ARGV.length == 1
+		warn "#{$0} an_annotation_json_file.json" 
+		exit
+	end
+	annotations = JSON.parse File.read(ARGV[0]).strip, symbolize_names: true
+	denotations = annotations[:denotations] || []
+	if denotations.nil? && annotations[:tracks]
+		denotations = annotations[:tracks].first[:denotations]
+	end
+
+	text_mapping = TextAlignment::CharMapping.new(annotations[:text])
+	text_mapped = text_mapping.mapped_text
+	denotations_mapped = text_mapping.enmap_denotations(denotations)
+	new_annotations = {text:text_mapped, denotations:denotations_mapped}
 
+	puts new_annotations.to_json
 end