text_alignment 0.7.3 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0788bdb6d161499f5a5258757b9f61faee96b60246b422b57b17ba953b4a2c87'
-  data.tar.gz: 4564fd15e1e1d673932438206989dc706aa67f2467698f16946e2635c562ec90
+  metadata.gz: 4d626b64acdca0630dc344e2f0f4c2152e481ca7c8209ac1aa8c025fb1ae7c0f
+  data.tar.gz: 972222f1f3a575cdb30cd83f886d6d3f36109b79a35cc2c970aa89cbe8fd007d
 SHA512:
-  metadata.gz: c93882cc28e8bfdbdeed325b282b63fd3f3644d1739ed979ef92a7d8a133e26f4a8ffd0bda22da0fc2e0a31c77c6d49ce89f8f0fae802c9ae2041a7db60a2a4e
-  data.tar.gz: 238ed44ac0c0a178a64743846550639c626d3a300a79f05fedb016e71a890f11025b52d260ce2f46f00d68b1c4b9c13c48a8be8712f737bb2b57b0274b174b8b
+  metadata.gz: 288691e9ee650af3b9dfe5b47f1a3d05ff952292958ef733a5a5f47aeeaa73e6e1b7b0591afe3549b6cbabf04817bf406d3ca2ee55faf65b7e3816dfe52be644
+  data.tar.gz: 7f45baecc36f00310f868a596ca60bce709078bdc51a5f9438a916cd42062bb93dacadafb64e90e42276d0633914c978045fed4b597ea7411c8762761be6bd36

data/bin/align_annotations

@@ -26,33 +26,44 @@ def read_text(filename)
 	end
 end
 
-def align_mdoc(source_annotations, target_annotations)
+def align_denotations(denotations, source_text, alignment, debug = false)
+	alignment.align(source_text, denotations)
+	new_denotations = alignment.transform_hdenotations(denotations)
+
+	if debug
+		warn "[block alignment]"
+		warn alignment.alignment_show
+		warn "-----"
+	end
+
+	lost_annotations = alignment.lost_annotations
+	unless lost_annotations.empty?
+		warn "\n[lost annotations] #{lost_annotations.length}"
+		lost_annotations.each do |a|
+			warn "#{a}"
+		end
+		warn "====="
+	end
+	warn
+
+	# return target annotations
+	new_denotations
+end
+
+def align_mannotations(source_annotations, reference_text, alignment, debug = false)
+	target_annotations = {text:reference_text}
+
 	idnum_denotations = 0
 	idnum_relations = 0
 	idnum_attributes = 0
 	idnum_modifications = 0
 
-	source_annotations.each do |annotations|
-		alignment = TextAlignment::TextAlignment.new(annotations[:text], target_annotations[:text])
-
-		puts alignment.alignment_show
-		puts "-----"
-		puts
-
-		# alignment.block_alignments.each do |a|
-			# p {source:a[:source], target:a[:target]}
-			# puts "--"
-			# p a[:alignment] if a[:alignment].nil? || a[:alignment] == :empty
-			# puts "--"
-			# puts annotations[:text][a[:source][:begin] ... a[:source][:end]]
-			# puts "--"
-			# puts target_text[a[:target][:begin] ... a[:target][:end]]
-			# puts "======"
-		# end
-
+	source_annotations.each_with_index do |annotations, i|
 		if annotations.has_key?(:denotations) && !annotations[:denotations].empty?
 			ididx = {}
-			denotations = alignment.transform_hdenotations(annotations[:denotations])
+			warn "[#{i}]-=-=-=-=-"
+			denotations = align_denotations(annotations[:denotations], annotations[:text], alignment, debug)
+
 			denotations.each do |d|
 				reid = 'T' + (idnum_denotations += 1).to_s
 				ididx[d[:id]] = reid
@@ -99,128 +110,16 @@ unless ARGV.length == 2
 end
 
 source_annotations = read_annotations(ARGV[0])
-target_text = read_text(ARGV[1])
+reference_text = read_text(ARGV[1])
 
-lost_annotations = []
-target_annotations = if source_annotations.class == Array
-	align_mdoc(source_annotations, {text: target_text})
-else
-	alignment = TextAlignment::TextAlignment.new(source_annotations[:text], target_text, source_annotations[:denotations])
-
-	# verification
-	# source_text = source_annotations[:text]
-	# puts "=====BEGIN"
-	# (0 ... source_text.rstrip.length).each do |p|
-	# 	t = alignment.transform_begin_position(p)
-	# 	if t.nil?
-	# 		print source_text[p]
-	# 	else
-	# 		print '.'
-	# 	end
-	# end
-	# puts
-	# puts "=====END"
-
-	# puts "=====BEGIN"
-	# (0 .. source_text.rstrip.length).each do |p|
-	# 	t = alignment.transform_end_position(p)
-	# 	if t.nil?
-	# 		print source_text[p]
-	# 	else
-	# 		print '.'
-	# 	end
-	# end
-	# puts
-	# puts "=====END"
-
-	source_text = source_annotations[:text]
-
-	puts "[block alignment]"
-	puts alignment.alignment_show
-	puts "====="
-	# exit
-
-	denotations = alignment.transform_hdenotations(source_annotations[:denotations])
-	lost_annotations += alignment.lost_annotations if alignment.lost_annotations
-
-	source_annotations.merge({text:target_text, denotations:denotations})
-end
+alignment = TextAlignment::TextAlignment.new(reference_text, true)
 
-num_denotations_source, num_relations_source, num_attributes_source, num_modifications_source = if source_annotations.class == Array
-	num_denotations_source, num_relations_source, num_attributes_source, num_modifications_source = 0, 0, 0, 0
-	source_annotations.each do |annotations|
-		num_denotations_source += annotations[:denotations].nil? ? 0 : annotations[:denotations].length
-		num_relations_source += annotations[:relations].nil? ? 0 : annotations[:relations].length
-		num_attributes_source += annotations[:attributes].nil? ? 0 : annotations[:attributes].length
-		num_modifications_source += annotations[:modifications].nil? ? 0 : annotations[:modifications].length
-	end
-	[num_denotations_source, num_relations_source, num_attributes_source, num_modifications_source]
+target_annotations = if source_annotations.class == Array
+	align_mannotations(source_annotations, reference_text, alignment, false)
 else
-	num_denotations_source = source_annotations[:denotations].nil? ? 0 : source_annotations[:denotations].length
-	num_relations_source = source_annotations[:relations].nil? ? 0 : source_annotations[:relations].length
-	num_attributes_source = source_annotations[:attributes].nil? ? 0 : source_annotations[:attributes].length
-	num_modifications_source = source_annotations[:modifications].nil? ? 0 : source_annotations[:modifications].length
-	[num_denotations_source, num_relations_source, num_attributes_source, num_modifications_source]
-end
-
-warn "[source]"
-warn "denotations:\t#{num_denotations_source}"
-# warn "relations:\t#{num_relations_source}"
-# warn "attributes:\t#{num_attributes_source}"
-# warn "modifications:\t#{num_modifications_source}"
-
-warn "\n[target]"
-warn "denotations:\t#{target_annotations[:denotations].nil? ? 0 : target_annotations[:denotations].length}"
-# warn "relations:\t#{target_annotations[:relations].nil? ? 0 : target_annotations[:relations].length}"
-# warn "attributes:\t#{target_annotations[:attributes].nil? ? 0 : target_annotations[:attributes].length}"
-# warn "modifications:\t#{target_annotations[:modifications].nil? ? 0 : target_annotations[:modifications].length}"
-
-if lost_annotations
-	warn "\n[lost annotations]"
-	lost_annotations.each do |a|
-		p a
-	end
+	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment)
+	# denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], reference_text, true)
+	source_annotations.merge({text:reference_text, denotations:denotations})
 end
 
-#puts target_annotations.to_json
-
-# denotations = anns1[:denotations]
-
-# puts "[Alignment1]====="
-# align = TextAlignment::TextAlignment.new(str1, str2, TextAlignment::MAPPINGS)
-
-# align.alignment.each do |a|
-# 	p [a[:target][:begin], a[:target][:end], a[:source][:begin], a[:source][:end]]
-# end
-
-# puts TextAlignment::sdiff2cdiff(align.sdiff)
-# puts
-# puts "[Similarity]\n#{align.similarity}"
-# puts
-# puts '[Denotations original]'
-# pp denotations
-# puts
-# puts '[Denotations transformed]'
-# new_denotations = align.transform_hdenotations(denotations)
-# pp new_denotations
-# puts
-# puts "[Alignment2 (downcased)]====="
-# align = TextAlignment::TextAlignment.new(str1.downcase, str2.downcase, TextAlignment::MAPPINGS)
-# puts TextAlignment::sdiff2cdiff(align.sdiff)
-# puts
-# puts "[Similarity]\n#{align.similarity}"
-# puts
-# puts '[Denotations original]'
-# pp denotations
-# puts
-# puts '[Denotations transformed]'
-# new_denotations = align.transform_hdenotations(denotations)
-# pp new_denotations
-# puts
-# puts '[Annotations transformed]'
-# anns2[:denotations] = new_denotations
-# puts anns2.to_json
-
-# p align.common_elements
-# puts "---------------"
-# p align.mapped_elements
+puts target_annotations.to_json

data/lib/text_alignment/anchor_finder.rb

@@ -6,76 +6,65 @@ module TextAlignment; end unless defined? TextAlignment
 
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
-		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
-		@size_window = _size_window || TextAlignment::SIZE_WINDOW
-		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
-
-		@reverse = (target_str.length < source_str.length)
-
-		@s1, @s2 = if @reverse
+	def initialize(source_str, target_str, cultivation_map)
+		@s1, @s2 = if reverse?(source_str, target_str)
 			[target_str.downcase, source_str.downcase]
 		else
 			[source_str.downcase, target_str.downcase]
 		end
 
-		# current position in s1
-		@beg_s1 = 0
-		@end_s1_prev = 0
-		@end_s2_prev = 0
-	end
-
-	def get_next_anchor
-		# find the position of an anchor ngram in s1 and s2
-		while @beg_s1 < (@s1.length - @size_ngram)
-			if [' ', "\n", "\t"].include? @s1[@beg_s1]
-				@beg_s1 += 1
-				next
-			end
-			anchor = @s1[@beg_s1, @size_ngram]
+		@cultivation_map = cultivation_map
 
-			# search_position = 0
-			search_position = @end_s2_prev
-			while @beg_s2 = @s2.index(anchor, search_position)
-				# if both the begining points are sufficiantly close to the end points of the last match
-				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
+		@size_ngram  = TextAlignment::SIZE_NGRAM
+		@size_window = TextAlignment::SIZE_WINDOW
+		@sim_threshold = TextAlignment::TEXT_SIMILARITY_THRESHOLD
+		@pos_s1_final_possible_begin = @s1.length - @size_ngram - 1
 
-				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
+		# positions of last match
+		@pos_s1_last_match = 0
+		@pos_s2_last_match = 0
+	end
 
-				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
+	def reverse?(source_str = nil, target_str = nil)
+		unless source_str.nil?
+			@reverse_p = target_str.length < source_str.length
+		end
+		@reverse_p
+	end
 
-				search_position = @beg_s2 + 1
-			end
+	def get_next_anchor
+		# To find the beginning positions of an anchor ngram in s1 and s2, beginning from the last positions matched
+		beg_s2 = for beg_s1 in @pos_s1_last_match .. @pos_s1_final_possible_begin
 
-			break unless @beg_s2.nil?
+			# To skip whitespace letters
+			next if [' ', "\n", "\t"].include? @s1[beg_s1]
 
-			@beg_s1 += 1
+			_beg_s2 = get_beg_s2(beg_s1)
+			break _beg_s2 unless _beg_s2.nil?
 		end
 
-		return nil if @beg_s1 >= (@s1.length - @size_ngram)
+		# To return nil when it fails to find an anchor
+		return nil if beg_s2.class == Range
 
-		# extend the block
-		b1 = @beg_s1
-		b2 = @beg_s2
-		while b1 >= @end_s1_prev && b2 >= @end_s2_prev && @s1[b1] == @s2[b2]
+		# To extend the block to the left
+		b1 = beg_s1
+		b2 = beg_s2
+		while b1 >= @pos_s1_last_match && b2 >= @pos_s2_last_match && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
-
 		b1 += 1; b2 += 1
 
-		e1 = @beg_s1 + @size_ngram
-		e2 = @beg_s2 + @size_ngram
+		# To extend the block to the right
+		e1 = beg_s1 + @size_ngram
+		e2 = beg_s2 + @size_ngram
 		while @s1[e1] && @s1[e1] == @s2[e2]
 			e1 += 1; e2 += 1
 		end
 
-		@end_s1_prev = e1
-		@end_s2_prev = e2
-		@beg_s1 = e1
+		@pos_s1_last_match = e1
+		@pos_s2_last_match = e2
 
-		if @reverse
+		if reverse?
 			{source:{begin:b2 , end:e2}, target:{begin:b1, end:e1}}
 		else
 			{source:{begin:b1 , end:e1}, target:{begin:b2, end:e2}}
@@ -84,14 +73,92 @@ class TextAlignment::AnchorFinder
 
 	private
 
-	def get_left_windows
-		# commend below with the assumption that the beginning of a document gives a significant locational information
-		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
+	def get_beg_s2(beg_s1)
+		# to get the anchor to search for in s2
+		anchor = @s1[beg_s1, @size_ngram]
+
+		# comment out below with the assumption that texts are in the same order
+		# search_position = 0
+		search_position = @pos_s2_last_match
+
+		beg_s2_candidates = find_beg_s2_candidates(anchor, search_position)
+		return nil if beg_s2_candidates.empty?
+
+		find_valid_beg_s2(beg_s1, beg_s2_candidates)
+	end
+
+	# To find beg_s2 which match to the anchor
+	# return nil if the anchor is too much frequent
+	def find_beg_s2_candidates(anchor, search_position)
+		candidates = []
+		while _beg_s2 = @s2.index(anchor, search_position)
+			search_again_position = @cultivation_map.search_again_position(_beg_s2)
+			unless search_again_position.nil?
+				search_position = search_again_position
+				next
+			end
+
+			candidates << _beg_s2
+
+			# for speed, skip anchor of high frequency
+			if candidates.length > 5
+				candidates.clear
+				break
+			end
+
+			search_position = _beg_s2 + 1
+		end
+		candidates
+	end
+
+	def find_valid_beg_s2(beg_s1, beg_s2_candidates)
+		valid_beg_s2 = nil
+
+		(10 .. 30).step(10).each do |size_window|
+			valid_beg_s2 = nil
+
+			r = beg_s2_candidates.each do |beg_s2|
+				# if both the begining points are sufficiantly close to the end points of the last match
+				# break if beg_s1 > 0 && beg_s2 > 0 && (beg_s1 - @pos_s1_last_match < 5) && (beg_s2 >= @pos_s2_last_match) && (beg_s2 - @pos_s2_last_match < 5)
+				if beg_s1 > 0 && beg_s2 > 0 && (beg_s1 - @pos_s1_last_match < 5) && (beg_s2 - @pos_s2_last_match < 5)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+
+				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2)
+				if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+
+				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2)
+				if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
+					break unless valid_beg_s2.nil?
+					valid_beg_s2 = beg_s2
+					next
+				end
+			end
+
+			# r == nil means that the inner loop was broken (multiple candidates had passed the tests)
+			# r != nil means that the inner loop was completed (with or w/o a valid beg_s2 found)
+			break unless r.nil?
+		end
+
+		valid_beg_s2
+	end
+
+	def get_left_windows(beg_s1, beg_s2, size_window = nil)
+		size_window ||= @size_window
+
+		# comment out below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < size_window || @beg_s2 < size_window
 
 		window_s1 = ''
-		loc = @beg_s1 - 1
+		loc = beg_s1 - 1
 		count = 0
-		while count < @size_window && loc >= 0
+		while count < size_window && loc >= 0
 			if @s1[loc] =~ /[0-9a-zA-Z]/
 				window_s1 += @s1[loc]
 				count += 1
@@ -100,9 +167,9 @@ class TextAlignment::AnchorFinder
 		end
 
 		window_s2 = ''
-		loc = @beg_s2 - 1
+		loc = beg_s2 - 1
 		count = 0
-		while count < @size_window && loc >= 0
+		while count < size_window && loc >= 0
 			if @s2[loc] =~ /[0-9a-zA-Z]/
 				window_s2 += @s2[loc]
 				count += 1
@@ -113,15 +180,17 @@ class TextAlignment::AnchorFinder
 		[window_s1, window_s2]
 	end
 
-	def get_right_windows
+	def get_right_windows(beg_s1, beg_s2, size_window = nil)
+		size_window ||= @size_window
+
 		# commend below with the assumption that the end of a document gives a significant locational
-		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# return if (@beg_s1 + @size_ngram > (@s1.length - size_window)) || (@beg_s2 + @size_ngram > (@s2.length - size_window))
 
 		window_s1 = ''
-		loc = @beg_s1 + @size_ngram
+		loc = beg_s1 + @size_ngram
 		len_s1 = @s1.length
 		count = 0
-		while count < @size_window && loc < len_s1
+		while count < size_window && loc < len_s1
 			if @s1[loc] =~ /[0-9a-zA-Z]/
 				window_s1 += @s1[loc]
 				count += 1
@@ -130,10 +199,10 @@ class TextAlignment::AnchorFinder
 		end
 
 		window_s2 = ''
-		loc = @beg_s2 + @size_ngram
+		loc = beg_s2 + @size_ngram
 		len_s2 = @s2.length
 		count = 0
-		while count < @size_window && loc < len_s2
+		while count < size_window && loc < len_s2
 			if @s2[loc] =~ /[0-9a-zA-Z]/
 				window_s2 += @s2[loc]
 				count += 1
@@ -148,5 +217,4 @@ class TextAlignment::AnchorFinder
 		return 0 if str1.nil? || str2.nil?
 		String::Similarity.cosine(str1, str2, ngram:ngram_order)
 	end
-
-end
+end