text_alignment 0.11.1 → 0.11.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef59c0cd578ed453a67edeb3b29059f4b76c4c541f777fa35a06e76b299e2564
-  data.tar.gz: 5594c0f6eb1d52cc331c210fdf936e7cb09e30277f21933027e55b0c8cfa0e24
+  metadata.gz: 52ad7c8b5822308ae15153e35626e13b5ec60d77428c80cbe3f1dd69f36edac1
+  data.tar.gz: efbcbbea9eb87606b1614661187356e15f65965b85c30ebf7d349d2f7e1028b0
 SHA512:
-  metadata.gz: ef5bae492d7e7b840c00943ac24e571392c4c992a085c6a63c8cb6db126ed9137ba94f8cd7af42a91e17aa327cb9a5ce24b909944c157f21beb8a88c8ce7528b
-  data.tar.gz: b6d84dcdc35399f91a0f6a5a24d84b11169de9d861a249acca52f0fec86e6f1fbf1bb4dbb47a5d43654fa43735b181aa64034447fb71ca675090818dcf67133a
+  metadata.gz: 59a2a8dd68066271b61b950c18f513b03d2b674e86c0d6d065e265367e079c99bfa973cb41c75bdc2f2d19443a9b6f8e46dcca27bb71eecd27ed00d00c07533e
+  data.tar.gz: 4779c1cbe899021125f56204d87462a8c1222c961f547e1ddc9ba4c46e29211da564bdc6492346baaa16ad4eaaedce4601f6afdcc0a16031a29be79f624c4935

data/bin/align_annotations

@@ -2,6 +2,7 @@
 require 'text_alignment'
 require 'json'
 require 'pp'
+require 'optparse'
 
 def read_annotations(filename)
 	case File.extname(filename)
@@ -37,7 +38,7 @@ def align_denotations(denotations, source_text, alignment, debug = false)
 	end
 
 	lost_annotations = alignment.lost_annotations
-	unless lost_annotations.empty?
+	unless lost_annotations.nil? || lost_annotations.empty?
 		warn "\n[lost annotations] #{lost_annotations.length}"
 		lost_annotations.each do |a|
 			warn "#{a}"
@@ -108,24 +109,46 @@ def align_mannotations(source_annotations, reference_text, alignment, debug = fa
 end
 
 
+## Options
+overlap_p = false
+debug_p = false
+
+## command line option processing
+require 'optparse'
+optparse = OptionParser.new do |opts|
+	opts.banner = "Usage: align_annotations [options] target_annotations(.json|.txt) reference_text(.json|.txt)"
+
+	opts.on('-o', '--overlap', 'tells it to assume there may be overlapping texts.') do
+		overlap_p = true
+	end
+
+	opts.on('-d', '--debug', 'tells it to show debugging information.') do
+		debug_p = true
+	end
+
+	opts.on('-h', '--help', 'displays this screen.') do
+		puts opts
+		exit
+	end
+end
+
+optparse.parse!
+
 unless ARGV.length == 2
-	warn "align_annotations target_annotations(.json|.txt) reference_annotations(.json|.txt)"
-	exit
+	puts optparse.help
+	exit 1
 end
 
 source_annotations = read_annotations(ARGV[0])
 reference_text = read_text(ARGV[1])
 
-alignment = TextAlignment::TextAlignment.new(reference_text, true)
+alignment = TextAlignment::TextAlignment.new(reference_text, !overlap_p)
 
 target_annotations = if source_annotations.class == Array
-	# align_mannotations(source_annotations, reference_text, alignment, true)
-	align_mannotations(source_annotations, reference_text, alignment, false)
+	align_mannotations(source_annotations, reference_text, alignment, debug_p)
 else
-	# denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment)
-	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, false)
+	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, debug_p)
 	source_annotations.merge({text:reference_text, denotations:denotations})
 end
 
-# pp alignment.block_alignment
-puts target_annotations.to_json
+# puts target_annotations.to_json

data/lib/text_alignment/anchor_finder.rb

@@ -108,14 +108,14 @@ class TextAlignment::AnchorFinder
 					next
 				end
 
-				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2)
+				left_window_s1, left_window_s2 = get_left_windows(beg_s1, beg_s2, size_window)
 				if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
 					break unless valid_beg_s2.nil?
 					valid_beg_s2 = beg_s2
 					next
 				end
 
-				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2)
+				right_window_s1, right_window_s2 = get_right_windows(beg_s1, beg_s2, size_window)
 				if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
 					break unless valid_beg_s2.nil?
 					valid_beg_s2 = beg_s2
@@ -125,7 +125,11 @@ class TextAlignment::AnchorFinder
 
 			# r == nil means that the inner loop was broken (multiple candidates had passed the tests)
 			# r != nil means that the inner loop was completed (with or w/o a valid beg_s2 found)
-			break unless r.nil?
+			if r.nil?
+				valid_beg_s2 = nil
+			else
+				break
+			end
 		end
 
 		valid_beg_s2

data/lib/text_alignment/char_mapping.rb

@@ -61,6 +61,7 @@ TextAlignment::CHAR_MAPPING = [
 	["•", "*"],				#U+2022 (bullet)
 	[" ", " "],				#U+2009 (thin space)
 	[" ", " "],				#U+200A (hair space)
+	[" ", " "],				#U+202F (narrow no-break space)
 	[" ", " "],				#U+00A0 (Non-Breaking space)
 	["　", " "],				#U+3000 (ideographic space)
 	["‐", "-"],				#U+2010 (Hyphen)
@@ -94,10 +95,10 @@ class TextAlignment::CharMapping
 		@index_demap[position]
 	end
 
-	def enmap_denotations(_denotations)
-		return nil if _denotations.nil?
+	def enmap_denotations(denotations)
+		return nil if denotations.nil?
 
-		denotations = _denotations.map do |d|
+		denotations.map do |d|
 			d.dup.merge(span:{begin:enmap_position(d[:span][:begin]), end:enmap_position(d[:span][:end])})
 		end
 	end
@@ -107,7 +108,7 @@ class TextAlignment::CharMapping
 	def enmap_text(_text, char_mapping)
 		text = _text.dup
 
-		# To execute the single letter mapping
+		# To execute the single letter mapping replacement
 		char_mapping.each do |one, long|
 			text.gsub!(one, long) if long.length == 1
 		end
@@ -119,7 +120,7 @@ class TextAlignment::CharMapping
 
 			init_next = 0
 			while loc = text.index(long, init_next)
-				loc_len << [loc, long.length]
+				loc_len << [loc, long.length, 1]
 				init_next = loc + long.length
 			end
 
@@ -129,9 +130,9 @@ class TextAlignment::CharMapping
 
 		# To get the (location, length) index for consecutive whitespace sequences
 		init_next = 0
-		while loc = text.index(/\s{2,}/, init_next)
+		while loc = text.index(/\s{1,}/, init_next)
 			len = $~[0].length
-			loc_len << [loc, len]
+			loc_len << [loc, len, 0]
 			init_next = loc + len
 		end
 
@@ -142,17 +143,21 @@ class TextAlignment::CharMapping
 		init_next = 0
 		j = 0
 
-		loc_len.each do |loc, len|
+		loc_len.each do |loc, old_len, new_len|
 			offset_mapping += (init_next .. loc).map do |i|
+				m = [i, j]
 				j += 1
-				[i, j - 1]
+				m
 			end
-			init_next = loc + len
+
+			init_next = loc + old_len
+			j += (new_len - 1)
 		end
 
 		offset_mapping += (init_next .. text.length).map do |i|
+			m = [i, j]
 			j += 1
-			[i, j - 1]
+			m
 		end
 
 		# To execute the long letter mapping
@@ -161,7 +166,7 @@ class TextAlignment::CharMapping
 		end
 
 		# To replace multi whitespace sequences to a space
-		text.gsub!(/\s{2,}/, ' ')
+		text.gsub!(/\s{1,}/, '')
 
 		[text, offset_mapping]
 	end
@@ -175,7 +180,7 @@ if __FILE__ == $0
 		exit
 	end
 	annotations = JSON.parse File.read(ARGV[0]).strip, symbolize_names: true
-	denotations = annotations[:denotations]
+	denotations = annotations[:denotations] || []
 	if denotations.nil? && annotations[:tracks]
 		denotations = annotations[:tracks].first[:denotations]
 	end

data/lib/text_alignment/cultivation_map.rb

@@ -41,7 +41,7 @@ class TextAlignment::CultivationMap
 	end
 
 	def next_cultivated_position(position)
-		region = @map.bsearch{|r| position < r[0]}
+		region = @map.bsearch{|r| position <= r[0]}
 		region.nil? ? nil : region[0]
 	end
 
@@ -56,7 +56,7 @@ class TextAlignment::CultivationMap
 		else
 			if front_open?(region, closed_parts)
 				if rear_open?(region, closed_parts)
-					[:middle_closed, [closed_parts.first[1], closed_parts.last[0]]]
+					[:middle_closed, [closed_parts.first[0], closed_parts.last[1]]]
 				else
 					[:front_open, [region[0], closed_parts.first[0]]]
 				end
@@ -70,7 +70,7 @@ class TextAlignment::CultivationMap
 		end
 	end
 
-	def index(target, string, position)
+	def index(target, string, position = 0)
 		length = target.length
 		loop do
 			_begin = string.index(target, position)

data/lib/text_alignment/mixed_alignment.rb

@@ -147,13 +147,25 @@ class TextAlignment::MixedAlignment
 		# recoverbility
 		count_nws =	sdiff.count{|d| d.old_element =~ /\S/}
 		count_nws_match =	sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/}
-
 		coverage = count_nws_match.to_f / count_nws
 
 		# fragmentation rate
-		count_ofrag = sdiff.count{|d| d.old_element =~ /\s/}
-		count_frag = sdiff.collect{|d| (d.action == '=') && (d.old_element =~/\s/) ? ' ' : d.action}.join.scan(/=+/).count
-		rate_frag = count_ofrag.to_f / count_frag
+		frag_str = sdiff.collect do |d|
+			case d.action
+			when '='
+				'='
+			when '-'
+				''
+			when '+'
+				# (d.new_element =~ /\S/) ? '+' : ''
+				'+'
+			else
+				''
+			end
+		end.join.sub(/^[^=]++/, '').sub(/[^=]+$/, '')
+
+		count_frag = frag_str.scan(/=+/).count
+		rate_frag = 1.0 / count_frag
 
 		similarity = coverage * rate_frag
 	end

data/lib/text_alignment/text_alignment.rb

@@ -21,7 +21,7 @@ class TextAlignment::TextAlignment
 		@to_prevent_overlap = to_prevent_overlap
 
 		@original_text = nil
-		@block_alignment = nil
+		@blocks = nil
 		@cultivation_map = TextAlignment::CultivationMap.new
 	end
 
@@ -39,45 +39,20 @@ class TextAlignment::TextAlignment
 		denotations_mapped = @text_mapping.enmap_denotations(denotations)
 
 		## To generate the block_alignment of the input text against the reference text
-		# Initialization
-		# @block_alignment = {text: @original_text, reference_text: @original_reference_text, denotations: denotations}
-		@block_alignment = {text: @mapped_text, reference_text: @mapped_reference_text, denotations: denotations}
-
-		# Generation
-		@block_alignment[:blocks] = if r = whole_block_alignment(@mapped_text, @mapped_reference_text, @cultivation_map)
+		@blocks = if r = whole_block_alignment(@mapped_text, @mapped_reference_text, @cultivation_map)
 			r
 		else
 			find_block_alignment(@mapped_text, @mapped_reference_text, denotations_mapped, @cultivation_map)
 		end
-	end
-
-	def update_cultivation_map
-		return if @block_alignment.nil? || @block_alignment[:blocks].nil?
-
-		## To update the cultivation map
-		newly_cultivated_regions = @block_alignment[:blocks].collect do |b|
-			if b[:alignment] == :block || b[:alignment] == :term
-				[b[:target][:begin], b[:target][:end]]
-			else
-				nil
-			end
-		end.compact.inject([]) do |condensed, region|
-			if condensed.empty? || (condensed.last.last + 1 < region.first)
-				condensed.push region
-			else
-				condensed.last[1] = region.last
-			end
-			condensed
-		end
 
-		@cultivation_map.cultivate(newly_cultivated_regions)
+		@block_alignment = {text: @original_text, reference_text: @original_reference_text, denotations: denotations, blocks: demap_blocks(@blocks)}
 	end
 
 	def transform_begin_position(_begin_position)
 		begin_position = @text_mapping.enmap_position(_begin_position)
 
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] > begin_position}
-		block = @block_alignment[:blocks][i]
+		i = @blocks.index{|b| b[:source][:end] > begin_position}
+		block = @blocks[i]
 
 		b = if block[:alignment] == :block || block[:alignment] == :term
 			begin_position + block[:delta]
@@ -98,8 +73,8 @@ class TextAlignment::TextAlignment
 	def transform_end_position(_end_position)
 		end_position = @text_mapping.enmap_position(_end_position)
 
-		i = @block_alignment[:blocks].index{|b| b[:source][:end] >= end_position}
-		block = @block_alignment[:blocks][i]
+		i = @blocks.index{|b| b[:source][:end] >= end_position}
+		block = @blocks[i]
 
 		e = if block[:alignment] == :block || block[:alignment] == :term
 			end_position + block[:delta]
@@ -160,8 +135,8 @@ class TextAlignment::TextAlignment
 	end
 
 	def alignment_show
-		stext = @mapped_text
-		ttext = @mapped_reference_text
+		stext = @block_alignment[:text]
+		ttext = @block_alignment[:reference_text]
 
 		show = ''
 		@block_alignment[:blocks].each do |a|
@@ -276,20 +251,32 @@ class TextAlignment::TextAlignment
 					region_state, state_region = cultivation_map.region_state([b2, e2])
 					case region_state
 					when :closed
-						[]
+						[{source:{begin:b1, end:e1}, alignment: :empty}]
 					when :front_open
-						oe2 = state_region[1]
-						me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
-						local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+						if sum.empty? # when there is no preceding matched block
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							oe2 = state_region[1]
+							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+						end
 					when :rear_open
-						ob2 = state_region[0]
-						mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
-						local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
+						if cblock.nil? # when there is no following matched block
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							ob2 = state_region[0]
+							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
+							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
+						end
 					when :middle_closed
-						oe2 = state_region[0]
-						me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
-						attempt1 = local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
-						if attempt1.empty?
+						attempt1 = if sum.empty?
+							[{source:{begin:b1, end:e1}, alignment: :empty}]
+						else
+							oe2 = state_region[0]
+							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+						end
+						if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
 							ob2 = state_region[1]
 							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
 							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
@@ -298,8 +285,12 @@ class TextAlignment::TextAlignment
 						end
 					else # :open
 						if (e2 - b2) > len_buffer
-							attempt1 = local_alignment(str1, b1, e1, str2, b2, b2 + len_buffer, denotations, cultivation_map)
-							if attempt1.empty?
+							attempt1 = if sum.empty?
+								[{source:{begin:b1, end:e1}, alignment: :empty}]
+							else
+								local_alignment(str1, b1, e1, str2, b2, b2 + len_buffer, denotations, cultivation_map)
+							end
+							if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
 								local_alignment(str1, b1, e1, str2, e2 - len_buffer, e2, denotations, cultivation_map)
 							else
 								attempt1
@@ -318,10 +309,10 @@ class TextAlignment::TextAlignment
 	end
 
 	def whole_block_alignment(str1, str2, cultivation_map)
-		block_begin = cultivation_map.index(str1, str2, 0)
+		block_begin = cultivation_map.index(str1, str2)
 		return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}] unless block_begin.nil?
 
-		block_begin = cultivation_map.index(str1.downcase, str2.downcase, 0)
+		block_begin = cultivation_map.index(str1.downcase, str2.downcase)
 		return [{source:{begin:0, end:str1.length}, target:{begin:block_begin, end:block_begin + str1.length}, delta:block_begin, alignment: :block}] unless block_begin.nil?
 
 		nil
@@ -329,7 +320,7 @@ class TextAlignment::TextAlignment
 
 	def local_alignment(str1, b1, e1, str2, b2, e2, denotations = nil, cultivation_map)
 		tblocks = term_based_alignment(str1, b1, e1, str2, b2, e2, denotations, cultivation_map)
-		if tblocks.empty?
+		if tblocks.empty? || tblocks.first[:alignment] == :empty
 			lcs_alignment(str1, b1, e1, str2, b2, e2, cultivation_map)
 		else
 			tblocks
@@ -405,4 +396,38 @@ class TextAlignment::TextAlignment
 		end
 	end
 
+	def update_cultivation_map
+		return if @blocks.nil?
+
+		## To update the cultivation map
+		newly_cultivated_regions = @blocks.collect do |b|
+			if b[:alignment] == :block || b[:alignment] == :term
+				[b[:target][:begin], b[:target][:end]]
+			else
+				nil
+			end
+		end.compact.inject([]) do |condensed, region|
+			if condensed.empty? || (condensed.last.last + 1 < region.first)
+				condensed.push region
+			else
+				condensed.last[1] = region.last
+			end
+			condensed
+		end
+
+		@cultivation_map.cultivate(newly_cultivated_regions)
+	end
+
+	def demap_blocks(_blocks)
+		return nil if _blocks.nil?
+
+		blocks = _blocks.map{|b| b.dup}
+		blocks.each do |b|
+			b[:source] = {begin:@text_mapping.demap_position(b[:source][:begin]), end:@text_mapping.demap_position(b[:source][:end])} if b[:source]
+			b[:target] = {begin:@rtext_mapping.demap_position(b[:target][:begin]), end:@rtext_mapping.demap_position(b[:target][:end])} if b[:target]
+		end
+
+		blocks
+	end
+
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.11.1'
+	VERSION = '0.11.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.11.7
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-08 00:00:00.000000000 Z
+date: 2021-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary