text_alignment 0.11.8 → 0.11.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d07269e998620f3e8f1564a4b81d3710e1898058248377224be0c5398690872f
-  data.tar.gz: 7f457820b2d5a9a9dcbf00ceb89342ca7264e2ef85afd02bd2463256e71680dd
+  metadata.gz: 5f95c7fb8bfdeb768fbd1400f7e785a2ed18016322f2cdcba3ea7196aa4e86ac
+  data.tar.gz: cad0296a218108884703af07bcbf1b303a6e12c51ab14ad872586c50cfc8e82c
 SHA512:
-  metadata.gz: 299320641fb58973b89c64f1db0c79007f43103cb063bd171f386d9118cb49257765d0652024bbd7869daa88c4f18da7338b8d60285f3e0a95258049b9e62562
-  data.tar.gz: 3bdd474509910999521c52808a535227dbf5a6aa74345e39dc0e37eae7a000b51b1df3dbb50598abc865a036a4990cee0f3ccce35471d69cb2ef4bb9633fc83a
+  metadata.gz: 5d18a8d142974967fcfe358b561c83d76d1c5d8d2a277092ed4a8b42de14cd791b004830ff9f145735e6bc0a1265295f4fd6190c674ba62b57190d7e035a863b
+  data.tar.gz: c155ff9780d5f82893825787ee39ba319ddb1f16396d6a933334a83e485ff4d2eb2840d5fc1399f9873cd69a2458f029ced0d128d9099c4f08137d360e3e2007

data/bin/align_annotations

@@ -110,20 +110,28 @@ end
 
 
 ## Options
-overlap_p = false
-debug_p = false
+options = {}
+verbose = false
 
 ## command line option processing
 require 'optparse'
 optparse = OptionParser.new do |opts|
 	opts.banner = "Usage: align_annotations [options] target_annotations(.json|.txt) reference_text(.json|.txt)"
 
-	opts.on('-o', '--overlap', 'tells it to assume there may be overlapping texts.') do
-		overlap_p = true
+	opts.on('-d', '--duplicate', 'tells it to assume there may be duplicate texts.') do
+		options[:duplicate_texts] = true
 	end
 
-	opts.on('-d', '--debug', 'tells it to show debugging information.') do
-		debug_p = true
+	opts.on('-w', '--no-whitespaces', 'tells it to ignore whitespaces.') do
+		options[:to_ignore_whitespaces] = true
+	end
+
+	opts.on('-o', '--no-order', 'tells it to ignore the order of the texts.') do
+		options[:to_ignore_text_order] = true
+	end
+
+	opts.on('-v', '--verbose', 'tells it to show the state verbosely for debugging.') do
+		verbose = true
 	end
 
 	opts.on('-h', '--help', 'displays this screen.') do
@@ -142,12 +150,12 @@ end
 source_annotations = read_annotations(ARGV[0])
 reference_text = read_text(ARGV[1])
 
-alignment = TextAlignment::TextAlignment.new(reference_text, !overlap_p)
+alignment = TextAlignment::TextAlignment.new(reference_text, options)
 
 target_annotations = if source_annotations.class == Array
-	align_mannotations(source_annotations, reference_text, alignment, debug_p)
+	align_mannotations(source_annotations, reference_text, alignment, verbose)
 else
-	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, debug_p)
+	denotations = align_denotations(source_annotations[:denotations], source_annotations[:text], alignment, verbose)
 	source_annotations.merge({text:reference_text, denotations:denotations})
 end
 

data/lib/text_alignment/anchor_finder.rb

@@ -6,17 +6,18 @@ module TextAlignment; end unless defined? TextAlignment
 
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, cultivation_map, squeeze_ws = true)
-		@method_get_left_windows, @method_get_right_windows = if squeeze_ws
-			[method(:get_left_windows), method(:get_right_windows)]
-		else
+	def initialize(source_str, target_str, cultivation_map, to_ignore_whitespaces = false, to_ignore_text_order = false)
+		@method_get_left_windows, @method_get_right_windows = if to_ignore_whitespaces
 			[method(:get_left_windows_no_squeeze_ws), method(:get_right_windows_no_squeeze_ws)]
+		else
+			[method(:get_left_windows), method(:get_right_windows)]
 		end
 
 		@s1 = source_str.downcase
 		@s2 = target_str.downcase
 
 		@cultivation_map = cultivation_map
+		@to_ignore_text_order = to_ignore_text_order
 
 		@size_ngram  = TextAlignment::SIZE_NGRAM
 		@size_window = TextAlignment::SIZE_WINDOW
@@ -71,10 +72,7 @@ class TextAlignment::AnchorFinder
 		# to get the anchor to search for in s2
 		anchor = @s1[beg_s1, @size_ngram]
 
-		# comment out below with the assumption that texts are in the same order
-		# search_position = 0
-		search_position = @pos_s2_last_match
-
+		search_position = @to_ignore_text_order ? 0 : @pos_s2_last_match
 		beg_s2_candidates = find_beg_s2_candidates(anchor, search_position)
 		return nil if beg_s2_candidates.empty?
 

data/lib/text_alignment/char_mapping.rb

@@ -80,10 +80,10 @@ TextAlignment::CHAR_MAPPING = [
 
 
 class TextAlignment::CharMapping
-	attr_reader :mapped_text
+	attr_reader :mapped_text, :index_enmap
 
-	def initialize(_text, char_mapping = nil, squeeze_ws_to = 1)
-		if squeeze_ws_to == 0
+	def initialize(_text, char_mapping = nil, to_ignore_whitespaces = false)
+		if to_ignore_whitespaces
 			@method_get_positions_squeeze_ws = method(:get_positions_squeeze_ws_0)
 			@method_squeeze_ws = method(:squeeze_ws_0!)
 		else
@@ -140,29 +140,25 @@ class TextAlignment::CharMapping
 
 		# To get the replacement positions, (position, old_length, new_length), for consecutive whitespaces
 		rpositions += @method_get_positions_squeeze_ws.call(text)
-
 		rpositions.sort!{|a, b| a[0] <=> b[0]}
 
 		# To get the offset_mapping before and after replacement
-		offset_mapping = []
-		init_next = 0
-		j = 0
-
-		rpositions.each do |loc, old_len, new_len|
-			offset_mapping += (init_next .. loc).map do |i|
-				m = [i, j]
-				j += 1
+		offset_mapping = begin
+			i, j = 0, 0
+
+			offset_mappings = rpositions.map do |loc, old_len, new_len|
+				pre_len = loc - i
+				m = (0 .. pre_len).map{|c| [i + c, j + c]}
+				i = loc + old_len
+				j += pre_len + new_len
+
 				m
 			end
 
-			init_next = loc + old_len
-			j += (new_len - 1)
-		end
+			pre_len = text.length - i
+			offset_mappings << (0 .. pre_len).map{|c| [i + c, j + c]}
 
-		offset_mapping += (init_next .. text.length).map do |i|
-			m = [i, j]
-			j += 1
-			m
+			offset_mappings.reduce(:+)
 		end
 
 		# To execute the long letter mapping
@@ -179,7 +175,7 @@ class TextAlignment::CharMapping
 	# To get squeeze positions of whitespaces to one
 	def get_positions_squeeze_ws_1(text)
 		rpositions = []
-		text.scan(/s{2,}/) do |s|
+		text.scan(/\s{2,}/) do |s|
 			loc = $~.begin(0)
 			len = $~.end(0) - loc
 			rpositions << [loc, len, 1]
@@ -189,13 +185,7 @@ class TextAlignment::CharMapping
 
 	# To get squeeze positions of whitespaces to zero
 	def get_positions_squeeze_ws_0(text)
-		rpositions = []
-		text.scan(/\s+/) do |s|
-			loc = $~.begin(0)
-			len = $~.end(0) - loc
-			rpositions << [loc, len, 0]
-		end
-		rpositions
+		text.enum_for(:scan, /\s+/).map{[b = $~.begin(0), $~.end(0) - b, 0]}
 	end
 
 	def squeeze_ws_1!(text)
@@ -210,6 +200,7 @@ end
 
 if __FILE__ == $0
 	require 'json'
+	# require 'profile'
 
 	unless ARGV.length == 1
 		warn "#{$0} an_annotation_json_file.json" 
@@ -221,7 +212,8 @@ if __FILE__ == $0
 		denotations = annotations[:tracks].first[:denotations]
 	end
 
-	text_mapping = TextAlignment::CharMapping.new(annotations[:text])
+	text_mapping = TextAlignment::CharMapping.new(annotations[:text], nil, false)
+	# text_mapping = TextAlignment::CharMapping.new(annotations[:text], nil, true)
 	text_mapped = text_mapping.mapped_text
 	denotations_mapped = text_mapping.enmap_denotations(denotations)
 	new_annotations = {text:text_mapped, denotations:denotations_mapped}

data/lib/text_alignment/text_alignment.rb

@@ -16,11 +16,12 @@ class TextAlignment::TextAlignment
 		raise ArgumentError, "nil text" if reference_text.nil?
 
 		options ||= {}
-		@to_prevent_overlap = options[:to_prevent_overlap] || false
-		@squeeze_ws_to = options[:squeeze_ws_to] || 0
+		@duplicate_texts = options[:duplicate_texts] || false
+		@to_ignore_whitespaces = options[:to_ignore_whitespaces] || false
+		@to_ignore_text_order = options[:to_ignore_text_order] || false
 
 		@original_reference_text = reference_text
-		@rtext_mapping = TextAlignment::CharMapping.new(reference_text, nil, @squeeze_ws_to)
+		@rtext_mapping = TextAlignment::CharMapping.new(reference_text, nil, @to_ignore_whitespaces)
 		@mapped_reference_text = @rtext_mapping.mapped_text
 
 		@original_text = nil
@@ -30,12 +31,12 @@ class TextAlignment::TextAlignment
 
 	def align(text, denotations = nil)
 		# To maintain the cultivation map
-		update_cultivation_map if @to_prevent_overlap
+		update_cultivation_map unless @duplicate_texts
 
 		# In case the input text is the same as the previous one, reuse the previous text mapping
 		unless @original_text && @original_text == text
 			@original_text = text
-			@text_mapping = TextAlignment::CharMapping.new(text, nil, @squeeze_ws_to)
+			@text_mapping = TextAlignment::CharMapping.new(text, nil, @to_ignore_whitespaces)
 		end
 
 		@mapped_text = @text_mapping.mapped_text
@@ -205,7 +206,7 @@ class TextAlignment::TextAlignment
 
 	def find_block_alignment(str1, str2, denotations, cultivation_map)
 		## to find block alignments
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, cultivation_map, @squeeze_ws_to == 1)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, cultivation_map, @to_ignore_whitespaces, @to_ignore_text_order)
 
 		blocks = []
 		while block = anchor_finder.get_next_anchor
@@ -241,68 +242,75 @@ class TextAlignment::TextAlignment
 			b1 = lblock.nil? ? 0 : lblock[:source][:end]
 			e1 = cblock.nil? ? str1.length : cblock[:source][:begin]
 
-			if b1 < e1
+			if b1 <= e1
+				_str1 = str1[b1 ... e1]
+
 				b2 = lblock.nil? ? 0 : lblock[:target][:end]
 				e2 = cblock.nil? ? str2.length : cblock[:target][:begin]
-				_str1 = str1[b1 ... e1]
-				_str2 = str2[b2 ... e2]
 
-				sum += if _str1.strip.empty? || _str2.strip.empty?
-					[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
-				else
-					len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
-					region_state, state_region = cultivation_map.region_state([b2, e2])
-					case region_state
-					when :closed
-						[{source:{begin:b1, end:e1}, alignment: :empty}]
-					when :front_open
-						if sum.empty? # when there is no preceding matched block
-							[{source:{begin:b1, end:e1}, alignment: :empty}]
-						else
-							oe2 = state_region[1]
-							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
-							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
-						end
-					when :rear_open
-						if cblock.nil? # when there is no following matched block
-							[{source:{begin:b1, end:e1}, alignment: :empty}]
-						else
-							ob2 = state_region[0]
-							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
-							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
-						end
-					when :middle_closed
-						attempt1 = if sum.empty?
+				if b2 < e2
+					_str2 = str2[b2 ... e2]
+
+					sum += if _str1.strip.empty? || _str2.strip.empty?
+						[{source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}]
+					else
+						len_buffer = ((e1 - b1) * (1 + TextAlignment::BUFFER_RATE)).to_i + TextAlignment::BUFFER_MIN
+						region_state, state_region = cultivation_map.region_state([b2, e2])
+						case region_state
+						when :closed
 							[{source:{begin:b1, end:e1}, alignment: :empty}]
-						else
-							oe2 = state_region[0]
-							me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
-							local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
-						end
-						if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
-							ob2 = state_region[1]
-							mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
-							local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
-						else
-							attempt1
-						end
-					else # :open
-						if (e2 - b2) > len_buffer
+						when :front_open
+							if sum.empty? # when there is no preceding matched block
+								[{source:{begin:b1, end:e1}, alignment: :empty}]
+							else
+								oe2 = state_region[1]
+								me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+								local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
+							end
+						when :rear_open
+							if cblock.nil? # when there is no following matched block
+								[{source:{begin:b1, end:e1}, alignment: :empty}]
+							else
+								ob2 = state_region[0]
+								mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
+								local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
+							end
+						when :middle_closed
 							attempt1 = if sum.empty?
 								[{source:{begin:b1, end:e1}, alignment: :empty}]
 							else
-								local_alignment(str1, b1, e1, str2, b2, b2 + len_buffer, denotations, cultivation_map)
+								oe2 = state_region[0]
+								me2 = (oe2 - b2) > len_buffer ? b2 + len_buffer : oe2
+								local_alignment(str1, b1, e1, str2, b2, me2, denotations, cultivation_map)
 							end
 							if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
-								local_alignment(str1, b1, e1, str2, e2 - len_buffer, e2, denotations, cultivation_map)
+								ob2 = state_region[1]
+								mb2 = (e2 - ob2) > len_buffer ? e2 - len_buffer : ob2
+								local_alignment(str1, b1, e1, str2, mb2, e2, denotations, cultivation_map)
 							else
 								attempt1
 							end
-						else
-							local_alignment(str1, b1, e1, str2, b2, e2, denotations, cultivation_map)
+						else # :open
+							if (e2 - b2) > len_buffer
+								attempt1 = if sum.empty?
+									[{source:{begin:b1, end:e1}, alignment: :empty}]
+								else
+									local_alignment(str1, b1, e1, str2, b2, b2 + len_buffer, denotations, cultivation_map)
+								end
+								if (attempt1.empty? || attempt1.first[:alignment] == :empty) && !cblock.nil?
+									local_alignment(str1, b1, e1, str2, e2 - len_buffer, e2, denotations, cultivation_map)
+								else
+									attempt1
+								end
+							else
+								local_alignment(str1, b1, e1, str2, b2, e2, denotations, cultivation_map)
+							end
 						end
 					end
+				elsif b2 > e2 # when out of order
+					# ToDo
 				end
+
 			end
 
 			lblock = cblock

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.11.8'
+	VERSION = '0.11.10'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.11.8
+  version: 0.11.10
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-06 00:00:00.000000000 Z
+date: 2021-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -111,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.0.9
 signing_key: 
 specification_version: 4
 summary: Ruby class for aligning two character strings