text_alignment 0.3.21 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65e1d9b45ff59ac0a233b7656d2aca99d7e4e1051b1a03a0c7726521d4f2b280
-  data.tar.gz: 710a3b68c5263f26572727e6e9591ebd5fdb095af4633bd5037c61eae0bb5cb6
+  metadata.gz: d56b44ee3cc3451b78ddb87134fefd6c043b4730f1170b07069a417bb162bad1
+  data.tar.gz: 67318d70bee5bb82127c98e79e0b45e5e337832a2b8d23469474215168579f20
 SHA512:
-  metadata.gz: 598df22e41bbbe0a84b6e1a6a4e631ab0d8166810afd652086595feecbf0808a886685f42e5466626cbb1d6950dd9f1181be776b9938d6174dc7735c3ace24cd
-  data.tar.gz: f7dedfb7e64919129f816fbba24dbd1c2e2a056c242a0865915b8a611f594399b17d051d004a846796bba1c2e89c6fb2f17116cd118ca6217cf1a5dff4f6d4d8
+  metadata.gz: 60ae2292556d15d860e0c7377be9c1fd5aec6741b26f562fee8c0c2a9a123d0b8c656b2f7f34b8f74f83ea9ac4878b53f17454e3b25d27d00cd98ecf8175a387
+  data.tar.gz: fe8fe545101cddb638c00ffb6f6785f8b2f5ddb109b4010739e399a2c50fe41164d791fe4a5599a76fca5553c494f78fd49630368d48e406a8211fc0516916f5

data/bin/align_annotations

@@ -108,7 +108,7 @@ else
 	# verification
 	source_text = source_annotations[:text]
 	puts "=====BEGIN"
-	(0 ... source_text.length).each do |p|
+	(0 ... source_text.rstrip.length).each do |p|
 		t = alignment.transform_begin_position(p)
 		if t.nil?
 			print source_text[p]
@@ -120,7 +120,7 @@ else
 	puts "=====END"
 
 	puts "=====BEGIN"
-	(0 .. source_text.length).each do |p|
+	(0 .. source_text.rstrip.length).each do |p|
 		t = alignment.transform_end_position(p)
 		if t.nil?
 			print source_text[p]

data/lib/text_alignment/anchor_finder.rb

@@ -1,17 +1,15 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
-TextAlignment::SIZE_WINDOW = 40 unless defined? TextAlignment::SIZE_WINDOW
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.8 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
-
 class TextAlignment::AnchorFinder
 
-	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil)
+	def initialize(source_str, target_str, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		@size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
 		@size_window = _size_window || TextAlignment::SIZE_WINDOW
+		@sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
 
 		@reverse = (target_str.length < source_str.length)
 
@@ -30,6 +28,10 @@ class TextAlignment::AnchorFinder
 	def get_next_anchor
 		# find the position of an anchor ngram in s1 and s2
 		while @beg_s1 < (@s1.length - @size_ngram)
+			if [' ', "\n", "\t"].include? @s1[@beg_s1]
+				@beg_s1 += 1
+				next
+			end
 			anchor = @s1[@beg_s1, @size_ngram]
 
 			# search_position = 0
@@ -39,10 +41,10 @@ class TextAlignment::AnchorFinder
 				break if @beg_s1 > 0 && @beg_s2 > 0 && (@beg_s1 - @end_s1_prev < 5) && (@beg_s2 >= @end_s2_prev) && (@beg_s2 - @end_s2_prev < 5)
 
 				left_window_s1, left_window_s2 = get_left_windows
-				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if left_window_s1 && (text_similarity(left_window_s1, left_window_s2) > @sim_threshold)
 
 				right_window_s1, right_window_s2 = get_right_windows
-				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > TextAlignment::TEXT_SIMILARITY_TRESHOLD)
+				break if right_window_s2 && (text_similarity(right_window_s1, right_window_s2) > @sim_threshold)
 
 				search_position = @beg_s2 + 1
 			end
@@ -57,9 +59,10 @@ class TextAlignment::AnchorFinder
 		# extend the block
 		b1 = @beg_s1
 		b2 = @beg_s2
-		while b1 >= @end_s1_prev && b2 > -1 && @s1[b1] == @s2[b2]
+		while b1 >= @end_s1_prev && b2 >= @end_s2_prev && @s1[b1] == @s2[b2]
 			b1 -= 1; b2 -= 1
 		end
+
 		b1 += 1; b2 += 1
 
 		e1 = @beg_s1 + @size_ngram
@@ -82,7 +85,8 @@ class TextAlignment::AnchorFinder
 	private
 
 	def get_left_windows
-		return if @beg_s1 < @size_window || @beg_s2 < @size_window
+		# commend below with the assumption that the beginning of a document gives a significant locational information
+		# return if @beg_s1 < @size_window || @beg_s2 < @size_window
 
 		window_s1 = ''
 		loc = @beg_s1 - 1
@@ -110,7 +114,8 @@ class TextAlignment::AnchorFinder
 	end
 
 	def get_right_windows
-		return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
+		# commend below with the assumption that the end of a document gives a significant locational
+		# return if (@beg_s1 + @size_ngram > (@s1.length - @size_window)) || (@beg_s2 + @size_ngram > (@s2.length - @size_window))
 
 		window_s1 = ''
 		loc = @beg_s1 + @size_ngram

data/lib/text_alignment/approximate_fit.rb

@@ -1,13 +1,11 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'string-similarity'
 
 module TextAlignment; end unless defined? TextAlignment
 
 # approximate the location of str1 in str2
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
 TextAlignment::MIN_LENGTH_FOR_APPROXIMATION = 50 unless defined? TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::TEXT_SIMILARITY_TRESHOLD = 0.7 unless defined? TextAlignment::TEXT_SIMILARITY_TRESHOLD
 
 class << TextAlignment
 
@@ -16,8 +14,8 @@ class << TextAlignment
 		raise ArgumentError, 'nil string' if str1.nil? || str2.nil?
 		return 0, str2.length if str2.length < TextAlignment::MIN_LENGTH_FOR_APPROXIMATION
 
-		ngram1 = (0 .. str1.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str1[i, TextAlignment::SIGNATURE_NGRAM]}
-		ngram2 = (0 .. str2.length - TextAlignment::SIGNATURE_NGRAM).collect{|i| str2[i, TextAlignment::SIGNATURE_NGRAM]}
+		ngram1 = (0 .. str1.length - TextAlignment::SIZE_NGRAM).collect{|i| str1[i, TextAlignment::SIZE_NGRAM]}
+		ngram2 = (0 .. str2.length - TextAlignment::SIZE_NGRAM).collect{|i| str2[i, TextAlignment::SIZE_NGRAM]}
 		ngram_shared = ngram1 & ngram2
 
 		# If there is no shared n-gram found, it may mean there is no serious overlap between the two strings
@@ -45,7 +43,7 @@ class << TextAlignment
 			text_similarity = text_similarity(str1, str2[fit_begin ... fit_end])
 			cache["#{fit_begin}-#{fit_end}"] = text_similarity
 
-			break if text_similarity > TextAlignment::TEXT_SIMILARITY_TRESHOLD
+			break if text_similarity > TextAlignment::TEXT_SIMILARITY_THRESHOLD
 			fit_begin, fit_end = nil, nil
 		end
 		return fit_begin, fit_end if fit_begin && fit_end && fit_begin < fit_end

data/lib/text_alignment/constants.rb

@@ -0,0 +1,7 @@
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::SIZE_NGRAM = 8 unless defined? TextAlignment::SIZE_NGRAM
+TextAlignment::SIZE_WINDOW = 60 unless defined? TextAlignment::SIZE_WINDOW
+TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
+TextAlignment::BUFFER_MIN = 20 unless defined? TextAlignment::BUFFER_MIN
+TextAlignment::TEXT_SIMILARITY_THRESHOLD = 0.87 unless defined? TextAlignment::TEXT_SIMILARITY_THRESHOLD

data/lib/text_alignment/glcs_alignment_fast.rb

@@ -9,8 +9,6 @@ require 'text_alignment/mappings'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 5 unless defined? TextAlignment::SIGNATURE_NGRAM
-
 class TextAlignment::GLCSTextAlignment
 	attr_reader :position_map_begin, :position_map_end
 	attr_reader :common_elements, :mapped_elements

data/lib/text_alignment/text_alignment.rb

@@ -1,22 +1,22 @@
 #!/usr/bin/env ruby
+require 'text_alignment/constants'
 require 'text_alignment/anchor_finder'
 require 'text_alignment/mixed_alignment'
 
 module TextAlignment; end unless defined? TextAlignment
 
-TextAlignment::SIGNATURE_NGRAM = 7 unless defined? TextAlignment::SIGNATURE_NGRAM
-TextAlignment::BUFFER_RATE = 0.1 unless defined? TextAlignment::BUFFER_RATE
-TextAlignment::BUFFER_MIN = 10 unless defined? TextAlignment::BUFFER_MIN
-
-
 class TextAlignment::TextAlignment
 	attr_reader :block_alignments
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(str1, str2, mappings = nil)
+	def initialize(str1, str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
 		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
 
+		size_ngram  = _size_ngram  || TextAlignment::SIZE_NGRAM
+		size_window = _size_window || TextAlignment::SIZE_WINDOW
+		sim_threshold = _text_similiarity_threshold || TextAlignment::TEXT_SIMILARITY_THRESHOLD
+
 		mappings ||= TextAlignment::MAPPINGS
 
 		# try exact match
@@ -26,7 +26,7 @@ class TextAlignment::TextAlignment
 			return @block_alignments
 		end
 
-		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2)
+		anchor_finder = TextAlignment::AnchorFinder.new(str1, str2, size_ngram, size_window, sim_threshold)
 
 		# To collect matched blocks
 		mblocks = []

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.3.21'
+	VERSION = '0.4.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.3.21
+  version: 0.4.2
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-22 00:00:00.000000000 Z
+date: 2020-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-dictionary
@@ -77,6 +77,7 @@ files:
 - lib/text_alignment.rb
 - lib/text_alignment/anchor_finder.rb
 - lib/text_alignment/approximate_fit.rb
+- lib/text_alignment/constants.rb
 - lib/text_alignment/find_divisions.rb
 - lib/text_alignment/glcs_alignment.rb
 - lib/text_alignment/glcs_alignment_fast.rb