text_alignment 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4b2cdf0c257b74c6bec90b93d1907787f3c102108046731c2755684a1b156e9
-  data.tar.gz: 85334dad09a046432503183e3d3ad83841612299038f2f2dac1f9d5d208e1939
+  metadata.gz: 6bed1eba72da626227ab727ce22129d226539bcfae5ca22006ac26258b184d8c
+  data.tar.gz: d2c121ea072186fd25fd61fb90c5ffacb886c1d109b82c044a1666220b8f7d8b
 SHA512:
-  metadata.gz: 9272bdd6c56717b53d39b3f2009259accb608ea86b99758b6a7ee9cee1e7b275330db55af4e0eba1eba80ee69275a21a3179243394d24139b3018996f659abe1
-  data.tar.gz: a6a9d97d2bf81ac0c2972fd6e9d5202116156d8ff2e5e81a9bf0306e313dbc601522f887bcbcebff8b9d888cc06826a8ce69ba908dce29fa8decad85d53008af
+  metadata.gz: 6e526995325e79fdde8ecd729c04e2e6a21e13f0166acc39b341133055275a1bbd5a3318f78dd5af4a72237c140fa8eb06270441a16e2426e58a57183b91ca6a
+  data.tar.gz: ec423d59036b1ee5595141428fe320f0e9ca16b8b2660d46a0f59f376c3845ad70196d006c2f83390ac12f98b35ff14a1098fcd24cda0ee1c6534f36915def81

data/lib/text_alignment/mixed_alignment.rb

@@ -17,9 +17,10 @@ class TextAlignment::MixedAlignment
 	attr_reader :similarity
 	attr_reader :str1_match_initial, :str1_match_final, :str2_match_initial, :str2_match_final
 
-	def initialize(str1, str2, mappings = [])
-		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
-		mappings ||= []
+	def initialize(_str1, _str2)
+		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
+
+		str1, str2, mappings = string_preprocessing(_str1, _str2)
 
 		_compute_mixed_alignment(str1, str2, mappings)
 	end
@@ -62,7 +63,7 @@ class TextAlignment::MixedAlignment
 		end
 
 		cmp = TextAlignment::LCSComparison.new(str1, str2, lcs, @sdiff)
-		@similarity         = cmp.similarity
+		@similarity         = compute_similarity(str1, str2, @sdiff)
 		@str1_match_initial = cmp.str1_match_initial
 		@str1_match_final   = cmp.str1_match_final
 		@str2_match_initial = cmp.str2_match_initial
@@ -137,4 +138,73 @@ class TextAlignment::MixedAlignment
 		@position_map_begin = posmap_begin.sort.to_h
 		@position_map_end = posmap_end.sort.to_h
 	end
+
+	private
+
+	def string_preprocessing(_str1, _str2)
+		str1 = _str1.dup
+		str2 = _str2.dup
+		mappings = TextAlignment::MAPPINGS.dup
+
+		## single character mappings
+		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+		characters_from = character_mappings.collect{|m| m[0]}.join
+		characters_to   = character_mappings.collect{|m| m[1]}.join
+		characters_to.gsub!(/-/, '\-')
+
+		str1.tr!(characters_from, characters_to)
+		str2.tr!(characters_from, characters_to)
+
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+		## long to one character mappings
+		pletters = TextAlignment::PADDING_LETTERS
+
+		# find the padding letter for str1
+		@padding_letter1 = begin
+			i = pletters.index{|l| str2.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# find the padding letter for str2
+		@padding_letter2 = begin
+			i = pletters.index{|l| l != @padding_letter1 && str1.index(l).nil?}
+			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
+			TextAlignment::PADDING_LETTERS[i]
+		end
+
+		# ASCII foldings
+		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+		ascii_foldings.each do |f|
+			from = f[1]
+
+			if str2.index(f[0])
+				to   = f[0] + (@padding_letter1 * (f[1].length - 1))
+				str1.gsub!(from, to)
+			end
+
+			if str1.index(f[0])
+				to   = f[0] + (@padding_letter2 * (f[1].length - 1))
+				str2.gsub!(from, to)
+			end
+		end
+		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+		[str1, str2, mappings]
+	end
+
+	def compute_similarity(_s1, _s2, sdiff)
+		return 0 if sdiff.nil?
+
+		# compute the lcs only with non-whitespace letters
+		lcs = sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
+		return 0 if lcs == 0
+
+		s1 = _s1.tr(@padding_letter1, ' ')
+		s2 = _s2.tr(@padding_letter2, ' ')
+
+		similarity = lcs / [s1.scan(/\S/).count, s2.scan(/\S/).count].min.to_f
+	end
+
 end

data/lib/text_alignment/text_alignment.rb

@@ -12,12 +12,10 @@ class TextAlignment::TextAlignment
 	attr_reader :similarity
 	attr_reader :lost_annotations
 
-	def initialize(_str1, _str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
-		raise ArgumentError, "nil string" if _str1.nil? || _str2.nil?
+	def initialize(str1, str2, _size_ngram = nil, _size_window = nil, _text_similiarity_threshold = nil)
+		raise ArgumentError, "nil string" if str1.nil? || str2.nil?
 
-		@block_alignment = {source_text:_str1, target_text:_str2}
-
-		str1, str2, mappings = string_preprocessing(_str1, _str2)
+		@block_alignment = {source_text:str1, target_text:str2}
 
 		# try exact match
 		block_begin = str2.index(str1)
@@ -90,12 +88,11 @@ class TextAlignment::TextAlignment
 
 						_str1 = str1[b1 ... e1]
 						_str2 = str2[b2 ... e2]
-						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-						similarity = alignment_similarity(_str1, _str2, alignment)
-						if similarity < 0.6
-							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty, similarity: similarity}
+						alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase)
+						if alignment.similarity < 0.5
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment: :empty, similarity: alignment.similarity}
 						else
-							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment}
+							@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:0, end:e2}, alignment:alignment, similarity: alignment.similarity}
 						end
 					end
 				end
@@ -114,12 +111,11 @@ class TextAlignment::TextAlignment
 				if _str2.strip.empty?
 					@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty}
 				else
-					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					similarity = alignment_similarity(_str1, _str2, alignment)
-					if similarity < 0.6
-						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
+					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase)
+					if alignment.similarity < 0.5
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: alignment.similarity}
 					else
-						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment, similarity: alignment.similarity}
 					end
 				end
 			end
@@ -144,12 +140,11 @@ class TextAlignment::TextAlignment
 					_str1 = str1[b1 ... e1]
 					_str2 = str2[b2 ... e2]
 
-					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase, mappings)
-					similarity = alignment_similarity(_str1, _str2, alignment)
-					if similarity < 0.6
-						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: similarity}
+					alignment = TextAlignment::MixedAlignment.new(_str1.downcase, _str2.downcase)
+					if alignment.similarity < 0.5
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment: :empty, similarity: alignment.similarity}
 					else
-						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment}
+						@block_alignment[:blocks] << {source:{begin:b1, end:e1}, target:{begin:b2, end:e2}, alignment:alignment, similarity: alignment.similarity}
 					end
 
 					@block_alignment[:blocks] << {source:{begin:e1, end:-1}, target:{begin:e2, end:-1}, alignment: :empty} if e1 < str1.length
@@ -250,13 +245,13 @@ class TextAlignment::TextAlignment
 		@block_alignment[:blocks].each do |a|
 			show += case a[:alignment]
 			when :block
-				"===== common =====\n" +
+				"===== common ===== [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
 				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n"
 			when :empty
 				"xxxxx disparate texts (similarity: #{a[:similarity]})\n" +
-				"<<<<< string 1\n" +
+				"<<<<< string 1 [#{a[:source][:begin]} - #{a[:source][:end]}]\n" +
 				stext[a[:source][:begin] ... a[:source][:end]] + "\n\n" +
-				">>>>> string 2\n" +
+				">>>>> string 2 [#{a[:target][:begin]} - #{a[:target][:end]}]\n" +
 				ttext[a[:target][:begin] ... a[:target][:end]] + "\n\n"
 			else
 				astr1 = ''
@@ -290,7 +285,7 @@ class TextAlignment::TextAlignment
 					end
 				end.join('')
 
-				"***** local mismatch\n" +
+				"***** local mismatch [#{a[:source][:begin]} - #{a[:source][:end]}] [#{a[:target][:begin]} - #{a[:target][:end]}] (similarity: #{a[:similarity]})\n" +
 				"[#{astr1}]\n" +
 				"[#{astr2}]\n\n"
 			end
@@ -298,71 +293,4 @@ class TextAlignment::TextAlignment
 		show
 	end
 
-	private
-
-	def string_preprocessing(_str1, _str2)
-		str1 = _str1.dup
-		str2 = _str2.dup
-		mappings = TextAlignment::MAPPINGS.dup
-
-		## single character mappings
-		character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
-		characters_from = character_mappings.collect{|m| m[0]}.join
-		characters_to   = character_mappings.collect{|m| m[1]}.join
-		characters_to.gsub!(/-/, '\-')
-
-		str1.tr!(characters_from, characters_to)
-		str2.tr!(characters_from, characters_to)
-
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
-
-		## long to one character mappings
-		pletters = TextAlignment::PADDING_LETTERS
-
-		# find the padding letter for str1
-		@padding_letter1 = begin
-			i = pletters.index{|l| str2.index(l).nil?}
-			raise RuntimeError, "Could not find a padding letter for str1" if i.nil?
-			TextAlignment::PADDING_LETTERS[i]
-		end
-
-		# find the padding letter for str2
-		@padding_letter2 = begin
-			i = pletters.index{|l| l != @padding_letter1 && str1.index(l).nil?}
-			raise RuntimeError, "Could not find a padding letter for str2" if i.nil?
-			TextAlignment::PADDING_LETTERS[i]
-		end
-
-		# ASCII foldings
-		ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
-		ascii_foldings.each do |f|
-			from = f[1]
-
-			if str2.index(f[0])
-				to   = f[0] + (@padding_letter1 * (f[1].length - 1))
-				str1.gsub!(from, to)
-			end
-
-			if str1.index(f[0])
-				to   = f[0] + (@padding_letter2 * (f[1].length - 1))
-				str2.gsub!(from, to)
-			end
-		end
-		mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
-
-		[str1, str2, mappings]
-	end
-
-	def alignment_similarity(_s1, _s2, alignment)
-		return 0 if alignment.sdiff.nil?
-
-		# compute the lcs only with non-whitespace letters
-		lcs = alignment.sdiff.count{|d| d.action == '=' && d.old_element =~ /\S/ && d.new_element =~ /\S/}
-
-		s1 = _s1.tr(@padding_letter1, ' ')
-		s2 = _s2.tr(@padding_letter2, ' ')
-
-		similarity = 2 * lcs / (s1.scan(/\S/).count + s2.scan(/\S/).count).to_f
-	end
-
 end

data/lib/text_alignment/version.rb

@@ -1,3 +1,3 @@
 class TextAlignment
-	VERSION = '0.6.2'
+	VERSION = '0.6.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: text_alignment
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Jin-Dong Kim