text_sentencer 1.0.3 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8dda6323a11356aa2f6c78fa0281658ea677f09fbf0bfb19b19330f732e72c1
-  data.tar.gz: a41e5cc5ae3e1e294e9ba4921993e2d98003d74435fd5bc099c6c6fbf188554a
+  metadata.gz: 13537fd663911db81cc5d4068c4c3d8439d332c4742e9d027c6687031f44f32f
+  data.tar.gz: dbd6aacd4e7dfca434c27608fa747bb7fbd8d43bf62af682f0e55784fcccf748
 SHA512:
-  metadata.gz: b39ec569c1e988f4e7936385395ff1e34c15cb7721ffba6fb2453f633db360f092b5225d6ec4603b2beb391e2660291e9d48e912359eb33db9d2ccd1906d806f
-  data.tar.gz: 2780a4c3f6ac1fc7f0ab06e658a293b7e9a7d3f5a2df0a0904fa20aea187857c9f3bad7cc7adfb6f1eba4c57897f433c03d77a958559e09225b60c154c9488b3
+  metadata.gz: 9758707a8204ebe212e699f2f3dd2011d3ad0fdf355cde4c85df5521d17aa6763df593915b88890aad9c29c10903bbada40be9ec66cc082305fdf5b614c252bb
+  data.tar.gz: '09938a12984f7d33c6337adcaf59af9087ce2969d5d98b37b3e73d833d015a49950b6d7b65bc845dc0820727ea4d2b5af61a8bb0af7ce5201b7531748f8772a8'

data/bin/text_sentencer

@@ -38,8 +38,10 @@ annotations = sentencer.annotate(text)
 if output_mode == :json
 	puts JSON.pretty_generate(annotations)
 else
-	annotations[:denotations].each do |d|
-		span = d[:span]
-		puts text[span[:begin]...span[:end]]
+	if annotations.has_key?(:blocks)
+		annotations[:blocks].each do |d|
+			span = d[:span]
+			puts '[' + text[span[:begin]...span[:end]] + ']'
+		end
 	end
 end

data/lib/text_sentencer/text_sentencer.rb

@@ -7,111 +7,101 @@ class TextSentencer
 
 	DEFAULT_RULES = {
 		# All the positions of new line characters always take sentence break.
-		break_pattern: "([ \t]*\n+)+[ \t]*",
+		break_pattern: /([ \t]*\n+)+[ \t]*/,
 
 		# All the positions of space and tab characters are candiates of sentence break.
-		candidate_pattern: "[ \t]+",
+		candidate_pattern: /[ \t\n]+/,
 
 		# First, positive rules are applied to the break candidates to make initial segmantations.
 		positive_rules: [
-			['[.!?]', '[0-9A-Z]'],
-			['[:]', '[0-9]'],
-			['[:]', '[A-Z][a-z]']
+			[/[.!?]\z/, /\A[0-9A-Z]/],
+			[/[:]\z/, /\A[0-9]/],
+			[/[:]\z/, /\A[A-Z][a-z]/]
 		],
 
 		# Then, negative rules are applied to cancel some initial segmentations.
 		negative_rules: [
 			# Titles before names
-			['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
+			[/(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.\z/, /\A[A-Z][a-z]/],
 
 			# Titles usually before names, but ..
-			['(Sr|Jr)\.', '[A-Z][a-z]'],
+			[/(Sr|Jr)\.\z/, /\A[A-Z][a-z]/],
 
 			# Single letter abbriveations, e.g. middle name
-			# ['\b[A-Z]\.', '[A-Z][a-z]'],
+			# [/\b[A-Z]\.\z/, /\A[A-Z][a-z]/],
 
 			# Abbriveations, e.g. middle name
-			['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
+			[/\b[A-Z][a-z]*\.\z/, /\A[0-9A-Z]/],
 
 			# Frequent abbreviations that will never appear in the end of a sentence
-			['(cf|vs)\.', ''],
-			['e\.g\.', ''],
-			['i\.e\.', ''],
+			[/(cf|vs)\.\z/, //],
+			[/e\.g\.\z/, //],
+			[/i\.e\.\z/, //],
 
 			# Others
-			['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
+			[/(Sec|Chap|Fig|Eq)\.\z/, /\A[0-9A-Z]/]
 		]
 	}
 
 	def initialize(rules = nil)
 		rules ||= DEFAULT_RULES
 		@rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
-		@rules[:break_pattern] ||= ""
-		@rules[:candidate_pattern] ||= ""
+		@rules[:break_pattern] ||= nil
+		@rules[:candidate_pattern] ||= nil
 		@rules[:positive_rules] ||= []
 		@rules[:negative_rules] ||= []
 	end
 
 	def annotate(text)
-		return nil if text.nil?
-
 		sentences = segment(text)
-		denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
-		{text:text, denotations:denotations}
+		blocks = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
+		{text:text, blocks:blocks}
 	end
 
 	def segment(text)
-		breaks = if @rules[:break_pattern].empty?
+		# hard sentence breaks
+		breaks = if @rules[:break_pattern].nil?
 			[]
 		else
-			text.scan_offset(/#{@rules[:break_pattern]}/).map{|m| m.offset(0)}
+			text.scan_offset(@rules[:break_pattern]).map{|m| m.offset(0)}
 		end
 
-		candidates = if @rules[:candidate_pattern].empty?
+		# candidates of sentence breaks
+		candidates = if @rules[:candidate_pattern].nil?
 			[]
 		else
-			text.scan_offset(/#{@rules[:candidate_pattern]}/).map{|m| m.offset(0)}
+			text.scan_offset(@rules[:candidate_pattern]).map{|m| m.offset(0)}
 		end
 
-		# breaks take precedent
+		# hard sentence breaks are already in
 		candidates -= breaks
 
-		candidates.each do |c|
-			last_end, next_begin = c
-
-			if (last_end == 0) || (next_begin == text.length)
-				breaks << c
-				next
+		# filter using the positive rules. The first and last breaks remain.
+		candidates.select! do |c|
+			c[0] == 0 ||
+			c[1] == text.length ||
+			@rules[:positive_rules].any? do |r|
+				(text[0...c[0]] =~ r[0]) && (text[c[1]...-1] =~ r[1])
 			end
+		end
 
-			last_text = text[0...last_end]
-			next_text = text[next_begin..-1]
-
-			@rules[:positive_rules].each do |p|
-				if (last_text =~ /#{p[0]}\Z/) && (next_text =~ /\A#{p[1]}/)
-					break_p = true
-					@rules[:negative_rules].each do |n|
-						if (last_text =~ /#{n[0]}\Z/) && (next_text =~ /\A#{n[1]}/)
-							break_p = false
-							break
-						end
-					end
-					breaks << c if break_p
-					break
-				end
+		# filter using the negative rules
+		candidates.reject! do |c|
+			@rules[:negative_rules].any? do |r|
+				(text[0...c[0]] =~ r[0]) && (text[c[1]...-1] =~ r[1])
 			end
 		end
 
+		# add all the filtered candidates to sentence breaks
+		breaks += candidates
+		breaks.uniq!
 		breaks.sort!
 
-		sentences = []
-		lastbreak = 0
-		breaks.each do |b|
-			sentences << [lastbreak, b[0]] if b[0] > lastbreak
-			lastbreak = b[1]
-		end
-		sentences << [lastbreak, text.length] if lastbreak < text.length
+		# add the initial and final breaks unless already exist
+		breaks.unshift([0, 0]) if breaks.empty? || breaks[0][0] != 0
+		breaks.push([text.length, text.length]) unless breaks[-1][1] == text.length
 
-		sentences
+		# conver the breaks into sentences
+		sentences = (1 ... breaks.length).map{|i| [breaks[i-1][1], breaks[i][0]]}
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_sentencer
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 3.0.0
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-17 00:00:00.000000000 Z
+date: 2024-08-03 00:00:00.000000000 Z
 dependencies: []
 description: TextSentencer is a simple rule-based system for segmenting text into
   sentences.
@@ -41,7 +41,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: A simple, rule-based script to find sentence boundaries in text.