text_sentencer 1.0.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7008e00d8e1554e27e82bbf744eb74bf10704234
-  data.tar.gz: e863a486fe7f932c6354abd314c6e9690f4b1ece
+SHA256:
+  metadata.gz: 13537fd663911db81cc5d4068c4c3d8439d332c4742e9d027c6687031f44f32f
+  data.tar.gz: dbd6aacd4e7dfca434c27608fa747bb7fbd8d43bf62af682f0e55784fcccf748
 SHA512:
-  metadata.gz: a2762e82db9eece26fd97ec7ac3c44f13ad3fe9babe2910238d1ed5e4a616d3afbaac3ff421d6dda354d3ceb708ca0a5ce7877563f253bf726599f898f0a5f44
-  data.tar.gz: 9ee5c91045041905f61218944388157fdc7ffb53b6d7f8bd784ef8a286aa2acdf6cca524fb31f4a1e0725466bdbe130de55ef4b00535deb7bf5ae1b65c6ed49b
+  metadata.gz: 9758707a8204ebe212e699f2f3dd2011d3ad0fdf355cde4c85df5521d17aa6763df593915b88890aad9c29c10903bbada40be9ec66cc082305fdf5b614c252bb
+  data.tar.gz: '09938a12984f7d33c6337adcaf59af9087ce2969d5d98b37b3e73d833d015a49950b6d7b65bc845dc0820727ea4d2b5af61a8bb0af7ce5201b7531748f8772a8'

data/bin/text_sentencer

@@ -8,20 +8,20 @@ output_mode = :sentences
 ## command line option processing
 require 'optparse'
 optparse = OptionParser.new do |opts|
-  opts.banner = "Usage: text_sentencer [options]"
-
-  opts.on('-c', '--config=config_filename', 'specifies the configuration JSON file.') do |c|
-    config_filename = c
-  end
-    
-  opts.on('-j', '--json_output', 'outputs the result in JSON.') do
-    output_mode = :json
-  end
-
-  opts.on('-h', '--help', 'displays this screen.') do
-    puts opts
-    exit
-  end
+	opts.banner = "Usage: text_sentencer [options]"
+
+	opts.on('-c', '--config=config_filename', 'specifies the configuration JSON file.') do |c|
+		config_filename = c
+	end
+		
+	opts.on('-j', '--json_output', 'outputs the result in JSON.') do
+		output_mode = :json
+	end
+
+	opts.on('-h', '--help', 'displays this screen.') do
+		puts opts
+		exit
+	end
 end
 
 optparse.parse!
@@ -36,10 +36,12 @@ text = ARGF.read
 annotations = sentencer.annotate(text)
 
 if output_mode == :json
-  puts JSON.pretty_generate(annotations)
+	puts JSON.pretty_generate(annotations)
 else
-  annotations[:denotations].each do |d|
-    span = d[:span]
-    puts text[span[:begin]...span[:end]]
-  end
+	if annotations.has_key?(:blocks)
+		annotations[:blocks].each do |d|
+			span = d[:span]
+			puts '[' + text[span[:begin]...span[:end]] + ']'
+		end
+	end
 end

data/lib/text_sentencer/string_scan_offset.rb

@@ -1,9 +1,9 @@
 class String
-  def scan_offset(regex)
-    Enumerator.new do |y|
-      self.scan(regex) do
-        y << Regexp.last_match
-      end
-    end
-  end
+	def scan_offset(regex)
+		Enumerator.new do |y|
+			self.scan(regex) do
+				y << Regexp.last_match
+			end
+		end
+	end
 end

data/lib/text_sentencer/text_sentencer.rb

@@ -3,115 +3,105 @@ require 'text_sentencer/string_scan_offset'
 require 'pp'
 
 class TextSentencer
-  ## default rules
-
-  DEFAULT_RULES = {
-    # All the positions of new line characters always take sentence break.
-    break_pattern: "([ \t]*\n+)+[ \t]*",
-
-    # All the positions of space and tab characters are candiates of sentence break.
-    candidate_pattern: "[ \t]+",
-
-    # First, positive rules are applied to the break candidates to make initial segmantations.
-    positive_rules: [
-      ['[.!?]', '[0-9A-Z]'],
-      ['[:]', '[0-9]'],
-      ['[:]', '[A-Z][a-z]']
-    ],
-
-    # Then, negative rules are applied to cancel some initial segmentations.
-    negative_rules: [
-      # Titles before names
-      ['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
-
-      # Titles usually before names, but ..
-      ['(Sr|Jr)\.', '[A-Z][a-z]'],
-
-      # Single letter abbriveations, e.g. middle name
-      # ['\b[A-Z]\.', '[A-Z][a-z]'],
-
-      # Abbriveations, e.g. middle name
-      ['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
-
-      # Frequent abbreviations that will never appear in the end of a sentence
-      ['(cf|vs)\.', ''],
-      ['e\.g\.', ''],
-      ['i\.e\.', ''],
-
-      # Others
-      ['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
-    ]
-  }
-
-  def initialize(rules = nil)
-    rules ||= DEFAULT_RULES
-    @rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
-    @rules[:break_pattern] ||= ""
-    @rules[:candidate_pattern] ||= ""
-    @rules[:positive_rules] ||= []
-    @rules[:negative_rules] ||= []
-  end
-
-  def annotate(text)
-    return nil if text.nil?
-
-    sentences = segment(text)
-    denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
-    {text:text, denotations:denotations}
-  end
-
-  def segment(text)
-    breaks = if @rules[:break_pattern].empty?
-      []
-    else
-      text.scan_offset(/#{@rules[:break_pattern]}/).map{|m| m.offset(0)}
-    end
-
-    candidates = if @rules[:candidate_pattern].empty?
-      []
-    else
-      text.scan_offset(/#{@rules[:candidate_pattern]}/).map{|m| m.offset(0)}
-    end
-
-    # breaks take precedent
-    candidates -= breaks
-
-    candidates.each do |c|
-      last_end, next_begin = c
-
-      if (last_end == 0) || (next_begin == text.length)
-        breaks << c
-        next
-      end
-
-      last_text = text[0...last_end]
-      next_text = text[next_begin..-1]
-
-      @rules[:positive_rules].each do |p|
-        if (last_text =~ /#{p[0]}\Z/) && (next_text =~ /\A#{p[1]}/)
-          break_p = true
-          @rules[:negative_rules].each do |n|
-            if (last_text =~ /#{n[0]}\Z/) && (next_text =~ /\A#{n[1]}/)
-              break_p = false
-              break
-            end
-          end
-          breaks << c if break_p
-          break
-        end
-      end
-    end
-
-    breaks.sort!
-
-    sentences = []
-    lastbreak = 0
-    breaks.each do |b|
-      sentences << [lastbreak, b[0]] if b[0] > lastbreak
-      lastbreak = b[1]
-    end
-    sentences << [lastbreak, text.length] if lastbreak < text.length
-
-    sentences
-  end
+	## default rules
+
+	DEFAULT_RULES = {
+		# All the positions of new line characters always take sentence break.
+		break_pattern: /([ \t]*\n+)+[ \t]*/,
+
+		# All the positions of space and tab characters are candiates of sentence break.
+		candidate_pattern: /[ \t\n]+/,
+
+		# First, positive rules are applied to the break candidates to make initial segmantations.
+		positive_rules: [
+			[/[.!?]\z/, /\A[0-9A-Z]/],
+			[/[:]\z/, /\A[0-9]/],
+			[/[:]\z/, /\A[A-Z][a-z]/]
+		],
+
+		# Then, negative rules are applied to cancel some initial segmentations.
+		negative_rules: [
+			# Titles before names
+			[/(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.\z/, /\A[A-Z][a-z]/],
+
+			# Titles usually before names, but ..
+			[/(Sr|Jr)\.\z/, /\A[A-Z][a-z]/],
+
+			# Single letter abbriveations, e.g. middle name
+			# [/\b[A-Z]\.\z/, /\A[A-Z][a-z]/],
+
+			# Abbriveations, e.g. middle name
+			[/\b[A-Z][a-z]*\.\z/, /\A[0-9A-Z]/],
+
+			# Frequent abbreviations that will never appear in the end of a sentence
+			[/(cf|vs)\.\z/, //],
+			[/e\.g\.\z/, //],
+			[/i\.e\.\z/, //],
+
+			# Others
+			[/(Sec|Chap|Fig|Eq)\.\z/, /\A[0-9A-Z]/]
+		]
+	}
+
+	def initialize(rules = nil)
+		rules ||= DEFAULT_RULES
+		@rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
+		@rules[:break_pattern] ||= nil
+		@rules[:candidate_pattern] ||= nil
+		@rules[:positive_rules] ||= []
+		@rules[:negative_rules] ||= []
+	end
+
+	def annotate(text)
+		sentences = segment(text)
+		blocks = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
+		{text:text, blocks:blocks}
+	end
+
+	def segment(text)
+		# hard sentence breaks
+		breaks = if @rules[:break_pattern].nil?
+			[]
+		else
+			text.scan_offset(@rules[:break_pattern]).map{|m| m.offset(0)}
+		end
+
+		# candidates of sentence breaks
+		candidates = if @rules[:candidate_pattern].nil?
+			[]
+		else
+			text.scan_offset(@rules[:candidate_pattern]).map{|m| m.offset(0)}
+		end
+
+		# hard sentence breaks are already in
+		candidates -= breaks
+
+		# filter using the positive rules. The first and last breaks remain.
+		candidates.select! do |c|
+			c[0] == 0 ||
+			c[1] == text.length ||
+			@rules[:positive_rules].any? do |r|
+				(text[0...c[0]] =~ r[0]) && (text[c[1]...-1] =~ r[1])
+			end
+		end
+
+		# filter using the negative rules
+		candidates.reject! do |c|
+			@rules[:negative_rules].any? do |r|
+				(text[0...c[0]] =~ r[0]) && (text[c[1]...-1] =~ r[1])
+			end
+		end
+
+		# add all the filtered candidates to sentence breaks
+		breaks += candidates
+		breaks.uniq!
+		breaks.sort!
+
+		# add the initial and final breaks unless already exist
+		breaks.unshift([0, 0]) if breaks.empty? || breaks[0][0] != 0
+		breaks.push([text.length, text.length]) unless breaks[-1][1] == text.length
+
+		# conver the breaks into sentences
+		sentences = (1 ... breaks.length).map{|i| [breaks[i-1][1], breaks[i][0]]}
+	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_sentencer
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 3.0.0
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-07 00:00:00.000000000 Z
+date: 2024-08-03 00:00:00.000000000 Z
 dependencies: []
 description: TextSentencer is a simple rule-based system for segmenting text into
   sentences.
@@ -41,8 +41,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: A simple, rule-based script to find sentence boundaries in text.