text_sentencer 0.2.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 85aab334688ecac5dc4b3307c58a2bf0058e0aad
-  data.tar.gz: 0255396f5d925c06023111bca178157b56765586
+SHA256:
+  metadata.gz: b8dda6323a11356aa2f6c78fa0281658ea677f09fbf0bfb19b19330f732e72c1
+  data.tar.gz: a41e5cc5ae3e1e294e9ba4921993e2d98003d74435fd5bc099c6c6fbf188554a
 SHA512:
-  metadata.gz: a014eb537f0902018a4a71991110cea6350e26dadc108bc076f7673cd7ace1ea46a8ae8e07b76c4a22d7e94bbd0d7d94e96be5123e2ddf98b3536daba2a69b4c
-  data.tar.gz: 54885c68f05f96de55bb94ad6375cee45fb4fd6d9c41000cb7fa7091f0e3c0792ac63494c6b1d788088bc41a80559d3c711ccd8a82511a47c24c9fb7126a58c8
+  metadata.gz: b39ec569c1e988f4e7936385395ff1e34c15cb7721ffba6fb2453f633db360f092b5225d6ec4603b2beb391e2660291e9d48e912359eb33db9d2ccd1906d806f
+  data.tar.gz: 2780a4c3f6ac1fc7f0ab06e658a293b7e9a7d3f5a2df0a0904fa20aea187857c9f3bad7cc7adfb6f1eba4c57897f433c03d77a958559e09225b60c154c9488b3

data/bin/text_sentencer

@@ -1,47 +1,45 @@
 #!/usr/bin/env ruby
+require 'json'
 require 'text_sentencer'
 
 config_filename = nil
+output_mode = :sentences
 
 ## command line option processing
 require 'optparse'
 optparse = OptionParser.new do |opts|
-  opts.banner = "Usage: text_sentencer [options]"
-
-  opts.on('-c', '--config', 'specifies the configuration JSON file.') do |f|
-    config_filename = f
-  end
-    
-  opts.on('-h', '--help', 'displays this screen.') do
-    puts opts
-    exit
-  end
+	opts.banner = "Usage: text_sentencer [options]"
+
+	opts.on('-c', '--config=config_filename', 'specifies the configuration JSON file.') do |c|
+		config_filename = c
+	end
+		
+	opts.on('-j', '--json_output', 'outputs the result in JSON.') do
+		output_mode = :json
+	end
+
+	opts.on('-h', '--help', 'displays this screen.') do
+		puts opts
+		exit
+	end
 end
 
 optparse.parse!
 
-config = if config_filename
-	JSON.parse File.read(config_filename) if File.file?(config_filename)
+config = if config_filename && File.file?(config_filename)
+	JSON.parse File.read(config_filename)
 end
 
 sentencer = TextSentencer.new(config)
 
 text = ARGF.read
-
-## Preprocessing
-# It should be removed later
-text.gsub!(/ +/, ' ')
-text.gsub!(/\n+/, "\n")
-text.gsub!(/\t+/, "\t")
-text.gsub!(/\n /, "\n")
-text.gsub!(/ \n/, "\n")
-text.gsub!(/\t /, "\t")
-text.gsub!(/ \t/, "\t")
-text.gsub!(/\n\t/, "\n")
-text.gsub!(/\t\n/, "\n")
-
 annotations = sentencer.annotate(text)
-annotations[:denotations].each do |d|
-	span = d[:span]
-	puts text[span[:begin]...span[:end]]
+
+if output_mode == :json
+	puts JSON.pretty_generate(annotations)
+else
+	annotations[:denotations].each do |d|
+		span = d[:span]
+		puts text[span[:begin]...span[:end]]
+	end
 end

data/lib/text_sentencer/string_scan_offset.rb

@@ -0,0 +1,9 @@
+class String
+	def scan_offset(regex)
+		Enumerator.new do |y|
+			self.scan(regex) do
+				y << Regexp.last_match
+			end
+		end
+	end
+end

data/lib/text_sentencer/text_sentencer.rb

@@ -1,150 +1,117 @@
 #!/usr/bin/env ruby
+require 'text_sentencer/string_scan_offset'
+require 'pp'
 
 class TextSentencer
-  ## default rules
-
-  # All the positions of space and tab characters are candiates of sentence break.
-  BREAK_CANDIDATES = [
-    " ", "\t"
-  ]
-
-  # All the positions of new line characters always take sentence break.
-  BREAK_CHARACTERS = [
-    "\n"
-  ]
-
-  # First, positive rules are applied to the break candidates to make initial segmantations.
-  POSITIVE_RULES = [
-    ['[\.!?]', '[0-9A-Z]'],
-    ['[:]', '[0-9]'],
-    ['[:]', '[A-Z][a-z]']
-  ]
-
-  # Then, negative rules are applied to cancel some initial segmentations.
-  NEGATIVE_RULES = [
-    # Titles before names
-    ['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
-
-    # Titles usually before names, but ..
-    ['(Sr|Jr)\.', '[A-Z][a-z]'],
-
-    # Single letter abbriveations, e.g. middle name
-    # ['\b[A-Z]\.', '[A-Z][a-z]'],
-
-    # Abbriveations, e.g. middle name
-    ['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
-
-    # Frequent abbreviations that will never appear in the end of a sentence
-    ['(cf|vs)\.', ''],
-    ['e\.g\.', ''],
-    ['i\.e\.', ''],
-
-    # Others
-    ['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
-  ]
-
-  def initialize(rules = {})
-    rules ||= {}
-    @break_candidates = rules[:break_candidates] || BREAK_CANDIDATES
-    @break_characters = rules[:break_characters] || BREAK_CHARACTERS
-    @positive_rules = rules[:positive_rules] || POSITIVE_RULES
-    @negative_rules = rules[:negative_rules] || NEGATIVE_RULES
-  end
-
-  def annotate(text)
-    return nil if text.nil? || text.empty?
-    sentences = segment(text)
-    denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
-    denotations.empty? ? {text:text} : {text:text, denotations:denotations}
-  end
-
-  private
-
-  def segment(text)
-    original_text = text
-    text = original_text.strip
-    start = original_text.index(text)
-
-    # sentence breaks
-    breaks = []
-
-    # breaks by positive rules
-    pbreaks = []
-
-    # canceled breaks by negative rules
-    nbreaks = []
-
-    for l in 0..text.length
-
-      ## apply the positive rules to the places of break candidates
-      if @break_candidates.include?(text[l])
-        POSITIVE_RULES.each do |r|
-          if (text[0...l] =~ /#{r[0]}\Z/) && (text[l+1..-1] =~ /\A#{r[1]}/)
-            pbreaks << l
-            break
-          end
-        end
-      elsif @break_characters.include?(text[l])
-        breaks << l
-      end
-    end
-
-    ## apply the negative rules to the places of break candidates
-    pbreaks.each do |l|
-      NEGATIVE_RULES.each do |r|
-        if (text[0...l] =~ /#{r[0]}\Z/) && (text[l+1..-1] =~ /\A#{r[1]}/)
-          nbreaks << l
-          break
-        end
-      end
-    end
-    breaks += pbreaks - nbreaks
-    breaks.sort!
-
-    sentences = []
-    lastbreak = -1
-    breaks.each do |b|
-      sentences.push([lastbreak+1, b])
-      lastbreak = b
-    end
-    sentences.push([lastbreak+1, text.length])
-
-    ## filter out empty segments
-    sentences.delete_if {|b, e| text[b...e] !~ /[a-zA-Z0-9]/}
-
-    ## adjust offsets for the in text
-    sentences.collect!{|b, e| [b + start, e + start]}
-
-    sentences
-  end
-end
-
-if __FILE__ == $0
-  rules = {
-    break_candidates: [
-      " ", "\t"
-    ],
-
-    break_characters: [
-      "\n"
-    ],
-
-    positive_rules: [
-      ['[\.!?]', '[0-9A-Z]'],
-      ['[:]', '[0-9]'],
-      ['[:]', '[A-Z][a-z]']
-    ],
-
-    negative_rules: []
-  }
-
-  sentencer = TextSentencer.new
-
-  text = ''
-  ARGF.each do |line|
-    text += line
-  end
-
-  sen_so = sentencer.annotate(text)
-  p(sen_so)
+	## default rules
+
+	DEFAULT_RULES = {
+		# All the positions of new line characters always take sentence break.
+		break_pattern: "([ \t]*\n+)+[ \t]*",
+
+		# All the positions of space and tab characters are candiates of sentence break.
+		candidate_pattern: "[ \t]+",
+
+		# First, positive rules are applied to the break candidates to make initial segmantations.
+		positive_rules: [
+			['[.!?]', '[0-9A-Z]'],
+			['[:]', '[0-9]'],
+			['[:]', '[A-Z][a-z]']
+		],
+
+		# Then, negative rules are applied to cancel some initial segmentations.
+		negative_rules: [
+			# Titles before names
+			['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
+
+			# Titles usually before names, but ..
+			['(Sr|Jr)\.', '[A-Z][a-z]'],
+
+			# Single letter abbriveations, e.g. middle name
+			# ['\b[A-Z]\.', '[A-Z][a-z]'],
+
+			# Abbriveations, e.g. middle name
+			['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
+
+			# Frequent abbreviations that will never appear in the end of a sentence
+			['(cf|vs)\.', ''],
+			['e\.g\.', ''],
+			['i\.e\.', ''],
+
+			# Others
+			['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
+		]
+	}
+
+	def initialize(rules = nil)
+		rules ||= DEFAULT_RULES
+		@rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
+		@rules[:break_pattern] ||= ""
+		@rules[:candidate_pattern] ||= ""
+		@rules[:positive_rules] ||= []
+		@rules[:negative_rules] ||= []
+	end
+
+	def annotate(text)
+		return nil if text.nil?
+
+		sentences = segment(text)
+		denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
+		{text:text, denotations:denotations}
+	end
+
+	def segment(text)
+		breaks = if @rules[:break_pattern].empty?
+			[]
+		else
+			text.scan_offset(/#{@rules[:break_pattern]}/).map{|m| m.offset(0)}
+		end
+
+		candidates = if @rules[:candidate_pattern].empty?
+			[]
+		else
+			text.scan_offset(/#{@rules[:candidate_pattern]}/).map{|m| m.offset(0)}
+		end
+
+		# breaks take precedent
+		candidates -= breaks
+
+		candidates.each do |c|
+			last_end, next_begin = c
+
+			if (last_end == 0) || (next_begin == text.length)
+				breaks << c
+				next
+			end
+
+			last_text = text[0...last_end]
+			next_text = text[next_begin..-1]
+
+			@rules[:positive_rules].each do |p|
+				if (last_text =~ /#{p[0]}\Z/) && (next_text =~ /\A#{p[1]}/)
+					break_p = true
+					@rules[:negative_rules].each do |n|
+						if (last_text =~ /#{n[0]}\Z/) && (next_text =~ /\A#{n[1]}/)
+							break_p = false
+							break
+						end
+					end
+					breaks << c if break_p
+					break
+				end
+			end
+		end
+
+		breaks.sort!
+
+		sentences = []
+		lastbreak = 0
+		breaks.each do |b|
+			sentences << [lastbreak, b[0]] if b[0] > lastbreak
+			lastbreak = b[1]
+		end
+		sentences << [lastbreak, text.length] if lastbreak < text.length
+
+		sentences
+	end
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: text_sentencer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.3
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-20 00:00:00.000000000 Z
+date: 2021-01-17 00:00:00.000000000 Z
 dependencies: []
-description: TextSentencer is a simple rule-based system for segmenting a text block
-  into sentences.
+description: TextSentencer is a simple rule-based system for segmenting text into
+  sentences.
 email: jindong.kim@gmail.com
 executables:
 - text_sentencer
@@ -20,8 +20,9 @@ extra_rdoc_files: []
 files:
 - bin/text_sentencer
 - lib/text_sentencer.rb
+- lib/text_sentencer/string_scan_offset.rb
 - lib/text_sentencer/text_sentencer.rb
-homepage: http://rubygems.org/gems/text_sentencer
+homepage: http://bionlp.dbcls.jp/text_sentencer
 licenses:
 - MIT
 metadata: {}
@@ -40,8 +41,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: A simple, rule-based script to find sentence boundaries in text.