text_sentencer 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7008e00d8e1554e27e82bbf744eb74bf10704234
-  data.tar.gz: e863a486fe7f932c6354abd314c6e9690f4b1ece
+SHA256:
+  metadata.gz: b8dda6323a11356aa2f6c78fa0281658ea677f09fbf0bfb19b19330f732e72c1
+  data.tar.gz: a41e5cc5ae3e1e294e9ba4921993e2d98003d74435fd5bc099c6c6fbf188554a
 SHA512:
-  metadata.gz: a2762e82db9eece26fd97ec7ac3c44f13ad3fe9babe2910238d1ed5e4a616d3afbaac3ff421d6dda354d3ceb708ca0a5ce7877563f253bf726599f898f0a5f44
-  data.tar.gz: 9ee5c91045041905f61218944388157fdc7ffb53b6d7f8bd784ef8a286aa2acdf6cca524fb31f4a1e0725466bdbe130de55ef4b00535deb7bf5ae1b65c6ed49b
+  metadata.gz: b39ec569c1e988f4e7936385395ff1e34c15cb7721ffba6fb2453f633db360f092b5225d6ec4603b2beb391e2660291e9d48e912359eb33db9d2ccd1906d806f
+  data.tar.gz: 2780a4c3f6ac1fc7f0ab06e658a293b7e9a7d3f5a2df0a0904fa20aea187857c9f3bad7cc7adfb6f1eba4c57897f433c03d77a958559e09225b60c154c9488b3

data/bin/text_sentencer

@@ -8,20 +8,20 @@ output_mode = :sentences
 ## command line option processing
 require 'optparse'
 optparse = OptionParser.new do |opts|
-  opts.banner = "Usage: text_sentencer [options]"
-
-  opts.on('-c', '--config=config_filename', 'specifies the configuration JSON file.') do |c|
-    config_filename = c
-  end
-    
-  opts.on('-j', '--json_output', 'outputs the result in JSON.') do
-    output_mode = :json
-  end
-
-  opts.on('-h', '--help', 'displays this screen.') do
-    puts opts
-    exit
-  end
+	opts.banner = "Usage: text_sentencer [options]"
+
+	opts.on('-c', '--config=config_filename', 'specifies the configuration JSON file.') do |c|
+		config_filename = c
+	end
+		
+	opts.on('-j', '--json_output', 'outputs the result in JSON.') do
+		output_mode = :json
+	end
+
+	opts.on('-h', '--help', 'displays this screen.') do
+		puts opts
+		exit
+	end
 end
 
 optparse.parse!
@@ -36,10 +36,10 @@ text = ARGF.read
 annotations = sentencer.annotate(text)
 
 if output_mode == :json
-  puts JSON.pretty_generate(annotations)
+	puts JSON.pretty_generate(annotations)
 else
-  annotations[:denotations].each do |d|
-    span = d[:span]
-    puts text[span[:begin]...span[:end]]
-  end
+	annotations[:denotations].each do |d|
+		span = d[:span]
+		puts text[span[:begin]...span[:end]]
+	end
 end

data/lib/text_sentencer/string_scan_offset.rb

@@ -1,9 +1,9 @@
 class String
-  def scan_offset(regex)
-    Enumerator.new do |y|
-      self.scan(regex) do
-        y << Regexp.last_match
-      end
-    end
-  end
+	def scan_offset(regex)
+		Enumerator.new do |y|
+			self.scan(regex) do
+				y << Regexp.last_match
+			end
+		end
+	end
 end

data/lib/text_sentencer/text_sentencer.rb

@@ -3,115 +3,115 @@ require 'text_sentencer/string_scan_offset'
 require 'pp'
 
 class TextSentencer
-  ## default rules
-
-  DEFAULT_RULES = {
-    # All the positions of new line characters always take sentence break.
-    break_pattern: "([ \t]*\n+)+[ \t]*",
-
-    # All the positions of space and tab characters are candiates of sentence break.
-    candidate_pattern: "[ \t]+",
-
-    # First, positive rules are applied to the break candidates to make initial segmantations.
-    positive_rules: [
-      ['[.!?]', '[0-9A-Z]'],
-      ['[:]', '[0-9]'],
-      ['[:]', '[A-Z][a-z]']
-    ],
-
-    # Then, negative rules are applied to cancel some initial segmentations.
-    negative_rules: [
-      # Titles before names
-      ['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
-
-      # Titles usually before names, but ..
-      ['(Sr|Jr)\.', '[A-Z][a-z]'],
-
-      # Single letter abbriveations, e.g. middle name
-      # ['\b[A-Z]\.', '[A-Z][a-z]'],
-
-      # Abbriveations, e.g. middle name
-      ['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
-
-      # Frequent abbreviations that will never appear in the end of a sentence
-      ['(cf|vs)\.', ''],
-      ['e\.g\.', ''],
-      ['i\.e\.', ''],
-
-      # Others
-      ['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
-    ]
-  }
-
-  def initialize(rules = nil)
-    rules ||= DEFAULT_RULES
-    @rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
-    @rules[:break_pattern] ||= ""
-    @rules[:candidate_pattern] ||= ""
-    @rules[:positive_rules] ||= []
-    @rules[:negative_rules] ||= []
-  end
-
-  def annotate(text)
-    return nil if text.nil?
-
-    sentences = segment(text)
-    denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
-    {text:text, denotations:denotations}
-  end
-
-  def segment(text)
-    breaks = if @rules[:break_pattern].empty?
-      []
-    else
-      text.scan_offset(/#{@rules[:break_pattern]}/).map{|m| m.offset(0)}
-    end
-
-    candidates = if @rules[:candidate_pattern].empty?
-      []
-    else
-      text.scan_offset(/#{@rules[:candidate_pattern]}/).map{|m| m.offset(0)}
-    end
-
-    # breaks take precedent
-    candidates -= breaks
-
-    candidates.each do |c|
-      last_end, next_begin = c
-
-      if (last_end == 0) || (next_begin == text.length)
-        breaks << c
-        next
-      end
-
-      last_text = text[0...last_end]
-      next_text = text[next_begin..-1]
-
-      @rules[:positive_rules].each do |p|
-        if (last_text =~ /#{p[0]}\Z/) && (next_text =~ /\A#{p[1]}/)
-          break_p = true
-          @rules[:negative_rules].each do |n|
-            if (last_text =~ /#{n[0]}\Z/) && (next_text =~ /\A#{n[1]}/)
-              break_p = false
-              break
-            end
-          end
-          breaks << c if break_p
-          break
-        end
-      end
-    end
-
-    breaks.sort!
-
-    sentences = []
-    lastbreak = 0
-    breaks.each do |b|
-      sentences << [lastbreak, b[0]] if b[0] > lastbreak
-      lastbreak = b[1]
-    end
-    sentences << [lastbreak, text.length] if lastbreak < text.length
-
-    sentences
-  end
+	## default rules
+
+	DEFAULT_RULES = {
+		# All the positions of new line characters always take sentence break.
+		break_pattern: "([ \t]*\n+)+[ \t]*",
+
+		# All the positions of space and tab characters are candiates of sentence break.
+		candidate_pattern: "[ \t]+",
+
+		# First, positive rules are applied to the break candidates to make initial segmantations.
+		positive_rules: [
+			['[.!?]', '[0-9A-Z]'],
+			['[:]', '[0-9]'],
+			['[:]', '[A-Z][a-z]']
+		],
+
+		# Then, negative rules are applied to cancel some initial segmentations.
+		negative_rules: [
+			# Titles before names
+			['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
+
+			# Titles usually before names, but ..
+			['(Sr|Jr)\.', '[A-Z][a-z]'],
+
+			# Single letter abbriveations, e.g. middle name
+			# ['\b[A-Z]\.', '[A-Z][a-z]'],
+
+			# Abbriveations, e.g. middle name
+			['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
+
+			# Frequent abbreviations that will never appear in the end of a sentence
+			['(cf|vs)\.', ''],
+			['e\.g\.', ''],
+			['i\.e\.', ''],
+
+			# Others
+			['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
+		]
+	}
+
+	def initialize(rules = nil)
+		rules ||= DEFAULT_RULES
+		@rules = Hash[rules.map{|(k,v)| [k.to_sym,v]}]
+		@rules[:break_pattern] ||= ""
+		@rules[:candidate_pattern] ||= ""
+		@rules[:positive_rules] ||= []
+		@rules[:negative_rules] ||= []
+	end
+
+	def annotate(text)
+		return nil if text.nil?
+
+		sentences = segment(text)
+		denotations = sentences.inject([]){|c, s| c << {:span => {:begin => s[0], :end => s[1]}, :obj => 'Sentence'}}
+		{text:text, denotations:denotations}
+	end
+
+	def segment(text)
+		breaks = if @rules[:break_pattern].empty?
+			[]
+		else
+			text.scan_offset(/#{@rules[:break_pattern]}/).map{|m| m.offset(0)}
+		end
+
+		candidates = if @rules[:candidate_pattern].empty?
+			[]
+		else
+			text.scan_offset(/#{@rules[:candidate_pattern]}/).map{|m| m.offset(0)}
+		end
+
+		# breaks take precedent
+		candidates -= breaks
+
+		candidates.each do |c|
+			last_end, next_begin = c
+
+			if (last_end == 0) || (next_begin == text.length)
+				breaks << c
+				next
+			end
+
+			last_text = text[0...last_end]
+			next_text = text[next_begin..-1]
+
+			@rules[:positive_rules].each do |p|
+				if (last_text =~ /#{p[0]}\Z/) && (next_text =~ /\A#{p[1]}/)
+					break_p = true
+					@rules[:negative_rules].each do |n|
+						if (last_text =~ /#{n[0]}\Z/) && (next_text =~ /\A#{n[1]}/)
+							break_p = false
+							break
+						end
+					end
+					breaks << c if break_p
+					break
+				end
+			end
+		end
+
+		breaks.sort!
+
+		sentences = []
+		lastbreak = 0
+		breaks.each do |b|
+			sentences << [lastbreak, b[0]] if b[0] > lastbreak
+			lastbreak = b[1]
+		end
+		sentences << [lastbreak, text.length] if lastbreak < text.length
+
+		sentences
+	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: text_sentencer
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Jin-Dong Kim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-07 00:00:00.000000000 Z
+date: 2021-01-17 00:00:00.000000000 Z
 dependencies: []
 description: TextSentencer is a simple rule-based system for segmenting text into
   sentences.
@@ -41,8 +41,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: A simple, rule-based script to find sentence boundaries in text.