text_sentencer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1ec3b6b9d2b3596397c952d6e905c75f4667a73d
+  data.tar.gz: f094203047168fae9a682d1344904c301c9b62ae
+SHA512:
+  metadata.gz: 1142ded7a5be0e72cb840f8f0f6b294196b4718f1e793935304068c6dd4763eedcd55db26ed70e24cb7477d5b3f3fcb24f4c0c57b9a6751a325e4a26c429d553
+  data.tar.gz: 44ef4ff3c18c18623b340a32331153da3f24e3e992f7b481d682d05ec1778e05f75f85ce1992182a8c6cc383d28a3e14e0d553d3f2df20853928400da73a13a0

data/lib/text_sentencer.rb

@@ -0,0 +1 @@
+require 'text_sentencer/text_sentencer'

data/lib/text_sentencer/rules.rb

@@ -0,0 +1,33 @@
+module TextSentencer; end unless defined? TextSentencer
+
+# All the positions of whitespace characters are candiate of sentence boundary.
+
+# First, positive rules are applied to find make initial segmantations.
+TextSentencer::POSITIVE_RULES = [
+  ['[\.!?]', '[0-9A-Z]'],
+  ['[:]', '[0-9]'],
+  ['[:]', '[A-Z][a-z]']
+]
+
+# Then, negative rules are applied to cancel some initial segmentations.
+TextSentencer::NEGATIVE_RULES = [
+  # Titles before names
+  ['(Mrs|Mmes|Mr|Messrs|Ms|Prof|Dr|Drs|Rev|Hon|Sen|St)\.', '[A-Z][a-z]'],
+
+  # Titles usually before names, but ..
+  ['(Sr|Jr)\.', '[A-Z][a-z]'],
+
+  # Single letter abbriveations, e.g. middle name
+  # ['\b[A-Z]\.', '[A-Z][a-z]'],
+
+  # Abbriveations, e.g. middle name
+  ['\b[A-Z][a-z]*\.', '[0-9A-Z]'],
+
+  # Frequent abbreviations that will never appear in the end of a sentence
+  ['(cf|vs)\.', ''],
+  ['e\.g\.', ''],
+  ['i\.e\.', ''],
+
+  # Others
+  ['(Sec|Chap|Fig|Eq)\.', '[0-9A-Z]']
+]

data/lib/text_sentencer/text_sentencer.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+require 'text_sentencer/rules'
+
+module TextSentencer; end unless defined? TextSentencer
+
+module TextSentencer
+  def TextSentencer.segment(text)
+    original_text = text
+    text = original_text.strip
+    start = original_text.index(text)
+
+    ## apply the positive rules to the places of space and newline characters
+    pbreaks = []                # breaks by positive rules
+    for l in 0..text.length
+
+      case text[l]
+      when ' '                   # space
+        POSITIVE_RULES.each do |r|
+          if (text[0...l] =~ /#{r[0]}\Z/) && (text[l+1..-1] =~ /\A#{r[1]}/)
+            pbreaks << l
+            break
+          end
+        end
+      when "\n"                   # newline
+        pbreaks << l
+      end
+    end
+
+    ## apply the negative rules to the places of space characters
+    nbreaks = []                # breaks by negative rules
+    pbreaks.each do |l|
+      if text[l] == ' '
+        NEGATIVE_RULES.each do |r|
+          if (text[0...l] =~ /#{r[0]}\Z/) && (text[l+1..-1] =~ /\A#{r[1]}/)
+            nbreaks << l
+            break
+          end
+        end
+      end
+    end
+    breaks = pbreaks - nbreaks
+
+    sentences = []
+    lastbreak = -1
+    breaks.each do |b|
+      sentences.push([lastbreak+1, b])
+      lastbreak = b
+    end
+    sentences.push([lastbreak+1, text.length])
+
+    ## filter out empty segments
+    sentences.delete_if {|b, e| text[b...e] !~ /[a-zA-Z0-9]/}
+
+    ## adjust offsets for the in text
+    sentences.collect!{|b, e| [b + start, e + start]}
+
+    sentences
+  end
+end
+
+if __FILE__ == $0
+  text = ''
+  ARGF.each do |line|
+    text += line
+  end
+
+  sen_so = TextSentencer.segment(text)
+  p(sen_so)
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: text_sentencer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jin-Dong Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-11 00:00:00.000000000 Z
+dependencies: []
+description: Text sentencer finds sentence boundaries of a given text. It is a simple,
+  rule-based system.
+email: jindong.kim@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/text_sentencer.rb
+- lib/text_sentencer/rules.rb
+- lib/text_sentencer/text_sentencer.rb
+homepage: http://rubygems.org/gems/text_sentencer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: To find sentences in text.
+test_files: []