sentence 1.0.0

data/README

@@ -0,0 +1,22 @@
+= Read Me
+
+by Tomasz Bak
+
+== Description
+
+Welcome to Sentence.
+
+This library helps spliting text into its constituent sentences, based on a regular expressions
+(packed into SplitPattern class) and a list of abbreviations.
+
+It was designed to be flexiple and clean, and it has three primary goals:
+
+1.  Be easely applicable in wide range of usage scenarios (i.e. splitting sentences
+     generated by html2text, what involved support for simple formating understanding,
+     see examples/example.rb that parses corpora/medicine.txt).
+     
+2.  Be library language aware, what is particulary important in building abbreviations and
+    conjunctions base (see lib/conjunctions.rb and lib/abbreviations.rb).
+
+3.  Use extesive testing not to break previously working code with new abbreviations and
+    patterns, which hopefully will be extended in future with patches sent by users.

data/lib/abbreviations.rb

@@ -0,0 +1,18 @@
+module Sentence
+  module Abbreviations
+    ROMAN = ['i','ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x']
+
+    def self.EN
+      people = ['jr', 'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', "sens?", "reps?", 'gov', 'supt',  'det', 'rev']
+      army = ['col','gen', 'lt', 'cmdr', 'adm', 'capt', 'sgt', 'cpl', 'maj']
+      month = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec','sept']
+      misc = ['vs', 'etc', 'no', 'esp', 'e.g', 'cf','i.e']
+    
+      people + army + month + misc + ROMAN
+    end
+    
+    def self.PL
+      ROMAN
+    end
+  end
+end

data/lib/conjunctions.rb

@@ -0,0 +1,11 @@
+module Sentence
+  module Conjunctions
+    def self.EN
+      ['and', 'or', 'but', 'so', 'as', 'of']
+    end
+    
+    def self.PL
+      ['i', 'lub', 'oraz', 'za']
+    end
+  end
+end

data/lib/sentence.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/abbreviations'
+require File.dirname(__FILE__) + '/conjunctions'
+require File.dirname(__FILE__) + '/sentence_splitter'
+require File.dirname(__FILE__) + '/split_pattern'
+
+module Sentence
+  # English language rules
+  def self.EN
+    SentenceSplitter.new(:EN)
+  end
+  
+  # Polish language rules
+  def self.PL
+    SentenceSplitter.new(:PL)
+  end
+end

data/lib/sentence_splitter.rb

@@ -0,0 +1,94 @@
+module Sentence
+  EOS = "\001" # End Of Sentence
+
+  class SentenceSplitter
+
+    include Abbreviations
+    include Conjunctions
+    
+    def initialize(lang)
+      @lang = lang
+      @discarded_text_as_sentence = []
+      @split_patterns = []
+    end
+    
+    def discard(texts)
+      discard_texts_as_sentence(texts)
+    end
+    
+    def patterns(paterns)
+      @split_patterns = paterns
+      self
+    end
+    
+    def discard_texts_as_sentence(texts)
+      @discarded_text_as_sentence = texts
+      self
+    end
+
+    # get_sentences - takes text input and splits it into sentences.
+    def get_sentences( text )
+      return [] if text.nil? or text.empty?
+      
+      @split_patterns.each do |pattern|
+        pattern.process(text)
+      end
+      
+      marked_text = first_sentence_breaking(text)
+      fixed_marked_text = split_unsplit_stuff( remove_false_end_of_sentence( marked_text ) )
+      sentences = fixed_marked_text.split(EOS)
+      clean_sentences(sentences)
+    end
+    
+    def first_sentence_breaking(text)
+      # each match means a different sentence
+      text.gsub!(/\n\s*\n/,EOS)	# double new-line
+      text.gsub!(/\|\s*([A-Z])/,EOS+'\1')	# break on | before A-Z
+      text.gsub!(/\n\s*([A-Z])/,EOS+'\1') # new line and new world starts with A-Z
+      text.gsub!(/^\s*[\*\o\#]\s(.*)/,EOS+'\1') # list items starts with [*,o,#]
+      text.gsub!(/([\.\!\?])(\s)/,'\1'+EOS+'\2') # punctuation 
+      text
+    end
+    
+    def remove_false_end_of_sentence( text )
+      Abbreviations.send(@lang).each do |abb|
+        text.gsub!(/(#{abb}\.)#{EOS}/i,'\1')
+      end
+      Conjunctions.send(@lang).each do |con|
+        text.gsub!(/(#{con}\s*)#{EOS}/i,'\1')
+      end
+      text
+    end
+    
+    def split_unsplit_stuff(text)
+      text.gsub!(/\n#{EOS}(\s*[a-z])/,'\1')
+      text.gsub!(/([A-Z0-9]\.)#{EOS}/,'\1') # single upcase letter or number with .
+      text
+    end
+    
+    def clean_sentences(sentences)
+      cleaned_sentences = []
+      sentences.each do |sentence|
+        
+        subsentences = [ sentence ]
+        if sentence.index(/\./) == nil
+          subsentences = sentence.split('*') # split * if no . in sentence
+        end
+        
+        subsentences.each do |sentence|
+          case sentence.strip
+          when '', "\n", "..."
+          else
+            if sentence.index(/\w/) != nil
+              sentence_whithout_ws = sentence.split.join(' ')
+              unless @discarded_text_as_sentence.include? sentence_whithout_ws.downcase
+                cleaned_sentences << sentence_whithout_ws 
+              end
+            end
+          end
+        end
+      end
+      cleaned_sentences
+    end
+  end
+end

data/lib/split_pattern.rb

@@ -0,0 +1,31 @@
+module Sentence
+  class SplitPattern
+    attr_reader :name, :pattern, :replace
+    @@extracted_patterns = {}
+    
+    def initialize(name, pattern, replace)
+      @name = name
+      @pattern = pattern
+      @replace = replace
+    end
+    
+    def process(text)
+      #@@extracted_patterns[@name].each do |caption| = text.scan(@pattern)
+      
+      text.scan(@pattern).each do |caption|
+        @@extracted_patterns[@name] = [] if @@extracted_patterns[@name].nil?
+        caption = [ caption ] if caption.class != 'Array'
+        Sentence.EN.get_sentences(caption.flatten.join(Sentence::EOS)).each do |sentence|
+          #puts "sentence = #{sentence.inspect}"
+          @@extracted_patterns[@name] << sentence unless sentence.empty?
+        end
+      end
+      
+      text.gsub!(@pattern, @replace)
+    end
+    
+    def self.get_extracted_patterns
+      @@extracted_patterns
+    end
+  end
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: sentence
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2008-01-13 00:00:00 +01:00
+summary: Splits text into its constituent sentences
+require_paths: 
+- lib
+email: hellolinus@gmail.com
+homepage: http://rubyforge.org/projects/sentence
+rubyforge_project: 
+description: 
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Tomasz Bak
+files: 
+- lib/abbreviations.rb
+- lib/conjunctions.rb
+- lib/sentence.rb
+- lib/sentence_splitter.rb
+- lib/split_pattern.rb
+- README
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+