rbbt-text 1.2.0 → 1.3.0

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -1,5 +1,5 @@
 require 'rbbt/nlp/nlp'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 module NLP
   Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
 

data/lib/rbbt/nlp/nlp.rb

@@ -2,8 +2,8 @@ require 'rbbt'
 require 'rbbt/util/tmpfile'
 require 'rbbt/persist'
 require 'rbbt/resource'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/segmented'
+require 'rbbt/segment'
+require 'rbbt/segment/segmented'
 require 'rbbt/nlp/genia/sentence_splitter'
 require 'digest/md5'
 
@@ -101,7 +101,7 @@ module NLP
     input = sentences.collect{|sentence| sentence.gsub(/\n/, NEW_LINE_MASK)} * "\n"
     sentence_tokens = TmpFile.with_file(input) do |fin|
       out = local_persist(Digest::MD5.hexdigest(input), :Chunks, :string) do
-        CMD.cmd("cd #{Rbbt.software.opt.Gdep.find}; ./gdep #{ fin }").read
+        CMD.cmd("cd #{Rbbt.software.opt.Gdep.produce.find}; ./gdep #{ fin }").read
       end
 
       out.split(/^$/).collect do |sentence|
@@ -120,10 +120,10 @@ module NLP
 
 
   def self.gdep_parse_sentences_extension(sentences)
-    require Rbbt.software.opt.Gdep.ruby["Gdep.so"].find
+    require Rbbt.software.opt.Gdep.produce.ruby["Gdep.so"].find
     gdep = Gdep.new
     if not gdep.gdep_is_loaded
-      Misc.in_dir Rbbt.software.opt.Gdep.find do
+      Misc.in_dir Rbbt.software.opt.Gdep.produce.find do
         gdep.load_gdep 
       end
     end

data/lib/rbbt/nlp/open_nlp/sentence_splitter.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/resource'
 
 module OpenNLP
@@ -33,48 +33,49 @@ module OpenNLP
   def self.sentence_splitter(text)
     return [] if text.nil? or text.empty?
 
-    text = Misc.to_utf8(text)
-    last = 0
-    begin
-      sentence_split_detector = self.sentence_split_detector
-      
-      sentences = nil
-      TmpFile.with_file do |tmpfile|
-        start_time = Time.now
-
-        begin
-          pid = Process.fork do
-            sent = sentence_split_detector.sentDetect(text)
-            Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
-          end
+    Segment.ascii(text, "?") do
+      last = 0
+      begin
+        sentence_split_detector = self.sentence_split_detector
 
-          while not Process.waitpid(pid)
-            if Time.now - start_time > MAX
-              Process.kill(9, pid)
-              raise "Taking to long (> #{MAX} seconds)"
-            end
-            sleep 0.1
-          end
+        sentences = nil
+        TmpFile.with_file do |tmpfile|
+          start_time = Time.now
 
           begin
-            Process.waitpid(pid)
+            pid = Process.fork do
+              sent = sentence_split_detector.sentDetect(text)
+              Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
+            end
+
+            while not Process.waitpid(pid)
+              if Time.now - start_time > MAX
+                Process.kill(9, pid)
+                raise "Taking to long (> #{MAX} seconds)"
+              end
+              sleep 0.1
+            end
+
+            begin
+              Process.waitpid(pid)
+            end
+          rescue Errno::ECHILD
           end
-        rescue Errno::ECHILD
+
+          sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
         end
 
-        sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
+        sentences.collect{|sentence|
+          sentence = Misc.to_utf8(sentence)
+          start = text.index(sentence, last)
+          Segment.setup sentence, start
+          last = start + sentence.length - 1
+          sentence
+        }
+      rescue Exception
+        raise $!
+        raise "Sentence splitter raised exception: #{$!.message}"
       end
-
-      sentences.collect{|sentence|
-        sentence = Misc.to_utf8(sentence)
-        start = text.index(sentence, last)
-        Segment.setup sentence, start
-        last = start + sentence.length - 1
-        sentence
-      }
-    rescue Exception
-      raise $!
-      raise "Sentence splitter raised exception: #{$!.message}"
     end
   end
 end

data/lib/rbbt/segment.rb

@@ -0,0 +1,177 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+
+module SegID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def range
+    @range ||= Range.new(*_parts.last.split("..").map(&:to_i))
+  end
+
+  def docid
+    @docid ||= _parts[0..3] * ":"
+  end
+
+  def offset
+    range.begin
+  end
+
+  def segment_length
+    range.end - range.begin + 1
+  end
+
+  property :segment do
+    document = DocID.setup(docid, :corpus => corpus).document
+
+    text = document[range]
+
+    Segment.setup(text, docid)
+  end
+
+  property :segid do
+    self
+  end
+
+end
+
+module Segment
+  extend Entity
+  self.annotation :offset, :docid 
+
+  def segment_length
+    length
+  end
+
+  def eend
+    offset.to_i + length - 1
+  end
+
+  def range
+    (offset.to_i..eend)
+  end
+
+  property :segid do |corpus=nil|
+    SegID.setup([docid, range] * ":", :corpus => corpus)
+  end
+
+  alias id segid
+
+  property :segment do
+    self
+  end
+
+  def self.sort(segments, inline = true)
+    if inline
+      segments.sort do |a,b| 
+        case
+        when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
+          0
+        when (a.nil? or a.offset.nil?)
+          -1
+        when (b.nil? or b.offset.nil?)
+          +1
+        when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i)
+          a.offset.to_i <=> b.offset.to_i
+        else
+          a.segment_length <=> b.segment_length
+        end
+      end
+    else
+      segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse
+    end
+  end
+
+  def self.overlaps(sorted_segments)
+    last = nil
+    overlaped = []
+
+    sorted_segments.reverse.each do |segment| 
+      overlaped << segment if (not last.nil?) and segment.range.end > last 
+      last = segment.range.begin
+    end
+
+    overlaped
+  end
+
+  def self.clean_sort(segments)
+    sorted = sort(segments).reject{|s| s.offset.nil?}
+    overlaps = overlaps(sorted)
+    overlaps.each do |s|
+      sorted.delete s
+    end
+
+    sorted
+  end
+
+  def self.split(text, segments, skip_segments = false)
+    sorted_segments = clean_sort segments
+
+    chunks      = []
+    segment_end = 0
+    text_offset = 0
+    sorted_segments.each do |segment|
+      return chunks if text.nil? or text.empty?
+      next if segment.offset.nil?
+      offset = segment.offset - text_offset
+
+      # Consider segment offset. Save pre, or skip if overlap
+      case
+      when offset < 0 # Overlap, skip
+        next
+      when offset > 0 # Save pre
+        chunk = text[0..offset - 1]
+        Segment.setup(chunk, text_offset)
+        chunks << chunk
+      end
+
+      segment_end = offset + segment.segment_length - 1
+
+      if not skip_segments
+        chunk = text[offset..segment_end]
+        Segment.setup(chunk, text_offset + offset)
+        chunks << chunk
+      end
+
+      text_offset += segment_end + 1
+      text = text[segment_end + 1..-1]
+
+    end
+
+    if not text.nil? and not text.empty?
+      chunk = text.dup
+      Segment.setup(chunk, text_offset)
+      chunks << chunk
+    end
+
+    chunks
+  end
+
+  def self.align(text, parts)
+    pre_offset = 0
+    docid = text.respond_to?(:docid) ? text.docid : nil
+    parts.each do |part|
+      offset = text.index part
+      next if offset.nil?
+      Segment.setup(part, pre_offset + offset, docid)
+      pre_offset += offset + part.segment_length - 1
+      text = text[(offset + part.segment_length - 1)..-1]
+    end
+  end
+
+  def self.index(*args)
+    Segment::RangeIndex.index(*args)
+  end
+
+end
+
+require 'rbbt/segment/range_index'
+require 'rbbt/segment/overlaps'
+require 'rbbt/segment/transformed'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/encoding'
+

data/lib/rbbt/segment/annotation.rb

@@ -0,0 +1,58 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+require 'rbbt/segment'
+
+module AnnotID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def segid
+    @segid ||= _parts[0..4] * ":"
+  end
+
+  def type
+    @type ||= _parts[5]
+  end
+
+  property :annotation do
+    segment = SegID.setup(segid, :corpus => corpus).segment
+
+    SegmentAnnotation.setup(segment, :type => type)
+  end
+
+  property :annotid do
+    self
+  end
+
+end
+
+module SegmentAnnotation
+  extend Entity
+  include Segment
+  self.annotation :type
+
+  property :segid do
+    case self
+    when SegID
+      self
+    when Segment
+      super()
+    else
+      raise "Unknown object: #{self}"
+    end
+  end
+
+  property :annotid do |corpus=nil|
+    AnnotID.setup([segid, type] * ":", :corpus => corpus)
+  end
+
+  alias id annotid
+
+  property :annotation do
+    self
+  end
+end

data/lib/rbbt/segment/encoding.rb

@@ -0,0 +1,18 @@
+require 'rbbt/segment'
+module Segment
+  def self.bad_chars(text)
+    segments = []
+    text.chars.each_with_index do |c,i|
+      if ! c.ascii_only?
+        segments << Segment.setup(c, :offset => i)
+      end
+    end
+    segments
+  end
+
+  def self.ascii(text, replace = nil, &block)
+    bad = bad_chars(text)
+    replace = "?" if replace.nil?
+    Transformed.with_transform(text, bad, replace, &block)
+  end
+end

data/lib/rbbt/{text/segment → segment}/named_entity.rb

@@ -1,17 +1,18 @@
-require 'rbbt/text/segment'
-require 'rbbt/entity'
+require 'rbbt/segment'
+require 'rbbt/segment/annotation'
 
-module NamedEntity 
+module NamedEntity
   extend Entity
   include Segment
+  include SegmentAnnotation
 
-  self.annotation :type, :code, :score
+  self.annotation :entity_type, :code, :score
 
   def report
     <<-EOF
 String: #{ self }
 Offset: #{ offset.inspect }
-Type: #{type.inspect}
+Type: #{entity_type.inspect}
 Code: #{code.inspect}
 Score: #{score.inspect}
     EOF
@@ -20,7 +21,7 @@ Score: #{score.inspect}
   def html
     text = <<-EOF
 <span class='Entity'\
-#{type.nil? ? "" : " attr-entity-type='#{Array === type ? type * " " : type}'"}\
+#{entity_type.nil? ? "" : " attr-entity-type='#{Array === entity_type ? entity_type * " " : entity_type}'"}\
 #{code.nil?  ? "" : " attr-entity-code='#{Array === code ? code * " " : code}'"}\
 #{score.nil? ? "" : " attr-entity-score='#{Array === score ? score * " " : score}'"}\
 >#{ self }</span>
@@ -29,14 +30,14 @@ Score: #{score.inspect}
   end
 
   def entity(params = nil)
-    code = self.dup
+    code = self.code || self.dup
     format, entity = code.split(":")
     entity, format = format, nil if entity.nil?
-    
-    if defined?(Entity) && Entity.formats.include?(type) or Entity.formats.include?(format)
+
+    if defined?(Entity) && Entity.formats.include?(entity_type) or Entity.formats.include?(format)
       params ||= {}
       params[:format] = format if format and params[:format].nil?
-      mod = (Entity.formats[type] || Entity.format[entity])
+      mod = (Entity.formats[entity_type] || Entity.format[entity])
       mod.setup(entity, params)
     end
 
@@ -44,4 +45,3 @@ Score: #{score.inspect}
   end
 
 end
-

data/lib/rbbt/segment/overlaps.rb

@@ -0,0 +1,63 @@
+module Segment
+  def pull(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset += offset 
+    end
+
+    self
+  end
+
+  def push(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset -= offset 
+    end
+
+    self
+  end
+
+  def make_relative(segments, &block)
+    if block_given?
+      segments.each{|s| s.push offset}
+      yield(segments)
+      segments.each{|s| s.pull offset}
+    else
+      segments.each{|s| s.push offset}
+    end
+  end
+
+  def range_in(container = nil)
+    raise "No offset specified" if offset.nil?
+    case
+    when (Segment === container and not container.offset.nil?)
+      ((offset - container.offset)..(self.eend - container.offset))
+    when Integer === container
+      ((offset - container)..(self.eend - container))
+    else
+      range
+    end
+  end
+
+  def includes?(segment)
+    (segment.offset.to_i >= self.offset.to_i) and
+    (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i)
+  end
+
+  def overlaps?(segment)
+    segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.eend || 
+    self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.eend
+  end
+
+  def overlaps(segments)
+    segments.select{|s| self.overlaps?(s) }
+  end
+
+  def self.collisions(main, secondary)
+    secondary.select do |ss|
+      main.select{|ms| ms.overlaps? ss }.any?
+    end
+  end
+end