rbbt-text 1.1.9 → 1.3.3

data/lib/rbbt/segment.rb

@@ -0,0 +1,179 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+require 'rbbt/document'
+
+module SegID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def range
+    @range ||= Range.new(*_parts[4].split("..").map(&:to_i))
+  end
+
+  def docid
+    @docid ||= DocID.setup(_parts[0..3] * ":")
+  end
+
+  def offset
+    range.begin
+  end
+
+  def segment_length
+    range.end - range.begin + 1
+  end
+
+  property :segment => :single do
+    docid = self.docid
+    document = DocID.setup(docid, :corpus => corpus).document
+
+    text = document[range]
+
+    Segment.setup(text, :docid => docid, :offset => offset)
+  end
+
+  property :segid do
+    self
+  end
+
+end
+
+module Segment
+  extend Entity
+  self.annotation :offset, :docid 
+
+  def segment_length
+    length
+  end
+
+  def eend
+    offset.to_i + length - 1
+  end
+
+  def range
+    (offset.to_i..eend)
+  end
+
+  property :segid do |corpus=nil|
+    SegID.setup([docid, range] * ":", :corpus => corpus)
+  end
+
+  alias id segid
+
+  property :segment do
+    self
+  end
+
+  def self.sort(segments, inline = true)
+    if inline
+      segments.sort do |a,b| 
+        case
+        when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
+          0
+        when (a.nil? or a.offset.nil?)
+          -1
+        when (b.nil? or b.offset.nil?)
+          +1
+        when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i)
+          a.offset.to_i <=> b.offset.to_i
+        else
+          a.segment_length <=> b.segment_length
+        end
+      end
+    else
+      segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse
+    end
+  end
+
+  def self.overlaps(sorted_segments)
+    last = nil
+    overlaped = []
+
+    sorted_segments.reverse.each do |segment| 
+      overlaped << segment if (not last.nil?) and segment.range.end > last 
+      last = segment.range.begin
+    end
+
+    overlaped
+  end
+
+  def self.clean_sort(segments)
+    sorted = sort(segments).reject{|s| s.offset.nil?}
+    overlaps = overlaps(sorted)
+    overlaps.each do |s|
+      sorted.delete s
+    end
+
+    sorted
+  end
+
+  def self.split(text, segments, skip_segments = false)
+    sorted_segments = clean_sort segments
+
+    chunks      = []
+    segment_end = 0
+    text_offset = 0
+    sorted_segments.each do |segment|
+      return chunks if text.nil? or text.empty?
+      next if segment.offset.nil?
+      offset = segment.offset - text_offset
+
+      # Consider segment offset. Save pre, or skip if overlap
+      case
+      when offset < 0 # Overlap, skip
+        next
+      when offset > 0 # Save pre
+        chunk = text[0..offset - 1]
+        Segment.setup(chunk, text_offset)
+        chunks << chunk
+      end
+
+      segment_end = offset + segment.segment_length - 1
+
+      if not skip_segments
+        chunk = text[offset..segment_end]
+        Segment.setup(chunk, text_offset + offset)
+        chunks << chunk
+      end
+
+      text_offset += segment_end + 1
+      text = text[segment_end + 1..-1]
+
+    end
+
+    if not text.nil? and not text.empty?
+      chunk = text.dup
+      Segment.setup(chunk, text_offset)
+      chunks << chunk
+    end
+
+    chunks
+  end
+
+  def self.align(text, parts)
+    pre_offset = 0
+    docid = text.respond_to?(:docid) ? text.docid : nil
+    parts.each do |part|
+      offset = text.index part
+      next if offset.nil?
+      Segment.setup(part, pre_offset + offset, docid)
+      pre_offset += offset + part.segment_length - 1
+      text = text[(offset + part.segment_length - 1)..-1]
+    end
+  end
+
+  def self.index(*args)
+    Segment::RangeIndex.index(*args)
+  end
+
+end
+
+require 'rbbt/segment/range_index'
+require 'rbbt/segment/overlaps'
+require 'rbbt/segment/transformed'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/encoding'
+

data/lib/rbbt/segment/annotation.rb

@@ -0,0 +1,58 @@
+require 'rbbt-util'
+require 'rbbt/segment'
+require 'rbbt/entity'
+
+module AnnotID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def segid
+    @segid ||= _parts[0..4] * ":"
+  end
+
+  def type
+    @type ||= _parts[5]
+  end
+
+  property :annotation do
+    segment = SegID.setup(segid, :corpus => corpus).segment
+
+    SegmentAnnotation.setup(segment, :type => type)
+  end
+
+  property :annotid do
+    self
+  end
+
+end
+
+module SegmentAnnotation
+  extend Entity
+  include Object::Segment
+  self.annotation :type
+
+  property :segid do
+    case self
+    when SegID
+      self
+    when Segment
+      super()
+    else
+      raise "Unknown object: #{self}"
+    end
+  end
+
+  property :annotid do |corpus=nil|
+    AnnotID.setup([segid, type, Misc.obj2digest(self.info)] * ":", :corpus => corpus)
+  end
+
+  alias id annotid
+
+  property :annotation do
+    self
+  end
+end

data/lib/rbbt/segment/encoding.rb

@@ -0,0 +1,18 @@
+require 'rbbt/segment'
+module Segment
+  def self.bad_chars(text)
+    segments = []
+    text.chars.each_with_index do |c,i|
+      if ! c.ascii_only?
+        segments << Segment.setup(c, :offset => i)
+      end
+    end
+    segments
+  end
+
+  def self.ascii(text, replace = nil, &block)
+    bad = bad_chars(text)
+    replace = "?" if replace.nil?
+    Transformed.with_transform(text, bad, replace, &block)
+  end
+end

data/lib/rbbt/{text/segment → segment}/named_entity.rb

@@ -1,17 +1,18 @@
-require 'rbbt/text/segment'
-require 'rbbt/entity'
+require 'rbbt/segment'
+require 'rbbt/segment/annotation'
 
-module NamedEntity 
+module NamedEntity
   extend Entity
   include Segment
+  include SegmentAnnotation
 
-  self.annotation :type, :code, :score
+  self.annotation :entity_type, :code, :score
 
   def report
     <<-EOF
 String: #{ self }
 Offset: #{ offset.inspect }
-Type: #{type.inspect}
+Type: #{entity_type.inspect}
 Code: #{code.inspect}
 Score: #{score.inspect}
     EOF
@@ -20,7 +21,7 @@ Score: #{score.inspect}
   def html
     text = <<-EOF
 <span class='Entity'\
-#{type.nil? ? "" : " attr-entity-type='#{Array === type ? type * " " : type}'"}\
+#{entity_type.nil? ? "" : " attr-entity-type='#{Array === entity_type ? entity_type * " " : entity_type}'"}\
 #{code.nil?  ? "" : " attr-entity-code='#{Array === code ? code * " " : code}'"}\
 #{score.nil? ? "" : " attr-entity-score='#{Array === score ? score * " " : score}'"}\
 >#{ self }</span>
@@ -29,13 +30,14 @@ Score: #{score.inspect}
   end
 
   def entity(params = nil)
+    code = self.code || self.dup
     format, entity = code.split(":")
     entity, format = format, nil if entity.nil?
-    
-    if defined?(Entity) && Entity.formats.include?(type) or Entity.formats.include?(format)
+
+    if defined?(Entity) && Entity.formats.include?(entity_type) or Entity.formats.include?(format)
       params ||= {}
       params[:format] = format if format and params[:format].nil?
-      mod = (Entity.formats[type] || Entity.format[entity])
+      mod = (Entity.formats[entity_type] || Entity.format[entity])
       mod.setup(entity, params)
     end
 
@@ -43,4 +45,3 @@ Score: #{score.inspect}
   end
 
 end
-

data/lib/rbbt/segment/overlaps.rb

@@ -0,0 +1,63 @@
+module Segment
+  def pull(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset += offset 
+    end
+
+    self
+  end
+
+  def push(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset -= offset 
+    end
+
+    self
+  end
+
+  def make_relative(segments, &block)
+    if block_given?
+      segments.each{|s| s.push offset}
+      yield(segments)
+      segments.each{|s| s.pull offset}
+    else
+      segments.each{|s| s.push offset}
+    end
+  end
+
+  def range_in(container = nil)
+    raise "No offset specified" if offset.nil?
+    case
+    when (Segment === container and not container.offset.nil?)
+      ((offset - container.offset)..(self.eend - container.offset))
+    when Integer === container
+      ((offset - container)..(self.eend - container))
+    else
+      range
+    end
+  end
+
+  def includes?(segment)
+    (segment.offset.to_i >= self.offset.to_i) and
+    (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i)
+  end
+
+  def overlaps?(segment)
+    segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.eend || 
+    self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.eend
+  end
+
+  def overlaps(segments)
+    segments.select{|s| self.overlaps?(s) }
+  end
+
+  def self.collisions(main, secondary)
+    secondary.select do |ss|
+      main.select{|ms| ms.overlaps? ss }.any?
+    end
+  end
+end

data/lib/rbbt/segment/range_index.rb

@@ -0,0 +1,35 @@
+module Segment::RangeIndex
+  attr_accessor :corpus
+
+  def [](*args)
+    res = super(*args)
+    SegID.setup(res, :corpus => corpus)
+  end
+
+  def self.index(segments, corpus, persist_file = :memory)
+    segments = segments.values.flatten if Hash === segments
+
+    annotation_index = 
+      Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do
+
+        value_size = 0
+        index_data = segments.collect{|segment| 
+          next if segment.offset.nil?
+          range = segment.range
+          value_size = [segment.segid.length, value_size].max
+          [segment.segid, [range.begin, range.end]]
+        }.compact
+
+        fwt = FixWidthTable.get :memory, value_size, true
+        fwt.add_range index_data
+
+        fwt
+      end
+
+    annotation_index.extend Segment::RangeIndex
+    annotation_index.corpus = corpus
+    annotation_index
+  end
+
+end
+  

data/lib/rbbt/segment/relationship.rb

@@ -0,0 +1,7 @@
+module Relationship
+  extend Entity
+
+  self.annotation :segments
+  self.annotation :type
+
+end

data/lib/rbbt/{text/segment → segment}/segmented.rb

@@ -1,5 +1,5 @@
 require 'rbbt/annotations'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 
 module Segmented
   extend Annotation

data/lib/rbbt/segment/token.rb

@@ -0,0 +1,23 @@
+require 'rbbt/segment'
+
+module Token
+  extend Entity
+  include Segment
+
+  self.annotation :original
+
+  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+
+    tokens = []
+    while matchdata = text.match(split_at)
+      tokens << Token.setup(matchdata.pre_match, :offset => start) unless matchdata.pre_match.empty?
+      tokens << Token.setup(matchdata.captures.first, :offset => start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      start += matchdata.end(0)
+      text = matchdata.post_match
+    end
+
+    tokens << Token.setup(text, :offset => start) unless text.empty?
+
+    tokens
+  end
+end