rbbt-text 0.2.1 → 0.5.0

data/lib/rbbt/corpus/document_repo.rb

@@ -0,0 +1,115 @@
+require 'rbbt/util/misc'
+require 'tokyocabinet'
+
+class DocumentRepo < TokyoCabinet::BDB
+  class OpenError < StandardError;end
+  class KeyFormatError < StandardError;end
+
+  CONNECTIONS = {} unless defined? CONNECTIONS
+
+  def self.get(path, write = false)
+
+    if !File.exists?(path) or not CONNECTIONS.include? path
+      CONNECTIONS[path] = self.new(path, true)
+    end
+
+    d = CONNECTIONS[path] 
+    
+    if write and not d.write?
+      d.write
+    else
+      d.read if d.write?
+    end
+
+    d
+  end
+
+
+  alias original_open open
+  def open(write = false)
+    flags = (write ? TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT : TokyoCabinet::BDB::OREADER)
+
+    FileUtils.mkdir_p File.dirname(@path_to_db) unless File.exists?(File.dirname(@path_to_db))
+    if !self.original_open(@path_to_db, flags)
+      ecode = self.ecode
+      raise OpenError, "Open error: #{self.errmsg(ecode)}. Trying to open file #{@path_to_db}"
+    end
+
+    @write = write
+
+  end
+
+  def write?
+    @write
+  end
+
+  def write
+    self.close
+    self.open(true)
+  end
+
+  def read
+    self.close
+    self.open(false)
+  end
+
+  def initialize(path, write = false)
+    super()
+
+    @path_to_db = path
+
+    if write || ! File.exists?(@path_to_db)
+      self.setcache(100000) or raise "Error setting cache"
+      self.open(true)
+    else
+      self.open(false)
+    end
+  end
+
+  def docid2fields(docid)
+    docid.split(":", -1).values_at 0,1,2,3
+  end
+
+  def fields2docid(namespace = nil, id = nil, type = nil, hash = nil)
+    [namespace, id, type, hash] * ":"
+  end
+
+  def docid(docid)
+    get(docid)
+  end
+
+  def add(text, namespace, id, type, hash)
+    write unless write?
+    docid = fields2docid(namespace, id, type, hash)
+    self[docid] = text unless self.include? docid
+    docid
+  end
+
+  def find(namespace=nil, id = nil, type = nil, hash = nil)
+    case
+    when namespace.nil?
+      self.keys
+    when id.nil?
+      range_start = [namespace] * ":" + ':'
+      range_end   = [namespace] * ":" + ';'
+      self.range(range_start, true, range_end, false)
+    when (type and hash)
+      [[namespace, id, type, hash] * ":"]
+    when hash
+      [[namespace, id, "", hash] * ":"]
+    when type
+      range_start = [namespace, id, type] * ":" + ':'
+      range_end   = [namespace, id, type] * ":" + ';'
+      self.range(range_start, true, range_end, false)
+    else
+      range_start = [namespace, id] * ":" + ':'
+      range_end   = [namespace, id] * ":" + ';'
+      self.range(range_start, true, range_end, false)
+    end
+  end
+
+  def find_docid(docid)
+    find(*docid2fields(docid))
+  end
+
+end

data/lib/rbbt/corpus/sources/pubmed.rb

@@ -0,0 +1,26 @@
+require 'rbbt/sources/pubmed'
+
+class Corpus
+
+  NAMESPACES = {} unless defined? NAMESPACES
+  NAMESPACES[:pubmed] = :add_pmid
+
+  def add_pmid(pmid, type = nil)
+    pmids = Array === pmid ? pmid : [pmid]
+    type = nil if String === type and type.empty?
+
+    PubMed.get_article(pmids).collect do |pmid, article|
+      if (type.nil? and article.pdf_url.nil?) or (not type.nil? and type.to_sym === :abstract)
+        add_document(article.text, :pubmed, pmid, :abstract)
+      else
+        raise "No FullText available for #{ pmid }" if article.pdf_url.nil?
+        add_document(article.full_text, :pubmed, pmid, :fulltext)
+      end
+    end
+  end
+  
+  def add_pubmed_query(query, max, type = nil)
+    pmids = PubMed.query(query, max)
+    add_pmid(pmids, type)
+  end
+end

data/lib/rbbt/ner/NER.rb

@@ -1,18 +1,20 @@
 require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/named_entity'
+require 'rbbt/ner/annotations/annotated'
 
 class NER
-  def entities(text, overlap = true, *args)
+  def entities(text, protect = false, *args)
     case
     when Array === text
       text.collect do |element|
-        matches = entities(element, overlap, *args)
+        matches = entities(element, protect, *args)
         matches.each{|match|
-          match.offset += element.offset if match.offset
+          match.offset += element.offset if match.offset and element.offset
         }
         matches
       end.flatten
-    when (Annotated === text and not overlap)
-      entities(text.split, overlap, *args)
+    when (Annotated === text and protect)
+      entities(text.split_segments(true), protect, *args)
     else
       match(text, *args)
     end

data/lib/rbbt/ner/abner.rb

@@ -7,7 +7,7 @@ require 'rbbt/ner/NER'
 # in Java Abner[http://www.cs.wisc.edu/~bsettles/abner/].
 class Abner < NER
 
-  Rbbt.add_software "ABNER" => ['','']
+  Rbbt.software.opt.ABNER.define_as_install Rbbt.share.install.software.ABNER.find
 
   @@JFile   = Rjb::import('java.io.File')
   @@Tagger  = Rjb::import('abner.Tagger')
@@ -27,14 +27,25 @@ class Abner < NER
   # returns all the mentions found, regardless of type, to be coherent
   # with the rest of NER packages in Rbbt.
   def match(text)
+    return [] if text.nil? or text.empty?
 
     res = @tagger.getEntities(text)
     types = res[1]
     strings = res[0]
 
+    global_offset = 0
     strings.zip(types).collect do |mention, type| 
       mention = mention.to_s; 
-      NamedEntity.annotate(mention, nil, type.to_s)
+      offset = text.index(mention)
+      if offset.nil?
+        NamedEntity.annotate(mention, nil, type.to_s)
+      else
+        NamedEntity.annotate(mention, offset + global_offset, type.to_s)
+        text = text[offset + mention.length..-1]
+        global_offset += offset + mention.length
+      end
+
+      mention
     end
   end
 

data/lib/rbbt/ner/annotations.rb

@@ -1,5 +1,63 @@
 module Segment 
-  attr_accessor :offset
+  attr_accessor :offset, :docid
+
+  def self.included(base)
+    if base.instance_methods.include? "segment_types"
+      class << base
+        self.module_eval do 
+          define_method "extended" do |object|
+            object.segment_types ||= []
+            object.segment_types << self.to_s unless object.segment_types.include? self.to_s
+          end
+        end
+      end
+    end
+  end
+
+  def self.annotate(string, offset = nil, docid = nil)
+    string.extend Segment
+    string.offset = offset
+    string.docid = docid
+    string
+  end
+
+  def id
+    new = info.dup
+    Digest::MD5.hexdigest(Misc.hash2string(new) << self << (offset || 0).to_s)
+  end
+ 
+  SKIP = %w(docid offset)
+  def info
+    equal_ascii = "="[0]
+    info = {}
+    singleton_methods.select{|method| method[-1] == equal_ascii}.
+      collect{|m| m[(0..-2)]}.each{|m| info[m] = self.send(m) if self.respond_to?(m) and not SKIP.include? m.to_s}
+    info
+    info.delete_if{|k,v| v.nil?}
+    info
+  end
+
+  def self.load(text, start, eend, info, docid = nil)
+    string = text[start.to_i..eend.to_i] if start and eend
+    string ||= info[:literal]
+    string.extend Segment
+
+    # add types
+    types = info.delete("segment_types")|| info.delete(:segment_types) || []
+    types.each do |type| string.extend Misc.string2const(type) end
+
+    # set info data
+    info.each do |key,value|
+      string.send key + '=', value if string.respond_to? key.to_sym
+    end
+
+    string.docid = docid
+    string.offset = start.to_i
+
+    string
+  end
+
+  # {{{ Sorting and splitting
 
   def self.sort(segments, inline = true)
     if inline
@@ -14,21 +72,43 @@ module Segment
         when (not a.range.include? b.offset and not b.range.include? a.offset)
           a.offset <=> b.offset
         else
-          b.length <=> a.length
+          a.length <=> b.length
         end
-      end.reverse
+      end
     else
-      segments.sort_by do |segment| segment.offset || 0 end
+      segments.sort_by do |segment| segment.offset || 0 end.reverse
+    end
+  end
+
+  def self.overlaps(sorted_segments)
+
+    last = nil
+    overlaped = []
+    sorted_segments.reverse.each do |segment| 
+      overlaped << segment if (not last.nil?) and segment.range.end > last 
+      last = segment.range.begin
     end
+
+    overlaped
+  end
+
+  def self.clean_sort(segments)
+    sorted = sort(segments).reject{|s| s.offset.nil?}
+    overlaps = overlaps(sorted)
+    overlaps.each do |s|
+      sorted.delete s
+    end
+
+    sorted
   end
 
-  def self.split(text, segments)
-    sorted_segments = sort segments
+  def self.split(text, segments, skip_segments = false)
+    sorted_segments = clean_sort segments
 
     chunks      = []
     segment_end = 0
     text_offset = 0
-    sorted_segments.each do |segment|
+    sorted_segments.reverse.each do |segment|
       return chunks if text.nil? or text.empty?
       next if segment.offset.nil?
       offset = segment.offset - text_offset
@@ -45,12 +125,15 @@ module Segment
 
       segment_end = offset + segment.length - 1
 
-      chunk = text[offset..segment_end]
-      Segment.annotate(chunk, text_offset + offset)
-      chunks << chunk
+      if not skip_segments
+        chunk = text[offset..segment_end]
+        Segment.annotate(chunk, text_offset + offset)
+        chunks << chunk
+      end
 
       text_offset += segment_end + 1
       text = text[segment_end + 1..-1]
+
     end
 
     if not text.nil? and text.any?
@@ -62,62 +145,110 @@ module Segment
     chunks
   end
 
-  def self.annotate(string, offset = nil)
-    string.extend Segment
-    string.offset = offset
-    string
+  # {{{ Ranges and manipulation
+
+  def pull(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset += offset 
+    end
+
+    self
+  end
+
+  def push(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset -= offset 
+    end
+
+    self
+  end
+
+  def make_relative(segments)
+    segments.collect{|s| s.push offset}
+  end
+
+  def end
+    return nil if offset.nil?
+    offset + length - 1
   end
 
   def range
-    (offset..offset + length - 1)
+    raise "No offset specified" if offset.nil?
+    (offset..self.end)
   end
-end
 
-module Annotated
-  attr_accessor :annotations
-  def self.annotate(string)
-    string.extend Annotated
-    string.annotations = []
-    string
+  def range_in(container = nil)
+    raise "No offset specified" if offset.nil?
+    case
+    when (Segment === container and not container.offset.nil?)
+      ((offset - container.offset)..(self.end - container.offset))
+    when Integer === container
+      ((offset - container)..(self.end - container))
+    else
+      range
+    end
   end
 
-  def split
-    Segment.split(self, @annotations)
+  def self.align(text, parts)
+    pre_offset = 0
+    parts.each do |part|
+      offset = text.index part
+      next if offset.nil?
+      Segment.annotate(part, pre_offset + offset)
+      pre_offset += offset + part.length - 1
+      text = text[(offset + part.length - 1)..-1]
+    end
   end
-end
 
-module NamedEntity 
-  include Segment
-  attr_accessor :type, :code, :score
+  class Index
+    attr_accessor :index, :data
+    def initialize(index, data)
+      @index = index
+      @data = data
+    end
 
-  def self.annotate(string, offset = nil, type = nil, code = nil, score = nil)
-    string.extend NamedEntity
-    string.offset = offset
-    string.type  = type
-    string.code  = code
-    string.score = score
-    string
+    def [](pos)
+      index[pos].collect{|id| data[id]}
+    end
   end
 
-  def to_s
-    <<-EOF
-String: #{ self }
-Offset: #{ offset.inspect }
-Type: #{type.inspect}
-Code: #{code.inspect}
-Score: #{score.inspect}
-    EOF
+  def self.index(segments, persistence_file = :memory)
+
+    segments = segments.values.flatten if Hash === segments
+
+    annotation_index = 
+      Persistence.persist("Index", :Index, :fwt, :persistence => (! (persistence_file.nil? or persistence_file == :memory)), :persistence_file => persistence_file, :range => true) do
+
+        value_size = 0
+        index_data = segments.collect{|segment| 
+          next if segment.offset.nil?
+          range = segment.range
+          value_size = [segment.id.length, value_size].max
+          [segment.id, [range.begin, range.end]]
+        }.compact
+
+        fwt = FixWidthTable.get :memory, value_size, true
+        fwt.add_range index_data
+        fwt
+      end
+
+    data = {}
+    segments.each do |segment| data[segment.id] = segment end
+    Index.new annotation_index, data
   end
+
 end
 
-module Token
+module Comment
   include Segment
-  attr_accessor :original
-  def self.annotate(string, offset = nil, original = nil)
-    string.extend Token
-    string.offset   = offset
-    string.original = original
-    string
+  attr_accessor :comment
+  def self.annotate(text, comment = nil)
+    text.extend Comment
+    text.comment = (comment.nil? ? text : comment)
+    text
   end
 end
-