rbbt-text 1.1.8 → 1.1.9

data/lib/rbbt/text/corpus/document_repo.rb

@@ -0,0 +1,68 @@
+require 'rbbt/util/misc'
+require 'tokyocabinet'
+
+class Corpus
+  module DocumentRepo
+    class OpenError < StandardError;end
+    class KeyFormatError < StandardError;end
+
+    TC_CONNECTIONS = {}
+    def self.open_tokyocabinet(path, write)
+      database = Persist.open_tokyocabinet(path, write, :single, TokyoCabinet::BDB)
+      database.extend DocumentRepo
+      database
+    end
+
+    def docid2fields(docid)
+      docid.split(":", -1).values_at 0,1,2,3
+    end
+
+    def fields2docid(namespace = nil, id = nil, type = nil, hash = nil)
+      [namespace, id, type, hash] * ":"
+    end
+
+    def docid(docid)
+      get(docid)
+    end
+
+    def add(text, namespace, id, type, hash)
+      docid = fields2docid(namespace, id, type, hash)
+
+      return docid if self.include?(docid)
+
+      write_and_close do
+        self[docid] = text
+      end
+
+      docid
+    end
+
+    def find(namespace=nil, id = nil, type = nil, hash = nil)
+      case
+      when namespace.nil?
+        self.keys
+      when id.nil?
+        range_start = [namespace] * ":" + ':'
+        range_end   = [namespace] * ":" + ';'
+        self.range(range_start, true, range_end, false)
+      when (type and hash)
+        [[namespace, id, type, hash] * ":"]
+      when hash
+        [[namespace, id, "", hash] * ":"]
+      when type
+        range_start = [namespace, id, type] * ":" + ':'
+        range_end   = [namespace, id, type] * ":" + ';'
+        self.range(range_start, true, range_end, false)
+      else
+        range_start = [namespace, id] * ":" + ':'
+        range_end   = [namespace, id] * ":" + ';'
+        self.range(range_start, true, range_end, false)
+      end
+    end
+
+    def find_docid(docid)
+      find(*docid2fields(docid))
+    end
+
+  end
+end

data/lib/rbbt/text/corpus/sources/pmid.rb

@@ -0,0 +1,34 @@
+require 'rbbt/sources/pubmed'
+
+class Corpus
+
+  NAMESPACES = {} unless defined? NAMESPACES
+  NAMESPACES[:pubmed] = :add_pmid
+
+  def add_pmid(pmid, type = nil)
+    pmids = Array === pmid ? pmid : [pmid]
+    type = nil if String === type and type.empty?
+
+    PubMed.get_article(pmids).collect do |pmid, article|
+      Log.debug "Loading pmid #{pmid}"
+      if type.nil? || type.to_sym == :abstract
+        add_document(article.abstract || "", :PMID, pmid, :abstract)
+      elsif type.to_sym == :title
+        add_document(article.title, :PMID, pmid, :title)
+      else
+        raise "No FullText available for #{ pmid }" if article.full_text.nil?
+        add_document(article.full_text, :PMID, pmid, :fulltext)
+      end
+    end
+  end
+  
+  def add_pubmed_query(query, max = 3000, type = nil)
+    pmids = PubMed.query(query, max)
+    add_pmid(pmids, type)
+  end
+
+  self.claim "PMID" do |id, type|
+    Log.debug "Claiming #{id}"
+    self.add_pmid(id, type)
+  end
+end

data/lib/rbbt/text/document.rb

@@ -0,0 +1,39 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+
+require 'rbbt/text/corpus'
+
+module Document
+  extend Entity
+  class << self
+    attr_accessor :corpus
+  end
+
+  property :document => :single do
+    Document.corpus.docid(self)
+  end
+
+  property :type => :single do |type|
+    self.annotate((self.split(":").values_at(0,1)) * ":" + ":" + type.to_s)
+  end
+
+  property :title => :single do
+    type(:title).text
+  end
+
+  property :full_text => :single do
+    type(:full_text).text
+  end
+
+  property :abstract => :single do
+    type(:abstract).text
+  end
+
+  property :text => :single do
+    document.text
+  end
+
+  property :entities => :single do |type,*args|
+    document.method(type).call *args
+  end
+end

data/lib/rbbt/{ner → text}/segment.rb

@@ -77,9 +77,14 @@ module Segment
     self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.end
   end
 
+  def overlaps(segments)
+    segments.select{|s| self.overlaps?(s)}
+  end
+
+
   def self.collisions(main, secondary)
-    collisions = secondary.select do |ss|
-      collisions = main.select{|ms| ms.overlaps? ss }.any?
+    secondary.select do |ss|
+      main.select{|ms| ms.overlaps? ss }.any?
     end
   end
 
@@ -320,7 +325,7 @@ module Segment
     tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
 
     segments.each do |segment|
-      tsv[segment.id] = self.tsv_values_for_segment(segment, fields)
+      tsv[segment.object_id.to_s] = self.tsv_values_for_segment(segment, fields)
     end
 
     tsv
@@ -343,8 +348,8 @@ module Segment
     [offset, self.end] * ".."
   end
 
-  def ==(other)
-    self.id == other.id
-  end
+  #def ==(other)
+  #  self.text == other.text
+  #end
 end
 

data/lib/rbbt/{ner → text}/segment/docid.rb

@@ -1,4 +1,4 @@
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
 module SegmentWithDocid 
   extend Annotation

data/lib/rbbt/{ner → text}/segment/named_entity.rb

@@ -1,4 +1,4 @@
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/entity'
 
 module NamedEntity 
@@ -32,7 +32,7 @@ Score: #{score.inspect}
     format, entity = code.split(":")
     entity, format = format, nil if entity.nil?
     
-    if defined? Entity and Entity.formats.include? type or Entity.formats.include? format
+    if defined?(Entity) && Entity.formats.include?(type) or Entity.formats.include?(format)
       params ||= {}
       params[:format] = format if format and params[:format].nil?
       mod = (Entity.formats[type] || Entity.format[entity])

data/lib/rbbt/{ner → text}/segment/relationship.rb

@@ -1,4 +1,4 @@
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
 module Relationship
   extend Annotation

data/lib/rbbt/{ner → text}/segment/segmented.rb

@@ -1,5 +1,5 @@
 require 'rbbt/annotations'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
 module Segmented
   extend Annotation

data/lib/rbbt/{ner → text}/segment/token.rb

@@ -1,5 +1,5 @@
 require 'rbbt/annotations'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
 module Token
   attr_accessor :offset, :original

data/lib/rbbt/{ner → text}/segment/transformed.rb

@@ -1,10 +1,12 @@
 require 'rbbt/util/misc'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
 module Transformed
 
   def self.transform(text, segments, replacement = nil, &block)
 
+    block = replacement if Proc === replacement
+
     text.extend Transformed
     text.replace_segments(segments, replacement, &block)
 
@@ -24,68 +26,44 @@ module Transformed
   end
 
   attr_accessor :transformed_segments, :transformation_stack
- 
+
   def shift(segment_o)
     begin_shift = 0
     end_shift = 0
 
+    text_offset = self.respond_to?(:offset)? self.offset.to_i : 0
     @transformed_segments.sort_by{|id, info| info.last}.each{|id,info| 
-      pseg_o, diff = info
+      pseg_o, diff, utext, pseg_u, index  = info
+
+      pseg_u = ((pseg_u.begin + text_offset)..(pseg_u.last + text_offset))
 
       case
         # Before
-      when segment_o.last + end_shift < pseg_o.begin
+      when segment_o.last + end_shift < pseg_u.begin
         # After
-      when (segment_o.begin + begin_shift > pseg_o.last)
+      when (segment_o.begin + begin_shift > pseg_u.last)
         begin_shift += diff
         end_shift += diff
         # Includes
-      when (segment_o.begin + begin_shift <= pseg_o.begin and segment_o.last + end_shift >= pseg_o.last)
+      when (segment_o.begin + begin_shift <= pseg_u.begin and segment_o.last + end_shift >= pseg_u.last)
         end_shift += diff
         # Inside
-      when (segment_o.begin + begin_shift >= pseg_o.begin and segment_o.last + end_shift <= pseg_o.last)
+      when (segment_o.begin + begin_shift >= pseg_u.begin and segment_o.last + end_shift <= pseg_u.last)
         return nil
         # Overlaps start
-      when (segment_o.begin + begin_shift <= pseg_o.begin and segment_o.last + end_shift <= pseg_o.last)
+      when (segment_o.begin + begin_shift <= pseg_u.begin and segment_o.last + end_shift <= pseg_u.last)
         return nil
         # Overlaps end
-      when (segment_o.begin + begin_shift >= pseg_o.begin and segment_o.last + end_shift >= pseg_o.last)
+      when (segment_o.begin + begin_shift >= pseg_u.begin and segment_o.last + end_shift >= pseg_u.last)
         return nil
-     else
-        raise "Unknown overlaps: #{segment_o.inspect} - #{pseg_o.inspect}"
+      else
+        raise "Unknown overlaps: #{segment_o.inspect} - #{pseg_u.inspect}"
       end
     }
 
     [begin_shift, end_shift]
   end
 
-  #def self.sort(segments)
-  #  segments.compact.sort do |a,b|
-  #    case
-  #    when ((a.nil? && b.nil?) || (a.offset.nil? && b.offset.nil?))
-  #      0
-  #    when (a.nil? || a.offset.nil?)
-  #      -1
-  #    when (b.nil? || b.offset.nil?)
-  #      +1
-  #      # Non-overlap
-  #    when (a.end < b.offset.to_i || b.end < a.offset.to_i)
-  #      b.offset <=> a.offset
-  #      # b includes a
-  #    when (a.offset.to_i >= b.offset.to_i && a.end <= b.end)
-  #      -1
-  #      # b includes a
-  #    when (b.offset.to_i >= a.offset.to_i && b.end <= a.end)
-  #      +1
-  #      # Overlap
-  #    when (a.offset.to_i > b.offset.to_i && a.end > b.end || b.offset.to_i > a.offset.to_i && b.end > a.end)
-  #      b.length <=> a.length
-  #    else
-  #      raise "Unexpected case in sort: #{a.range} - #{b.range}"
-  #    end
-  #  end
-  #end
-
   def replace_segments(segments, replacement = nil, &block)
     @transformed_segments ||= {}
     @transformation_stack ||= []
@@ -93,8 +71,9 @@ module Transformed
 
     segments = [segments] unless Array === segments 
     orig_length = self.length
-    Segment.sort(segments).each do |segment|
+    Segment.clean_sort(segments).each do |segment|
       next if segment.offset.nil?
+
       shift = shift segment.range
 
       next if shift.nil?
@@ -102,6 +81,7 @@ module Transformed
       shift_begin, shift_end = shift
 
       text_offset = self.respond_to?(:offset)? self.offset.to_i : 0
+
       updated_begin = segment.offset.to_i + shift_begin - text_offset
       updated_end   = segment.range.last + shift_end - text_offset
 
@@ -113,6 +93,8 @@ module Transformed
         next
       end
 
+      #raise "error '#{segment}' => '#{updated_text}'" if updated_text != segment
+
       original_text = segment.dup
       segment.replace updated_text
 
@@ -177,8 +159,31 @@ module Transformed
     end
   end
 
-  def self.ansi(text, entities, colors = nil)
-
+  #def self.sort(segments)
+  #  segments.compact.sort do |a,b|
+  #    case
+  #    when ((a.nil? && b.nil?) || (a.offset.nil? && b.offset.nil?))
+  #      0
+  #    when (a.nil? || a.offset.nil?)
+  #      -1
+  #    when (b.nil? || b.offset.nil?)
+  #      +1
+  #      # Non-overlap
+  #    when (a.end < b.offset.to_i || b.end < a.offset.to_i)
+  #      b.offset <=> a.offset
+  #      # b includes a
+  #    when (a.offset.to_i >= b.offset.to_i && a.end <= b.end)
+  #      -1
+  #      # b includes a
+  #    when (b.offset.to_i >= a.offset.to_i && b.end <= a.end)
+  #      +1
+  #      # Overlap
+  #    when (a.offset.to_i > b.offset.to_i && a.end > b.end || b.offset.to_i > a.offset.to_i && b.end > a.end)
+  #      b.length <=> a.length
+  #    else
+  #      raise "Unexpected case in sort: #{a.range} - #{b.range}"
+  #    end
+  #  end
+  #end
 
-  end
 end

data/test/rbbt/entity/test_document.rb

@@ -8,6 +8,7 @@ Workflow.require_workflow "TextMining"
 
 require 'rbbt/entity/pmid'
 require 'rbbt/entity/document'
+require 'rbbt/corpus'
 require 'test/unit'
 
 

data/test/rbbt/ner/test_abner.rb

@@ -13,6 +13,7 @@ class TestAbner < Test::Unit::TestCase
         assert(mentions.include? mention)
       }
     rescue
+      Log.exception $!
     end
   end
 end

data/test/rbbt/ner/test_linnaeus.rb

@@ -11,6 +11,7 @@ class TestLinnaeus < Test::Unit::TestCase
         assert(mentions.include? mention)
       }
     rescue
+      Log.exception $!
     end
   end
 end

data/test/rbbt/nlp/open_nlp/test_sentence_splitter.rb

@@ -29,7 +29,6 @@ sentence. This is
 another sentence. 
     EOF
 
-    iii OpenNLP.sentence_split_detector.sentDetect(text)
     assert_equal 5, OpenNLP.sentence_split_detector.sentDetect(text).length
 
     assert_equal 5, OpenNLP.sentence_splitter(text).length

data/test/rbbt/text/corpus/sources/test_pmid.rb

@@ -0,0 +1,33 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../../..', 'test_helper.rb')
+require 'rbbt/text/document'
+require 'rbbt/text/corpus'
+require 'rbbt/text/corpus/sources/pmid'
+
+class TestCorpusPMID < Test::Unit::TestCase
+  def setup
+    Log.severity = 0
+    Document.corpus = Corpus.new Rbbt.tmp.test.document_corpus
+
+    Corpus::Document.define :words do
+      words = self.text.split(" ")
+      Segment.align(self.text, words)
+    end
+
+    Corpus::Document.define :genes do
+      require 'rbbt/ner/banner'
+      Banner.new.match(self.text)
+    end
+
+    Corpus::Document.persist_in_global_tsv("genes")
+    Corpus::Document.persist_in_global_tsv(:words)
+  end
+
+  def test_query
+    docids = Document.corpus.add_pubmed_query("SARS-Cov-2", 2000, :abstract)
+
+    docids.each do |docid|
+      iif Document.corpus.docid(docid).text
+    end
+  end
+end
+