rbbt-text 1.2.0 → 1.3.4

data/lib/rbbt/segment/annotation.rb

@@ -0,0 +1,58 @@
+require 'rbbt-util'
+require 'rbbt/segment'
+require 'rbbt/entity'
+
+module AnnotID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def segid
+    @segid ||= _parts[0..4] * ":"
+  end
+
+  def type
+    @type ||= _parts[5]
+  end
+
+  property :annotation do
+    segment = SegID.setup(segid, :corpus => corpus).segment
+
+    SegmentAnnotation.setup(segment, :type => type)
+  end
+
+  property :annotid do
+    self
+  end
+
+end
+
+module SegmentAnnotation
+  extend Entity
+  include Object::Segment
+  self.annotation :type
+
+  property :segid do
+    case self
+    when SegID
+      self
+    when Segment
+      super()
+    else
+      raise "Unknown object: #{self}"
+    end
+  end
+
+  property :annotid do |corpus=nil|
+    AnnotID.setup([segid, type, Misc.obj2digest(self.info)] * ":", :corpus => corpus)
+  end
+
+  alias id annotid
+
+  property :annotation do
+    self
+  end
+end

data/lib/rbbt/segment/encoding.rb

@@ -0,0 +1,18 @@
+require 'rbbt/segment'
+module Segment
+  def self.bad_chars(text)
+    segments = []
+    text.chars.each_with_index do |c,i|
+      if ! c.ascii_only?
+        segments << Segment.setup(c, :offset => i)
+      end
+    end
+    segments
+  end
+
+  def self.ascii(text, replace = nil, &block)
+    bad = bad_chars(text)
+    replace = "?" if replace.nil?
+    Transformed.with_transform(text, bad, replace, &block)
+  end
+end

data/lib/rbbt/{text/segment → segment}/named_entity.rb

@@ -1,42 +1,46 @@
-require 'rbbt/text/segment'
-require 'rbbt/entity'
+require 'rbbt/segment'
+require 'rbbt/segment/annotation'
 
-module NamedEntity 
+module NamedEntity
   extend Entity
   include Segment
+  include SegmentAnnotation
 
-  self.annotation :type, :code, :score
+  self.annotation :entity_type, :code, :score
 
   def report
     <<-EOF
 String: #{ self }
 Offset: #{ offset.inspect }
-Type: #{type.inspect}
+Type: #{entity_type.inspect}
 Code: #{code.inspect}
 Score: #{score.inspect}
     EOF
   end
 
   def html
+    title = code.nil? ? entity_type : [entity_type, code].compact * ":"
+
     text = <<-EOF
 <span class='Entity'\
-#{type.nil? ? "" : " attr-entity-type='#{Array === type ? type * " " : type}'"}\
+#{entity_type.nil? ? "" : " attr-entity-type='#{Array === entity_type ? entity_type * " " : entity_type}'"}\
 #{code.nil?  ? "" : " attr-entity-code='#{Array === code ? code * " " : code}'"}\
 #{score.nil? ? "" : " attr-entity-score='#{Array === score ? score * " " : score}'"}\
+#{title.nil? ? "" : " title='#{Array === title ? title * " " : title}'"}\
 >#{ self }</span>
     EOF
     text.chomp
   end
 
   def entity(params = nil)
-    code = self.dup
+    code = self.code || self.dup
     format, entity = code.split(":")
     entity, format = format, nil if entity.nil?
-    
-    if defined?(Entity) && Entity.formats.include?(type) or Entity.formats.include?(format)
+
+    if defined?(Entity) && Entity.formats.include?(entity_type) or Entity.formats.include?(format)
       params ||= {}
       params[:format] = format if format and params[:format].nil?
-      mod = (Entity.formats[type] || Entity.format[entity])
+      mod = (Entity.formats[entity_type] || Entity.format[entity])
       mod.setup(entity, params)
     end
 
@@ -44,4 +48,3 @@ Score: #{score.inspect}
   end
 
 end
-

data/lib/rbbt/segment/overlaps.rb

@@ -0,0 +1,63 @@
+module Segment
+  def pull(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset += offset 
+    end
+
+    self
+  end
+
+  def push(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset -= offset 
+    end
+
+    self
+  end
+
+  def make_relative(segments, &block)
+    if block_given?
+      segments.each{|s| s.push offset}
+      yield(segments)
+      segments.each{|s| s.pull offset}
+    else
+      segments.each{|s| s.push offset}
+    end
+  end
+
+  def range_in(container = nil)
+    raise "No offset specified" if offset.nil?
+    case
+    when (Segment === container and not container.offset.nil?)
+      ((offset - container.offset)..(self.eend - container.offset))
+    when Integer === container
+      ((offset - container)..(self.eend - container))
+    else
+      range
+    end
+  end
+
+  def includes?(segment)
+    (segment.offset.to_i >= self.offset.to_i) and
+    (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i)
+  end
+
+  def overlaps?(segment)
+    segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.eend || 
+    self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.eend
+  end
+
+  def overlaps(segments)
+    segments.select{|s| self.overlaps?(s) }
+  end
+
+  def self.collisions(main, secondary)
+    secondary.select do |ss|
+      main.select{|ms| ms.overlaps? ss }.any?
+    end
+  end
+end

data/lib/rbbt/segment/range_index.rb

@@ -0,0 +1,35 @@
+module Segment::RangeIndex
+  attr_accessor :corpus
+
+  def [](*args)
+    res = super(*args)
+    SegID.setup(res, :corpus => corpus)
+  end
+
+  def self.index(segments, corpus, persist_file = :memory)
+    segments = segments.values.flatten if Hash === segments
+
+    annotation_index = 
+      Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do
+
+        value_size = 0
+        index_data = segments.collect{|segment| 
+          next if segment.offset.nil?
+          range = segment.range
+          value_size = [segment.segid.length, value_size].max
+          [segment.segid, [range.begin, range.end]]
+        }.compact
+
+        fwt = FixWidthTable.get :memory, value_size, true
+        fwt.add_range index_data
+
+        fwt
+      end
+
+    annotation_index.extend Segment::RangeIndex
+    annotation_index.corpus = corpus
+    annotation_index
+  end
+
+end
+  

data/lib/rbbt/segment/relationship.rb

@@ -0,0 +1,7 @@
+module Relationship
+  extend Entity
+
+  self.annotation :segments
+  self.annotation :type
+
+end

data/lib/rbbt/{text/segment → segment}/segmented.rb

@@ -1,5 +1,5 @@
 require 'rbbt/annotations'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 
 module Segmented
   extend Annotation

data/lib/rbbt/segment/token.rb

@@ -0,0 +1,23 @@
+require 'rbbt/segment'
+
+module Token
+  extend Entity
+  include Segment
+
+  self.annotation :original
+
+  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+
+    tokens = []
+    while matchdata = text.match(split_at)
+      tokens << Token.setup(matchdata.pre_match, :offset => start) unless matchdata.pre_match.empty?
+      tokens << Token.setup(matchdata.captures.first, :offset => start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      start += matchdata.end(0)
+      text = matchdata.post_match
+    end
+
+    tokens << Token.setup(text, :offset => start) unless text.empty?
+
+    tokens
+  end
+end

data/lib/rbbt/{text/segment → segment}/transformed.rb

@@ -1,6 +1,3 @@
-require 'rbbt/util/misc'
-require 'rbbt/text/segment'
-
 module Transformed
 
   def self.transform(text, segments, replacement = nil, &block)
@@ -71,6 +68,10 @@ module Transformed
 
     segments = [segments] unless Array === segments 
     orig_length = self.length
+
+    offset = self.respond_to?(:offset) ? self.offset.to_i : 0
+    segments = segments.select{|s| s.offset.to_i >= offset && s.offset.to_i <= offset + self.length - 1 }
+
     Segment.clean_sort(segments).each do |segment|
       next if segment.offset.nil?
 
@@ -89,7 +90,7 @@ module Transformed
 
       updated_text = self[updated_begin..updated_end]
       if updated_text.nil?
-        Log.warn "Range outside of segment: #{self.length} #{segment.locus} (#{updated_range})"
+        Log.warn "Range outside of segment: #{self.length} #{segment.range} (#{updated_range})"
         next
       end
 
@@ -111,10 +112,10 @@ module Transformed
 
       self[updated_begin..updated_end] = new
 
-      @transformed_segments[segment.segment_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
+      @transformed_segments[segment.object_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
 
       segment.replace original_text
-      stack << segment.segment_id
+      stack << segment.object_id
     end
     @transformation_stack << stack
   end
@@ -122,13 +123,13 @@ module Transformed
   def fix_segment(segment, range, diff)
     case
       # Before
-    when segment.end < range.begin
+    when segment.eend < range.begin
       # After
     when segment.offset.to_i > range.end + diff
       segment.offset = segment.offset.to_i - diff
       # Includes
-    when (segment.offset.to_i <= range.begin and segment.end >= range.end + diff)
-      segment.replace self[segment.offset.to_i..segment.end - diff]
+    when (segment.offset.to_i <= range.begin and segment.eend >= range.end + diff)
+      segment.replace self[segment.offset.to_i..segment.eend - diff]
     else
       raise "Segment Overlaps"
     end
@@ -141,7 +142,8 @@ module Transformed
 
     if first_only
       @transformation_stack.pop.reverse.each do |id|
-        orig_range, diff, text, range = @transformed_segments.delete id
+        segment_info = @transformed_segments.delete id
+        orig_range, diff, text, range = segment_info 
 
         new_range = (range.begin..range.last + diff)
         self[new_range] = text

data/lib/rbbt/segment/tsv.rb

@@ -0,0 +1,41 @@
+#module Segment
+#
+#  def self.set_tsv_fields(fields, segments)
+#    tsv_fields = []
+#    add_types = ! (fields.delete(:no_types) || fields.delete("no_types") || fields.include?(:JSON) || fields.include?("JSON"))
+#    literal = (fields.delete(:literal) || fields.delete("literal"))
+#    tsv_fields << "Start" << "End"
+#    tsv_fields << :annotation_types if add_types
+#    tsv_fields << :literal if literal
+#
+#    if fields.any? and not (fields == [:all] or fields == ["all"])
+#      tsv_fields.concat fields 
+#    else
+#      tsv_fields.concat segments.first.annotations if segments.any?
+#    end
+#    tsv_fields
+#    tsv_fields.collect!{|f| f.to_s}
+#    tsv_fields.delete "offset"
+#    tsv_fields
+#  end
+#
+#  def self.tsv(segments, *fields)
+#    fields = set_tsv_fields fields, segments
+#    tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
+#
+#    segments.each do |segment|
+#      tsv[segment.segment_id] = self.tsv_values_for_segment(segment, fields)
+#    end
+#
+#    tsv
+#  end
+#
+#  def self.load_tsv(tsv)
+#    fields = tsv.fields
+#    tsv.with_unnamed do
+#      tsv.collect do |id, values|
+#        Annotated.load_tsv_values(id, values, fields)
+#      end
+#    end
+#  end
+#end

data/share/install/software/Linnaeus

@@ -12,7 +12,7 @@ pkg_dir="`opt_dir \"$name\"`"
 build_dir=`build_dir`
 mv "$build_dir" "$pkg_dir"
 tmp_file="~/.rbbt/tmp/species-proxy-properties.tmp"
-mkdir -p $(basename "$tmp_file")
+mkdir -p $(dirname "$tmp_file")
 cat  "$pkg_dir/species-proxy/properties.conf" |grep -v "^.dir =" >> $tmp_file
 echo "\$dir = $pkg_dir/species-proxy/" > "$pkg_dir/species-proxy/properties.conf"
 cat $tmp_file | grep -v "^#" >>  "$pkg_dir/species-proxy/properties.conf"

data/share/install/software/OpenNLP

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 name='OpenNLP'
-url="http://apache.rediris.es/opennlp/opennlp-1.9.1/apache-opennlp-1.9.1-bin.tar.gz"
+url="http://apache.rediris.es/opennlp/opennlp-1.9.2/apache-opennlp-1.9.2-bin.tar.gz"
 
 get_src "$name" "$url"
 move_opt "$name" 

data/test/rbbt/document/corpus/test_pubmed.rb

@@ -0,0 +1,15 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/document/corpus/pubmed'
+
+class TestCorpusPubmed < Test::Unit::TestCase
+  def test_add_pmid
+    corpus = Document::Corpus.setup({})
+
+    document = corpus.add_pmid("32299157", :abstract).first
+    title = document.to(:title)
+    assert title.include?("COVID-19")
+  end
+end
+

data/test/rbbt/document/test_annotation.rb

@@ -0,0 +1,140 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/segment'
+require 'rbbt/document/annotation'
+require 'rbbt/segment/named_entity'
+
+class TestAnnotation < Test::Unit::TestCase
+  class CalledOnce < Exception; end
+  def setup
+    Document.define :words do
+      self.split(" ")
+    end
+
+    $called_once = false
+    Document.define :persisted_words do
+      raise CalledOnce if $called_once
+      $called_once = true
+      self.split(" ")
+    end
+
+    Document.define_multiple :multiple_words do |list|
+      list.collect{|doc| doc.words}
+    end
+
+    Document.define :ner do
+      $called_once = true
+      self.split(" ").collect{|e| NamedEntity.setup(e, :code => Misc.digest(e)) }
+    end
+
+    Document.persist :ner
+  end
+
+  def test_define
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    Document::Corpus.setup corpus
+
+    corpus.add_document(text)
+
+    assert_equal text[text.words[1].range], text.words[1]
+  end
+
+  def test_define_multiple
+    text1 = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    text2 = "This is another sentence"
+    Document.setup(text1, "TEST", "test_doc1", nil)
+    Document.setup(text2, "TEST", "test_doc2", nil)
+
+    corpus = {}
+    Document::Corpus.setup corpus
+
+    corpus.add_document(text1)
+    corpus.add_document(text2)
+
+    assert_equal 2, Document.setup([text1, text2]).multiple_words.length
+    assert_equal text1.split(" "), text1.multiple_words
+
+    #Document.persist :multiple_words, :annotations, :annotation_repo => Rbbt.tmp.test.multiple_words
+    #assert_equal 2, Document.setup([text1, text2]).multiple_words.length
+    #assert_equal text1.split(" "), text1.multiple_words
+  end
+
+  def test_persist
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    Document::Corpus.setup corpus
+
+    corpus.add_document(text)
+
+    assert_equal "persisted_words", text.persisted_words.first.type
+
+    assert_raise CalledOnce do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+
+    Log.severity = 0
+    Document.persist :persisted_words, :annotations, :file => Rbbt.tmp.test.persisted_words.find(:user)
+
+    $called_once = false
+    text.persisted_words
+    assert $called_once
+
+    assert_nothing_raised  do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+  end
+
+  def test_persist_annotation_repo
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    Document::Corpus.setup corpus
+
+    corpus.add_document(text)
+
+    assert_equal "persisted_words", text.persisted_words.first.type
+
+    assert_raise CalledOnce do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+
+    Log.severity = 0
+    Document.persist :persisted_words, :annotations, :annotation_repo => Rbbt.tmp.test.persisted_words_repo.find(:user)
+
+    $called_once = false
+    text.persisted_words
+    assert $called_once
+
+    assert_nothing_raised  do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+  end
+
+  def test_persist_ner
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    Document::Corpus.setup corpus
+
+    corpus.add_document(text)
+
+
+    text.ner
+
+    $called_once = false
+    text.ner
+
+    assert ! $called_once
+    
+    assert  text.ner.first.segid.include?("TEST:")
+  end
+end
+