rbbt-text 0.6.2 → 0.6.3

data/lib/rbbt/bow/dictionary.rb

@@ -90,6 +90,7 @@ class Dictionary::TF_IDF
        @terms[term].to_f / num_docs * Math::log(1.0/df_value)
       ]
     }
+
     if limit
       Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
     else

data/lib/rbbt/corpus/document.rb

@@ -148,7 +148,9 @@ class Document
         fields = data.fields if fields.nil? and data.respond_to? :fields
 
 
-        data.filter
+        if data.respond_to? :persistence_path and String === data.persistence_path
+          data.filter(data.persistence_path + '.filters')
+        end
         data.add_filter("field:#{ doc_field }", @docid)
         data.add_filter("field:#{ entity_field }", "#{ entity }")
         keys = data.keys
@@ -157,7 +159,7 @@ class Document
 
         if keys.empty?
           segments = produce_#{entity}
-          segments << Segment.setup("No #{entity} found in document #{ @docid }", -1) if segments.empty?
+          segments << Segment.setup("No #{entity} found in document " + @docid.to_s, -1) if segments.empty?
           tsv = Segment.tsv(segments, *fields.reject{|f| ["#{doc_field}", "#{entity_field}", "Start", "End", "annotation_types"].include? f})
 
           tsv.add_field "#{ doc_field }" do
@@ -178,6 +180,7 @@ class Document
           data.pop_filter
           data.pop_filter
           data.read
+
         else
           if raw == :check
             data.close

data/lib/rbbt/corpus/document_repo.rb

@@ -99,7 +99,8 @@ module DocumentRepo
   end
 
   def add(text, namespace, id, type, hash)
-    write
+    read
+    write unless write?
     docid = fields2docid(namespace, id, type, hash)
     self[docid] = text unless self.include? docid
     read

data/lib/rbbt/entity/document.rb

@@ -0,0 +1,40 @@
+require 'rbbt/entity'
+
+module Document
+  extend Entity
+
+  class << self
+    attr_accessor :corpus
+  end
+
+  property :text => :array2single do
+    article_text = {}
+    missing = []
+
+    self.each do |doc|
+      Document.corpus.read if Document.corpus.respond_to? :read
+      if Document.corpus.include?(doc) 
+        article_text[doc] =  Document.corpus[doc] 
+      else
+        missing << doc
+      end
+    end
+
+    if missing.any?
+      missing.first.annotate missing
+      missing_text = Misc.process_to_hash(missing){|list| list._get_text}
+
+      Misc.lock Document.corpus.persistence_path do
+        Document.corpus.write if Document.corpus.respond_to? :write
+        missing_text.each do |doc, text|
+          article_text[doc] = text
+          Document.corpus[doc] = text
+        end
+        Document.corpus.read if Document.corpus.respond_to? :read
+      end
+    end
+
+    article_text.values_at *self
+  end
+
+end

data/lib/rbbt/ner/segment.rb

@@ -5,6 +5,10 @@ module Segment
   extend Annotation
   self.annotation :offset
 
+  def offset=(offset)
+    @offset = offset.nil? ? nil : offset.to_i
+  end
+
   #{{{ Ranges
 
   def end
@@ -297,8 +301,11 @@ module Segment
   end
 
   def self.load_tsv(tsv)
-    tsv.collect do |id, values|
-      Annotated.load_tsv_values(id, values, tsv.fields)
+    fields = tsv.fields
+    tsv.with_unnamed do
+      tsv.collect do |id, values|
+        Annotated.load_tsv_values(id, values, fields)
+      end
     end
   end
 

data/lib/rbbt/ner/segment/named_entity.rb

@@ -1,9 +1,11 @@
 require 'rbbt/ner/segment'
+require 'rbbt/entity'
 
 module NamedEntity 
-  extend Annotation
+  extend Entity
   include Segment
-  self.annotation :type, :code, :score
+
+  self.annotation :type, :code, :score, :docid
 
   def report
     <<-EOF

data/lib/rbbt/ner/segment/token.rb

@@ -2,9 +2,34 @@ require 'rbbt/annotations'
 require 'rbbt/ner/segment'
 
 module Token
-  extend Annotation
-  include Segment
-  self.annotation :original
+  attr_accessor :offset, :original
+  
+  def self.all_annotations
+    [:offset, :original]
+  end
+
+  def self.setup(text, start, original = nil)
+    text.extend Token
+    text.offset = start
+    text.original = original
+    text
+  end
+  
+  def info
+    {:original => original, :offset => offset}
+  end
+
+  def id
+    Misc.hash2md5 info.merge :self => self
+  end
+
+  def end
+    offset + self.length - 1
+  end
+
+  def range
+    (offset..self.end)
+  end
 
   def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
 

data/lib/rbbt/ner/segment/transformed.rb

@@ -1,9 +1,9 @@
+require 'rbbt/util/misc'
 require 'rbbt/ner/segment'
+
 module Transformed
-  attr_accessor :transformation_offset_differences, :transformation_original
 
   def self.transform(text, segments, replacement = nil, &block)
-    require 'rbbt/util/misc'
 
     text.extend Transformed
     text.replace(segments, replacement, &block)
@@ -12,7 +12,6 @@ module Transformed
   end
  
   def self.with_transform(text, segments, replacement)
-    require 'rbbt/util/misc'
 
     text.extend Transformed
     text.replace(segments, replacement)
@@ -24,147 +23,149 @@ module Transformed
     text.restore(segments, true)
   end
 
-  def transform_pos(pos)
-    return pos if transformation_offset_differences.nil?
-    # tranformation_offset_differences are assumed to be sorted in reverse
-    # order
-    transformation_offset_differences.reverse.each do |trans_diff|
-      acc = 0
-      trans_diff.reverse.each do |offset, diff, orig_length, trans_length|
-        break if offset >=  pos
-        acc += diff 
+  attr_accessor :transformed_segments, :transformation_stack
+ 
+  def shift(segment_o)
+    begin_shift = 0
+    end_shift = 0
+
+    @transformed_segments.sort_by{|id, info| info.last}.each{|id,info| 
+      pseg_o, diff = info
+
+      case
+        # Before
+      when segment_o.last + end_shift < pseg_o.begin
+        # After
+      when (segment_o.begin + begin_shift > pseg_o.last)
+        begin_shift += diff
+        end_shift += diff
+        # Includes
+      when (segment_o.begin + begin_shift <= pseg_o.begin and segment_o.last + end_shift >= pseg_o.last)
+        end_shift += diff
+        # Inside
+      when (segment_o.begin + begin_shift >= pseg_o.begin and segment_o.last + end_shift <= pseg_o.last)
+        return nil
+        # Overlaps start
+      when (segment_o.begin + begin_shift <= pseg_o.begin and segment_o.last + end_shift <= pseg_o.last)
+        return nil
+        # Overlaps end
+      when (segment_o.begin + begin_shift >= pseg_o.begin and segment_o.last + end_shift >= pseg_o.last)
+        return nil
+     else
+        raise "Unknown overlaps: #{segment_o.inspect} - #{pseg_o.inspect}"
       end
-      pos = pos - acc 
-    end
+    }
 
-    pos
+    [begin_shift, end_shift]
   end
 
-  def transform_range(range)
-    (transform_pos(range.begin)..transform_pos(range.end))
+  def self.sort(segments)
+    segments.compact.sort do |a,b|
+      case
+      when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
+        0
+      when (a.nil? or a.offset.nil?)
+        -1
+      when (b.nil? or b.offset.nil?)
+        +1
+        # Non-overlap
+      when (a.end < b.offset or b.end < a.offset)
+        b.offset <=> a.offset
+        # b includes a
+      when (a.offset >= b.offset and a.end <= b.end)
+        -1
+        # b includes a
+      when (b.offset >= a.offset and b.end <= a.end)
+        +1
+        # Overlap
+      when (a.offset > b.offset and a.end > b.end or b.offset < a.offset and b.end > a.end)
+        a.length <=> b.length
+      else
+        raise "Unexpected case in sort: #{a.range} - #{b.range}"
+      end
+    end
   end
 
-  def transformed_set(pos, value)
-    transformed_pos = case
-                when Range === pos
-                  transform_range(pos)
-                when Integer === pos
-                  transform_pos(pos)
-                else
-                  raise "Text position not understood '#{pos.inspect}'. Not Range or Integer"
-                end
-
-    self[transformed_pos] = value
-  end
+  def replace(segments, replacement = nil, &block)
+    @transformed_segments ||= {}
+    @transformation_stack ||= []
+    stack = []
 
-  def transformed_get(pos)
-    transformed_pos = case
-                when Range === pos
-                  transform_range(pos)
-                when Integer === pos
-                  transform_pos(pos)
-                else
-                  raise "Text position not understood '#{pos.inspect}'. Not Range or Integer"
-                end
-
-    self[transformed_pos]
-  end
+    Transformed.sort(segments).each do |segment|
+      next if segment.offset.nil?
+      shift = shift segment.range
 
-  def conflict?(segment_range)
-    return false if @transformation_offset_differences.nil? or @transformation_offset_differences.empty?
-    transformation_offset_difference = @transformation_offset_differences.last
+      next if shift.nil?
 
-    transformation_offset_difference.each do |info|
-      offset, diff, orig_length, trans_length = info
-      return true if segment_range.begin > offset and segment_range.begin < offset + trans_length or
-      segment_range.end   > offset and segment_range.end   < offset + trans_length
-    end
+      shift_begin, shift_end = shift
 
-    return false
-  end
+      text_offset = self.respond_to?(:offset)? self.offset : 0
+      updated_begin = segment.offset + shift_begin - text_offset
+      updated_end   = segment.range.last + shift_end - text_offset
 
-  def replace(segments, replacement = nil, &block)
-    replacement ||= block
-    raise "No replacement given" if replacement.nil?
-    transformation_offset_differences = []
-    transformation_original = []
+      updated_range = (updated_begin..updated_end)
 
-    Segment.clean_sort(segments).reverse.each do |segment|
-      untransformed_segment_range_here= segment.range_in(self)
-      transformed_segment_range  = self.transform_range(untransformed_segment_range_here)
-      next if conflict?(transformed_segment_range)
+      updated_text = self[updated_begin..updated_end]
 
-      text_before_transform = self[transformed_segment_range]
+      original_text = segment.dup
+      segment.replace updated_text
 
       case
+      when block_given?
+        new =  block.call(segment)
       when String === replacement
-        transformed_text = replacement
+        new = replacement
       when Proc === replacement
+        new = replacement.call(segment)
+      end
 
-        # Prepare segment with new text
-        save_segment_text = segment.dup
-        save_offset = segment.offset
-        segment.replace text_before_transform
-        segment.offset = transformed_segment_range.begin
+      diff = new.length - segment.length
 
-        transformed_text = replacement.call segment
+      self[updated_begin..updated_end] = new
 
-        # Restore segment with original text
-        segment.replace save_segment_text
-        segment.offset = save_offset
-      else
-        raise "Replacemente not String nor Proc"
-      end
-      diff = segment.length - transformed_text.length
-      self[transformed_segment_range] = transformed_text
+      @transformed_segments[segment.object_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
 
-      transformation_offset_differences << [untransformed_segment_range_here.begin, diff, text_before_transform.length, transformed_text.length]
-      transformation_original << text_before_transform
+      segment.replace original_text
+      stack << segment.object_id
     end
+    @transformation_stack << stack
+  end
 
-    @transformation_offset_differences ||= []
-    @transformation_offset_differences << transformation_offset_differences
-    @transformation_original ||= []
-    @transformation_original << transformation_original
+  def fix_segment(segment, range, diff)
+    case
+      # Before
+    when segment.end < range.begin
+      # After
+    when segment.offset > range.end + diff
+      segment.offset -= diff
+      # Includes
+    when (segment.offset <= range.begin and segment.end >= range.end + diff)
+      segment.replace self[segment.offset..segment.end - diff]
+    else
+      raise "Segment Overlaps"
+    end
   end
 
-  def restore(segments = nil, first_only = false)
-    stop = false
-    while self.transformation_offset_differences.any? and not stop
-      transformation_offset_differences = self.transformation_offset_differences.pop
-      transformation_original           = self.transformation_original.pop
+  def restore(segments, first_only = false)
+    return segments if @transformation_stack.empty?
 
-      ranges = transformation_offset_differences.collect do |offset,diff,orig_length,rep_length|
-        (offset..(offset + rep_length - 1))
-      end
+    if first_only
+      @transformation_stack.pop.reverse.each do |id|
+        orig_range, diff, text, range = @transformed_segments.delete id
 
-      ranges.zip(transformation_original).reverse.each do |range,text|
-        self.transformed_set(range, text)
+        new_range = (range.begin..range.last + diff)
+        self[new_range] = text
+        segments.each do |segment|
+          next unless Segment === segment
+          fix_segment(segment, range, diff)
+        end if Array === segments
       end
-
-      stop = true if first_only
-
-      next if segments.nil?
-
-      segment_ranges = segments.each do |segment|
-        r = segment.range
-
-        s = r.begin
-        e = r.end
-        sdiff = 0
-        ediff = 0
-        transformation_offset_differences.reverse.each do |offset,diff,orig_length,rep_length|
-          sdiff += diff if offset < s
-          ediff += diff if offset + rep_length - 1 < e
-        end
-
-        segment.offset = s + sdiff
-        segment.replace self[(s+sdiff)..(e + ediff)]
+      segments
+    else
+      while @transformation_stack.any?
+        restore(segments, true)
       end
+      segments
     end
-
-    segments
   end
 end
-
-

data/lib/rbbt/ner/token_trieNER.rb

@@ -110,7 +110,7 @@ class TokenTrieNER < NER
   end
 
   def self.merge(index1, index2)
-    index1.write if index1.respond_to? :write
+    index1.write if index1.respond_to? :write and not index1.write?
     index2.each do |key, new_index2|
       case
       when key == :END
@@ -119,7 +119,8 @@ class TokenTrieNER < NER
         end1.uniq!
         index1[:END] = end1
       when index1.include?(key)
-        index1[key] = merge(index1[key], new_index2)
+        new = merge(index1[key], new_index2)
+        index1[key] = new
       else
         index1[key] = new_index2
       end
@@ -148,7 +149,10 @@ class TokenTrieNER < NER
         tokens = Array === name ? name : tokenize(name, false, split_at, no_clean) 
         tokens.extend EnumeratedArray
 
-        tmp_index = merge(tmp_index, index_for_tokens(tokens, code, type, slack)) unless tokens.empty?
+        token_index = index_for_tokens(tokens, code, type, slack)
+
+        tmp_index = merge(tmp_index, token_index) unless tokens.empty?
+
         items_in_chunk += 1
 
         if items_in_chunk > chunk_size
@@ -267,22 +271,22 @@ class TokenTrieNER < NER
       TokenTrieNER.merge(@index, new.index)
     when TSV === new
       Log.debug "TokenTrieNER merging TSV"
-      old_unnamed = new.unnamed
-      old_monitor = new.monitor
-      new.unnamed = true
-      new.monitor = {:step => 1000, :desc => "Processing TSV into TokenTrieNER"}
-      TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
-      new.unnamed = old_unnamed
-      new.monitor = old_monitor
+      new.with_unnamed do
+        new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+        end
+      end
     when Hash === new
       Log.debug "TokenTrieNER merging Hash"
       TokenTrieNER.merge(@index, new)
     when String === new
       Log.debug "TokenTrieNER merging file: #{ new }"
       new = TSV.open(new, :flat)
-      new.unnamed = true
-      new.monitor = {:step => 1000, :desc => "Processing TSV into TokenTrieNER"}
-      TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+      new.with_unnamed do
+        new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+        end
+      end
     end
   end
 

data/test/rbbt/corpus/test_document.rb

@@ -2,7 +2,19 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.r
 require 'rbbt/corpus/document'
 require 'test/unit'
 
+module TokenEntity
+  extend Annotation
+  include Segment
+  self.annotation :original
+end
 class Document
+
+  def tokenize(text)
+    Token.tokenize(text).collect do |token|
+      TokenEntity.setup(token.dup, token.offset, token.original)
+    end
+  end
+
   define :sentences do 
     require 'rbbt/nlp/nlp'
     NLP.geniass_sentence_splitter(text)
@@ -10,22 +22,22 @@ class Document
 
   define :tokens do
     require 'rbbt/ner/segment/token'
-    Token.tokenize(text)
+    tokenize(text)
   end
  
   define :long_words do
     require 'rbbt/ner/segment/token'
-    Token.tokenize(text).select{|tok| tok.length > 5}
+    tokenize(text).select{|tok| tok.length > 5}
   end
 
   define :short_words do
     require 'rbbt/ner/segment/token'
-    Token.tokenize(text).select{|tok| tok.length < 5}
+    tokenize(text).select{|tok| tok.length < 5}
   end
  
   define :even_words do
     require 'rbbt/ner/segment/token'
-    Token.tokenize(text).select{|tok| tok.length % 2 == 0}
+    tokenize(text).select{|tok| tok.length % 2 == 0}
   end
 
   define :missing do
@@ -110,7 +122,7 @@ another sentence.
       doc = Document.new(dir)
       doc.text = text
 
-      sentence = doc.sentences.last
+      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
       doc.load_into sentence, :tokens 
 
       assert_equal 5, sentence.tokens.length
@@ -134,7 +146,7 @@ another sentence.
       doc = Document.new(dir)
       doc.text = text
 
-      sentence = doc.sentences.last
+      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
       Misc.benchmark(1) do
         doc = Document.new(dir)
         doc.text = text
@@ -166,7 +178,7 @@ another sentence.
       doc = Document.new(dir)
       doc.text = text * 10
 
-      sentence = doc.sentences.last
+      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
 
       doc.load_into sentence, :tokens, :long_words
 
@@ -178,9 +190,9 @@ another sentence.
       doc = Document.new(dir)
       doc.text = text * 10
       doc.sentences
-      assert_equal sentence, doc.sentences.last
+      assert_equal sentence, doc.sentences.sort_by{|sentence| sentence.offset}.last
 
-      sentence = doc.sentences.last
+      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
       doc.load_into sentence, :tokens, :long_words
 
       assert_equal 2, sentence.long_words.length
@@ -211,7 +223,7 @@ another sentence.
       doc.text = text * 10
       doc.docid = "TEST"
 
-      sentence = doc.sentences.last
+      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
 
       doc.load_into sentence, :tokens, :long_words, :short_words, :even_words
 

data/test/rbbt/ner/segment/test_named_entity.rb

@@ -4,7 +4,7 @@ require 'rbbt/ner/segment/named_entity'
 
 class TestClass < Test::Unit::TestCase
   def test_info
-    a = "test"
+    a = ["test"]
     NamedEntity.setup a
     assert(! a.info.keys.include?(:code))
     a.code = 10

data/test/rbbt/ner/segment/test_transformed.rb

@@ -1,9 +1,11 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/ner/segment/transformed'
 require 'rbbt/ner/segment/named_entity'
+require 'rexml/document'
+require 'rand'
 
 class TestClass < Test::Unit::TestCase
-  def test_transform
+  def tttest_transform
     a = "This sentence mentions the TP53 gene and the CDK5 protein"
     original = a.dup
 
@@ -56,11 +58,13 @@ class TestClass < Test::Unit::TestCase
     Transformed.with_transform(a, [gene1], "GN") do 
       assert_equal original.sub("TP53", 'GN'), a
     end
+
     assert_equal original, a
 
-    Transformed.with_transform(a, [gene1,gene2], "GN") do 
+    Transformed.with_transform(a, [gene1, gene2], "GN") do 
       assert_equal original.gsub(/TP53|CDK5R1/, 'GN'), a
     end
+
     assert_equal original, a
 
     Transformed.with_transform(a, [gene1], "GN") do 
@@ -69,6 +73,7 @@ class TestClass < Test::Unit::TestCase
       end
       assert_equal original.gsub(/TP53/, 'GN'), a
     end
+
     assert_equal original, a
 
     exp1, exp2 = nil, nil
@@ -169,7 +174,37 @@ class TestClass < Test::Unit::TestCase
         assert_equal one, a
       end
     end
-
   end
+
+  def test_error
+    a = "Do not have a diagnosis of another hereditary APC resistance/Factor V Leiden, Protein S or C deficiency, prothrombin gene mutation (G20210A), or acquired (lupus anticoagulant) thrombophilic disorder"
+
+    entity1 = "gene"
+    entity1.extend NamedEntity
+    entity1.offset = a.index entity1
+    entity1.type = "Gene"
+
+    entity2 = "prothrombin gene mutation"
+    entity2.extend NamedEntity
+    entity2.offset = a.index entity2
+    entity2.type = "Mutation"
+
+    entity3 = "Protein S or C"
+    entity3.extend NamedEntity
+    entity3.offset = a.index entity3
+    entity3.type = "Gene"
+
+    entity4 = "prothrombin gene mutation"
+    entity4.extend NamedEntity
+    entity4.offset = a.index entity2
+    entity4.type = "Disease"
+
+
+    Transformed.with_transform(a, [entity1].sort_by{rand}, Proc.new{|e| e.html}) do 
+      Transformed.with_transform(a, [entity3, entity2, entity4].sort_by{rand}, Proc.new{|e| e.html}) do 
+        assert_nothing_raised{REXML::Document.new "<xml>"+ a + "</xml>"}
+      end
+    end
+   end
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-text
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 1
   prerelease: 
   segments: 
   - 0
   - 6
-  - 2
-  version: 0.6.2
+  - 3
+  version: 0.6.3
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-03 00:00:00 +02:00
-default_executable: get_ppis.rb
+date: 2012-02-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rbbt-util
@@ -106,6 +105,7 @@ files:
 - lib/rbbt/corpus/document.rb
 - lib/rbbt/corpus/document_repo.rb
 - lib/rbbt/corpus/sources/pubmed.rb
+- lib/rbbt/entity/document.rb
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
 - lib/rbbt/ner/banner.rb
@@ -161,7 +161,6 @@ files:
 - test/rbbt/corpus/test_corpus.rb
 - test/rbbt/corpus/test_document.rb
 - bin/get_ppis.rb
-has_rdoc: true
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []
 
@@ -191,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)