rbbt-text 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8373a3408f7b9cbc481ef108d899de4283513115
-  data.tar.gz: e431a61729bd0f67a4129384a06a5b3a81824afc
+SHA256:
+  metadata.gz: ea1646b5f32644bb5872f57422534b49955f988df26df4a65c8dda592515eac3
+  data.tar.gz: 3f6bc60546b79c76b6b35840712453616c377fcc088f321e95847f116776bef1
 SHA512:
-  metadata.gz: 575313a7d598cbec0ec05827bebee1f4ccc8b56ccdac89478beb4993b5188356384172cf762e6ca58f0c0953fae0f29fcd46423d1291391d9958b66ce62230d0
-  data.tar.gz: 4609a5ce9448f0a0a3ad480bc9f0cc6f3dfc728eacb0c6bf291d8b29b23d084f56a76c8776d421d935e50ef097fb10009d41121ef046b95c85177060335a1629
+  metadata.gz: 9376c68bad67733b5771b57ead7c962d45ff29c44362d1c51bf3480d3c3bf9f1f75284e40044fc4ed95bd94a03ab0759b3b7320bf1e3da00a0cdd82255c9395c
+  data.tar.gz: cd25a9cd91fde366be195801d45238d555edfc94f2b06391db7db2d9f4781b34dd599514385782d6c7e22af2841c5f3322ba74bf0a3a9c1fdbe308a255f00098

data/bin/get_ppis.rb

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
 
 require 'rbbt-util'
-require 'rbbt/annotations/corpus'
-require 'rbbt/annotations/corpus/pubmed'
-require 'rbbt/annotations/relationships/ppi'
+require 'rbbt/corpus/corpus'
+require 'rbbt/corpus/sources/pubmed'
+#require 'rbbt/annotations/relationships/ppi'
 require 'rbbt/sources/pubmed'
-require 'rbbt/ner/annotations'
+#require 'rbbt/ner/annotations'
 require 'rbbt/ner/token_trieNER'
-require 'rbbt/ner/annotations/transformed'
+#require 'rbbt/ner/annotations/transformed'
 require 'rbbt/ner/chemical_tagger'
 
 Corpus.define_entity_ner "Compounds", false do |doc|

data/lib/rbbt/bow/dictionary.rb

@@ -182,7 +182,4 @@ class Dictionary::KL
   def weights(options = {})
     best(options)
   end
-
-   
-    
 end

data/lib/rbbt/corpus/document.rb

@@ -8,10 +8,10 @@ require 'json'
 
 class Document
 
-  attr_accessor :text, :docid, :namespace, :id, :type, :hash, :segments, :segment_indeces, :persist_dir, :global_persistence
+  attr_accessor :text, :docid, :namespace, :id, :type, :hash, :segments, :segment_indices, :persist_dir, :global_persistence
   def initialize(persist_dir = nil, docid = nil, text = nil, global_persistence = nil)
     @segments = {}
-    @segment_indeces = {}
+    @segment_indices = {}
 
     if not persist_dir.nil?
       @persist_dir = persist_dir
@@ -236,7 +236,7 @@ class Document
   end
 
   def segment_index(name, persist_dir = nil)
-    @segment_indeces[name] ||= Segment.index(self.send(name), persist_dir.nil? ? :memory : File.join(persist_dir, name + '.range'))
+    @segment_indices[name] ||= Segment.index(self.send(name), persist_dir.nil? ? :memory : File.join(persist_dir, name + '.range'))
   end
 
   def load_into(segment, *annotations)

data/lib/rbbt/corpus/sources/pubmed.rb

@@ -10,8 +10,9 @@ class Corpus
     type = nil if String === type and type.empty?
 
     PubMed.get_article(pmids).collect do |pmid, article|
+      add_document(article.title, :pubmed, pmid, :title)
       if (type.nil? and article.pdf_url.nil?) or (not type.nil? and type.to_sym === :abstract)
-        add_document(article.text, :pubmed, pmid, :abstract)
+        add_document(article.abstract || "", :pubmed, pmid, :abstract)
       else
         raise "No FullText available for #{ pmid }" if article.pdf_url.nil?
         add_document(article.full_text, :pubmed, pmid, :fulltext)

data/lib/rbbt/ner/abner.rb

@@ -11,6 +11,7 @@ class Abner < NER
   Rbbt.claim Rbbt.software.opt.ABNER, :install, Rbbt.share.install.software.ABNER.find
 
   def self.init
+    Rbbt.software.opt.ABNER.produce
     @@JFile   ||= Rjb::import('java.io.File')
     @@Tagger  ||= Rjb::import('abner.Tagger')
     @@Trainer ||= Rjb::import('abner.Trainer')

data/lib/rbbt/ner/banner.rb

@@ -10,6 +10,7 @@ class Banner < NER
   Rbbt.claim Rbbt.software.opt.BANNER, :install, Rbbt.share.install.software.BANNER.find
 
   def self.init
+    Rbbt.software.opt.BANNER.produce
     @@JFile                    ||= Rjb::import('java.io.File')
     @@SimpleTokenizer          ||= Rjb::import('banner.tokenization.SimpleTokenizer')
     @@CRFTagger                ||= Rjb::import('banner.tagging.CRFTagger')

data/lib/rbbt/ner/brat.rb

@@ -0,0 +1,30 @@
+require 'rbbt/ner/segment/named_entity'
+require 'rbbt/ner/segment/relationship'
+module Brat
+  Rbbt.claim Rbbt.software.opt.Brat, :install, "https://github.com/nlplab/brat.git"
+
+  def self.load(file)
+    entities = {}
+    relationships = {}
+    entity_ids = {}
+    TSV.traverse file, :type => :array do |line|
+      id, info, literal = line.split("\t")
+      case id[0] 
+      when "T"
+        type, start, eend = info.split(" ")
+        entities[id] = NamedEntity.setup(literal, :offset => start.to_i, :type => type)
+      when "#"
+        type, id = info.split(" ")
+        entities[id].code = literal unless entities[id].nil?
+      when "R"
+        type, *args = info.split(" ")
+        tf, tg = args.collect{|e| e.split(":").last }
+        tf = entities[tf]
+        tg = entities[tg]
+        relationship = Relationship.setup([tf,tg] * "~" + "#" + type, :terms => [tf,tg], :type => type)
+        relationships[id] = relationship
+      end
+    end
+    [entities.values, relationships.values]
+  end
+end

data/lib/rbbt/ner/g_norm_plus.rb

@@ -0,0 +1,80 @@
+require 'rbbt-util'
+module GNormPlus
+
+  Rbbt.claim Rbbt.software.opt.GNormPlus, :install do
+    url = "https://www.ncbi.nlm.nih.gov/CBBresearch/Lu/Demo/tmTools/download/GNormPlus/GNormPlusJava.zip"
+    script =<<-EOF
+(cd $(opt_dir $name); sh Installation.sh; chmod +x Ab3P identify_abbr)
+    EOF
+    {:src => url, :commands => script}
+  end
+
+  CONFIG =<<-EOF
+
+#===Annotation
+#Attribution setting:
+#FocusSpecies = Taxonomy ID
+#	All: All species
+#	9606: Human
+#	4932: yeast
+#	7227: Fly
+#	10090: Mouse
+#	10116: Rat
+#	7955: Zebrafish
+#	3702: Arabidopsis thaliana
+#open: True
+#close: False
+
+[Focus Species]
+	FocusSpecies = All
+[Dictionary & Model]
+	DictionaryFolder = ./Dictionary
+	GNRModel = ./Dictionary/GNR.Model
+	SCModel = ./Dictionary/SimConcept.Model
+	GeneIDMatch = True
+	Normalization2Protein = False
+	DeleteTmp = True
+EOF
+
+  def self.process(texts)
+    TmpFile.with_file do |tmpdir|
+      Open.mkdir tmpdir
+      Misc.in_dir tmpdir do
+        Open.ln_s Rbbt.software.opt.GNormPlus.Dictionary.find, '.'
+        Open.ln_s Rbbt.software.opt.GNormPlus["BioC.dtd"].find, '.'
+        Open.ln_s Rbbt.software.opt.GNormPlus["Ab3P"].find, '.'
+        Open.ln_s Rbbt.software.opt.GNormPlus["CRF"].find, '.'
+        Open.mkdir 'input'
+        Open.mkdir 'output'
+        Open.mkdir 'tmp'
+
+        texts.each do |name,text|
+          Open.write("input/#{name}.txt") do |f|
+            f.puts "#{name}|a|" << text
+            f.puts
+          end
+        end
+        Open.write('config', CONFIG)
+        CMD.cmd_log("java -Xmx20G -Xms20G  -jar '#{Rbbt.software.opt.GNormPlus.find}/GNormPlus.jar' 'input' 'output' 'config'")
+
+        if texts.respond_to? :key_field
+          key_field = texts.key_field
+        else
+          key_field = "ID"
+        end
+        tsv = TSV.setup({}, :key_field => key_field, :fields => ["Entities"], :type => :flat)
+        Dir.glob("output/*.txt").each do |file|
+          name = File.basename(file).sub(".txt",'')
+          entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '.').split("\t")[1..-1] * ":"}
+          tsv[name] = entities
+        end
+        tsv
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  Log.severity = 0
+  Rbbt.software.opt.GNormPlus.produce
+end

data/lib/rbbt/ner/linnaeus.rb

@@ -8,8 +8,8 @@ module Linnaeus
 
   ARGS = ["--properties", Rbbt.software.opt.Linnaeus["species-proxy/properties.conf"].find]
 
-  Rjb::load(nil, jvmargs = ['-Xms2G','-Xmx2G']) unless Rjb.loaded?
 
+  Rjb::load(nil, jvmargs = ['-Xms2G','-Xmx2G']) unless Rjb.loaded?
   def self.init
     begin
       @@ArgParser    = Rjb::import('martin.common.ArgParser')

data/lib/rbbt/ner/segment.rb

@@ -72,6 +72,17 @@ module Segment
     (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i)
   end
 
+  def overlaps?(segment)
+    segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.end || 
+    self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.end
+  end
+
+  def self.collisions(main, secondary)
+    collisions = secondary.select do |ss|
+      collisions = main.select{|ms| ms.overlaps? ss }.any?
+    end
+  end
+
   #{{{ Sorting
 
   def self.sort(segments, inline = true)
@@ -84,14 +95,14 @@ module Segment
           -1
         when (b.nil? or b.offset.nil?)
           +1
-        when (not a.range.include? b.offset and not b.range.include? a.offset)
-          a.offset <=> b.offset
+        when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i)
+          a.offset.to_i <=> b.offset.to_i
         else
           a.segment_length <=> b.segment_length
         end
       end
     else
-      segments.sort_by do |segment| segment.offset || 0 end.reverse
+      segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse
     end
   end
 
@@ -282,7 +293,7 @@ module Segment
 
     info[:annotation_types] = [Segment] unless info.include? :annotation_types
 
-    Annotated.load(object, info)
+    Annotated.load_entity(object, info)
   end
 
   def self.set_tsv_fields(fields, segments)
@@ -324,5 +335,16 @@ module Segment
     end
   end
 
+  def ansi(color)
+    Log.color color, self
+  end
+
+  def locus
+    [offset, self.end] * ".."
+  end
+
+  def ==(other)
+    self.id == other.id
+  end
 end
 

data/lib/rbbt/ner/segment/named_entity.rb

@@ -41,5 +41,6 @@ Score: #{score.inspect}
 
     entity
   end
+
 end
 

data/lib/rbbt/ner/segment/relationship.rb

@@ -2,19 +2,23 @@ require 'rbbt/ner/segment'
 
 module Relationship
   extend Annotation
-  include Segment
+  self.annotation :segment
   self.annotation :terms
+  self.annotation :type
+
+  def text
+    if segment
+      segment
+    else
+      type + ": " + terms * ", "
+    end
+  end
 
   def html
     text = <<-EOF
 <span class='Relationship'\
->#{ self }</span>
+>#{ self.text }</span>
     EOF
     text.chomp
   end
-
-  def html_with_entities(*types)
-    annotations.values_at(*types).each do |segments|
-    end
-  end
 end

data/lib/rbbt/ner/segment/transformed.rb

@@ -6,7 +6,7 @@ module Transformed
   def self.transform(text, segments, replacement = nil, &block)
 
     text.extend Transformed
-    text.replace(segments, replacement, &block)
+    text.replace_segments(segments, replacement, &block)
 
     text
   end
@@ -14,11 +14,11 @@ module Transformed
   def self.with_transform(text, segments, replacement = nil)
 
     text.extend Transformed
-    text.replace(segments, replacement)
+    text.replace_segments(segments, replacement)
 
     segments = yield text
 
-    segments = nil unless Array === segments
+    segments = nil unless Array === segments && Segment === segments.first
 
     text.restore(segments, true)
   end
@@ -59,39 +59,41 @@ module Transformed
     [begin_shift, end_shift]
   end
 
-  def self.sort(segments)
-    segments.compact.sort do |a,b|
-      case
-      when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
-        0
-      when (a.nil? or a.offset.nil?)
-        -1
-      when (b.nil? or b.offset.nil?)
-        +1
-        # Non-overlap
-      when (a.end < b.offset.to_i or b.end < a.offset.to_i)
-        b.offset <=> a.offset
-        # b includes a
-      when (a.offset.to_i >= b.offset.to_i and a.end <= b.end)
-        -1
-        # b includes a
-      when (b.offset.to_i >= a.offset.to_i and b.end <= a.end)
-        +1
-        # Overlap
-      when (a.offset.to_i > b.offset.to_i and a.end > b.end or b.offset.to_i < a.offset.to_i and b.end > a.end)
-        a.length <=> b.length
-      else
-        raise "Unexpected case in sort: #{a.range} - #{b.range}"
-      end
-    end
-  end
-
-  def replace(segments, replacement = nil, &block)
+  #def self.sort(segments)
+  #  segments.compact.sort do |a,b|
+  #    case
+  #    when ((a.nil? && b.nil?) || (a.offset.nil? && b.offset.nil?))
+  #      0
+  #    when (a.nil? || a.offset.nil?)
+  #      -1
+  #    when (b.nil? || b.offset.nil?)
+  #      +1
+  #      # Non-overlap
+  #    when (a.end < b.offset.to_i || b.end < a.offset.to_i)
+  #      b.offset <=> a.offset
+  #      # b includes a
+  #    when (a.offset.to_i >= b.offset.to_i && a.end <= b.end)
+  #      -1
+  #      # b includes a
+  #    when (b.offset.to_i >= a.offset.to_i && b.end <= a.end)
+  #      +1
+  #      # Overlap
+  #    when (a.offset.to_i > b.offset.to_i && a.end > b.end || b.offset.to_i > a.offset.to_i && b.end > a.end)
+  #      b.length <=> a.length
+  #    else
+  #      raise "Unexpected case in sort: #{a.range} - #{b.range}"
+  #    end
+  #  end
+  #end
+
+  def replace_segments(segments, replacement = nil, &block)
     @transformed_segments ||= {}
     @transformation_stack ||= []
     stack = []
 
-    Transformed.sort(segments).each do |segment|
+    segments = [segments] unless Array === segments 
+    orig_length = self.length
+    Segment.sort(segments).each do |segment|
       next if segment.offset.nil?
       shift = shift segment.range
 
@@ -106,6 +108,10 @@ module Transformed
       updated_range = (updated_begin..updated_end)
 
       updated_text = self[updated_begin..updated_end]
+      if updated_text.nil?
+        Log.warn "Range outside of segment: #{self.length} #{segment.locus} (#{updated_range})"
+        next
+      end
 
       original_text = segment.dup
       segment.replace updated_text
@@ -137,7 +143,7 @@ module Transformed
     when segment.end < range.begin
       # After
     when segment.offset.to_i > range.end + diff
-      segment.offset.to_i -= diff
+      segment.offset = segment.offset.to_i - diff
       # Includes
     when (segment.offset.to_i <= range.begin and segment.end >= range.end + diff)
       segment.replace self[segment.offset.to_i..segment.end - diff]
@@ -170,4 +176,9 @@ module Transformed
       segments
     end
   end
+
+  def self.ansi(text, entities, colors = nil)
+
+
+  end
 end

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -1,6 +1,8 @@
 require 'rbbt/nlp/nlp'
 require 'rbbt/ner/segment'
 module NLP
+  Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
+
   def self.returnFeatures(prevWord, delimiter, nextWord)
     if nextWord.match(/__ss__/)
       nw = nextWord.sub(/__ss__/, "")
@@ -235,4 +237,67 @@ module NLP
     end
 
   end
+
+  def self.geniass_sentence_splitter(text)
+    offsets = []
+
+    cleaned = text.gsub("\n",NEW_LINE_MASK)
+    TmpFile.with_file(cleaned) do |fin|
+      TmpFile.with_file do |fout|
+        CMD.cmd("cd #{Rbbt.software.opt.Geniass.find}; ./geniass #{ fin } #{ fout }")
+
+        
+        Open.write(fin, Open.read(fin).gsub(NEW_LINE_MASK, "\n"))
+        Open.write(fout, Open.read(fout).gsub("\n", '|').gsub(NEW_LINE_MASK, "\n"))
+        # Addapted from sentence2standOff.rb in Geniass package
+
+        inTxtStrict = Open.open(fin)
+        inTxtNew = Open.open(fout)
+
+        marker = "|"[0]
+        position = 0
+        sentenceCount = 1
+        target = ''
+        targetNew = ''
+        start = 0
+        finish = 0
+
+        while(!inTxtNew.eof?) do
+          targetNew = inTxtNew.getc
+          target = inTxtStrict.getc
+          position += 1
+          if targetNew == marker
+            sentenceCount += 1
+            finish = position - 1
+            offsets << [start, finish] if finish - start > 10
+            if targetNew == target
+              start = position
+            else
+              targetNew = inTxtNew.getc
+              while targetNew != target do
+                target = inTxtStrict.getc
+                position += 1
+              end
+              start = position - 1
+            end
+          end
+        end
+
+        finish = position - 1
+        offsets << [start, finish] if finish > start
+
+        inTxtStrict.close
+        inTxtNew.close
+      end
+    end
+
+    offsets.collect do |s,e|
+      sentence = text[s..e]
+      next if sentence.nil?
+      #sentence.gsub!(NEW_LINE_MASK, "\n")
+      Segment.setup sentence, s
+      sentence
+    end
+  end
+
 end

data/lib/rbbt/nlp/nlp.rb

@@ -16,76 +16,10 @@ module NLP
   #Rbbt.software.opt.StanfordParser.define_as_install Rbbt.share.install.software.StanfordParser.find
   #Rbbt.software.opt.StanfordParser.produce
 
-  Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
-  Rbbt.software.opt.Geniass.produce
-
   Rbbt.claim Rbbt.software.opt.Gdep, :install, Rbbt.share.install.software.Gdep.find
-  Rbbt.software.opt.Gdep.produce
 
   NEW_LINE_MASK = "\t\t \t  \t"
 
-  def self.geniass_sentence_splitter(text)
-    offsets = []
-
-    cleaned = text.gsub("\n",NEW_LINE_MASK)
-    TmpFile.with_file(cleaned) do |fin|
-      TmpFile.with_file do |fout|
-        CMD.cmd("cd #{Rbbt.software.opt.Geniass.find}; ./geniass #{ fin } #{ fout }")
-
-        
-        Open.write(fin, Open.read(fin).gsub(NEW_LINE_MASK, "\n"))
-        Open.write(fout, Open.read(fout).gsub("\n", '|').gsub(NEW_LINE_MASK, "\n"))
-        # Addapted from sentence2standOff.rb in Geniass package
-
-        inTxtStrict = Open.open(fin)
-        inTxtNew = Open.open(fout)
-
-        marker = "|"[0]
-        position = 0
-        sentenceCount = 1
-        target = ''
-        targetNew = ''
-        start = 0
-        finish = 0
-
-        while(!inTxtNew.eof?) do
-          targetNew = inTxtNew.getc
-          target = inTxtStrict.getc
-          position += 1
-          if targetNew == marker
-            sentenceCount += 1
-            finish = position - 1
-            offsets << [start, finish] if finish - start > 10
-            if targetNew == target
-              start = position
-            else
-              targetNew = inTxtNew.getc
-              while targetNew != target do
-                target = inTxtStrict.getc
-                position += 1
-              end
-              start = position - 1
-            end
-          end
-        end
-
-        finish = position - 1
-        offsets << [start, finish] if finish > start
-
-        inTxtStrict.close
-        inTxtNew.close
-      end
-    end
-
-    offsets.collect do |s,e|
-      sentence = text[s..e]
-      next if sentence.nil?
-      #sentence.gsub!(NEW_LINE_MASK, "\n")
-      Segment.setup sentence, s
-      sentence
-    end
-  end
-
   module GdepToken
     extend Annotation
     include Segment
@@ -219,3 +153,8 @@ module NLP
     end
   end
 end
+
+if __FILE__ == $0
+  Log.severity = 0
+  Rbbt.software.opt.Gdep.produce
+end

data/lib/rbbt/nlp/open_nlp/sentence_splitter.rb

@@ -6,16 +6,20 @@ require 'rbbt/resource'
 module OpenNLP
   Rbbt.claim Rbbt.software.opt.OpenNLP, :install, Rbbt.share.install.software.OpenNLP.find
 
+
   Rbbt.claim Rbbt.software.opt.OpenNLP.models["da-sent.bin"], :url, "http://opennlp.sourceforge.net/models-1.5/de-sent.bin"
 
   MAX = 5
 
-  @@FileInputStream = Rjb::import('java.io.FileInputStream')
-  @@SentenceModel = Rjb::import('opennlp.tools.sentdetect.SentenceModel')
-  @@SentenceDetectorME = Rjb::import('opennlp.tools.sentdetect.SentenceDetectorME')
-
   def self.sentence_split_detector
     @@sentence_split_detector ||= begin
+                                    Rbbt.software.opt.OpenNLP.produce
+                                    Rbbt.software.opt.OpenNLP.models["da-sent.bin"].produce
+
+                                    @@FileInputStream = Rjb::import('java.io.FileInputStream')
+                                    @@SentenceModel = Rjb::import('opennlp.tools.sentdetect.SentenceModel')
+                                    @@SentenceDetectorME = Rjb::import('opennlp.tools.sentdetect.SentenceDetectorME')
+
                                     modelIn = @@FileInputStream.new(Rbbt.software.opt.OpenNLP.models["da-sent.bin"].produce.find);
 
                                     model = @@SentenceModel.new(modelIn);

data/share/install/software/GNormPlus

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+name='GNormPlus'
+url="https://www.ncbi.nlm.nih.gov/CBBresearch/Lu/Demo/tmTools/download/GNormPlus/GNormPlusJava.zip"
+
+prepare_pkg "$name" "$url"
+

data/share/install/software/Gdep

@@ -5,7 +5,7 @@ RBBT_SOFTWARE_DIR="$2"
 source "$INSTALL_HELPER_FILE"
 
 name='Gdep'
-url="http://people.ict.usc.edu/~sagae/parser/gdep/gdep-beta2.tgz"
+url="http://www.sagae.org/gdep/gdep-beta2.tgz"
 
 
 get_pkg "$name" "$url"

data/share/install/software/OpenNLP

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 name='OpenNLP'
-url="http://apache.rediris.es//opennlp/opennlp-1.5.3/apache-opennlp-1.5.3-bin.tar.gz"
+url="http://apache.rediris.es/opennlp/opennlp-1.9.1/apache-opennlp-1.9.1-bin.tar.gz"
 
 get_src "$name" "$url"
 move_opt "$name" 

data/test/rbbt/ner/segment/test_named_entity.rb

@@ -6,7 +6,7 @@ class TestClass < Test::Unit::TestCase
   def test_info
     a = ["test"]
     NamedEntity.setup a
-    assert(! a.info.keys.include?(:code))
+    assert(a.info[:code].nil?)
     a.code = 10
     a.offset = 100
     assert a.info.include? :code
@@ -26,4 +26,27 @@ class TestClass < Test::Unit::TestCase
     assert Segment.tsv([a], nil).fields.include? "code"
     assert Segment.tsv([a], "literal").fields.include? "code"
   end
+
+  def test_segment_brat
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Gene"
+
+    gene3 = "TP53 gene"
+    gene3.extend NamedEntity
+    gene3.offset = a.index gene3
+    gene3.type = "Gene"
+
+    segments = [gene1, gene2, gene3]
+    assert segments.collect{|s| s.to_brat}.include? "Gene 27 35"
+
+  end
 end

data/test/rbbt/ner/segment/test_transformed.rb

@@ -2,10 +2,23 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_he
 require 'rbbt/ner/segment/transformed'
 require 'rbbt/ner/segment/named_entity'
 require 'rexml/document'
-require 'rand'
 
 class TestClass < Test::Unit::TestCase
-  def tttest_transform
+  def test_sort
+    text = <<-EOF
+More recently, PPAR activators were shown to inhibit the activation of inflammatory response genes (such as IL-2, IL-6, IL-8, TNF alpha and metalloproteases) by negatively interfering with the NF-kappa B, STAT and AP-1 signalling pathways in cells of the vascular wall.
+    EOF
+
+    entities = ["PPAR", "IL-2", "IL-6", "IL-8", "TNF alpha", "NF-kappa B", "AP-1", "STAT"].reverse.collect do |literal|
+      NamedEntity.setup(literal, :offset => text.index(literal))
+    end
+
+    Transformed.with_transform(text, entities, Proc.new{|e| "[" + e.upcase + "]" }) do 
+      assert text.include? "such as [IL-2]"
+    end
+  end
+
+  def ___test_transform
     a = "This sentence mentions the TP53 gene and the CDK5 protein"
     original = a.dup
 
@@ -27,6 +40,8 @@ class TestClass < Test::Unit::TestCase
     c[gene1.range] = "GN"
     assert_equal c, Transformed.transform(a,[gene1], "GN")
 
+    iii a.transformation_offset_differences
+    raise
     assert_equal gene2.offset, a.transformation_offset_differences.first.first.first
     assert_equal gene1.offset, a.transformation_offset_differences.last.first.first
 
@@ -216,5 +231,60 @@ class TestClass < Test::Unit::TestCase
       end
     end
    end
+
+  def test_nested_transform
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Protein"
+
+    Transformed.with_transform(a, [gene1,gene2], "[G]") do 
+      assert_equal "This sentence mentions the [G] gene and the [G] protein", a
+    end
+    Transformed.with_transform(a, [gene1], "[G1]") do 
+      Transformed.with_transform(a, [gene2], "[G2]") do 
+        assert_equal "This sentence mentions the [G1] gene and the [G2] protein", a
+      end
+    end
+    Transformed.with_transform(a, [gene2], "[G2]") do 
+      Transformed.with_transform(a, [gene1], "[G1]") do 
+        assert_equal "This sentence mentions the [G1] gene and the [G2] protein", a
+      end
+    end
+  end
+
+  def test_offset_transform
+    a = "ILF can bind to purine-rich regulatory motifs such as the human T-cell leukemia virus-long terminal region and the interleukin-2 promoter."
+
+    gene1 = "ILF"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "interleukin-2"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Protein"
+
+    Transformed.with_transform(a, [gene1,gene2], "[G]") do 
+      assert_equal "[G] can bind to purine-rich regulatory motifs such as the human T-cell leukemia virus-long terminal region and the [G] promoter.", a
+    end
+
+    offset = 100
+    a = Segment.setup(a, :offset => offset)
+    gene1.offset += offset
+    gene2.offset += offset
+    Transformed.with_transform(a, [gene1,gene2], "[G]") do 
+      assert_equal "[G] can bind to purine-rich regulatory motifs such as the human T-cell leukemia virus-long terminal region and the [G] promoter.", a
+    end
+
+  end
 end
 

data/test/rbbt/ner/test_brat.rb

@@ -0,0 +1,64 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/brat'
+
+class TestBrat < Test::Unit::TestCase
+  def test_load
+    text =<<-EOF
+T2	DBTF 52 55	Nrl
+#2	AnnotatorNotes T2	4901
+T3	NONDBTF 80 89	rhodopsin
+#3	AnnotatorNotes T3	6010
+T4	BIOLOGICALPROCESS 90 105	gene expression
+#4	AnnotatorNotes T4	-
+T5	DBTF 127 130	Nrl
+#5	AnnotatorNotes T5	4901
+T7	MOLECULARFUNCTION 197 204	binding
+#7	AnnotatorNotes T7	-
+T8	PHENOTYPE 241 252	extended AP
+#8	AnnotatorNotes T8	-
+T10	DBTF 331 334	Nrl
+#10	AnnotatorNotes T10	4901
+T11	TISSUE 381 399	photoreceptor cell
+#11	AnnotatorNotes T11	-
+T12	NONDBTF 414 423	rhodopsin
+#12	AnnotatorNotes T12	6010
+T13	CELLULARCOMPONENT 494 501	nuclear
+#13	AnnotatorNotes T13	-
+T14	TISSUE 548 572	retinoblastoma cell line
+#14	AnnotatorNotes T14	-
+T17	NONDBTF 660 669	rhodopsin
+#17	AnnotatorNotes T17	6010
+T18	DBTF 676 679	Nrl
+#18	AnnotatorNotes T18	4901
+T19	CELLULARCOMPONENT 749 764	protein complex
+#19	AnnotatorNotes T19	-
+T20	DBTF 797 800	Nrl
+#20	AnnotatorNotes T20	4901
+T21	DBTF 853 856	Nrl
+#21	AnnotatorNotes T21	4901
+T22	MOLECULARFUNCTION 882 892	luciferase
+#22	AnnotatorNotes T22	-
+T23	DBTF 943 946	Nrl
+#23	AnnotatorNotes T23	4901
+T24	NONDBTF 989 998	rhodopsin
+#24	AnnotatorNotes T24	6010
+T26	DBTF 1110 1113	Nrl
+#26	AnnotatorNotes T26	4901
+T27	DBTF 1224 1227	Nrl
+#27	AnnotatorNotes T27	4901
+T28	DBTF 1271 1274	Nrl
+#28	AnnotatorNotes T28	4901
+T30	DBTF 1385 1388	Nrl
+#30	AnnotatorNotes T30	4901
+R1	ACTIVATION Arg1:T2 Arg2:T3
+R2	ACTIVATION Arg1:T10 Arg2:T12
+R3	ACTIVATION Arg1:T23 Arg2:T24
+T1	DBTF 250 254	AP-1
+    EOF
+
+    io = StringIO.new text
+    iii Brat.load io
+
+  end
+end
+

data/test/rbbt/ner/test_g_norm_plus.rb

@@ -0,0 +1,16 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/g_norm_plus'
+
+Log.severity = 0
+class TestGNormPlus < Test::Unit::TestCase
+  def test_match
+    text =<<-EOF
+We found that TP53 is regulated by MDM2 in Homo sapiens
+    EOF
+
+
+    mentions = GNormPlus.process({:file => text})
+    Log.tsv mentions
+  end
+end
+

data/test/rbbt/ner/test_segment.rb

@@ -96,6 +96,5 @@ Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors of
       assert_equal %w(CDK5R1), index[gene2.offset + 1]
     end
   end
-
 end
 

data/test/rbbt/nlp/genia/test_sentence_splitter.rb

@@ -0,0 +1,9 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/nlp/genia/sentence_splitter'
+
+class TestClass < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/rbbt/nlp/open_nlp/test_sentence_splitter.rb

@@ -29,11 +29,14 @@ sentence. This is
 another sentence. 
     EOF
 
+    iii OpenNLP.sentence_split_detector.sentDetect(text)
+    assert_equal 5, OpenNLP.sentence_split_detector.sentDetect(text).length
+
     assert_equal 5, OpenNLP.sentence_splitter(text).length
     assert_equal "This is a \nsentence.", OpenNLP.sentence_splitter(text)[3]
   end
 
-  def test_text_sentences
+  def _test_text_sentences
     Misc.benchmark(100) do
       OpenNLP.sentence_splitter($text).include? "Our
 findings highlight the role of SMARCA4 in the pathogenesis of SMARCB1-positive

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-04 00:00:00.000000000 Z
+date: 2020-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rjb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: 'Text mining tools: named entity recognition and normalization, document
   classification, bag-of-words, dictionaries, etc'
 email: miguel.vazquez@fdi.ucm.es
@@ -100,8 +86,10 @@ files:
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
 - lib/rbbt/ner/banner.rb
+- lib/rbbt/ner/brat.rb
 - lib/rbbt/ner/chemical_tagger.rb
 - lib/rbbt/ner/finder.rb
+- lib/rbbt/ner/g_norm_plus.rb
 - lib/rbbt/ner/linnaeus.rb
 - lib/rbbt/ner/ngram_prefix_dictionary.rb
 - lib/rbbt/ner/oscar3.rb
@@ -125,6 +113,7 @@ files:
 - share/install/software/ABNER
 - share/install/software/BANNER
 - share/install/software/ChemicalTagger
+- share/install/software/GNormPlus
 - share/install/software/Gdep
 - share/install/software/Geniass
 - share/install/software/Linnaeus
@@ -141,13 +130,16 @@ files:
 - test/rbbt/bow/test_misc.rb
 - test/rbbt/entity/test_document.rb
 - test/rbbt/ner/segment/test_named_entity.rb
+- test/rbbt/ner/segment/test_relationship.rb
 - test/rbbt/ner/segment/test_segmented.rb
 - test/rbbt/ner/segment/test_transformed.rb
 - test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/test_banner.rb
+- test/rbbt/ner/test_brat.rb
 - test/rbbt/ner/test_chemical_tagger.rb
 - test/rbbt/ner/test_finder.rb
+- test/rbbt/ner/test_g_norm_plus.rb
 - test/rbbt/ner/test_linnaeus.rb
 - test/rbbt/ner/test_ngram_prefix_dictionary.rb
 - test/rbbt/ner/test_oscar4.rb
@@ -156,6 +148,7 @@ files:
 - test/rbbt/ner/test_rnorm.rb
 - test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_token_trieNER.rb
+- test/rbbt/nlp/genia/test_sentence_splitter.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
 - test/rbbt/nlp/test_nlp.rb
 - test/test_helper.rb
@@ -177,14 +170,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/nlp/test_nlp.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
+- test/rbbt/nlp/genia/test_sentence_splitter.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_misc.rb
 - test/rbbt/bow/test_dictionary.rb
@@ -195,6 +188,8 @@ test_files:
 - test/rbbt/ner/test_rnorm.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_ngram_prefix_dictionary.rb
+- test/rbbt/ner/test_brat.rb
+- test/rbbt/ner/test_g_norm_plus.rb
 - test/rbbt/ner/test_chemical_tagger.rb
 - test/rbbt/ner/test_banner.rb
 - test/rbbt/ner/test_token_trieNER.rb
@@ -202,6 +197,7 @@ test_files:
 - test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_linnaeus.rb
 - test/rbbt/ner/segment/test_transformed.rb
+- test/rbbt/ner/segment/test_relationship.rb
 - test/rbbt/ner/segment/test_named_entity.rb
 - test/rbbt/ner/segment/test_segmented.rb
 - test/rbbt/ner/test_oscar4.rb