rbbt-text 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '009cfce2ce954c03db5c09d0bd6f5d25bf59d508776d7370bb6bd0fb3a135f36'
-  data.tar.gz: 3d11d2a5934512958d10dbdfad5e22a9a2481b332c985ab1e2c8e92427d6f375
+  metadata.gz: c2a24d8e7faf30d53e41a00a27f6145e8e9f18f0c10af57cdddaea0ee18c35d6
+  data.tar.gz: 3475006965110391e35151cd1b5368028dacf467aa276f8eb68fce3320be1122
 SHA512:
-  metadata.gz: e9338d4b54d2b66efda11dee3d37366c4f4ae78bde80f0abc1016b34c928e1db857ad73f33ba1da611ad232513498430736c46134a902b3930a8f832afed3e09
-  data.tar.gz: 0cdeeee67636d4e0b0714334b3c187cb0f5ea5c7363fe27fc84d438643a0d6f204413a4dd5d99c8c43d847539320c484fde2b5300b298cf9cc782148d98802ee
+  metadata.gz: da40a039a4792eb5e7fa00270870279221c74dcbf51df1b5278b247496fefbfa888a87b7ab19f05676644c51a01177eb49e229cb0156fe7f0190dd4933d41e24
+  data.tar.gz: a32fca5f21a987dcbb6b5541015cc33879330e6f1ef7c4a28e75debe5bdd1dc8bf7b98bfc91d828e605f29868aa972b55cd59bb4f86e66d2fb0cfea31fac2ae0

data/lib/rbbt/document.rb

@@ -0,0 +1,46 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+require 'rbbt/document/annotation'
+
+module DocID
+  extend Entity
+  self.annotation :corpus
+
+  class << self
+    attr_accessor :default_corpus
+  end
+
+  def corpus
+    annotation_values[:corpus] || DocID.default_corpus
+  end
+
+  property :to do |type|
+    namespace, code = self.split(":")
+    DocID.setup([namespace, code, "title"] * ":", :corpus => corpus)
+  end
+
+  def document
+    text = self.corpus[self]
+    namespace, id, type = self.split(":")
+    Document.setup(text, namespace, id, type, :corpus => corpus)
+  end
+end
+
+module Document
+  extend Entity
+  self.annotation :namespace, :code, :type, :corpus
+
+  property :docid do |corpus=nil|
+    digest = Misc.digest(self)
+    corpus = self.corpus if corpus.nil?
+
+    DocID.setup([namespace, code, type, digest] * ":", :corpus => corpus)
+  end
+
+  property :to do |type|
+    docid.to(type).document
+  end
+
+  alias id docid
+end
+

data/lib/rbbt/document/annotation.rb

@@ -0,0 +1,42 @@
+require 'rbbt/segment/annotation'
+
+module Document
+  def self.define(type, &block)
+    send :property, type do 
+      segments = self.instance_exec &block
+
+      Segment.align(self, segments) unless segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+
+      segments.each do |segment|
+        SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
+      end
+
+      docid = self.docid
+      segments.each{|s| s.docid = docid if s.docid.nil? }
+
+      segments
+    end
+  end
+
+  def self.define_multiple(type, &block)
+    send :property, type => :multiple do |list|
+      doc_segments = self.instance_exec list, &block
+
+      doc_segments = doc_segments.chunked_values_at(self) if Hash === doc_segments
+
+      doc_segments.each_with_index do |segments,i|
+        document = list[i]
+        Segment.align(document, segments) unless segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+
+        segments.each do |segment|
+          SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
+        end
+
+        docid = document.docid
+        segments.each{|s| s.docid = docid if s.docid.nil? }
+
+        segments
+      end
+    end
+  end
+end

data/lib/rbbt/document/corpus.rb

@@ -0,0 +1,38 @@
+require 'rbbt-util'
+
+module Document::Corpus
+
+  def self.setup(corpus)
+    corpus.extend Document::Corpus
+  end
+
+  def add_document(document)
+    self[document.docid] = document
+  end
+
+  def [](*args)
+    docid, *rest = args
+    res = super(*args)
+    return res if args.length > 1
+    namespace, id, type  = docid.split(":")
+
+    if res.nil?
+      if Document::Corpus.claims.include?(namespace.to_s)
+        res = self.instance_exec(id, type, &Document::Corpus.claims[namespace.to_s])
+      end
+    end
+
+    Document.setup(res, namespace, id, type, self) unless res.nil?
+    
+    res
+  end
+
+  class << self
+    attr_accessor :claims
+    def claim(namespace, &block)
+      @claims = {}
+      @claims[namespace.to_s] = block
+    end
+  end
+
+end

data/lib/rbbt/document/corpus/pubmed.rb

@@ -0,0 +1,33 @@
+require 'rbbt/sources/pubmed'
+
+module Document::Corpus
+  def add_pmid(pmid, type = nil)
+    pmids = Array === pmid ? pmid : [pmid]
+    type = nil if String === type and type.empty?
+
+    res = PubMed.get_article(pmids).collect do |pmid, article|
+      Log.debug "Loading pmid #{pmid}"
+      document = if type.nil? || type.to_sym == :abstract
+                   Document.setup(article.abstract || "", "PMID", pmid, :abstract, self, :corpus => self)
+                 elsif type.to_sym == :title
+                   Document.setup(article.title, :PMID, pmid, :title, self)
+                 else
+                   raise "No FullText available for #{ pmid }" if article.full_text.nil?
+                   Document.setup(article.full_text, :PMID, pmid, :fulltext, self, :corpus => self)
+                 end
+      add_document(document)
+    end
+
+    Document.setup(res)
+  end
+  
+  def add_pubmed_query(query, max = 3000, type = nil)
+    pmids = PubMed.query(query, max)
+    add_pmid(pmids, type)
+  end
+
+  self.claim "PMID" do |id, type|
+    Log.debug "Claiming #{id}"
+    self.add_pmid(id, type).first
+  end
+end

data/lib/rbbt/ner/NER.rb

@@ -1,6 +1,6 @@
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/named_entity'
-require 'rbbt/text/segment/segmented'
+require 'rbbt/segment'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/segmented'
 
 class NER
   def entities(text, protect = false, *args)

data/lib/rbbt/ner/abner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'rbbt/resource'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
 
 # Offers a Ruby interface to the Abner Named Entity Recognition Package

data/lib/rbbt/ner/banner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'rbbt/ner/NER'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 
 # Offers a Ruby interface to the Banner Named Entity Recognition Package
 # in Java. Banner[http://banner.sourceforge.net/].

data/lib/rbbt/ner/brat.rb

@@ -1,4 +1,4 @@
-require 'rbbt/text/segment/named_entity'
+require 'rbbt/segment/named_entity'
 require 'rbbt/text/segment/relationship'
 module Brat
   Rbbt.claim Rbbt.software.opt.Brat, :install, "https://github.com/nlplab/brat.git"

data/lib/rbbt/ner/chemical_tagger.rb

@@ -1,6 +1,5 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -8,7 +7,7 @@ class ChemicalTagger < NER
   Rbbt.claim Rbbt.software.opt.ChemicalTagger, :install, Rbbt.share.install.software.ChemicalTagger.find
 
   def self.init
-    ENV["CLASSPATH"] = ENV["CLASSPATH"].split(":").reverse * ":"
+    ENV["CLASSPATH"] = [ENV["CLASSPATH"].split(":"), Rbbt.software.opt.ChemicalTagger.produce.glob("*.jar").first].reverse * ":"
     Rjb::load(nil, jvmargs = ['-Xms1G','-Xmx2G']) unless Rjb.loaded?
     @@RbbtChemicalTagger ||= Rjb::import('RbbtChemicalTagger')
   end

data/lib/rbbt/ner/g_norm_plus.rb

@@ -1,4 +1,6 @@
 require 'rbbt-util'
+require 'rbbt/segment'
+require 'rbbt/segment/named_entity'
 module GNormPlus
 
   Rbbt.claim Rbbt.software.opt.GNormPlus, :install do
@@ -35,8 +37,8 @@ module GNormPlus
 	HomologeneID = False
 	Normalization2Protein = False
 	ShowUnNormalizedMention = False
+	IgnoreNER = False
 	DeleteTmp = True
-	IgnoreNER = True
 EOF
 
   def self.process(texts)
@@ -69,7 +71,7 @@ EOF
         tsv = TSV.setup({}, :key_field => key_field, :fields => ["Entities"], :type => :flat)
         Dir.glob("output/*.txt").each do |file|
           name = File.basename(file).sub(".txt",'')
-          entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '.').split("\t")[1..-1] * ":"}
+          entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '·').split("\t")[1..-1] * ":"}
           tsv[name] = entities
         end
 
@@ -79,6 +81,21 @@ EOF
       end
     end
   end
+
+  def self.entities(texts)
+    res = {}
+    process(texts).each do |name, entities|
+
+      segments = entities.collect do |entity|
+        start, eend, literal, type, code = entity.split(":")
+        literal.gsub!('·',':')
+
+        NamedEntity.setup(literal, :offset => start.to_i, :entity_type => type, :code => code)
+      end
+
+      res[name] = segments
+    end
+  end
 end
 
 if __FILE__ == $0

data/lib/rbbt/ner/linnaeus.rb

@@ -1,12 +1,12 @@
 require 'rjb'
 require 'rbbt'
-require 'rbbt/text/segment/named_entity'
+require 'rbbt/segment/named_entity'
 
 module Linnaeus
 
   Rbbt.claim Rbbt.software.opt.Linnaeus, :install, Rbbt.share.install.software.Linnaeus.find
 
-  ARGS = ["--properties", Rbbt.software.opt.Linnaeus["species-proxy/properties.conf"].find]
+  ARGS = ["--properties", Rbbt.software.opt.Linnaeus.produce["species-proxy/properties.conf"].find]
 
 
   Rjb::load(nil, jvmargs = ['-Xms2G','-Xmx2G']) unless Rjb.loaded?
@@ -31,7 +31,7 @@ module Linnaeus
     init unless defined? @@Matcher
 
     @@Matcher.match(text).toArray().collect do |mention|
-      NamedEntity.setup(mention.text(), mention.start(), "Organism", mention.ids(), mention.probabilities())
+      NamedEntity.setup(mention.text(), :offset => mention.start(), :entity_type => "Organism", :code => mention.ids(), :score => mention.probabilities())
     end
   end
 end

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -1,8 +1,8 @@
 require 'rbbt'
 require 'rbbt/util/misc'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
+require 'rbbt/segment/token'
 require 'rbbt/ner/NER'
 require 'inline'
 
@@ -150,7 +150,7 @@ VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
 
   def match(text)
     matches = NGramPrefixDictionary.match(index, (case_insensitive ? text.downcase : text)).collect{|name, code, offset|
-      NamedEntity.setup(name, offset, type, code)
+      NamedEntity.setup(name, :offset => offset, :entity_type => type, :code => code)
     }
 
     if case_insensitive

data/lib/rbbt/ner/oscar3.rb

@@ -1,7 +1,6 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -53,7 +52,7 @@ class OSCAR3 < NER
         next unless type.nil? or type.include? mention_type
         score  = memm ? entities.get(key).to_string.to_f : nil
 
-        NamedEntity.setup mention, rstart.to_i + offset, mention_type, nil, score
+        NamedEntity.setup mention, :offset => rstart.to_i + offset, :entity_type => mention_type, :score => score
         
         mentions << mention unless mentions.collect{|m| m.to_s}.include? mention.to_s
       end

data/lib/rbbt/ner/oscar4.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -25,7 +25,7 @@ class OSCAR4 < NER
     @@tagger ||= @@OSCAR.new()
   end
 
-  def self.match(text,  type = nil)
+  def self.match(text, protect = false,  type = nil)
     self.init
 
     return [] if text.nil? or text.strip.empty?
@@ -46,7 +46,7 @@ class OSCAR4 < NER
 
       next unless entity.getType.toString == type unless type.nil?
 
-      NamedEntity.setup mention, entity.getStart, entity.getType, inchi, entity.getConfidence
+      NamedEntity.setup mention, :offset => entity.getStart, :entity_type => entity.getType, :code => inchi, :score => entity.getConfidence
 
       result << mention
     end

data/lib/rbbt/ner/patterns.rb

@@ -1,7 +1,7 @@
-require 'rbbt/text/segment/named_entity'
-require 'rbbt/text/segment/segmented'
-require 'rbbt/text/segment/transformed'
-require 'rbbt/text/segment/relationship'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/transformed'
+#require 'rbbt/segment/relationship'
 require 'rbbt/ner/regexpNER'
 require 'rbbt/ner/token_trieNER'
 require 'rbbt/nlp/nlp'
@@ -14,7 +14,8 @@ class PatternRelExt
     regexpNER = RegExpNER.new type => patterns.collect{|p| /#{p}/}
     segments = sentence.segments
     segments = segments.values.flatten if Hash === segments
-    Transformed.with_transform(sentence, segments, Proc.new{|s| s.type.to_s.upcase}) do |sentence|
+    Transformed.with_transform(sentence, segments, Proc.new{|s| s.entity_type.to_s.upcase}) do |sentence|
+      ppp sentence
       regexpNER.entities(sentence)
     end
   end

data/lib/rbbt/ner/regexpNER.rb

@@ -1,4 +1,3 @@
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/simpleDSL'
 
@@ -23,7 +22,7 @@ class RegExpNER < NER
       end
 
       if match and not match.empty?
-        NamedEntity.setup(match, start + pre.length, type)
+        NamedEntity.setup(match, :offset => start + pre.length, :entity_type => type)
         matches << match
       end
 

data/lib/rbbt/ner/token_trieNER.rb

@@ -1,8 +1,8 @@
 require 'rbbt'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
+require 'rbbt/segment/token'
 
 class TokenTrieNER < NER
   def self.clean(token)
@@ -16,13 +16,13 @@ class TokenTrieNER < NER
   def self.prepare_token(token, start, extend_to_token = true, no_clean = false)
     if no_clean
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(token, :offset => start, :original => token)
       else
         token
       end
     else
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(clean(token), :offset => start, :original => token)
       else
         clean(token)
       end
@@ -137,7 +137,7 @@ class TokenTrieNER < NER
     tmp_index = {}
     hash.send(hash.respond_to?(:through)? :through : :each) do |code, names|
       names = Array === names ? names : [names]
-      names.flatten! if Array === names.first and not Token === names.first.first
+      names.flatten! if Array === names.first and not Segment === names.first.first
 
       if names.empty?
         names.unshift code unless TSV === hash and not (hash.fields.nil? or hash.fields.empty?)
@@ -237,7 +237,7 @@ class TokenTrieNER < NER
       match << ((t.respond_to?(:original) and not t.original.nil?) ? t.original : t)
     }
 
-    NamedEntity.setup(match, match_tokens.first.offset, type, codes)
+    NamedEntity.setup(match, :offset => match_tokens.first.offset, :entity_type => type, :code => codes)
   end
 
   attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean