rbbt-text 1.2.0 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '009cfce2ce954c03db5c09d0bd6f5d25bf59d508776d7370bb6bd0fb3a135f36'
-  data.tar.gz: 3d11d2a5934512958d10dbdfad5e22a9a2481b332c985ab1e2c8e92427d6f375
+  metadata.gz: 496288d7d3ff1215ded1fd210192d5887a6a071eea5f322295a669a5d648d77b
+  data.tar.gz: 47996496009cbcdaab38a9dc9bf6efbbe7fc0145f315b0a48bfab0f543742f94
 SHA512:
-  metadata.gz: e9338d4b54d2b66efda11dee3d37366c4f4ae78bde80f0abc1016b34c928e1db857ad73f33ba1da611ad232513498430736c46134a902b3930a8f832afed3e09
-  data.tar.gz: 0cdeeee67636d4e0b0714334b3c187cb0f5ea5c7363fe27fc84d438643a0d6f204413a4dd5d99c8c43d847539320c484fde2b5300b298cf9cc782148d98802ee
+  metadata.gz: 36e7415ad06207066844a30001c8541865f066d1e83a4a2ddc5182c54b704cd3d442cbccce219bd2114717a83656d07558c42725eca75597fea239b6e13244ab
+  data.tar.gz: 988eff4d242d0425910b96fac4188df079c8c53c3abea2825cc97d5af5118841680705fa33461a5b4cfa7b8d6b32a486465e44b75f20fad324e4623c6c8083d8

data/lib/rbbt/bow/bow.rb

@@ -69,6 +69,11 @@ module BagOfWords
     count = bigrams ? count(bigrams(text)) : count(words(text))
     count.values_at(*terms)
   end
+
+  def self.weighted_features(text, weights)
+    features = features(text, weights.keys)
+    features.zip(weights.values).collect{|f,w| f * w } 
+  end
 end
 
 class String
@@ -82,5 +87,3 @@ class String
     BagOfWords.bigrams(self)
   end
 end
-
-

data/lib/rbbt/bow/dictionary.rb

@@ -74,28 +74,32 @@ class Dictionary::TF_IDF
   end
 
   def best(options = {})
-    high, low, limit = {
-      :low   => 0,
-      :high    => 1,
-    }.merge(options).
-    values_at(:high, :low, :limit)
-
-    num_docs = @num_docs.to_f
-    best = df.select{|term, value|
-      value >= low && value <= high
-    }.collect{|p| 
-      term     = p.first
-      df_value = p.last
-      [term,
-       @terms[term].to_f / num_docs * Math::log(1.0/df_value)
-      ]
-    }
-
-    if limit
-      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
-    else
-      Hash[*best.flatten]
-    end
+    key = Misc.obj2digest(options)
+    @best ||= {}
+    @best[key] ||= begin
+                     high, low, limit = {
+                       :low   => 0,
+                       :high    => 1,
+                     }.merge(options).
+                     values_at(:high, :low, :limit)
+
+                     num_docs = @num_docs.to_f
+                     best = df.select{|term, value|
+                       value >= low && value <= high
+                     }.collect{|p| 
+                       term     = p.first
+                       df_value = p.last
+                       [term,
+                        @terms[term].to_f / num_docs * Math::log(1.0/df_value)
+                       ]
+                     }
+
+                     if limit
+                       Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit-1).flatten]
+                     else
+                       Hash[*best.flatten]
+                     end
+                   end
   end
 
   def weights(options = {})
@@ -173,7 +177,7 @@ class Dictionary::KL
       best[term] = pos * Math::log(pos / neg) + neg * Math::log(neg / pos)
     }
     if limit
-      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
+      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit-1).flatten]
     else
       best
     end

data/lib/rbbt/document.rb

@@ -0,0 +1,55 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+
+module DocID
+  extend Entity
+  self.annotation :corpus
+
+  class << self
+    attr_accessor :default_corpus
+  end
+
+  def corpus
+    annotation_values[:corpus] || DocID.default_corpus
+  end
+
+  property :to do |type|
+    namespace, code = self.split(":")
+    DocID.setup([namespace, code, "title"] * ":", :corpus => corpus)
+  end
+
+  property :document => :both do
+    if Array === self
+      namespace, id, type = nil, nil, nil
+      docs = self.collect do |docid|
+        text = self.corpus[docid]
+        namespace, id, type = docid.split(":")
+        text
+      end
+      Document.setup(docs, :corpus => corpus)
+    else
+      text = self.corpus[self]
+      namespace, id, type = self.split(":")
+      Document.setup(text, :namespace => namespace, :code => id, :type => type, :corpus => corpus)
+    end
+  end
+end
+
+module Document
+  extend Entity
+  self.annotation :namespace, :code, :type, :corpus
+
+  property :docid do |corpus=nil|
+    digest = Misc.digest(self)
+    corpus = self.corpus if corpus.nil?
+
+    DocID.setup([namespace, code, type, digest] * ":", :corpus => corpus)
+  end
+
+  property :to do |type|
+    docid.to(type).document
+  end
+
+  alias id docid
+end
+

data/lib/rbbt/document/annotation.rb

@@ -0,0 +1,45 @@
+require 'rbbt/segment'
+require 'rbbt/segment/annotation'
+
+module Document
+  def self.define(type, &block)
+    send :property, type do 
+      segments = self.instance_exec &block
+
+      Segment.align(self, segments) unless segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+
+      segments.each do |segment|
+        SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
+      end
+
+      docid = self.docid
+      segments.each{|s| s.docid = docid if s.docid.nil? }
+
+      segments
+    end
+  end
+
+  def self.define_multiple(type, &block)
+    send :property, type => :multiple do |list|
+      doc_segments = self.instance_exec list, &block
+
+      doc_segments = doc_segments.chunked_values_at(list) if Hash === doc_segments
+
+      doc_segments.each_with_index do |segments,i|
+        next if segments.nil?
+        document = list[i]
+        Segment.align(document, segments) unless segments.nil? || segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+
+        segments.each do |segment|
+          SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
+        end
+
+        docid = document.docid
+
+        segments.each{|s| s.docid = docid if s.docid.nil? }
+
+        segments
+      end
+    end
+  end
+end

data/lib/rbbt/document/corpus.rb

@@ -0,0 +1,63 @@
+require 'rbbt-util'
+
+module Document::Corpus
+
+  def self.setup(corpus)
+    corpus.extend Document::Corpus unless Document::Corpus === corpus
+    corpus.extend Persist::TSVAdapter unless Persist::TSVAdapter === corpus
+    corpus
+  end
+
+  def add_document(document)
+    docid = document.docid
+    return self[docid] if self.include?(docid)
+    self.write_and_close do
+      self[docid] = document
+    end
+  end
+  
+  def docids(prefix)
+    prefix += ":" unless prefix == :all || prefix[-1] == ":"
+    docids = self.read_and_close do
+      prefix == :all ? self.keys : self.prefix(prefix)
+    end
+    DocID.setup(docids, :corpus => self)
+  end
+
+  def documents(prefix)
+    self.docids(prefix).document
+  end
+
+  def [](*args)
+    docid, *rest = args
+
+    res = self.read_and_close do
+      super(*args)
+    end
+    
+    res.force_encoding(Encoding.default_external) if res 
+    return res if args.length > 1
+
+    namespace, id, type  = docid.split(":")
+
+    if res.nil?
+      if Document::Corpus.claims.include?(namespace.to_s)
+        res = self.instance_exec(id, type, &Document::Corpus.claims[namespace.to_s])
+      end
+    end
+
+    res.force_encoding(Encoding.default_external) if res 
+    Document.setup(res, namespace, id, type, self) unless res.nil?
+    
+    res
+  end
+
+  class << self
+    attr_accessor :claims
+    def claim(namespace, &block)
+      @claims = {}
+      @claims[namespace.to_s] = block
+    end
+  end
+
+end

data/lib/rbbt/document/corpus/pubmed.rb

@@ -0,0 +1,33 @@
+require 'rbbt/sources/pubmed'
+
+module Document::Corpus
+  def add_pmid(pmid, type = nil)
+    pmids = Array === pmid ? pmid : [pmid]
+    type = nil if String === type and type.empty?
+
+    res = PubMed.get_article(pmids).collect do |pmid, article|
+      document = if type.nil? || type.to_sym == :abstract
+                   Document.setup(article.abstract || "", "PMID", pmid, :abstract, self, :corpus => self)
+                 elsif type.to_sym == :title
+                   Document.setup(article.title, :PMID, pmid, :title, self)
+                 else
+                   raise "No FullText available for #{ pmid }" if article.full_text.nil?
+                   Document.setup(article.full_text, :PMID, pmid, :fulltext, self, :corpus => self)
+                 end
+      Log.debug "Loading pmid #{pmid}"
+      add_document(document)
+    end
+
+    Document.setup(res)
+  end
+  
+  def add_pubmed_query(query, max = 3000, type = nil)
+    pmids = PubMed.query(query, max)
+    add_pmid(pmids, type)
+  end
+
+  self.claim "PMID" do |id, type|
+    Log.debug "Claiming #{id}"
+    self.add_pmid(id, type).first
+  end
+end

data/lib/rbbt/ner/NER.rb

@@ -1,6 +1,6 @@
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/named_entity'
-require 'rbbt/text/segment/segmented'
+require 'rbbt/segment'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/segmented'
 
 class NER
   def entities(text, protect = false, *args)

data/lib/rbbt/ner/abner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'rbbt/resource'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
 
 # Offers a Ruby interface to the Abner Named Entity Recognition Package

data/lib/rbbt/ner/banner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'rbbt/ner/NER'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 
 # Offers a Ruby interface to the Banner Named Entity Recognition Package
 # in Java. Banner[http://banner.sourceforge.net/].

data/lib/rbbt/ner/brat.rb

@@ -1,4 +1,4 @@
-require 'rbbt/text/segment/named_entity'
+require 'rbbt/segment/named_entity'
 require 'rbbt/text/segment/relationship'
 module Brat
   Rbbt.claim Rbbt.software.opt.Brat, :install, "https://github.com/nlplab/brat.git"

data/lib/rbbt/ner/chemical_tagger.rb

@@ -1,6 +1,5 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -8,7 +7,7 @@ class ChemicalTagger < NER
   Rbbt.claim Rbbt.software.opt.ChemicalTagger, :install, Rbbt.share.install.software.ChemicalTagger.find
 
   def self.init
-    ENV["CLASSPATH"] = ENV["CLASSPATH"].split(":").reverse * ":"
+    ENV["CLASSPATH"] = [ENV["CLASSPATH"].split(":"), Rbbt.software.opt.ChemicalTagger.produce.glob("*.jar").first].reverse * ":"
     Rjb::load(nil, jvmargs = ['-Xms1G','-Xmx2G']) unless Rjb.loaded?
     @@RbbtChemicalTagger ||= Rjb::import('RbbtChemicalTagger')
   end

data/lib/rbbt/ner/g_norm_plus.rb

@@ -1,4 +1,6 @@
 require 'rbbt-util'
+require 'rbbt/segment'
+require 'rbbt/segment/named_entity'
 module GNormPlus
 
   Rbbt.claim Rbbt.software.opt.GNormPlus, :install do
@@ -35,8 +37,8 @@ module GNormPlus
 	HomologeneID = False
 	Normalization2Protein = False
 	ShowUnNormalizedMention = False
+	IgnoreNER = False
 	DeleteTmp = True
-	IgnoreNER = True
 EOF
 
   def self.process(texts)
@@ -53,11 +55,16 @@ EOF
         Open.mkdir 'tmp'
 
         texts.each do |name,text|
+          text = Misc.fixutf8(text)
+
+          text = text.gsub('|', '#').gsub("\n", " ").gsub(/\t/,' ')
+
           Open.write("input/#{name}.txt") do |f|
-            f.puts "#{name}|a|" << text.gsub("\n\n", "\n·")
+            f.puts "#{name}|a|" << text
             f.puts
           end
         end
+
         Open.write('config', CONFIG)
         CMD.cmd_log("java -Xmx20G -Xms20G  -jar '#{Rbbt.software.opt.GNormPlus.produce.find}/GNormPlus.jar' 'input' 'output' 'config'")
 
@@ -69,7 +76,7 @@ EOF
         tsv = TSV.setup({}, :key_field => key_field, :fields => ["Entities"], :type => :flat)
         Dir.glob("output/*.txt").each do |file|
           name = File.basename(file).sub(".txt",'')
-          entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '.').split("\t")[1..-1] * ":"}
+          entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '·').split("\t")[1..-1] * ":"}
           tsv[name] = entities
         end
 
@@ -79,6 +86,22 @@ EOF
       end
     end
   end
+
+  def self.entities(texts)
+    res = {}
+    process(texts).each do |name, entities|
+
+      segments = entities.collect do |entity|
+        start, eend, literal, type, code = entity.split(":")
+        literal.gsub!('·',':')
+
+        NamedEntity.setup(literal, :offset => start.to_i, :entity_type => type, :code => code)
+      end
+
+      res[name] = segments
+    end
+    res
+  end
 end
 
 if __FILE__ == $0

data/lib/rbbt/ner/linnaeus.rb

@@ -1,12 +1,12 @@
 require 'rjb'
 require 'rbbt'
-require 'rbbt/text/segment/named_entity'
+require 'rbbt/segment/named_entity'
 
 module Linnaeus
 
   Rbbt.claim Rbbt.software.opt.Linnaeus, :install, Rbbt.share.install.software.Linnaeus.find
 
-  ARGS = ["--properties", Rbbt.software.opt.Linnaeus["species-proxy/properties.conf"].find]
+  ARGS = ["--properties", Rbbt.software.opt.Linnaeus.produce["species-proxy/properties.conf"].find]
 
 
   Rjb::load(nil, jvmargs = ['-Xms2G','-Xmx2G']) unless Rjb.loaded?
@@ -31,7 +31,7 @@ module Linnaeus
     init unless defined? @@Matcher
 
     @@Matcher.match(text).toArray().collect do |mention|
-      NamedEntity.setup(mention.text(), mention.start(), "Organism", mention.ids(), mention.probabilities())
+      NamedEntity.setup(mention.text(), :offset => mention.start(), :entity_type => "Organism", :code => mention.ids(), :score => mention.probabilities())
     end
   end
 end

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -1,8 +1,8 @@
 require 'rbbt'
 require 'rbbt/util/misc'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
+require 'rbbt/segment/token'
 require 'rbbt/ner/NER'
 require 'inline'
 
@@ -150,7 +150,7 @@ VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
 
   def match(text)
     matches = NGramPrefixDictionary.match(index, (case_insensitive ? text.downcase : text)).collect{|name, code, offset|
-      NamedEntity.setup(name, offset, type, code)
+      NamedEntity.setup(name, :offset => offset, :entity_type => type, :code => code)
     }
 
     if case_insensitive