rbbt-text 1.3.9 → 1.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92c6b4b9d3452c6b495fc9f291b551a73c8c150faee05053b7ecadc62ccbbd53
-  data.tar.gz: 70e341cf31466628c42b9947c882b64ff592e4703e06c8629ab56d513fe0a975
+  metadata.gz: 2f10312d9b6598ddc9b6fa98b38909afdd575b33a497ae1ff3f17c7a9c6e37bf
+  data.tar.gz: f79c61c7e34dd113a2c5002342c0c2df92a4a28c770394bf2c456a34a2730cc7
 SHA512:
-  metadata.gz: 2e8fdff40dd93072c3c377c59ff02d7374f3f81961dfc0f2596386776408c623543eb2e1f0da0112b3a8384d865c8331c659c650f2f2288a3d6282eca80e804e
-  data.tar.gz: d7b335d138eb48de51af8922d80d01715c4c61c025b525a9f63fcae789de329eedc2557cb10ba369f5987205ef87ffd85fc407fc3a29f6a75ef0b41951e4962b
+  metadata.gz: a9fb4dc49c538a58a8aa04e81947df212668c5ef9097434fa7d3eff54dd17a8657f581451b64e6b247cb64428436823a305dd64ae6a5fed2126b92285c02ad81
+  data.tar.gz: 0d31423660cd232102aa2b9914dab61ff929cf02a37b5094bd58481cac733c167d0e4fcdb4b3025e41a4775bd8033566ed3f402f66c317b3955406d1a3d3eb6f

data/lib/rbbt/document/annotation.rb

@@ -6,7 +6,9 @@ module Document
     send :property, type do 
       segments = self.instance_exec &block
 
-      Segment.align(self, segments) unless segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+      Segment.align(self, segments) unless segments.empty? || 
+          (Segment === segments && segments.offset) || 
+          (Array === segments && Segment === segments.first && segments.first.offset)
 
       segments.each do |segment|
         SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
@@ -17,6 +19,36 @@ module Document
 
       segments
     end
+
+    DocID.property type do
+      self.document.send(type)
+    end
+
+    SegID.property type do
+      self.overlaps(self.docid.send(type))
+    end
+
+    Segment.property type do
+      self.overlaps(self.docid.send(type))
+    end
+
+    seg_type = "segids_for_" + type.to_s
+
+    send :property, seg_type do 
+      SegID.setup(self.send(type).collect{|s| s.segid })
+    end
+
+    DocID.property seg_type do
+      self.document.send(seg_type)
+    end
+
+    SegID.property seg_type do
+      self.overlaps(self.docid.send(seg_type))
+    end
+
+    Segment.property seg_type do
+      self.overlaps(self.docid.send(seg_type))
+    end
   end
 
   def self.define_multiple(type, &block)
@@ -28,7 +60,10 @@ module Document
       doc_segments.each_with_index do |segments,i|
         next if segments.nil?
         document = list[i]
-        Segment.align(document, segments) unless segments.nil? || segments.empty? || (Segment === segments && segments.offset) || (Segment === segments.first && segments.first.offset)
+        Segment.align(document, segments) unless segments.nil? || 
+          segments.empty? || 
+          (Segment === segments && segments.offset) || 
+          (Array === segments && Segment === segments.first && segments.first.offset)
 
         segments.each do |segment|
           SegmentAnnotation.setup(segment, :type => type.to_s) unless SegmentAnnotation === segment && segment.type
@@ -41,5 +76,35 @@ module Document
         segments
       end
     end
+
+    DocID.property type do
+      self.document.send(type)
+    end
+
+    SegID.property type do
+      self.overlaps(self.docid.send(type))
+    end
+
+    Segment.property type do
+      self.overlaps(self.docid.send(type))
+    end
+
+    seg_type = "segids_for_" + type.to_s
+
+    send :property, seg_type do 
+      SegID.setup(self.send(type).collect{|s| s.segid })
+    end
+
+    DocID.property seg_type do
+      self.document.send(seg_type)
+    end
+
+    SegID.property seg_type do
+      self.overlaps(self.docid.send(seg_type))
+    end
+
+    Segment.property seg_type do
+      self.overlaps(self.docid.send(seg_type))
+    end
   end
 end

data/lib/rbbt/document/corpus/pubmed.rb

@@ -2,8 +2,8 @@ require 'rbbt/sources/pubmed'
 
 module Document::Corpus
   PUBMED_NAMESPACE="PMID"
-  def add_pmid(pmid, type = nil, update = false)
-    type = :abstract if type.nil?
+  def add_pmid(pmid, type = :title_and_abstract, update = false)
+    type = :title_and_abstract if type.nil?
 
     if ! (update || Array === pmid)
       id = [PUBMED_NAMESPACE, pmid, type].collect{|e| e.to_s}*":"
@@ -16,9 +16,11 @@ module Document::Corpus
 
     res = PubMed.get_article(pmids).collect do |pmid, article|
       document = if type.to_sym == :abstract
-                   Document.setup(article.abstract || "", PUBMED_NAMESPACE, pmid, :abstract, self, :corpus => self)
+                   Document.setup(article.abstract || "", PUBMED_NAMESPACE, pmid, type.to_sym , self, :corpus => self)
                  elsif type.to_sym == :title
-                   Document.setup(article.title, PUBMED_NAMESPACE, pmid, :title, self)
+                   Document.setup(article.title || "", PUBMED_NAMESPACE, pmid, type.to_sym, self)
+                 elsif type.to_sym == :title_and_abstract
+                   Document.setup((article.title || "") + "\n\n" + (article.abstract || ""), PUBMED_NAMESPACE, pmid, type.to_sym, self)
                  else
                    raise "No FullText available for #{ pmid }" if article.full_text.nil?
                    Document.setup(article.full_text, PUBMED_NAMESPACE, pmid, :fulltext, self, :corpus => self)

data/lib/rbbt/document.rb

@@ -9,6 +9,10 @@ module DocID
     attr_accessor :default_corpus
   end
 
+  def id
+    self
+  end
+
   def corpus
     annotation_values[:corpus] || DocID.default_corpus
   end

data/lib/rbbt/ner/g_norm_plus.rb

@@ -66,7 +66,8 @@ EOF
         end
 
         Open.write('config', CONFIG)
-        CMD.cmd_log("java -Xmx20G -Xms20G  -jar '#{Rbbt.software.opt.GNormPlus.produce.find}/GNormPlus.jar' 'input' 'output' 'config'")
+        mem = Rbbt::Config.get(:java_mem, :GNormPlus, :g_norm_plus, :gnormplus, :gnp, :default => "2G")
+        CMD.cmd_log("java -Xmx#{mem} -Xms#{mem}  -jar '#{Rbbt.software.opt.GNormPlus.produce.find}/GNormPlus.jar' 'input' 'output' 'config'")
 
         if texts.respond_to? :key_field
           key_field = texts.key_field

data/lib/rbbt/ner/regexpNER.rb

@@ -11,7 +11,15 @@ class RegExpNER < NER
       pre   = matchdata.pre_match
       post  = matchdata.post_match
 
-      if matchdata.captures.any?
+      if matchdata.named_captures.any?
+        match = matchdata[0]
+        code = matchdata.named_captures.collect{|k,v| [k,v] * "=" } * ";"
+        NamedEntity.setup(match, :offset => pre.length + start, :entity_type => type, :code => code)
+        matches << match
+        eend = match.length + pre.length
+        text = text[eend..-1] 
+        start += match.length + pre.length
+      elsif matchdata.captures.any?
         match = matchdata.captures.first
         offset, eend = matchdata.offset(1)
         NamedEntity.setup(match, :offset => start + offset, :entity_type => type)
@@ -88,7 +96,7 @@ class RegExpNER < NER
   def match(text)
     matches = RegExpNER.match_regexp_hash(text, @regexps, @split_on_matches)
     matches.collect do |m|
-      NamedEntity.setup(m, :offset => m.offset, :type =>  m.type, :code => m)
+      NamedEntity.setup(m, :offset => m.offset, :type =>  m.type, :code => m.code || m)
     end
   end
 

data/lib/rbbt/segment/annotation.rb

@@ -4,6 +4,7 @@ require 'rbbt/entity'
 
 module AnnotID
   extend Entity
+  include SegID
   self.annotation :corpus
 
   def _parts

data/lib/rbbt/segment/overlaps.rb

@@ -1,4 +1,4 @@
-module Segment
+module SegmentRanges
   def pull(offset)
     if self.offset.nil? or offset.nil?
       self.offset = nil
@@ -61,3 +61,11 @@ module Segment
     end
   end
 end
+
+module Segment
+  include SegmentRanges
+end
+
+module SegID
+  include SegmentRanges
+end

data/lib/rbbt/segment.rb

@@ -22,6 +22,10 @@ module SegID
     range.begin
   end
 
+  def eend
+    offset.to_i + length - 1
+  end
+
   def segment_length
     range.end - range.begin + 1
   end

data/test/rbbt/document/test_annotation.rb

@@ -13,6 +13,10 @@ class TestAnnotation < Test::Unit::TestCase
       self.split(" ")
     end
 
+    Document.define :lines do
+      self.split("\n")
+    end
+
     $called_once = false
     Document.define :persisted_words do
       raise CalledOnce if $called_once
@@ -145,5 +149,22 @@ class TestAnnotation < Test::Unit::TestCase
 
     assert  text.ner.first.segid.include?("TEST:")
   end
+
+  def test_sentence_words
+    text =<<-EOF
+This is sentence 1
+This is sentence 2
+    EOF
+
+    Document.setup(text)
+
+    words = text.words
+    numbers = words.select{|w| w =~ /\d/}
+    text.lines.each do |sentence|
+      Transformed.with_transform(sentence, numbers, "[NUM]") do
+        puts sentence
+      end
+    end
+  end
 end
 

data/test/rbbt/ner/test_g_norm_plus.rb

@@ -12,6 +12,7 @@ We found that TP53 is regulated by MDM2 in Homo
 sapiens
     EOF
 
+    Rbbt::Config.add_entry :java_mem, "2G", :gnp
     mentions = GNormPlus.process({:file => text})
 
     assert_equal 1, mentions.length
@@ -23,6 +24,7 @@ sapiens
 We found that TP53 is regulated by MDM2 in Homo sapiens
     EOF
 
+    Rbbt::Config.add_entry :java_mem, "2G", :gnp
     mentions = GNormPlus.entities({:file => text})
     assert mentions["file"].include?("TP53")
     mentions["file"].each do |mention|

data/test/rbbt/ner/test_regexpNER.rb

@@ -88,6 +88,14 @@ class TestRegExpNER < Test::Unit::TestCase
   end
 
 
+  def test_entities_named_captures
+    sentence = "In a sentence I should find not this but this"
+
+    ner = RegExpNER.new({:this => /(?<who>I) should find not this but (this)/})
+    matches = ner.entities(sentence)
+  end
+
+
 
   def test_regexp_order
     text =<<-EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.3.9
+  version: 1.3.10
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-12 00:00:00.000000000 Z
+date: 2023-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -175,45 +175,45 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/nlp/test_nlp.rb
-- test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
-- test/rbbt/nlp/genia/test_sentence_splitter.rb
+- test/test_spaCy.rb
+- test/test_helper.rb
+- test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_misc.rb
-- test/rbbt/bow/test_dictionary.rb
-- test/rbbt/test_document.rb
-- test/rbbt/document/test_annotation.rb
+- test/rbbt/segment/test_encoding.rb
+- test/rbbt/segment/test_transformed.rb
+- test/rbbt/segment/test_overlaps.rb
+- test/rbbt/segment/test_named_entity.rb
+- test/rbbt/segment/test_corpus.rb
+- test/rbbt/segment/test_range_index.rb
+- test/rbbt/segment/test_annotation.rb
+- test/rbbt/entity/test_document.rb
 - test/rbbt/document/corpus/test_pubmed.rb
 - test/rbbt/document/test_corpus.rb
-- test/rbbt/entity/test_document.rb
+- test/rbbt/document/test_annotation.rb
+- test/rbbt/test_document.rb
 - test/rbbt/ner/test_patterns.rb
-- test/rbbt/ner/test_NER.rb
-- test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/rnorm/test_tokens.rb
-- test/rbbt/ner/test_rnorm.rb
-- test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_ngram_prefix_dictionary.rb
+- test/rbbt/ner/test_token_trieNER.rb
+- test/rbbt/ner/test_finder.rb
 - test/rbbt/ner/test_brat.rb
+- test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_g_norm_plus.rb
+- test/rbbt/ner/test_rnorm.rb
+- test/rbbt/ner/test_linnaeus.rb
 - test/rbbt/ner/test_chemical_tagger.rb
-- test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/test_token_trieNER.rb
-- test/rbbt/ner/test_finder.rb
+- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/test_rner.rb
-- test/rbbt/ner/test_linnaeus.rb
 - test/rbbt/ner/test_oscar4.rb
+- test/rbbt/ner/test_banner.rb
 - test/rbbt/test_segment.rb
-- test/rbbt/segment/test_transformed.rb
-- test/rbbt/segment/test_overlaps.rb
-- test/rbbt/segment/test_annotation.rb
-- test/rbbt/segment/test_named_entity.rb
-- test/rbbt/segment/test_encoding.rb
-- test/rbbt/segment/test_range_index.rb
-- test/rbbt/segment/test_corpus.rb
-- test/test_spaCy.rb
-- test/test_helper.rb
+- test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
+- test/rbbt/nlp/test_nlp.rb
+- test/rbbt/nlp/genia/test_sentence_splitter.rb