rbbt-text 1.5.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a1a38b7a9c9f9fe0ce8fd7b8fd19a3ca39f483e8ccaaa1412af023262181fd8
-  data.tar.gz: ddec2f95b5c6fe9a69e67cf79a29c3ef2d9c37301e442d8664c7bbff1298a365
+  metadata.gz: 80e662635b01507c60638565bb8f00f789b4e6423227b98cee0ac38d8dd4b344
+  data.tar.gz: 6b9a1be5fdea2bb56a770ca2fe6838528102dfe2d15f27abb5f0e5b1849e6574
 SHA512:
-  metadata.gz: b554f4db313a65e0b682e2fd4456aaf9d1b4a489ca025ad3dc89cf71df772d97807b8b2ec62e3753eb62cf93bd88d7d0555c7ca5f9d76bd52a6c6d5fd56b313d
-  data.tar.gz: c419af9eb52723c0a761ddc025aadad99cebd1891cbd178aff466d509b84fdf76f449dfa62ba574a0b9dbeb0c519a4341b53375f18c09e52d19fd912cf5c1188
+  metadata.gz: 383affa64fa2b1e6d54817e343b3710d134c01d68867699490c604258324a6806bd829492873e819a5666780e0282ab1a375ae03147691aa64ef78705d7d0097
+  data.tar.gz: 67a52fca24335775faacf2a68cd101b24ff19d0fbff9560b4468c61546cee62aa02cdf8c5a1d3db39e5784856bd4df4573c9a00f901e8ca368cf169f8a88101d

data/lib/rbbt/ner/pubtator.rb

@@ -0,0 +1,67 @@
+
+require 'rbbt/segment'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/transformed'
+require 'rbbt/text/misc'
+require 'rest-client'
+module Pubtator
+  PUBTATOR_URL="https://www.ncbi.nlm.nih.gov/research/pubtator-api/publications/export/pubtator"
+
+  def self.pubtator_entities(pmids, concepts = ['gene'], alignments = nil)
+
+    texts = {}
+    entities = {}
+
+    last = nil
+    Misc.chunk(pmids, 1000) do |chunk|
+      time = Time.now
+      if last
+        diff = time - last
+        if diff < 3
+          sleep(3 - diff)
+        end
+      end
+      last = time
+
+      response = RestClient.post(PUBTATOR_URL, {pmids: chunk, concepts: concepts}.to_json, {content_type: 'json', accept: 'json'}).body
+      response.split("\n").each do |line|
+        next if line.empty?
+        if line =~ /^\d+\|/
+          pmid, text_type, content = line.split("|")
+          texts[pmid] ||= []
+          texts[pmid] << content
+        else
+          pmid, start, eend, literal, type, code = line.split("\t")
+          ne = NamedEntity.setup(literal, code: code, type: type, offset: start.to_i)
+          entities[pmid] ||= [] 
+          entities[pmid] << ne 
+        end
+      end
+    end
+
+    if alignments
+      new_entities = {}
+      entities.each do |pmid,list|
+        text = texts[pmid] * " "
+        alignment = alignments[pmid]
+        raise "Alignment for #{pmid} not found" if alignment.nil?
+        greek_characters = Misc.greek_characters
+        new_list = Transformed.with_transform(alignment, greek_characters.keys, lambda{|k| greek_characters[k] }) do
+          list.collect do |entity|
+            begin
+              Segment.relocate(entity, text, alignment, 10)
+              entity
+            rescue Exception
+              Log.low "Entity #{entity} (#{entity.range}) not found in alignment text for #{pmid}"
+              next
+            end
+          end
+        end
+        new_entities[pmid] = new_list.compact
+      end
+      entities = new_entities
+    end
+
+    entities
+  end
+end

data/lib/rbbt/segment/transformed.rb

@@ -10,10 +10,10 @@ module Transformed
     text
   end
  
-  def self.with_transform(text, segments, replacement = nil)
+  def self.with_transform(text, replace_segments, replacement = nil)
 
     text.extend Transformed
-    text.replace_segments(segments, replacement)
+    text.replace_segments(replace_segments, replacement)
 
     segments = yield text
 
@@ -61,7 +61,7 @@ module Transformed
     [begin_shift, end_shift]
   end
 
-  def replace_segments(segments, replacement = nil, &block)
+  def replace_segments(segments, replacement = nil, strict = false, &block)
     @transformed_segments ||= {}
     @transformation_stack ||= []
     stack = []
@@ -71,6 +71,15 @@ module Transformed
 
     offset = self.respond_to?(:offset) ? self.offset.to_i : 0
 
+    segments = segments.collect do |s|
+      if Segment === s
+        s
+      elsif String === s
+        matches = self.scan(s)
+        Segment.align(self, matches)
+      end
+    end.flatten
+
     segments = segments.select do |s| 
       shift = shift s.range
       s_offset = s.offset.to_i
@@ -82,7 +91,6 @@ module Transformed
 
     Segment.clean_sort(segments).each do |segment|
       next if segment.offset.nil?
-
       shift = shift segment.range
 
       next if shift.nil?
@@ -139,7 +147,7 @@ module Transformed
     when (segment.offset.to_i <= range.begin and segment.eend >= range.end + diff)
       segment.replace self[segment.offset.to_i..segment.eend - diff]
     else
-      raise "Segment Overlaps"
+      raise "Segment overlaps with transformation: #{Misc.fingerprint(segment)} (#{segment.range} & #{range.begin}..#{range.end + diff})"
     end
   end
 
@@ -155,10 +163,16 @@ module Transformed
 
         new_range = (range.begin..range.last + diff)
         self[new_range] = text
-        segments.each do |segment|
-          next unless Segment === segment
-          fix_segment(segment, range, diff)
-        end if Array === segments
+        segments = segments.collect do |segment|
+          next segment unless Segment === segment
+          begin
+            fix_segment(segment, range, diff)
+            segment
+          rescue
+            Log.low "Skipped: " + $!.message
+            next
+          end
+        end.compact if Array === segments
       end
       segments
     else

data/lib/rbbt/text/misc.rb

@@ -0,0 +1,5 @@
+module Misc
+  def self.greek_characters
+    @@greek_characters ||= Rbbt.share.text.greek.tsv
+  end
+end

data/share/text/greek.tsv

@@ -0,0 +1,51 @@
+#: :type=:single
+#Character	Greek letter
+Α	Alpha
+α	alpha
+Β	Beta
+β	beta
+γ	gamma
+Γ	Gamma
+Δ	Delta
+δ	Delta
+Ε	Epsilon
+ε	epsilon
+Ζ	Zeta
+ζ	zeta
+η	eta
+Η	Eta
+Θ	Theta
+θ	theta
+ι	iota
+Ι	Iota
+Κ	Kappa
+κ	kappa
+λ	lamda
+Λ	Lamda
+Μ	Mu
+μ	mu
+ν	nu
+Ν	Nu
+Ξ	Xi
+ξ	xi
+ο	omicron
+Ο	Omicron
+π	pi
+Π	Pi
+Ρ	Rho
+ρ	rho
+ς	final sigma
+σ	sigma
+Σ	Sigma
+τ	tau
+Τ	Tau
+υ	upsilon
+Υ	Upsilon
+φ	phi
+Φ	Phi
+Χ	Chi
+χ	chi
+Ψ	Psi
+ψ	psi
+ω	omega
+Ω	Omega

data/test/rbbt/ner/test_pubtator.rb

@@ -0,0 +1,70 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'rbbt/ner/NER'
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/document/corpus/pubmed'
+class TestPubtator < Test::Unit::TestCase
+  def with_corpus(&block)
+    TmpFile.with_file do |corpus|
+      yield Document::Corpus.setup(corpus)
+    end
+  end
+
+  def _test_align
+    pmids = "19522013|20861254|38267746".split("|")
+    alignments = {} 
+    with_corpus do |corpus|
+      corpus.add_pmid(pmids).each do |document|
+        alignments[document.code] = document
+      end
+      entities = Pubtator.pubtator_entities(pmids, ['gene'], alignments)
+      entities.each do |pmid,list|
+        document = corpus.add_pmid(pmid)
+        list.each do |entity|
+          assert_equal entity, document[entity.range]
+        end
+      end
+    end
+  end
+
+  def test_pmid
+    Log.severity = 0
+    pmids = "22291955".split("|")
+    alignments = {} 
+    with_corpus do |corpus|
+      corpus.add_pmid(pmids).each do |document|
+        alignments[document.code] = document
+      end
+      entities = Pubtator.pubtator_entities(pmids, ['gene'], alignments)
+      entities.each do |pmid,list|
+        document = corpus.add_pmid(pmid)
+        list.each do |entity|
+          assert_equal entity, document[entity.range]
+        end
+      end
+    end
+  end
+
+  def _test_greek
+
+    pmids = "20861254".split("|")
+    alignments = {} 
+    with_corpus do |corpus|
+      corpus.add_pmid(pmids).each do |document|
+        alignments[document.code] = document
+      end
+      entities = Pubtator.pubtator_entities(pmids, ['gene'], alignments)
+      entities.each do |pmid,list|
+        document = corpus.add_pmid(pmid)
+        list.each do |entity|
+          assert_equal entity, document[entity.range]
+        end
+        assert list.select{|e| e.include? 'α' }.any?
+      end
+
+    end
+  end
+end
+

data/test/rbbt/segment/test_transformed.rb

@@ -279,6 +279,29 @@ More recently, PPAR activators were shown to inhibit the activation of inflammat
     end
   end
 
+  def test_string_transform
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+
+    gene2 = "CDK5R1"
+
+   Transformed.with_transform(a, [gene1,gene2], "[G]") do 
+      assert_equal "This sentence mentions the [G] gene and the [G] protein", a
+    end
+    Transformed.with_transform(a, [gene1], "[G1]") do 
+      Transformed.with_transform(a, [gene2], "[G2]") do 
+        assert_equal "This sentence mentions the [G1] gene and the [G2] protein", a
+      end
+    end
+    Transformed.with_transform(a, [gene2], "[G2]") do 
+      Transformed.with_transform(a, [gene1], "[G1]") do 
+        assert_equal "This sentence mentions the [G1] gene and the [G2] protein", a
+      end
+    end
+  end
+
+
   def test_offset_transform
     a = "ILF can bind to purine-rich regulatory motifs such as the human T-cell leukemia virus-long terminal region and the interleukin-2 promoter."
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-14 00:00:00.000000000 Z
+date: 2024-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -82,6 +82,7 @@ files:
 - lib/rbbt/ner/oscar3.rb
 - lib/rbbt/ner/oscar4.rb
 - lib/rbbt/ner/patterns.rb
+- lib/rbbt/ner/pubtator.rb
 - lib/rbbt/ner/regexpNER.rb
 - lib/rbbt/ner/rner.rb
 - lib/rbbt/ner/rnorm.rb
@@ -104,6 +105,7 @@ files:
 - lib/rbbt/segment/token.rb
 - lib/rbbt/segment/transformed.rb
 - lib/rbbt/segment/tsv.rb
+- lib/rbbt/text/misc.rb
 - share/install/software/ABNER
 - share/install/software/BANNER
 - share/install/software/ChemicalTagger
@@ -119,6 +121,7 @@ files:
 - share/rner/config.rb
 - share/rnorm/cue_default
 - share/rnorm/tokens_default
+- share/text/greek.tsv
 - share/wordlists/stopwords
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
@@ -139,6 +142,7 @@ files:
 - test/rbbt/ner/test_ngram_prefix_dictionary.rb
 - test/rbbt/ner/test_oscar4.rb
 - test/rbbt/ner/test_patterns.rb
+- test/rbbt/ner/test_pubtator.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_rner.rb
 - test/rbbt/ner/test_rnorm.rb
@@ -199,6 +203,7 @@ test_files:
 - test/rbbt/ner/test_ngram_prefix_dictionary.rb
 - test/rbbt/ner/test_oscar4.rb
 - test/rbbt/ner/test_patterns.rb
+- test/rbbt/ner/test_pubtator.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_rner.rb
 - test/rbbt/ner/test_rnorm.rb