rbbt-text 1.1.9 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77391b4691e4ea2a6e5da918bc40820bae8175ff1d82f9c96a1685986605dfd7
-  data.tar.gz: a83dd9236502d1787f1040fb4c60a6160086515713282283e434b589c1425743
+  metadata.gz: '009cfce2ce954c03db5c09d0bd6f5d25bf59d508776d7370bb6bd0fb3a135f36'
+  data.tar.gz: 3d11d2a5934512958d10dbdfad5e22a9a2481b332c985ab1e2c8e92427d6f375
 SHA512:
-  metadata.gz: f69d7eb10741d2b3c7735e8e29f29625567775647d16d0261b42cce108d2f8309a2e938dad3360842a964a9c5d4fd5a2197c72618ab40971f7a65306e9c6936a
-  data.tar.gz: dec802a15cfc7c8c9a90ee8ec0c83af88c881ee16e071776a995554aa0661603bdd6cb7bf30162c43beccf1a423a2e8d26afc15f92544ccc08284a87a038a1b2
+  metadata.gz: e9338d4b54d2b66efda11dee3d37366c4f4ae78bde80f0abc1016b34c928e1db857ad73f33ba1da611ad232513498430736c46134a902b3930a8f832afed3e09
+  data.tar.gz: 0cdeeee67636d4e0b0714334b3c187cb0f5ea5c7363fe27fc84d438643a0d6f204413a4dd5d99c8c43d847539320c484fde2b5300b298cf9cc782148d98802ee

data/lib/rbbt/ner/g_norm_plus.rb

@@ -10,35 +10,39 @@ module GNormPlus
   end
 
   CONFIG =<<-EOF
-
 #===Annotation
 #Attribution setting:
 #FocusSpecies = Taxonomy ID
-#	All: All species
-#	9606: Human
-#	4932: yeast
-#	7227: Fly
-#	10090: Mouse
-#	10116: Rat
-#	7955: Zebrafish
-#	3702: Arabidopsis thaliana
+#       All: All species
+#       9606: Human
+#       4932: yeast
+#       7227: Fly
+#       10090: Mouse
+#       10116: Rat
+#       7955: Zebrafish
+#       3702: Arabidopsis thaliana
 #open: True
 #close: False
 
 [Focus Species]
-	FocusSpecies = All
+	FocusSpecies = 9606
+	FilterAntibody = False
 [Dictionary & Model]
 	DictionaryFolder = ./Dictionary
 	GNRModel = ./Dictionary/GNR.Model
 	SCModel = ./Dictionary/SimConcept.Model
 	GeneIDMatch = True
+	HomologeneID = False
 	Normalization2Protein = False
+	ShowUnNormalizedMention = False
 	DeleteTmp = True
+	IgnoreNER = True
 EOF
 
   def self.process(texts)
     TmpFile.with_file do |tmpdir|
       Open.mkdir tmpdir
+
       Misc.in_dir tmpdir do
         Open.ln_s Rbbt.software.opt.GNormPlus.Dictionary.find, '.'
         Open.ln_s Rbbt.software.opt.GNormPlus["BioC.dtd"].find, '.'
@@ -50,12 +54,12 @@ EOF
 
         texts.each do |name,text|
           Open.write("input/#{name}.txt") do |f|
-            f.puts "#{name}|a|" << text
+            f.puts "#{name}|a|" << text.gsub("\n\n", "\n·")
             f.puts
           end
         end
         Open.write('config', CONFIG)
-        CMD.cmd_log("java -Xmx20G -Xms20G  -jar '#{Rbbt.software.opt.GNormPlus.find}/GNormPlus.jar' 'input' 'output' 'config'")
+        CMD.cmd_log("java -Xmx20G -Xms20G  -jar '#{Rbbt.software.opt.GNormPlus.produce.find}/GNormPlus.jar' 'input' 'output' 'config'")
 
         if texts.respond_to? :key_field
           key_field = texts.key_field
@@ -68,6 +72,9 @@ EOF
           entities = Open.read(file).split("\n")[1..-1].collect{|l| l.gsub(':', '.').split("\t")[1..-1] * ":"}
           tsv[name] = entities
         end
+
+        raise "GNormPlus failed: no results found" if tsv.size == 0 && texts.size > 0
+
         tsv
       end
     end

data/lib/rbbt/text/corpus/document.rb

@@ -1,5 +1,6 @@
 require 'rbbt/text/segment'
 require 'rbbt/text/segment/segmented'
+require 'rbbt/text/segment/docid'
 require 'rbbt/tsv'
 require 'rbbt/resource/path'
 require 'rbbt/persist/tsv'
@@ -15,6 +16,7 @@ class Corpus
     attr_accessor :text, :docid, :namespace, :id, :type, :hash, :segments, :segment_indices, :persist_dir, :global_persistence, :corpus
 
     attr_accessor :multiple_result
+
     def initialize(persist_dir = nil, docid = nil, text = nil, global_persistence = nil, corpus = nil)
       @segments = {}
       @segment_indices = {}
@@ -44,16 +46,22 @@ class Corpus
     end
 
     def self.define(entity, &block)
-      send :define_method, "produce_#{entity}", &block
+      send :define_method, "produce_#{entity}" do 
+        segments = self.instance_exec &block
 
-      self.class_eval <<-EOC, __FILE__, __LINE__
+        segments.each{|s| s.docid = docid }
+      end
+
+      self.class_eval <<-EOC, __FILE__, __LINE__ + 1
         def load_#{entity}(raw = false)
           return if segments.include? "#{ entity }"
           if self.respond_to?("load_with_persistence_#{entity}") and not @persist_dir.nil?
-            segments["#{entity}"] = load_with_persistence_#{entity}(raw)
+            entities = load_with_persistence_#{entity}(raw)
           else
-            segments["#{ entity }"] = produce_#{entity}
+            entities = produce_#{entity}
           end
+
+          segments["#{ entity }"] = entities
         end
 
         def #{entity}(raw = false)
@@ -77,7 +85,10 @@ class Corpus
 
     def self.define_multiple(entity, &block)
       send :define_method, "produce_#{entity}" do
-        return self.multiple_result[entity] if self.multiple_result && self.multiple_result[entity]
+        if self.multiple_result && self.multiple_result[entity]
+          segments = self.multiple_result[entity]
+          return segments.each{|s| s.docid = docid }
+        end
         raise MultipleEntity, "Entity #{entity} runs with multiple documents, please prepare beforehand with prepare_multiple: #{self.docid}"
       end
 
@@ -86,14 +97,16 @@ class Corpus
         self
       end.send :define_method, name, &block
 
-      self.class_eval <<-EOC, __FILE__, __LINE__
+      self.class_eval <<-EOC, __FILE__, __LINE__ + 1
         def load_#{entity}(raw = false)
           return if segments.include? "#{ entity }"
           if self.respond_to?("load_with_persistence_#{entity}") and not @persist_dir.nil?
-            segments["#{entity}"] = load_with_persistence_#{entity}(raw)
+            entities = load_with_persistence_#{entity}(raw)
           else
-            segments["#{ entity }"] = produce_#{entity}
+            entities = produce_#{entity}
           end
+
+          segments["#{ entity }"] = entities
         end
 
         def #{entity}(raw = false)
@@ -124,7 +137,7 @@ class Corpus
           missing << doc
         end
       end
-      res = self.send("multiple_produce_#{entity.to_s}", missing)
+      res = self.send("multiple_produce_#{entity.to_s}", missing) if missing.any?
       case res
       when Array
         res.each_with_index do |res,i|
@@ -142,7 +155,9 @@ class Corpus
           end
         end
       end
-      missing.each{|doc| doc.send entity }
+      missing.each{|doc|
+        doc.send entity 
+      }
     end
 
 
@@ -197,7 +212,7 @@ class Corpus
         FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields unless fields.nil?
       end
 
-      self.class_eval <<-EOC, __FILE__, __LINE__
+      self.class_eval <<-EOC, __FILE__, __LINE__ + 1
         def load_with_persistence_#{entity}(raw = false)
           repo = TSV_REPOS["#{ entity }"]
           if repo.nil?
@@ -253,7 +268,7 @@ class Corpus
 
       FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields 
 
-      self.class_eval <<-EOC, __FILE__, __LINE__
+      self.class_eval <<-EOC, __FILE__, __LINE__ + 1
         def load_with_persistence_#{entity}(raw = false)
           fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
 
@@ -261,20 +276,23 @@ class Corpus
           
           begin
 
-            data.read true
+            if data.respond_to? :persistence_path and String === data.persistence_path
+              data.filter(data.persistence_path + '.filters')
+            end
+
+            keys = data.read_and_close do
 
-            fields = data.fields if fields.nil? and data.respond_to? :fields
+              fields = data.fields if fields.nil? and data.respond_to? :fields
 
+              data.add_filter("field:#{ doc_field }", @docid) if fields.include?("#{doc_field}")
+              data.add_filter("field:#{ entity_field }", "#{ entity }") if fields.include?("#{entity_field}")
+              keys = data.keys
+              data.pop_filter if fields.include?("#{entity_field}")
+              data.pop_filter if fields.include?("#{doc_field}")
 
-            if data.respond_to? :persistence_path and String === data.persistence_path
-              data.filter(data.persistence_path + '.filters')
+              keys
             end
 
-            data.add_filter("field:#{ doc_field }", @docid) if data.fields.include?("#{doc_field}")
-            data.add_filter("field:#{ entity_field }", "#{ entity }") if data.fields.include?("#{entity_field}")
-            keys = data.keys
-            data.pop_filter if data.fields.include?("#{entity_field}")
-            data.pop_filter if data.fields.include?("#{doc_field}")
 
             if keys.empty?
               segments = produce_#{entity}
@@ -289,34 +307,38 @@ class Corpus
                 "#{ entity }"
               end
 
-              data.add_filter("field:#{ doc_field }", @docid) if data.fields.include?("#{doc_field}")
-              data.add_filter("field:#{ entity_field }", "#{ entity }") if data.fields.include?("#{entity_field}")
-              data.write true
-              keys = tsv.collect do |key, value|
-                data[key] = value
-                key
+              keys = data.write_and_close do
+                data.add_filter("field:#{ doc_field }", @docid) if fields.include?("#{doc_field}")
+                data.add_filter("field:#{ entity_field }", "#{ entity }") if fields.include?("#{entity_field}")
+                keys = tsv.collect do |key, value|
+                  data[key] = value
+                  key
+                end
+                data.pop_filter if fields.include?("#{entity_field}")
+                data.pop_filter if fields.include?("#{doc_field}")
+                keys
               end
-              data.pop_filter if data.fields.include?("#{entity_field}")
-              data.pop_filter if data.fields.include?("#{doc_field}")
-              data.read
 
             else
-              if raw == :check
-                data.close
-                return nil
-              end
+              return nil if raw == :check
             end
 
             return data.values if raw
 
             start_pos = data.identify_field "Start"
-            segments = data.values_at(*keys).collect{|annotation| 
+            data.read_and_close do 
+              data.chunked_values_at(keys).collect{|annotation| 
+                  begin
                 pos = annotation[start_pos]
-                Segment.load_tsv_values(text, annotation, data.fields) unless [-1, "-1", [-1], ["-1"]].include? pos
-            }.compact
-            data.close
-
-            segments
+                Segment.load_tsv_values(text, annotation, fields) unless [-1, "-1", [-1], ["-1"]].include?(pos)
+                  rescue
+                    Log.exception $!
+                    iif keys
+                    iif [text, annotation]
+                  end
+                  
+              }.compact
+            end
           ensure
             data.close
           end
@@ -348,7 +370,7 @@ class Corpus
         segment.segments[name] = annotations
         class << segment
           self
-        end.class_eval "def #{ name }; @segments['#{ name }']; end", __FILE__, __LINE__
+        end.class_eval "def #{ name }; @segments['#{ name }']; end", __FILE__, __LINE__ + 1
       end
 
       segment

data/lib/rbbt/text/segment.rb

@@ -3,7 +3,7 @@ require 'rbbt/fix_width_table'
 
 module Segment 
   extend Annotation
-  self.annotation :offset
+  self.annotation :offset, :docid
 
   def segment_length
     begin
@@ -325,7 +325,7 @@ module Segment
     tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
 
     segments.each do |segment|
-      tsv[segment.object_id.to_s] = self.tsv_values_for_segment(segment, fields)
+      tsv[segment.segment_id] = self.tsv_values_for_segment(segment, fields)
     end
 
     tsv
@@ -348,6 +348,14 @@ module Segment
     [offset, self.end] * ".."
   end
 
+  def segment_id
+    if self.respond_to?(:docid)
+      [docid, locus, Misc.obj2digest(info)] * ":"
+    else
+      Misc.obj2digest(info)
+    end
+  end
+
   #def ==(other)
   #  self.text == other.text
   #end

data/lib/rbbt/text/segment/docid.rb

@@ -1,46 +1,46 @@
 require 'rbbt/text/segment'
 
-module SegmentWithDocid 
-  extend Annotation
-
-  self.annotation :docid
-
-  def masked?
-    self[0..5] == "MASKED"
-  end
-
-  def mask
-    return self if masked?
-    raise "Cannot mask an array of elements, they must be masked individually" if Array === self
-    raise "Cannot mask a segment with no docid" if not self.respond_to? :docid or docid.nil?
-    raise "Cannot mask a segment with no offset" if offset.nil?
-    textual_position = ["MASKED", length] * ":"
-    self.replace(textual_position)
-    self
-  end
-
-  def unmasked_text
-    return self unless masked?
-    tag, length = self.split(":")
-    Document.setup(docid).text[offset.to_i..(offset.to_i+length.to_i-1)]
-  end
-
-  def unmask
-    return self unless masked?
-    self.replace(unmasked_text)
-    self
-  end
-
-  def str_length
-    self.length
-  end
-
-  def masked_length
-    self.split(":").last.to_i
-  end
-
-  def segment_length
-    masked? ? masked_length : str_length
-  end
-end
-
+#module SegmentWithDocid 
+#  extend Annotation
+#
+#  self.annotation :docid
+#
+#  def masked?
+#    self[0..5] == "MASKED"
+#  end
+#
+#  def mask
+#    return self if masked?
+#    raise "Cannot mask an array of elements, they must be masked individually" if Array === self
+#    raise "Cannot mask a segment with no docid" if not self.respond_to? :docid or docid.nil?
+#    raise "Cannot mask a segment with no offset" if offset.nil?
+#    textual_position = ["MASKED", length] * ":"
+#    self.replace(textual_position)
+#    self
+#  end
+#
+#  def unmasked_text
+#    return self unless masked?
+#    tag, length = self.split(":")
+#    Document.setup(docid).text[offset.to_i..(offset.to_i+length.to_i-1)]
+#  end
+#
+#  def unmask
+#    return self unless masked?
+#    self.replace(unmasked_text)
+#    self
+#  end
+#
+#  def str_length
+#    self.length
+#  end
+#
+#  def masked_length
+#    self.split(":").last.to_i
+#  end
+#
+#  def segment_length
+#    masked? ? masked_length : str_length
+#  end
+#end
+#

data/lib/rbbt/text/segment/named_entity.rb

@@ -29,6 +29,7 @@ Score: #{score.inspect}
   end
 
   def entity(params = nil)
+    code = self.dup
     format, entity = code.split(":")
     entity, format = format, nil if entity.nil?
     

data/lib/rbbt/text/segment/transformed.rb

@@ -111,10 +111,10 @@ module Transformed
 
       self[updated_begin..updated_end] = new
 
-      @transformed_segments[segment.object_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
+      @transformed_segments[segment.segment_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
 
       segment.replace original_text
-      stack << segment.object_id
+      stack << segment.segment_id
     end
     @transformation_stack << stack
   end

data/test/rbbt/ner/test_g_norm_plus.rb

@@ -8,7 +8,6 @@ class TestGNormPlus < Test::Unit::TestCase
 We found that TP53 is regulated by MDM2 in Homo sapiens
     EOF
 
-
     mentions = GNormPlus.process({:file => text})
     Log.tsv mentions
   end

data/test/rbbt/text/corpus/test_document.rb

@@ -10,10 +10,6 @@ class TestCorpusDocument < Test::Unit::TestCase
       Segment.align(self.text, words)
     end
 
-    Open.mkdir Rbbt.tmp.test.annotations.find
-    Corpus::Document.persist_in_global_tsv(:words, Rbbt.tmp.test.anotations.words.find)
-
-
     Corpus::Document.define_multiple :words2 do |documents|
       documents.collect do |doc|
         words = doc.text.split(" ")
@@ -21,32 +17,66 @@ class TestCorpusDocument < Test::Unit::TestCase
       end
     end
 
+    Open.mkdir Rbbt.tmp.test.annotations.find
+
+    Corpus::Document.persist_in_global_tsv(:words, Rbbt.tmp.test.anotations.words.find)
     Corpus::Document.persist_in_global_tsv(:words2, Rbbt.tmp.test.anotations.counts.find)
   end
 
   def test_words
     text = "This is a test document"
-    document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc", text)
+    document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", text)
     assert_equal Segment.sort(document.words), text.split(" ")
+    assert document.words.first.docid
+    assert document.words.first.segment_id.include?("TEST")
   end
 
   def test_words_multiple
     document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
-    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
+    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is another test document")
+    document3 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc3:3", "This is yet another test document")
 
-    docs = [document1, document2]
+    docs = [document1, document2, document3]
 
     Corpus::Document.prepare_multiple(docs, :words2)
-    
+
+    assert document1.words.first.docid
+    assert document1.words.first.segment_id.include?("TEST")
+
     assert_equal document1.words2, document1.text.split(" ")
     assert_equal document2.words2, document2.text.split(" ")
+    assert_equal document3.words2, document3.text.split(" ")
 
     document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
-    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
+    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is another test document")
 
     docs = [document1, document2]
 
     Corpus::Document.prepare_multiple(docs, :words2)
   end
+
+  def test_parallel
+    text =<<-EOF
+This is a test document number
+    EOF
+
+    docs = []
+    100.times do |i|
+      docs << text.chomp + " " + i.to_s
+    end
+
+    Log.with_severity 0 do
+      TSV.traverse docs, :cpus => 10, :bar => true do |doc|
+        hash = Misc.digest(doc)
+        document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:test:#{hash}", doc)
+        assert_equal Segment.sort(document.words), document.text.split(" ")
+      end
+      TSV.traverse docs, :cpus => 10, :bar => true do |doc|
+        hash = Misc.digest(doc)
+        document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:test:#{hash}", doc)
+        assert_equal Segment.sort(document.words), document.text.split(" ")
+      end
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.2.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-13 00:00:00.000000000 Z
+date: 2020-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util