rbbt-util 5.4.1 → 5.5.0

data/lib/rbbt/util/semaphore.rb

@@ -1,4 +1,10 @@
-require 'inline'
+begin
+  require 'inline'
+  continue = true
+rescue Exception
+  Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
+  continue = false
+end
 
 module RbbtSemaphore
   inline(:C) do |builder|
@@ -67,5 +73,5 @@ void post_semaphore(char* name){
       pids.each do |pid| Process.waitpid pid end
     end
   end
-end
+end if continue
 

data/lib/rbbt/workflow.rb

@@ -4,6 +4,7 @@ require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
 
 module Workflow
+
   def self.resolve_locals(inputs)
     inputs.each do |name, value|
       if value =~ /^local:(.*?):(.*)/ or 
@@ -33,66 +34,56 @@ module Workflow
     eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
   end
 
+  def self.load_workflow_file(filename)
+    begin
+      $LOAD_PATH.unshift(File.join(File.dirname(File.expand_path(filename)), 'lib'))
+      require filename
+      Log.info "Workflow loaded from: #{ filename }"
+      return true
+    rescue Exception
+      Log.warn "Error loading workflow: #{ filename }"
+      raise $!
+    end
+  end
+
   def self.require_local_workflow(wf_name)
+    filename = nil
+
     if Path === wf_name
       case
-
         # Points to workflow file
       when ((File.exists?(wf_name.find) and not File.directory?(wf_name.find)) or File.exists?(wf_name.find + '.rb')) 
-        $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name.find)), 'lib'))
-        require wf_name.find
-        Log.medium "Workflow loaded from file: #{ wf_name }"
-        return true
+        filename = wf_name.find
 
         # Points to workflow dir
       when (File.exists?(wf_name.find) and File.directory?(wf_name.find) and File.exists?(File.join(wf_name.find, 'workflow.rb')))
-        $LOAD_PATH.unshift(File.join(File.expand_path(wf_name.find), 'lib'))
-        require File.join(wf_name.find, 'workflow.rb')
-        Log.medium "Workflow loaded from directory: #{ wf_name }"
-        return true
-
-      else
-        raise "Workflow path was not resolved: #{ wf_name } (#{wf_name.find})"
+        filename = wf_name['workflow.rb'].find
       end
 
     else
       case
         # Points to workflow file
       when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb') or File.exists?(wf_name))
-        $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
-        wf_name = "./" << wf_name unless wf_name[0] == "/"
-        require wf_name
-        Log.medium "Workflow loaded from file: #{ wf_name }"
-        return true
-
+        filename = (wf_name =~ /\.?\//) ? wf_name : "./" << wf_name 
       when (defined?(Rbbt) and Rbbt.etc.workflow_dir.exists?)
         dir = Rbbt.etc.workflow_dir.read.strip
         dir = File.join(dir, wf_name)
-        $LOAD_PATH.unshift(File.join(File.expand_path(dir), 'lib'))
-        require File.join(dir, 'workflow.rb')
-        Log.medium "Workflow #{wf_name} loaded from workflow_dir: #{ dir }"
-        return true
-
+        filename = File.join(dir, 'workflow.rb')
       when defined?(Rbbt)
         path = Rbbt.workflows[wf_name].find
-        $LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
-        require File.join(path, 'workflow.rb')
-        Log.medium "Workflow #{wf_name} loaded from Rbbt.workflows: #{ path }"
-        return true
-
+        filename = File.join(dir, 'workflow.rb')
       else
         path = File.join(ENV['HOME'], '.workflows', wf_name)
-        $LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
-        require File.join(path, 'workflow.rb')
-        Log.medium "Workflow #{wf_name} loaded from .workflows: #{ path }"
-        return true
+        filename = File.join(dir, 'workflow.rb')
       end
     end
 
-    raise "Workflow not found our could not be loaded: #{ wf_name }"
+    return false if filename.nil? or not File.exists? filename
+    load_workflow_file filename
   end
-
   def self.require_workflow(wf_name)
+
+    # Already loaded
     begin
       Misc.string2const wf_name
       Log.debug "Workflow #{ wf_name } already loaded"
@@ -100,9 +91,10 @@ module Workflow
     rescue Exception
     end
 
+    # Load remotely
     if Rbbt.etc.remote_workflows.exists?
       remote_workflows = Rbbt.etc.remote_workflows.yaml
-      if remote_workflows.include? wf_name
+      if Hash === remote_workflows and remote_workflows.include?(wf_name)
         url = remote_workflows[wf_name]
         require_remote_workflow(wf_name, url)
         Log.debug "Workflow #{ wf_name } loaded remotely: #{ url }"
@@ -110,19 +102,12 @@ module Workflow
       end
     end
 
-    begin
-      require_local_workflow(wf_name) 
-    rescue Exception
-      raise "Workflow not found: #{ wf_name }" if wf_name == Misc.snake_case(wf_name)
-      begin
-        require_local_workflow(Misc.snake_case(wf_name))
-      rescue Exception
-        Log.error("Workflow not found: #{ wf_name }")
-        raise $!
-      end
-    end
+    # Load locally
+
+    require_local_workflow(wf_name) || require_local_workflow(Misc.snake_case(wf_name)) || raise("Workflow not found or could not be loaded: #{ wf_name }")
   end
 
+
   attr_accessor :description
   attr_accessor :libdir, :workdir 
   attr_accessor :helpers, :tasks

data/lib/rbbt/workflow/accessor.rb

@@ -37,7 +37,7 @@ class Step
 
   def set_info(key, value)
     return nil if @exec
-    value = Annotated.purge value 
+    value = Annotated.purge value if defined? Annotated
     Open.lock(info_file) do
       i = info
       i[key] = value

data/lib/rbbt/workflow/step.rb

@@ -4,6 +4,8 @@ require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/accessor'
 
+class ParameterException < Exception; end
+
 class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
@@ -176,10 +178,10 @@ class Step
         rescue Step::Aborted
           Log.debug("Forked process aborted: #{@path}")
           log :aborted, "Aborted"
-          exit -1
+          raise $!
         rescue Exception
-          Log.debug("Exception caught on forked process: #{$!.message}")
-          exit -1
+          Log.debug("Exception caught on forked process: #{@path}")
+          raise $!
         end
 
         begin

data/share/rbbt_commands/workflow/server

@@ -63,4 +63,5 @@ end
 WorkflowRest.add_workflow Workflow.workflows.last, true
 
 WorkflowRest.port = options[:port] || 4567
+WorkflowRest.bind = options[:bind] || "0.0.0.0"
 WorkflowRest.run!

data/share/rbbt_commands/workflow/task

@@ -4,7 +4,9 @@ require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
 
-def usage(workflow = nil, task = nil)
+YAML::ENGINE.yamler = 'syck' if defined? YAML::ENGINE and YAML::ENGINE.respond_to? :yamler
+
+def usage(workflow = nil, task = nil, error = nil)
   puts SOPT.doc
   puts "## WORKFLOW"
   puts
@@ -23,6 +25,10 @@ def usage(workflow = nil, task = nil)
     puts workflow.workflow_description
     puts
     workflow.doc(task)
+    if error
+        puts
+        puts "Error: " << error 
+    end
   end
 
   exit 0
@@ -104,7 +110,7 @@ options = SOPT.get <<EOF
 -as--array_separator* Change the character that separates elements of Arrays, ',', '|', or '\\n' by default:
 -cl--clean Clean the last step of the job so that it gets recomputed:
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely:
--n--name* Job name to use. The name 'Default' is used by default:
+-jn--jobname* Job name to use. The name 'Default' is used by default:
 -pn--printname Print the name of the job and exit without starting it:
 EOF
 
@@ -180,6 +186,7 @@ if recursive_clean and job.done?
 end
 
 # run
+begin
 if do_exec
   res = job.exec
   case
@@ -212,6 +219,9 @@ if options.delete(:printname)
 else
   Log.low "Job name: #{job.name}" 
 end
+rescue ParameterException
+    usage(workflow, task, $!.message)
+end
 
 if Step === res
   puts Open.read(res.path) if File.exists? res.path

data/test/rbbt/association/test_index.rb

@@ -0,0 +1,36 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+require 'rbbt/workflow'
+require 'rbbt/association'
+require 'rbbt/association/index'
+
+Workflow.require_workflow "Genomics"
+require 'rbbt/entity/gene'
+class TestAssociationIndex < Test::Unit::TestCase
+
+  def setup
+  end
+
+  def teardown
+  end
+
+
+  def test_subset
+    require 'rbbt/sources/pina'
+    require 'rbbt/sources/kegg'
+    tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013").ensembl
+    index = Association.index(Pina.protein_protein,
+                              {:namespace => tp53.organism, 
+                                :target => "Interactor UniProt/SwissProt Accession=~UniProt/SwissProt Accession",
+                                :format => "Ensembl Gene ID",
+                                :undirected => true}, 
+                                {:dir => '/tmp/test_association3', :update => false})
+    genes = tp53.pina_interactors.ensembl
+    genes << tp53
+
+    Misc.benchmark(10) do
+      index.subset_entities("Ensembl Gene ID" => genes).length
+    end
+
+    assert_equal 204, index.subset_entities("Ensembl Gene ID" => genes).select{|m| m.partition("~")[0] == tp53}.uniq.length
+  end
+end

data/test/rbbt/test_annotations.rb

@@ -91,10 +91,11 @@ class TestAnnotations < Test::Unit::TestCase
     AnnotatedString.setup(str1, annotation_str1)
     AnnotatedString.setup(str2, annotation_str2)
     
-    assert_equal annotation_str1, Annotated.tsv([str1, str2], :all)[str1.id + ":0"]["annotation_str"] 
-    assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id + ":0"]["literal"] 
-    assert_equal annotation_str1, Annotated.tsv([str1, str2], :all)[str1.id + ":0"]["annotation_str"] 
-    assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id + ":0"]["annotation_str"] 
+    assert_equal str1, Annotated.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.id + ":0"]["literal"] 
+    assert_equal annotation_str1, Annotated.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.id + ":0"]["annotation_str"] 
+    assert_equal str1, Annotated.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.id + ":0"]["literal"] 
+    assert_equal annotation_str1, Annotated.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.id + ":0"]["annotation_str"] 
+    assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON).tap{|t| t.unnamed = false}[str1.id + ":0"]["annotation_str"] 
   end
 
   def test_literal

data/test/rbbt/test_association.rb

@@ -1,6 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
 require 'rbbt/workflow'
 require 'rbbt/association'
+require 'rbbt/knowledge_base'
 require 'rbbt/entity'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
@@ -22,55 +23,81 @@ class TestAssociations < Test::Unit::TestCase
   
   FAssocs = ""
   DAssocs = ""
+
   def setup
     FAssocs.replace TmpFile.tmp_file
     DAssocs.replace TmpFile.tmp_file
     Open.write(FAssocs, TEST_ASSOCIATIONS)
   end
+
   def teardown
     FileUtils.rm FAssocs
     FileUtils.rm_rf DAssocs
   end
 
-  def test_simple_open
+  def _test_simple_open
     database = Association.open(FAssocs, {}, :dir => DAssocs)
     assert_equal ["C", "K"], database["c"]["Entity2"]
   end
  
-  def test_source_open
+  def _test_source_open
     database = Association.open(FAssocs, {:source => "Entity2", :zipped => true}, :dir => DAssocs)
     assert_equal ["c", "3", 'cc', "PTEN"], database["C"].flatten
     assert_equal ["c", "4", 'kk', "PTEN"], database["K"].flatten
   end
   
-  def test_target_open
+  def _test_target_open
     database = Association.open(FAssocs, {:source => "Entity2", :target => "Entity3", :zipped => true}, :dir => DAssocs)
     assert_equal ["cc", "c", "3", "PTEN"], database["C"].flatten
     assert_equal ["kk", "c", "4", "PTEN"], database["K"].flatten
   end
 
-  def test_gene_open
+  def test_target_open
+    require 'rbbt/sources/pina'
+
+    database = Association.open(Pina.protein_protein,{ 
+                                :undirected => false, 
+                                :target => "Interactor UniProt/SwissProt Accession=~UniProt/SwissProt Accession"}, :persist => false)
+
+    assert( ! database["Q13547"][0].include?("O15379"))
+
+    database = Association.open(Pina.protein_protein,{
+                                :undirected => true, 
+                                :target => "Interactor UniProt/SwissProt Accession=~UniProt/SwissProt Accession"}, :persist => false)
+
+    assert database["O15379"][0].include? "Q13547"
+
+    assert database["Q13547"][0].include? "O15379"
+  end
+
+
+  def _test_gene_open
     database = Association.open(FAssocs, {:source => "Gene=~Associated Gene Name", :target => "Entity3", :zipped => true}, :dir => DAssocs)
     assert_equal ["aa"], database["TP53"].first
   end
 
-  def test_gene_open_translate
+  def _test_gene_open_translate
     tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013")
-    database = Association.open(FAssocs, {:source => "Gene=~Associated Gene Name", :source_type => "Ensembl Gene ID", :target => "Entity3", :zipped => true}, :dir => DAssocs)
+    database = Association.open(FAssocs, {:source => "Gene=~Associated Gene Name", :source_format => "Ensembl Gene ID", :target => "Entity3", :zipped => true}, :dir => DAssocs)
     assert_equal ["aa"], database[tp53.ensembl].first
   end
 
-  def test_gene_target_open_translate
+  def _test_gene_target_open_translate
     tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013")
     database = Association.open(FAssocs, {:target => "Gene=~Associated Gene Name=>Ensembl Gene ID", :source => "Entity3", :zipped => true}, :dir => DAssocs)
     assert_equal [tp53.ensembl], database["aa"].first
   end
 
-  def test_undirected
-    require 'rbbt/sources/pina'
-    require 'rbbt/gene_associations'
-    tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013")
-    index = Association.index_database('pina', {:source_type => "Ensembl Gene ID", :target_type => "Ensembl Gene ID", :undirected => true}, {:dir => DAssocs})
-    assert Association.connections(index, "Gene" => tp53.pina_interactors.ensembl.compact).any?
+  def _test_ICGC
+    assoc = Association.open(Open.open('ftp://data.dcc.icgc.org/current/Chronic_Lymphocytic_Leukemia-ISC_MICINN-ES/simple_somatic_mutation.CLLE-ES.tsv.gz'),
+                         { :source => "gene_affected=~Ensembl Gene ID=>Associated Gene Name", :target => "icgc_donor_id=~Sample", 
+                           :fields => ['consequence_type'],  
+                           :namespace => 'Hsa/jan2013',
+                           :merge => true, :header_hash=>''}, :persist => false)
+
+    assert_equal 9, assoc["SF3B1"]["Sample"].uniq.length
+
   end
+
+
 end

data/test/rbbt/test_knowledge_base.rb

@@ -0,0 +1,103 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '../..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'test/unit'
+require 'rbbt/knowledge_base'
+require 'rbbt/sources/pina'
+
+require 'rbbt/workflow'
+Workflow.require_workflow "Genomics"
+require 'genomics_kb'
+
+class TestKnowledgeBase < Test::Unit::TestCase
+  def setup
+    KnowledgeBase.knowledge_base_dir = Rbbt.tmp.knowledge_base_test.find
+    @kb = Genomics.knowledge_base
+  end
+
+  def test_register
+    TmpFile.with_file do |dir|
+      kb = KnowledgeBase.new dir
+
+      kb.register :pina, Pina.protein_protein, :target => "Interactor UniProt/SwissProt Accession=~UniProt/SwissProt Accession"
+      assert_equal [Gene], kb.entity_types
+      assert kb.all_databases.include? :pina
+    end
+  end
+
+  def test_format_Gene
+    TmpFile.with_file do |dir|
+      kb = KnowledgeBase.new dir, "Hsa/jan2013"
+      kb.format["Gene"] = "Ensembl Gene ID"
+
+      kb.register 'nature', NCI.nature_pathways, :merge => true, :target => "UniProt/SwissProt Accession", :key_field => 0
+
+      assert kb.get_database('nature', :persist => false).slice("Ensembl Gene ID").values.flatten.uniq.length > 10
+    end
+  end
+
+  def test_fields
+    TmpFile.with_file do |dir|
+      kb = KnowledgeBase.new dir, "Hsa/jan2013"
+      kb.format["Gene"] = "Ensembl Gene ID"
+
+      kb.register 'nature', NCI.nature_pathways, :merge => true, :fields => [2], :key_field => 0
+      assert kb.get_database('nature', :persist => false).slice("Ensembl Gene ID").values.flatten.uniq.length > 10
+    end
+  end
+
+  def test_global
+    assert @kb.all_databases.include? "pina"
+  end
+
+  def test_benchmark
+    tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013").ensembl
+    kb = KnowledgeBase.new Rbbt.tmp.test.kb2
+    kb.namespace = "Hsa/jan2013"
+
+      require 'rbbt/sources/COSMIC'
+      require 'rbbt/entity/genomic_mutation'
+      mutations = tp53.COSMIC_mutations
+      Misc.benchmark(10) do 
+        name = "mutations"
+        kb.add_index name, "Ensembl Gene ID", "Genomic Mutation", "Change"
+        kb.write name do
+          mutations.each do |gm|
+            kb.add name, tp53, gm, gm.base
+          end
+        end
+      end
+  end
+
+  def test_items
+    tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013").ensembl
+    kb = KnowledgeBase.new Rbbt.tmp.test.kb2, "Hsa/jan2013"
+    kb.index('g2t', Organism.gene_transcripts("Hsa/jan2013"), :target => "Ensembl Transcript ID")
+  end
+
+  def test_benchmark2
+    tp53 = Gene.setup("TP53", "Associated Gene Name", "Hsa/jan2013").ensembl
+    kb = KnowledgeBase.new Rbbt.tmp.test.kb2, "Hsa/jan2013"
+    kb.index('g2t', Organism.gene_transcripts("Hsa/jan2013"), :target => "Ensembl Transcript ID")
+    l = nil
+    Misc.benchmark(1000) do
+     l = tp53.transcripts.length
+    end
+    assert l > 0
+  end
+
+  def __test_subset
+    gene = "TP53"
+    found = Genomics.knowledge_base.identify :pina, gene
+    p53_interactors = Misc.profile{ Genomics.knowledge_base.children(:pina, found).target_entity }
+
+    ddd p53_interactors.length
+
+    Misc.profile do
+      puts Genomics.knowledge_base.subset(:pina,{"Gene" => p53_interactors}).length
+    end
+    ddd 2
+    #assert Genomics.knowledge_base.subset(:pina,{"Gene" => p53_interactors}).target_entities.name.include? "MDM2"
+  end
+
+end
+