rbbt-util 5.33.12 → 5.33.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 729b93bb0b82b05a03eb9cdf99728b4793ecd002ab06a92c08dc899341d26810
-  data.tar.gz: 568b263ae5bbc16e18383bd0fc5fb42054d2954b674bb1df97f80280ade9e7b2
+  metadata.gz: 06dd3eb576050f8f635f49f02e4382ce2b65e23194fe3a5d601f2394c36941a0
+  data.tar.gz: cba8da17adfd6716a7ba870d67779e9a0618662b6d1fafdc9088a8669dc90db7
 SHA512:
-  metadata.gz: 1c56e3f171523bb412b391899b93ba88d80038a2a201ed4573897c3039526fbf1f05d2a2bcb81b837d6cb9ad31672c7c5a67572d7baa3789c10f3fc3929db274
-  data.tar.gz: b178ecce6c10f7933bdafd3cf495e3a185d6f0005b5111ed6b86f79f70b9608d613c837d7141c6ca42bf4504d352f46102d2ac628bc19d6e9a2001dbcc081284
+  metadata.gz: 25f4d416e5d10f8e813f510278de8c510861e881027b4bef94125d33ce431f2613c4b535c7c75025eaf2b14c2315d50d3e78bf181c02cdb24ab72f80e3d1906f
+  data.tar.gz: 9145fc107255622f68ae7e0b761d8f638bfb8dfeb4ef69205b74e3174ce190dd59bf4dcda85e090dc69bafc480dc302819d475b4685f6618975283f54ace870c

data/lib/rbbt/hpc/batch.rb

@@ -138,35 +138,37 @@ EOF
       batch_options = IndiferentHash.setup({})
 
       keys = [
-        :batch_dir,
+        :queue,
+        :account,
+        :partition,
+        :exclusive,
+        :highmem,
+        :time,
+        :nodes,
+        :task_cpus,
+        :mem,
+        :mem_per_cpu,
+        :gres,
         :lua_modules,
+        :contraints,
+        :licenses,
+        :batch_dir,
         :batch_name,
         :contain,
+        :sync,
         :contain_and_sync,
         :copy_image,
         :drbbt,
         :env_cmd,
-        :exclusive,
-        :highmem,
         :manifest,
-        :nodes,
-        :queue,
-        :singularity,
-        :singularity_img,
-        :singularity_opt_dir,
-        :singularity_ruby_inline,
-        :sync,
-        :task_cpus,
-        :gres,
-        :mem,
-        :mem_per_cpu,
-        :licenses,
-        :contraints,
-        :time,
         :user_group,
         :wipe_container,
         :workdir,
-        :purge_deps
+        :purge_deps,
+        :singularity,
+        :singularity_img,
+        :singularity_opt_dir,
+        :singularity_ruby_inline
       ]
 
       keys.each do |key|
@@ -224,7 +226,6 @@ EOF
       Misc.add_defaults batch_options, 
         :batch_name => batch_name,
         :inputs_dir => inputs_dir, 
-        :queue => 'debug',
         :nodes => 1, 
         :step_path => job.path,
         :task_cpus => 1,

data/lib/rbbt/hpc/slurm.rb

@@ -21,16 +21,18 @@ export BATCH_SYSTEM=SLURM
     def self.header(options = {})
       options = options.dup
 
-      queue          = Misc.process_options options, :queue
-      task_cpus      = Misc.process_options options, :task_cpus
-      time           = Misc.process_options options, :time
-      nodes          = Misc.process_options options, :nodes
-      workdir        = Misc.process_options options, :workdir
-      exclusive      = Misc.process_options options, :exclusive
-      highmem        = Misc.process_options options, :highmem
-      licenses       = Misc.process_options options, :licenses
-      constraint     = Misc.process_options options, :constraint
-      gres           = Misc.process_options options, :gres
+      queue      = Misc.process_options options, :queue
+      account    = Misc.process_options options, :account
+      partition  = Misc.process_options options, :partition
+      task_cpus  = Misc.process_options options, :task_cpus
+      time       = Misc.process_options options, :time
+      nodes      = Misc.process_options options, :nodes
+      workdir    = Misc.process_options options, :workdir
+      exclusive  = Misc.process_options options, :exclusive
+      highmem    = Misc.process_options options, :highmem
+      licenses   = Misc.process_options options, :licenses
+      constraint = Misc.process_options options, :constraint
+      gres       = Misc.process_options options, :gres
 
       constraint     = [constraint, "highmem"].compact * "&" if highmem
 
@@ -47,6 +49,8 @@ export BATCH_SYSTEM=SLURM
 
       sbatch_params = {"job-name" => batch_name,
                        "qos" => queue,
+                       "account" => account,
+                       "partition" => partition,
                        "output" => fout,
                        "error" => ferr,
                        "cpus-per-task" => task_cpus,

data/lib/rbbt/util/python/util.rb

@@ -0,0 +1,33 @@
+module RbbtPython
+  def self.py2ruby_a(array)
+    PyCall::List.(array).to_a
+  end
+
+  class << self
+    alias to_a py2ruby_a 
+  end
+
+
+  def self.tsv2df(tsv)
+    df = nil
+    RbbtPython.run 'pandas' do
+      df = pandas.DataFrame.new(tsv.values, columns: tsv.fields, index: tsv.keys)
+      df.columns.name = tsv.key_field
+    end
+    df
+  end
+
+  def self.df2tsv(tuple, options = {})
+    options = Misc.add_defaults options, :type => :list
+    tsv = TSV.setup({}, options)
+    tsv.key_field = tuple.columns.name
+    tsv.fields = py2ruby_a(tuple.columns.values)
+    keys = tuple.index.values
+    PyCall.len(tuple.index).times do |i|
+      k = keys[i]
+      tsv[k] = py2ruby_a(tuple.values[i])
+    end
+    tsv
+  end
+
+end

data/lib/rbbt/workflow/dependencies.rb

@@ -74,7 +74,10 @@ module Workflow
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
 
     dep.original_workflow ||= dep.workflow if dep.workflow
-    dep.original_task_name ||= dep.task_name if dep.workflow
+    dep.original_task_name ||= dep.task_name if dep.task_name
+
+    dep.original_task_name ||= dep.path.split("/")[-3]
+    dep.original_task_name ||= dep.path.split("/")[-2]
 
     dep.workflow = workflow
     dep.info[:name] = dep.name

data/lib/rbbt/workflow/step/dependencies.rb

@@ -368,7 +368,7 @@ class Step
   def run_dependencies
     dep_step = {}
 
-    rec_dependencies = self.rec_dependencies(true)
+    rec_dependencies = self.rec_dependencies(true) + input_dependencies
 
     return if rec_dependencies.empty?
 

data/lib/rbbt/workflow/usage.rb

@@ -25,6 +25,8 @@ module Task
           selects << [i, task.input_options[i][:select_options]] 
         end
 
+        next if new_inputs.empty?
+
         if task.workflow and task.workflow != workflow
           puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
         else

data/share/rbbt_commands/hpc/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/hpc/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/lsf/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/lsf/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/slurm/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/slurm/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/test/rbbt/util/python/test_util.rb

@@ -0,0 +1,17 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/python'
+require 'rbbt/util/python/util'
+
+class TestPythonUtil < Test::Unit::TestCase
+
+  def test_tuple
+    tsv = TSV.setup([], :key_field => "Key", :fields => %w(Value1 Value2), :type => :list)
+    tsv["k1"] = %w(V1_1 V2_1)
+    tsv["k2"] = %w(V1_2 V2_2)
+    df = RbbtPython.tsv2df(tsv)
+    new_tsv = RbbtPython.df2tsv(df)
+    assert_equal tsv, new_tsv
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.12
+  version: 5.33.13
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-20 00:00:00.000000000 Z
+date: 2022-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -312,6 +312,7 @@ files:
 - lib/rbbt/util/open.rb
 - lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
+- lib/rbbt/util/python/util.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
 - lib/rbbt/util/simpleopt.rb
@@ -539,6 +540,7 @@ files:
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/python/test_util.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/simpleopt/test_setup.rb
@@ -609,6 +611,7 @@ test_files:
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_procpath.rb
+- test/rbbt/util/python/test_util.rb
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb