rbbt-util 5.33.11 → 5.33.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5c26a161d8529f19d32ace8b5565042b4baa6fef3d69580f46aadc523e8cc51
-  data.tar.gz: 8ee1305d3a66a55273474da60e84cec3385391ad99c8c87e764fd72eaaf29285
+  metadata.gz: d2e19fbbde1862e9ee0c4758f0ed4fa4aaa74909454467409c298615b9d6db77
+  data.tar.gz: 4067a1b7bb47bb8f690f7b14a1db8222e46e0c43da4f4ba2497e4c265197d8fd
 SHA512:
-  metadata.gz: 6b9ddf9ff821584ef21f7c30be381965ea9a8cf1cce7e9913cc11c118fa1d31211fe321d72c5d8daabcb14aac5b20df45c527fefa1d142c7a08633c3818b79dd
-  data.tar.gz: c0ab742b23fe87fd7f52bfdfe75690a02566547a34a484912c74766cff70c83634434d5a67a9e665bfdffb28be143afdb20028356befe045e19b14bcc83a21ff
+  metadata.gz: 0aa8a4d7ba684aa381bb1ca9492b1b2a788bfec05f85c72e7a508000087db169fe8a7d03de2b356a927a8f15518e21c0c070facf0937a92ee361cdb27a013fa5
+  data.tar.gz: 4adf8ee16df7927701bff0aa342a3efef5c98c6e98ae556d6ebbda8dabba15a627f856fa6ddd3649b067d0c066bb1faa72ae2163c96512bc704d1628731aa58d

data/lib/rbbt/hpc/batch.rb

@@ -138,35 +138,37 @@ EOF
       batch_options = IndiferentHash.setup({})
 
       keys = [
-        :batch_dir,
+        :queue,
+        :account,
+        :partition,
+        :exclusive,
+        :highmem,
+        :time,
+        :nodes,
+        :task_cpus,
+        :mem,
+        :mem_per_cpu,
+        :gres,
         :lua_modules,
+        :contraints,
+        :licenses,
+        :batch_dir,
         :batch_name,
         :contain,
+        :sync,
         :contain_and_sync,
         :copy_image,
         :drbbt,
         :env_cmd,
-        :exclusive,
-        :highmem,
         :manifest,
-        :nodes,
-        :queue,
-        :singularity,
-        :singularity_img,
-        :singularity_opt_dir,
-        :singularity_ruby_inline,
-        :sync,
-        :task_cpus,
-        :gres,
-        :mem,
-        :mem_per_cpu,
-        :licenses,
-        :contraints,
-        :time,
         :user_group,
         :wipe_container,
         :workdir,
-        :purge_deps
+        :purge_deps,
+        :singularity,
+        :singularity_img,
+        :singularity_opt_dir,
+        :singularity_ruby_inline
       ]
 
       keys.each do |key|
@@ -224,7 +226,6 @@ EOF
       Misc.add_defaults batch_options, 
         :batch_name => batch_name,
         :inputs_dir => inputs_dir, 
-        :queue => 'debug',
         :nodes => 1, 
         :step_path => job.path,
         :task_cpus => 1,

data/lib/rbbt/hpc/slurm.rb

@@ -21,16 +21,18 @@ export BATCH_SYSTEM=SLURM
     def self.header(options = {})
       options = options.dup
 
-      queue          = Misc.process_options options, :queue
-      task_cpus      = Misc.process_options options, :task_cpus
-      time           = Misc.process_options options, :time
-      nodes          = Misc.process_options options, :nodes
-      workdir        = Misc.process_options options, :workdir
-      exclusive      = Misc.process_options options, :exclusive
-      highmem        = Misc.process_options options, :highmem
-      licenses       = Misc.process_options options, :licenses
-      constraint     = Misc.process_options options, :constraint
-      gres           = Misc.process_options options, :gres
+      queue      = Misc.process_options options, :queue
+      account    = Misc.process_options options, :account
+      partition  = Misc.process_options options, :partition
+      task_cpus  = Misc.process_options options, :task_cpus
+      time       = Misc.process_options options, :time
+      nodes      = Misc.process_options options, :nodes
+      workdir    = Misc.process_options options, :workdir
+      exclusive  = Misc.process_options options, :exclusive
+      highmem    = Misc.process_options options, :highmem
+      licenses   = Misc.process_options options, :licenses
+      constraint = Misc.process_options options, :constraint
+      gres       = Misc.process_options options, :gres
 
       constraint     = [constraint, "highmem"].compact * "&" if highmem
 
@@ -47,6 +49,8 @@ export BATCH_SYSTEM=SLURM
 
       sbatch_params = {"job-name" => batch_name,
                        "qos" => queue,
+                       "account" => account,
+                       "partition" => partition,
                        "output" => fout,
                        "error" => ferr,
                        "cpus-per-task" => task_cpus,

data/lib/rbbt/resource.rb

@@ -155,6 +155,8 @@ module Resource
       FileUtils.rm_rf final_path if File.exist? final_path
       return false
     end
+
+    final_path
   end
 
   def produce(path, force = false)
@@ -326,11 +328,15 @@ url='#{url}'
           rescue
             FileUtils.rm_rf final_path if File.exist? final_path
             raise $!
-          end unless (remote_server && get_from_server(path, final_path))
+          end unless (remote_server && final_path = get_from_server(path, final_path))
         end
       end
     end
 
+    # After producing a file, make sure we recheck all locations, the file
+    # might have appeared with '.gz' extension for instance
+    path.instance_variable_set("@path", {})
+
     path
   end
 

data/lib/rbbt/util/misc/communication.rb

@@ -0,0 +1,79 @@
+module Misc
+  PUSHBULLET_KEY=begin
+                   if ENV["PUSHBULLET_KEY"]
+                     ENV["PUSHBULLET_KEY"]
+                   else
+                     config_api = File.join(ENV['HOME'], 'config/apps/pushbullet/apikey')
+                     if File.exist? config_api
+                       File.read(config_api).strip
+                     else
+                       nil
+                     end
+                   end
+                 end
+
+  def self.notify(description, event='notification', key = nil)
+    if PUSHBULLET_KEY.nil? and key.nil?
+      Log.warn "Could not notify, no PUSHBULLET_KEY"
+      return
+    end
+
+    Thread.new do
+      application = 'rbbt'
+      event ||= 'notification'
+      key ||= PUSHBULLET_KEY
+      `curl -s --header "Authorization: Bearer #{key}" -X POST https://api.pushbullet.com/v2/pushes --header 'Content-Type: application/json' --data-binary '{"type": "note", "title": "#{event}", "body": "#{description}"}'`
+    end
+  end
+
+  def self.send_email_old(from, to, subject, message, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, :from_alias => nil, :to_alias => nil, :server => 'localhost', :port => 25, :user => nil, :pass => nil, :auth => :login
+
+    server, port, user, pass, from_alias, to_alias, auth = Misc.process_options options, :server, :port, :user, :pass, :from_alias, :to_alias, :auth
+
+    msg = <<-END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{to}>
+Subject: #{subject}
+
+#{message}
+END_OF_MESSAGE
+
+    Net::SMTP.start(server, port, server, user, pass, auth) do |smtp|
+      smtp.send_message msg, from, to
+    end
+  end
+
+  def self.send_email(from, to, subject, message, options = {})
+    require 'mail'
+
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, :from_alias => nil, :to_alias => nil, :server => 'localhost', :port => 25, :user => nil, :pass => nil, :auth => :login, :files => []
+
+    server, port, user, pass, from_alias, to_alias, auth, files = Misc.process_options options, :server, :port, :user, :pass, :from_alias, :to_alias, :auth, :files
+
+    files = [] if files.nil?
+    files = [files] unless Array === files
+
+    Mail.defaults do
+      delivery_method :smtp, address: server, port: port, user_name: user, password: pass
+    end
+
+    mail = Mail.deliver do 
+      from  "#{from_alias} <#{from}>"
+      to "#{to_alias} <#{to}>"
+      subject subject
+
+      text_part do 
+        body message
+      end
+
+      files.each do |file|
+        file = file.find if Path === file
+        file = file.path if Step === file
+      end
+    end
+  end
+
+end

data/lib/rbbt/util/misc/development.rb

@@ -385,32 +385,6 @@ def self.add_libdir(dir=nil)
     `ps -o rss -p #{pid || $$}`.strip.split.last.to_i
   end
 
-  PUSHBULLET_KEY=begin
-                   if ENV["PUSHBULLET_KEY"]
-                     ENV["PUSHBULLET_KEY"]
-                   else
-                     config_api = File.join(ENV['HOME'], 'config/apps/pushbullet/apikey')
-                     if File.exist? config_api
-                       File.read(config_api).strip
-                     else
-                       nil
-                     end
-                   end
-                 end
-
-  def self.notify(description, event='notification', key = nil)
-    if PUSHBULLET_KEY.nil? and key.nil?
-      Log.warn "Could not notify, no PUSHBULLET_KEY"
-      return
-    end
-
-    Thread.new do
-      application = 'rbbt'
-      event ||= 'notification'
-      key ||= PUSHBULLET_KEY
-      `curl -s --header "Authorization: Bearer #{key}" -X POST https://api.pushbullet.com/v2/pushes --header 'Content-Type: application/json' --data-binary '{"type": "note", "title": "#{event}", "body": "#{description}"}'`
-    end
-  end
 
   def self.unzip_in_dir(file, dir)
     raise "Target is not a directory: #{file}" if File.exist?(dir) and not File.directory?(dir)

data/lib/rbbt/util/misc/system.rb

@@ -16,25 +16,6 @@ module Misc
     end
   end
 
-  def self.send_email(from, to, subject, message, options = {})
-    IndiferentHash.setup(options)
-    options = Misc.add_defaults options, :from_alias => nil, :to_alias => nil, :server => 'localhost', :port => 25, :user => nil, :pass => nil, :auth => :login
-
-    server, port, user, pass, from_alias, to_alias, auth = Misc.process_options options, :server, :port, :user, :pass, :from_alias, :to_alias, :auth
-
-    msg = <<-END_OF_MESSAGE
-From: #{from_alias} <#{from}>
-To: #{to_alias} <#{to}>
-Subject: #{subject}
-
-#{message}
-END_OF_MESSAGE
-
-Net::SMTP.start(server, port, server, user, pass, auth) do |smtp|
-  smtp.send_message msg, from, to
-end
-  end
-
   def self.env_add(var, value, sep = ":", prepend = true)
     ENV[var] ||= ""
     return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/

data/lib/rbbt/util/misc.rb

@@ -20,6 +20,7 @@ require 'rbbt/util/misc/options'
 require 'rbbt/util/misc/system'
 require 'rbbt/util/misc/objects'
 require 'rbbt/util/misc/manipulation'
+require 'rbbt/util/misc/communication'
 
 require 'to_regexp'
 

data/lib/rbbt/util/open.rb

@@ -678,7 +678,14 @@ module Open
       if Open.exists? notification_file
         key = Open.read(notification_file).strip
         key = nil if key.empty?
-        Misc.notify("Wrote " << file, nil, key)
+        if key.include?("@")
+          to = from = key
+          subject = "Wrote " << file
+          message = "Content attached"
+          Misc.send_email(from, to, subject, message, :files => [file])
+        else
+          Misc.notify("Wrote " << file, nil, key)
+        end
         Open.rm notification_file
       end
     rescue

data/lib/rbbt/util/python/util.rb

@@ -0,0 +1,33 @@
+module RbbtPython
+  def self.py2ruby_a(array)
+    PyCall::List.(array).to_a
+  end
+
+  class << self
+    alias to_a py2ruby_a 
+  end
+
+
+  def self.tsv2df(tsv)
+    df = nil
+    RbbtPython.run 'pandas' do
+      df = pandas.DataFrame.new(tsv.values, columns: tsv.fields, index: tsv.keys)
+      df.columns.name = tsv.key_field
+    end
+    df
+  end
+
+  def self.df2tsv(tuple, options = {})
+    options = Misc.add_defaults options, :type => :list
+    tsv = TSV.setup({}, options)
+    tsv.key_field = tuple.columns.name
+    tsv.fields = py2ruby_a(tuple.columns.values)
+    keys = tuple.index.values
+    PyCall.len(tuple.index).times do |i|
+      k = keys[i]
+      tsv[k] = py2ruby_a(tuple.values[i])
+    end
+    tsv
+  end
+
+end

data/lib/rbbt/workflow/dependencies.rb

@@ -74,7 +74,10 @@ module Workflow
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
 
     dep.original_workflow ||= dep.workflow if dep.workflow
-    dep.original_task_name ||= dep.task_name if dep.workflow
+    dep.original_task_name ||= dep.task_name if dep.task_name
+
+    dep.original_task_name ||= dep.path.split("/")[-3]
+    dep.original_task_name ||= dep.path.split("/")[-2]
 
     dep.workflow = workflow
     dep.info[:name] = dep.name

data/lib/rbbt/workflow/step/dependencies.rb

@@ -368,7 +368,7 @@ class Step
   def run_dependencies
     dep_step = {}
 
-    rec_dependencies = self.rec_dependencies(true)
+    rec_dependencies = self.rec_dependencies(true) + input_dependencies
 
     return if rec_dependencies.empty?
 

data/lib/rbbt/workflow/usage.rb

@@ -25,6 +25,8 @@ module Task
           selects << [i, task.input_options[i][:select_options]] 
         end
 
+        next if new_inputs.empty?
+
         if task.workflow and task.workflow != workflow
           puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
         else

data/share/rbbt_commands/hpc/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/hpc/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/lsf/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/lsf/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/slurm/orchestrate

@@ -25,6 +25,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/share/rbbt_commands/slurm/task

@@ -24,6 +24,8 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
+-a--account* Account
+-p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
 -m--mem* SLURM minimum memory

data/test/rbbt/util/misc/test_communication.rb

@@ -0,0 +1,11 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/communication'
+
+class TestCommunication < Test::Unit::TestCase
+  def test_send_email
+    to = from = 'mvazque2@localhost'
+    subject = message = "Test"
+    iii Misc.send_email(to, from, subject, message)
+  end
+end
+

data/test/rbbt/util/python/test_util.rb

@@ -0,0 +1,17 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/python'
+require 'rbbt/util/python/util'
+
+class TestPythonUtil < Test::Unit::TestCase
+
+  def test_tuple
+    tsv = TSV.setup([], :key_field => "Key", :fields => %w(Value1 Value2), :type => :list)
+    tsv["k1"] = %w(V1_1 V2_1)
+    tsv["k2"] = %w(V1_2 V2_2)
+    df = RbbtPython.tsv2df(tsv)
+    new_tsv = RbbtPython.df2tsv(df)
+    assert_equal tsv, new_tsv
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.11
+  version: 5.33.14
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-05 00:00:00.000000000 Z
+date: 2022-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -292,6 +292,7 @@ files:
 - lib/rbbt/util/misc.rb
 - lib/rbbt/util/misc/annotated_module.rb
 - lib/rbbt/util/misc/bgzf.rb
+- lib/rbbt/util/misc/communication.rb
 - lib/rbbt/util/misc/concurrent_stream.rb
 - lib/rbbt/util/misc/development.rb
 - lib/rbbt/util/misc/exceptions.rb
@@ -311,6 +312,7 @@ files:
 - lib/rbbt/util/open.rb
 - lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
+- lib/rbbt/util/python/util.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
 - lib/rbbt/util/simpleopt.rb
@@ -531,12 +533,14 @@ files:
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/python/test_util.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/simpleopt/test_setup.rb
@@ -607,10 +611,12 @@ test_files:
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_procpath.rb
+- test/rbbt/util/python/test_util.rb
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_format.rb
+- test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb