rbbt-util 5.35.4 → 5.37.0

data/python/rbbt.py

@@ -1,7 +1,93 @@
 import warnings
 import sys
 import os
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 
 def rbbt():
     print("Rbbt")
+
+def path(subdir = None, base_dir = None):
+    from pathlib import Path
+    import os
+
+    if (base_dir == None):
+        base_dir = os.path.join(Path.home(), ".rbbt")
+    if (subdir == None):
+        return base_dir
+    else:
+        return os.path.join(base_dir, subdir)
+
+def inspect(obj):
+    print(dir(obj))
+
+def log_tsv(tsv):
+    print(tsv)
+    print(tsv.keys())
+
+def tsv_preamble(line, comment_char="#"):
+    import re
+    header = dict()
+    entries = re.sub(f"^{comment_char}:", '', line)
+    entries = re.sub(f"^{comment_char}:", '', line).split("#")
+    for entry in entries:
+        key, value = entry.split("=")
+        key = re.sub("^:","",key)
+        value = re.sub("^:","",value)
+        header[key] = value
+
+    return header
+
+
+def tsv_header(filename, sep="\t", comment_char="#"):
+    import re
+
+    f = open(filename)
+    line = f.readline().strip()
+
+    if (not line.startswith(comment_char)):
+        header = {"fields":None, "type":"list", "start": 0}
+    else:
+        header = dict()
+        start = 0
+        if (line.startswith(f"{comment_char}:")):
+            header["preamble"]=tsv_preamble(line, comment_char)
+            if ("type" in header["preamble"]):
+                header["type"] = header["preamble"]["type"]
+            line = f.readline().strip()
+            start = 1
+
+        if (line.startswith(comment_char)):
+            header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
+            header["key_field"] = header["all_fields"][0]
+            header["fields"] = header["all_fields"][1:]
+
+        header["start"] = start
+
+    f.close()
+    return header
+
+
+def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
+    import pandas
+
+    if (comment_char == ""):
+        tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
+    else:
+        header = tsv_header(filename, sep=sep, comment_char="#")
+
+        if ("type" in header and header["type"] == "flat"):
+            return None
+        else:
+            if ("sep" in header):
+                sep=header["sep"]
+
+            tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
+
+            if ("fields" in header):
+                tsv.columns = header["fields"]
+                tsv.index.name = header["key_field"]
+
+    return tsv
+
+def tsv(*args, **kwargs):
+    return tsv_pandas(*args, **kwargs)
+

data/share/rbbt_commands/tsv/write_excel

@@ -54,8 +54,9 @@ tsvfile, excelfile = ARGV
 
 excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
 
-raise ParameterException, "No files given" if tsvfile.nil? and excelfile.nil?
-raise ParameterException, "No excel file given" if excelfile.nil?
+raise ParameterException, "No TSV file given" if tsvfile.nil?
+
+excelfile = tsvfile.sub(/\.tsv(\.gz)?$/, '.xlsx') if excelfile.nil?
 
 require 'rbbt/tsv/excel'
 TSV.open(tsvfile, options).excel(excelfile, options)

data/share/rbbt_commands/workflow/task

@@ -213,11 +213,15 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -prov--provenance Report the jobs provenance
 -W--workflows* Load a list of workflows
 -R--requires* Require a list of files
--pre--prepare* Prepare dependencies
--prec--prepare_cpus* Number of dependencies prepared in parallel
+-pro--produce* Prepare dependencies
+-proc--produce_cpus* Number of dependencies prepared in parallel
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 -PERF--procpath_performance* Measure performance using procpath
+-r--relay* Relay job to SSH server
+-sr--slurm_relay* Relay job to SSH SLURM server
+-rdep--relay_dependencies* Relay dependencies instead of main job
+-pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
 EOF
 
 workflow = ARGV.shift
@@ -415,13 +419,57 @@ begin
     exit 0
   end
 
-  if tasks = options.delete(:prepare)
+  if tasks = options.delete(:produce)
     tasks = tasks.split(",")
-    prepare_cpus = (options[:prepare_cpus] || 1)
-    puts Step.prepare_dependencies(job, tasks,  prepare_cpus)
+    produce_cpus = (options[:produce_cpus] || 1)
+    puts Step.produce_dependencies(job, tasks,  prepare_cpus)
     exit 0
   end
 
+  def match_dependencies(queries, dependencies)
+    queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
+
+    matched = []
+    queries.each do |q|
+      matched += dependencies.select do |dep|
+        if Array === q
+          q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
+        else
+          q.to_s == dep.task_name.to_s
+        end
+      end
+    end
+
+    matched
+  end
+
+  if server = options.delete(:relay)
+    require 'rbbt/workflow/remote_workflow'
+    relay_dependencies = options.delete(:relay_dependencies).split(",")
+    produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
+    if relay_dependencies
+      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
+        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+      end
+    else
+      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    end
+  end
+
+  if server = options.delete(:slurm_relay)
+    require 'rbbt/workflow/remote_workflow'
+    relay_dependencies = options.delete(:relay_dependencies).split(",")
+    produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
+    if relay_dependencies
+      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
+        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+      end
+    else
+      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    end
+  end
+
+
   if options[:procpath_performance]
     require 'rbbt/util/procpath'
     current_pid = job.info[:pid]

data/test/rbbt/test_workflow.rb

@@ -513,4 +513,92 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "TLA", job.run
     assert job.overriden
   end
+
+  def test_anonymous_workflow
+    workflow = Module.new
+    workflow.extend Workflow
+    workflow.define_singleton_method(:to_s){"TestWorkflow"}
+
+    assert_equal "TestWorkflow", workflow.to_s
+
+    t1 = workflow.task :s1 => :string do
+      task_name.to_s
+    end
+
+    workflow.dep :s1
+    t2 = workflow.task :s2 => :string do
+      task_name.to_s
+    end
+
+    workflow.dep :s2
+    t3 = workflow.task :s3 => :string do
+      task_name.to_s
+    end
+
+    assert_equal "s3", workflow.job(:s3).run
+  end
+
+  def test_low_level_step
+    Log.with_severity 0 do
+      TmpFile.with_file do |tmpdir|
+        Path.setup tmpdir
+
+        s1 = Step.new tmpdir.s1 do "s1" end 
+        s1.task_name = "s1"
+
+        # run clean run
+        assert_equal "s1", s1.run
+        s1.clean
+        assert_equal "s1", s1.run
+        assert_equal "s1", s1.run
+
+        # is persisted because it raises RbbtException
+        s1.task = proc do raise RbbtException end
+        assert_equal "s1", s1.run
+        s1.clean
+        assert_raises RbbtException do s1.run end
+        assert RbbtException === s1.get_exception
+        assert s1.error?
+
+        s1.recursive_clean
+        s1.task = proc do "s1" end
+
+        # add dependencies
+        s2 = Step.new tmpdir.s2 do 
+          step(:s1).load + " -> s2"  
+        end 
+        s2.task_name = "s2"
+        s2.dependencies << s1
+
+        s3 = Step.new tmpdir.s3 do 
+          step(:s2).load + " -> s3"  
+        end 
+        s3.task_name = "s3"
+        s3.dependencies << s2
+
+        assert_equal "s1 -> s2", s2.run
+        assert_equal "s1 -> s2 -> s3", s3.run
+        
+        # Test recusive behaviour
+        s1.task = proc do raise RbbtException end
+        ComputeDependency.setup(s1, :produce)
+        s1.clean
+        s3.clean
+        s3.run
+        assert_equal "s1 -> s2 -> s3", s3.run
+
+        s1.task = proc do raise RbbtException end
+        ComputeDependency.setup(s1, :produce)
+        s3.recursive_clean
+        assert_raises RbbtException do s3.run end
+
+        s1.task = proc{|v| v }
+        s1.inputs = ["test"]
+        assert_equal "test", s1.run
+        s3.recursive_clean
+        assert_equal "test -> s2 -> s3", s3.run
+      end
+    end
+  end
+
 end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -294,5 +294,6 @@ class TestWorkflowDependency < Test::Unit::TestCase
     assert job.dependencies.first.file('foo').exists?
     assert_equal 'done'.reverse, job.run
   end
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.35.4
+  version: 5.37.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-09 00:00:00.000000000 Z
+date: 2023-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: progress-monitor
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: lockfile
   requirement: !ruby/object:Gem::Requirement
@@ -52,34 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: spreadsheet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubyXL
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: highline
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: to_regexp
   requirement: !ruby/object:Gem::Requirement
@@ -366,7 +310,7 @@ files:
 - lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
 - lib/rbbt/workflow/step/info.rb
-- lib/rbbt/workflow/step/prepare.rb
+- lib/rbbt/workflow/step/produce.rb
 - lib/rbbt/workflow/step/run.rb
 - lib/rbbt/workflow/step/save_load_inputs.rb
 - lib/rbbt/workflow/step/status.rb
@@ -613,113 +557,113 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_association.rb
-- test/rbbt/hpc/test_orchestrate.rb
-- test/rbbt/hpc/test_slurm.rb
-- test/rbbt/hpc/orchestrate/test_batches.rb
-- test/rbbt/hpc/orchestrate/test_rules.rb
-- test/rbbt/hpc/orchestrate/test_chains.rb
-- test/rbbt/hpc/test_batch.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/test_packed_index.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_fix_width_table.rb
 - test/rbbt/workflow/test_remote_workflow.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/step/test_save_load_inputs.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/util/test_data.rb
-- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/util/test_migrate.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/test_procpath.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_concurrency.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/python/test_util.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/test_python.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_development.rb
+- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_communication.rb
-- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_R.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/test_python.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_migrate.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_marshal.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
-- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_marshal.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_manipulate.rb
 - test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/orchestrate/test_chains.rb
+- test/rbbt/hpc/orchestrate/test_rules.rb
+- test/rbbt/hpc/orchestrate/test_batches.rb
+- test/rbbt/hpc/test_slurm.rb
+- test/rbbt/hpc/test_orchestrate.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/test_helper.rb