rbbt-util 6.0.3 → 6.0.5

data/lib/rbbt/workflow/refactor/task_info.rb

@@ -1,65 +1,66 @@
-require_relative 'export'
-require_relative 'recursive'
-module Workflow
-  def task_info(name)
-    name = name.to_sym
-    task = tasks[name]
-    raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
-    id = File.join(self.to_s, name.to_s)
-    @task_info ||= {}
-    @task_info[id] ||= begin 
-                         description = task.description
-                         result_description = task.result_description
-                         result_type = task.result_type
-
-                         inputs = rec_inputs(name).uniq
-                         input_types = rec_input_types(name)
-                         input_descriptions = rec_input_descriptions(name)
-                         input_use = rec_input_use(name)
-                         input_defaults = rec_input_defaults(name)
-                         input_options = rec_input_options(name)
-                         extension = task.extension
-                         export = case
-                                  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
-                                    :synchronous
-                                  when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
-                                    :asynchronous
-                                  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
-                                    :exec
-                                  when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
-                                    :stream
-                                  else
-                                    :none
-                                  end
-
-                         dependencies = tasks[name].deps
-                         { :id => id,
-                           :description => description,
-                           :export => export,
-                           :inputs => inputs,
-                           :input_types => input_types,
-                           :input_descriptions => input_descriptions,
-                           :input_defaults => input_defaults,
-                           :input_options => input_options,
-                           :input_use => input_use,
-                           :result_type => result_type,
-                           :result_description => result_description,
-                           :dependencies => dependencies,
-                           :extension => extension
-                         }
-                       end
-  end
-end
-
-module Task
-  def result_description
-    ""
-  end
-
-  def result_type
-    @returns
-  end
-
-end
-
-
+#require_relative 'export'
+#require_relative 'recursive'
+#module Workflow
+#  def task_info(name)
+#    name = name.to_sym
+#    task = tasks[name]
+#    raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
+#    id = File.join(self.to_s, name.to_s)
+#    @task_info ||= {}
+#    @task_info[id] ||= begin 
+#                         description = task.description
+#                         result_description = task.result_description
+#                         returns = task.returns
+#
+#                         inputs = rec_inputs(name).uniq
+#                         input_types = rec_input_types(name)
+#                         input_descriptions = rec_input_descriptions(name)
+#                         input_use = rec_input_use(name)
+#                         input_defaults = rec_input_defaults(name)
+#                         input_options = rec_input_options(name)
+#                         extension = task.extension
+#                         export = case
+#                                  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
+#                                    :synchronous
+#                                  when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
+#                                    :asynchronous
+#                                  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
+#                                    :exec
+#                                  when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
+#                                    :stream
+#                                  else
+#                                    :none
+#                                  end
+#
+#                         dependencies = tasks[name].deps
+#                         { :id => id,
+#                           :description => description,
+#                           :export => export,
+#                           :inputs => inputs,
+#                           :input_types => input_types,
+#                           :input_descriptions => input_descriptions,
+#                           :input_defaults => input_defaults,
+#                           :input_options => input_options,
+#                           :input_use => input_use,
+#                           :returns => returns,
+#                           #:result_type => result_type,
+#                           #:result_description => result_description,
+#                           :dependencies => dependencies,
+#                           :extension => extension
+#                         }
+#                       end
+#  end
+#end
+#
+#module Task
+#  def result_description
+#    ""
+#  end
+#
+#  def result_type
+#    @returns
+#  end
+#
+#end
+#
+#

data/lib/rbbt/workflow/refactor.rb

@@ -1,6 +1,3 @@
-require_relative 'refactor/export'
-require_relative 'refactor/recursive'
-require_relative 'refactor/task_info'
 require_relative 'refactor/inputs'
 require_relative 'refactor/entity'
 

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -146,7 +146,6 @@ class RemoteWorkflow
                              task_info = RemoteWorkflow::REST.get_json(File.join(url, task.to_s, 'info'))
                              task_info = RemoteWorkflow.fix_hash(task_info)
 
-                             task_info[:result_type] = task_info[:result_type].to_sym
                              task_info[:export] = task_info[:export].to_sym
                              task_info[:input_types] = RemoteWorkflow.fix_hash(task_info[:input_types], true)
                              task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
@@ -248,7 +247,6 @@ class RemoteWorkflow
       end
     end
 
-
     def task_info(task)
       RemoteWorkflow::REST.task_info(url, task)
     end
@@ -260,6 +258,7 @@ class RemoteWorkflow
       @exec_exports = (task_exports["exec"] || []).collect{|task| task.to_sym }
       @stream_exports = (task_exports["stream"] || []).collect{|task| task.to_sym }
       @can_stream = task_exports["can_stream"]
+      (@asynchronous_exports + @synchronous_exports + @exec_exports).uniq.each do |e| tasks[e] end
     end
   end
 end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -94,7 +94,7 @@ job = wf.job(task, jobname, job_inputs)
 STDOUT.write res.to_json
       EOF
 
-      json = Misc.ssh_run(server, script)
+      json = SSHLine.ruby(server, script)
       Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
       JSON.parse(json)
     end
@@ -107,7 +107,7 @@ STDOUT.write res.to_json
 STDOUT.write res
       EOF
 
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.post_job(url, inputs_id, jobname = nil)
@@ -119,7 +119,7 @@ STDOUT.write res
 job.init_info
 STDOUT.write job.path
       EOF
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.run_job(url, input_id, jobname = nil)
@@ -133,7 +133,7 @@ job.clean if job.error? and job.recoverable_error?
 job.run unless job.done? || job.error?
 STDOUT.write job.path
       EOF
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.run_batch_job(url, input_id, jobname = nil, batch_options = {})
@@ -149,7 +149,7 @@ job.clean if job.error? and job.recoverable_error?
 HPC::BATCH_MODULE.run_job(job, batch_options) unless job.done? || job.error?
 STDOUT.write job.path
       EOF
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.orchestrate_batch_job(url, input_id, jobname = nil, batch_options = {})
@@ -165,7 +165,7 @@ job.clean if job.error? and job.recoverable_error?
 HPC::BATCH_MODULE.orchestrate_job(job, batch_options) unless job.done? || job.error?
 STDOUT.write job.path
       EOF
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.clean(url, input_id, jobname = nil)
@@ -175,7 +175,7 @@ STDOUT.write job.path
       script +=<<-EOF
 job.clean
       EOF
-      Misc.ssh_run(server, script)
+      SSHLine.ruby(server, script)
     end
 
     def self.upload_inputs(server, inputs, input_types, input_id)
@@ -234,7 +234,7 @@ job.clean
       all_deps.each do |dep,jobs|
         next if dep.done?
         next if job_list.include?(dep)
-        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Misc.fingerprint jobs}"
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Log.fingerprint jobs}"
         dep.produce
         missing_deps << dep
       end if produce_dependencies
@@ -243,7 +243,7 @@ job.clean
 
       #migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
       migrate_dependencies = all_deps.keys.collect{|d| [d] + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
-      Log.low "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
+      Log.low "Migrating #{migrate_dependencies.length} dependencies from #{Log.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
 
@@ -287,7 +287,7 @@ job.clean
 
       if options[:migrate]
         rjob.produce
-        Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
+        Step.migrate(Path.identify(job.path), 'user', :source => server) 
       end
 
       rjob
@@ -337,7 +337,7 @@ job.clean
       if options[:migrate]
         rjobs_job.each do |rjob,job|
           rjob.produce
-          Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
+          Step.migrate(Path.identify(job.path), 'user', :source => server) 
         end
       end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -289,7 +289,7 @@ class RemoteStep < Step
 
     stream = true if res.respond_to? :read
     join unless stream
-    result_type ||= self.result_type
+    result_type ||= self.type
 
     case result_type.to_sym
     when :string

data/lib/rbbt/workflow/remote_workflow.rb

@@ -5,7 +5,8 @@ class RemoteWorkflow
 
   attr_accessor :url, :name, :exec_exports, :synchronous_exports, :asynchronous_exports, :stream_exports
 
-  def initialize(url, name)
+  def initialize(url, name = nil)
+    name = File.basename(url) if name.nil?
     Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
     @url, @name = url, name
 

data/lib/rbbt-util.rb

@@ -20,7 +20,7 @@ require_relative 'rbbt/tsv'
 require_relative 'rbbt/workflow'
 
 Persist.cache_dir     = Rbbt.var.cache.persistence
-FileCache.cachedir    = Rbbt.var.cache.filecache.find :user
-TmpFile.tmpdir        = Rbbt.tmp.find :user
+FileCache.cachedir    = Rbbt.var.cache.filecache.find
+TmpFile.tmpdir        = Rbbt.tmp.find
 Resource.default_resource = Rbbt
 

data/lib/rbbt.rb

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift File.join(__dir__, '../../lib')
 
 require 'scout/path'
 require 'scout/resource'
-Path.add_path :rbbt_util, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+Path.add_path :rbbt_util_lib, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 module Rbbt
   extend Resource
 

data/python/rbbt/__init__.py

@@ -1,22 +1,29 @@
-import warnings
 import sys
 import os
 import subprocess
+import tempfile
+import shutil
+import pandas
+import numpy
 
-def cmd(cmd = None):
+
+def cmd(cmd=None):
     if cmd is None:
         print("Rbbt")
     else:
         return subprocess.run('rbbt_exec.rb', input=cmd.encode('utf-8'), capture_output=True).stdout.decode()
 
+
 def libdir():
-    return rbbt('puts Rbbt.find(:lib)').rstrip()
+    return cmd('puts Rbbt.find(:lib)').rstrip()
+
 
 def add_libdir():
     pythondir = os.path.join(libdir(), 'python')
     sys.path.insert(0, pythondir)
 
-def path(subdir = None, base_dir = None):
+
+def path(subdir=None, base_dir=None):
     from pathlib import Path
     import os
 
@@ -144,4 +151,71 @@ def save_tsv(filename, df, key=None):
     key = "#" + key
     df.to_csv(filename, sep="\t", index_label=key)
 
+def save_job_inputs(data):
+    temp_dir = tempfile.mkdtemp()  # Create a temporary directory
+
+    for name, value in data.items():
+        file_path = os.path.join(temp_dir, name)
+
+        if isinstance(value, str):
+            file_path += ".txt"
+            with open(file_path, "w") as f:
+                f.write(value)
+
+        elif isinstance(value, (bool)):
+            with open(file_path, "w") as f:
+                if value:
+                    f.write('true')
+                else:
+                    f.write('false')
+
+        elif isinstance(value, (int, float)):
+            with open(file_path, "w") as f:
+                f.write(str(value))
+
+        elif isinstance(value, pandas.DataFrame):
+            file_path += ".tsv"
+            save_tsv(file_path, value)
 
+        elif isinstance(value, numpy.ndarray) or isinstance(value, list):
+            file_path += ".list"
+            with open(file_path, "w") as f:
+                f.write("\n".join(value))
+
+        else:
+            raise TypeError(f"Unsupported data type for argument '{name}': {type(value)}")
+
+    return temp_dir
+
+
+def run_job(workflow, task, name='Default', fork=False, clean=False, **kwargs):
+    inputs_dir = save_job_inputs(kwargs)
+    cmd = ['rbbt', 'workflow', 'task', workflow, task, '--jobname', name, '--load_inputs', inputs_dir, '--nocolor']
+
+    if fork:
+        cmd.append('--fork')
+        cmd.append('--detach')
+
+    if clean:
+        if clean == 'recursive':
+            cmd.append('--recursive_clean')
+        else:
+            cmd.append('--clean')
+
+    proc = subprocess.run(
+        cmd,
+        capture_output=True,  # Capture both stdout and stderr
+        text=True  # Automatically decode outputs to strings
+        )
+    shutil.rmtree(inputs_dir)
+    if proc.returncode != 0:
+        output = proc.stderr.strip()
+        if output == '' :
+            output = proc.stdout.strip()
+        raise RuntimeError(output)  # Raise error with cleaned stderr content
+    return proc.stdout.strip()
+
+if __name__ == "__main__":
+    import json
+    res = run_job('Baking', 'bake_muffin_tray', 'test', add_blueberries=True, fork=True)
+    print(res)

data/python/rbbt/workflow/remote.py

@@ -0,0 +1,104 @@
+import requests
+import logging
+import json
+from urllib.parse import urlencode, urljoin
+from time import sleep
+import itertools
+
+def request_post(url, params):
+    response = requests.post(url, params)
+    return response
+
+def request_get(url, params):
+    query = urlencode(params)
+    full_url = f"{url}?{query}"
+    response = requests.get(full_url)
+    return response
+
+def get_json(url, params={}):
+    params['_format'] = 'json'
+    response = request_get(url, params)
+    if response.status_code == 200:
+        return json.loads(response.content)  # parse the JSON content from the response
+    else:
+        logging.error("Failed to initialize remote tasks")
+
+def get_raw(url, params={}):
+    params['_format'] = 'raw'
+    response = request_get(url, params)
+    if response.status_code == 200:
+        return response.content  # parse the JSON content from the response
+    else:
+        logging.error("Failed to initialize remote tasks")
+
+def join(url, *subpaths):
+    return url + "/" + "/".join(subpaths)
+
+class RemoteStep:
+    def __init__(self, url):
+        self.url = url
+
+    def info(self):
+        return get_json(join(self.url, 'info'))
+    def status(self):
+        return self.info()['status']
+
+    def done(self):
+        return self.status() == 'done'
+
+    def error(self):
+        return self.status() == 'error' or self.status() == 'aborted'
+
+    def running(self):
+        return not (self.done() or self.error())
+
+    def wait(self, time=1):
+        while self.running():
+            sleep(time)
+
+
+    def raw(self):
+        return get_raw(self.url)
+
+    def json(self):
+        return get_json(self.url)
+
+class RemoteWorkflow:
+    def __init__(self, url):
+        self.url = url
+        self.task_exports = {}
+        self.init_remote_tasks()
+
+    def init_remote_tasks(self):
+        self.task_exports = get_json(self.url)
+        self.tasks = []
+        self.tasks += self.task_exports['asynchronous']
+        self.tasks += self.task_exports['synchronous']
+        self.tasks += self.task_exports['exec']
+
+    def task_info(self, name):
+        return get_json(join(self.url, name, '/info'))
+
+    def job(self, task, **kwargs):
+        kwargs['_format'] = 'jobname'
+        response = request_post(join(self.url, task), kwargs)
+        if response.status_code == 200:
+            jobname = response.content.decode('utf-8')
+            step_url = join(self.url, task, jobname)
+            print(step_url)
+            return RemoteStep(step_url)
+        else:
+            logging.error("Failed to initialize remote tasks")
+
+
+if __name__ == "__main__":
+    wf = RemoteWorkflow('http://localhost:1900/Baking')
+    print(wf.tasks)
+    print(wf.task_info('bake_muffin_tray'))
+
+    step = wf.job('bake_muffin_tray', add_blueberries=True)
+    step.wait()
+    print(step.json())
+
+
+

data/python/rbbt/workflow.py

@@ -0,0 +1,64 @@
+from . import cmd, run_job
+import subprocess
+import json
+import time
+
+def save_inputs(directory, inputs, types):
+    return
+
+class Workflow:
+    def __init__(self, name):
+        self.name = name
+
+    def tasks(self):
+        ruby=f'Workflow.require_workflow("{self.name}").tasks.keys * "\n"'
+        return cmd(ruby).strip().split("\n")
+
+    def task_info(self, name):
+        ruby=f'Workflow.require_workflow("{self.name}").task_info("{name}").to_json'
+        return cmd(ruby)
+
+    def run(self, task, **kwargs):
+        return run_job(self.name, task, **kwargs)
+
+    def fork(self, task, **kwargs):
+        path = run_job(self.name, task, fork=True, **kwargs)
+        return Step(path)
+
+class Step:
+    def __init__(self, path):
+        self.path = path
+        self.info_content = None
+
+    def info(self):
+        if self.info_content:
+            return self.info_content
+        ruby=f'puts Step.load("{self.path}").info.to_json'
+        txt = cmd(ruby)
+        info_content = json.loads(txt)
+        status = info_content["status"]
+        if status == "done" or status == "error" or status == "aborted":
+            self.info_content = info_content
+        return info_content
+
+    def status(self):
+        return self.info()["status"]
+
+    def done(self):
+        return self.status() == 'done'
+
+    def error(self):
+        return self.status() == 'error'
+
+    def aborted(self):
+        return self.status() == 'aborted'
+
+    def join(self):
+        while not (self.done() or self.error() or self.aborted()):
+            time.sleep(1)
+
+    def load(self):
+        ruby=f'puts Step.load("{self.path}").load.to_json'
+        txt = cmd(ruby)
+        return json.loads(txt)
+

data/python/test.py

@@ -0,0 +1,10 @@
+if __name__ == "__main__":
+    import rbbt
+    import rbbt.workflow
+    wf = rbbt.workflow.Workflow('Baking')
+    step = wf.fork('bake_muffin_tray', add_blueberries=True, clean='recursive')
+    step.join()
+    print(step.load())
+
+
+

data/share/rbbt_commands/hpc/list

@@ -338,7 +338,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies + [step]).to_a.reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/share/rbbt_commands/lsf/list

@@ -338,7 +338,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies + [step]).to_a.reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/share/rbbt_commands/pbs/list

@@ -338,7 +338,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies + [step]).to_a.reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/share/rbbt_commands/resource/find

@@ -74,7 +74,7 @@ end if resource
 
 path = (resource || Rbbt)[path]
 
-if where.nil? || where == 'all' || path.search_paths.include?(where.to_sym)
+if where.nil? || where == 'all' || path.map_maps.include?(where.to_sym)
   location = path.find(where)
 
   if Array === location

data/share/rbbt_commands/slurm/list

@@ -338,7 +338,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies + [step]).to_a.reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/share/rbbt_commands/system/status

@@ -36,7 +36,7 @@ inputs = (options[:inputs] || "").split(",")
 info_fields = (options[:info_fields] || "").split(",")
 
 def pid_msg(pid)
-  color = if pid and Misc.pid_exists? pid
+  color = if pid and Misc.pid_alive? pid
             :green
           else
             :red
@@ -211,7 +211,7 @@ workflows.sort.each do |workflow,tasks|
         status = :missing if status == :done and not (Open.exist?(file) && ! Open.broken_link?(file))
         status = :broken if Open.broken_link?(file)
         status = status.to_s
-        if status != "done" and pid and not Misc.pid_exists?(pid)
+        if status != "done" and pid and not Misc.pid_alive?(pid)
           if File.exist? file
             status << Log.color(:red, " (out of sync)") 
           else

data/share/rbbt_commands/workflow/info

@@ -56,7 +56,7 @@ def status_msg(status)
 end
 
 def pid_msg(pid, done = false)
-  color = if pid and (done or Misc.pid_exists? pid)
+  color = if pid and (done or Misc.pid_alive? pid)
             :green
           else
             :red