scout-rig 0.1.0

data/python/scout/__init__.py

@@ -0,0 +1,221 @@
+import sys
+import os
+import subprocess
+import tempfile
+import shutil
+import pandas
+import numpy
+
+
+def cmd(cmd=None):
+    if cmd is None:
+        print("Rbbt")
+    else:
+        return subprocess.run('rbbt_exec.rb', input=cmd.encode('utf-8'), capture_output=True).stdout.decode()
+
+
+def libdir():
+    return cmd('puts Rbbt.find(:lib)').rstrip()
+
+
+def add_libdir():
+    pythondir = os.path.join(libdir(), 'python')
+    sys.path.insert(0, pythondir)
+
+
+def path(subdir=None, base_dir=None):
+    from pathlib import Path
+    import os
+
+    if (base_dir == 'base'):
+        base_dir = os.path.join(Path.home(), ".rbbt")
+    elif (base_dir == 'lib'):
+        base_dir = libdir()
+    else:
+        for base_dir in ('lib', 'base'):
+            file = path(subdir, base_dir)
+            if os.path.exists(file):
+                return file
+        return path(subdir, 'base')
+
+    if (subdir == None):
+        return base_dir
+    else:
+        return os.path.join(base_dir, subdir)
+
+def read(subdir, base_dir = None, encoding='utf-8'):
+    file = path(subdir, base_dir)
+    with open(file, encoding=encoding) as f:
+        return f.read()
+
+def inspect(obj):
+    print(dir(obj))
+
+def rich(obj):
+    import rich
+    rich.inspect(obj)
+
+def log_tsv(tsv):
+    print(tsv)
+    print(tsv.keys())
+
+def benchmark():
+    import time
+    tic: float = time.perf_counter()
+    try:
+        yield
+    finally:
+        toc: float = time.perf_counter()
+    print(f"Computation time = {1000*(toc - tic):.3f}ms")
+
+def tsv_preamble(line, comment_char="#"):
+    import re
+    header = dict()
+    entries = re.sub(f"^{comment_char}:", '', line)
+    entries = re.sub(f"^{comment_char}:", '', line).split("#")
+    for entry in entries:
+        entry = entry.strip()
+        key, value = entry.split("=")
+        key = re.sub("^:","",key)
+        value = re.sub("^:","",value)
+        header[key] = value
+
+    return header
+
+
+def tsv_header(filename, sep="\t", comment_char="#", encoding='utf8'):
+    import re
+
+    f = open(filename, encoding=encoding)
+    line = f.readline().strip()
+
+    if (not line.startswith(comment_char)):
+        header = {"fields":None, "type":"list", "start": 0}
+    else:
+        header = dict()
+        start = 0
+        if (line.startswith(f"{comment_char}:")):
+            header["preamble"]=tsv_preamble(line, comment_char)
+            if ("type" in header["preamble"]):
+                header["type"] = header["preamble"]["type"]
+            line = f.readline().strip()
+            start = 1
+
+        if (line.startswith(comment_char)):
+            header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
+            header["key_field"] = header["all_fields"][0]
+            header["fields"] = header["all_fields"][1:]
+
+        header["start"] = start
+
+    f.close()
+    return header
+
+
+def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
+    import pandas
+
+    if (comment_char == ""):
+        tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
+    else:
+        header = tsv_header(filename, sep=sep, comment_char="#")
+
+        if ("type" in header and header["type"] == "flat"):
+            if ("sep" in header):
+                sep=header["sep"]
+
+            tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=None, skiprows=[0,1], **kwargs)
+
+            if ("key_field" in header):
+                tsv.index.name = header["key_field"]
+        else:
+            if ("sep" in header):
+                sep=header["sep"]
+
+            tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
+
+            if ("fields" in header):
+                tsv.columns = header["fields"]
+                tsv.index.name = header["key_field"]
+
+    return tsv
+
+def tsv(*args, **kwargs):
+    return tsv_pandas(*args, **kwargs)
+
+def save_tsv(filename, df, key=None):
+    if (key == None):
+        key = df.index.name
+    if (key == None):
+        key = "Key"
+    key = "#" + key
+    df.to_csv(filename, sep="\t", index_label=key)
+
+def save_job_inputs(data):
+    temp_dir = tempfile.mkdtemp()  # Create a temporary directory
+
+    for name, value in data.items():
+        file_path = os.path.join(temp_dir, name)
+
+        if isinstance(value, str):
+            file_path += ".txt"
+            with open(file_path, "w") as f:
+                f.write(value)
+
+        elif isinstance(value, (bool)):
+            with open(file_path, "w") as f:
+                if value:
+                    f.write('true')
+                else:
+                    f.write('false')
+
+        elif isinstance(value, (int, float)):
+            with open(file_path, "w") as f:
+                f.write(str(value))
+
+        elif isinstance(value, pandas.DataFrame):
+            file_path += ".tsv"
+            save_tsv(file_path, value)
+
+        elif isinstance(value, numpy.ndarray) or isinstance(value, list):
+            file_path += ".list"
+            with open(file_path, "w") as f:
+                f.write("\n".join(value))
+
+        else:
+            raise TypeError(f"Unsupported data type for argument '{name}': {type(value)}")
+
+    return temp_dir
+
+
+def run_job(workflow, task, name='Default', fork=False, clean=False, **kwargs):
+    inputs_dir = save_job_inputs(kwargs)
+    cmd = ['rbbt', 'workflow', 'task', workflow, task, '--jobname', name, '--load_inputs', inputs_dir, '--nocolor']
+
+    if fork:
+        cmd.append('--fork')
+        cmd.append('--detach')
+
+    if clean:
+        if clean == 'recursive':
+            cmd.append('--recursive_clean')
+        else:
+            cmd.append('--clean')
+
+    proc = subprocess.run(
+        cmd,
+        capture_output=True,  # Capture both stdout and stderr
+        text=True  # Automatically decode outputs to strings
+        )
+    shutil.rmtree(inputs_dir)
+    if proc.returncode != 0:
+        output = proc.stderr.strip()
+        if output == '' :
+            output = proc.stdout.strip()
+        raise RuntimeError(output)  # Raise error with cleaned stderr content
+    return proc.stdout.strip()
+
+if __name__ == "__main__":
+    import json
+    res = run_job('Baking', 'bake_muffin_tray', 'test', add_blueberries=True, fork=True)
+    print(res)

data/python/scout/workflow/remote.py

@@ -0,0 +1,103 @@
+import requests
+import logging
+import json
+from urllib.parse import urlencode, urljoin
+from time import sleep
+import itertools
+
+def request_post(url, params):
+    response = requests.post(url, params)
+    return response
+
+def request_get(url, params):
+    query = urlencode(params)
+    full_url = f"{url}?{query}"
+    response = requests.get(full_url)
+    return response
+
+def get_json(url, params={}):
+    params['_format'] = 'json'
+    response = request_get(url, params)
+    if response.status_code == 200:
+        return json.loads(response.content)  # parse the JSON content from the response
+    else:
+        logging.error("Failed to initialize remote tasks")
+
+def get_raw(url, params={}):
+    params['_format'] = 'raw'
+    response = request_get(url, params)
+    if response.status_code == 200:
+        return response.content  # parse the JSON content from the response
+    else:
+        logging.error("Failed to initialize remote tasks")
+
+def join(url, *subpaths):
+    return url + "/" + "/".join(subpaths)
+
+class RemoteStep:
+    def __init__(self, url):
+        self.url = url
+
+    def info(self):
+        return get_json(join(self.url, 'info'))
+    def status(self):
+        return self.info()['status']
+
+    def done(self):
+        return self.status() == 'done'
+
+    def error(self):
+        return self.status() == 'error' or self.status() == 'aborted'
+
+    def running(self):
+        return not (self.done() or self.error())
+
+    def wait(self, time=1):
+        while self.running():
+            sleep(time)
+
+
+    def raw(self):
+        return get_raw(self.url)
+
+    def json(self):
+        return get_json(self.url)
+
+class RemoteWorkflow:
+    def __init__(self, url):
+        self.url = url
+        self.task_exports = {}
+        self.init_remote_tasks()
+
+    def init_remote_tasks(self):
+        self.task_exports = get_json(self.url)
+        self.tasks = []
+        self.tasks += self.task_exports['asynchronous']
+        self.tasks += self.task_exports['synchronous']
+        self.tasks += self.task_exports['exec']
+
+    def task_info(self, name):
+        return get_json(join(self.url, name, '/info'))
+
+    def job(self, task, **kwargs):
+        kwargs['_format'] = 'jobname'
+        response = request_post(join(self.url, task), kwargs)
+        if response.status_code == 200:
+            jobname = response.content.decode('utf-8')
+            step_url = join(self.url, task, jobname)
+            return RemoteStep(step_url)
+        else:
+            logging.error("Failed to initialize remote tasks")
+
+
+if __name__ == "__main__":
+    wf = RemoteWorkflow('http://localhost:1900/Baking')
+    print(wf.tasks)
+    print(wf.task_info('bake_muffin_tray'))
+
+    step = wf.job('bake_muffin_tray', add_blueberries=True)
+    step.wait()
+    print(step.json())
+
+
+

data/python/scout/workflow.py

@@ -0,0 +1,64 @@
+from . import cmd, run_job
+import subprocess
+import json
+import time
+
+def save_inputs(directory, inputs, types):
+    return
+
+class Workflow:
+    def __init__(self, name):
+        self.name = name
+
+    def tasks(self):
+        ruby=f'Workflow.require_workflow("{self.name}").tasks.keys * "\n"'
+        return cmd(ruby).strip().split("\n")
+
+    def task_info(self, name):
+        ruby=f'Workflow.require_workflow("{self.name}").task_info("{name}").to_json'
+        return cmd(ruby)
+
+    def run(self, task, **kwargs):
+        return run_job(self.name, task, **kwargs)
+
+    def fork(self, task, **kwargs):
+        path = run_job(self.name, task, fork=True, **kwargs)
+        return Step(path)
+
+class Step:
+    def __init__(self, path):
+        self.path = path
+        self.info_content = None
+
+    def info(self):
+        if self.info_content:
+            return self.info_content
+        ruby=f'puts Step.load("{self.path}").info.to_json'
+        txt = cmd(ruby)
+        info_content = json.loads(txt)
+        status = info_content["status"]
+        if status == "done" or status == "error" or status == "aborted":
+            self.info_content = info_content
+        return info_content
+
+    def status(self):
+        return self.info()["status"]
+
+    def done(self):
+        return self.status() == 'done'
+
+    def error(self):
+        return self.status() == 'error'
+
+    def aborted(self):
+        return self.status() == 'aborted'
+
+    def join(self):
+        while not (self.done() or self.error() or self.aborted()):
+            time.sleep(1)
+
+    def load(self):
+        ruby=f'puts Step.load("{self.path}").load.to_json'
+        txt = cmd(ruby)
+        return json.loads(txt)
+

data/python/test.py

@@ -0,0 +1,12 @@
+if __name__ == "__main__":
+    import sys
+    sys.path.append('python')
+    import scout
+    import scout.workflow
+    wf = scout.workflow.Workflow('Baking')
+    step = wf.fork('bake_muffin_tray', add_blueberries=True, clean='recursive')
+    step.join()
+    print(step.load())
+
+
+

data/scout-rig.gemspec

@@ -0,0 +1,56 @@
+# Generated by juwelier
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: scout-rig 0.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "scout-rig".freeze
+  s.version = "0.1.0".freeze
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Miguel Vazquez".freeze]
+  s.date = "1980-01-02"
+  s.description = "Use other coding languages in your scout applications".freeze
+  s.email = "mikisvaz@gmail.com".freeze
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".vimproject",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/scout-rig.rb",
+    "lib/scout/python.rb",
+    "lib/scout/python/paths.rb",
+    "lib/scout/python/run.rb",
+    "lib/scout/python/script.rb",
+    "lib/scout/python/util.rb",
+    "python/scout/__init__.py",
+    "python/scout/__pycache__/__init__.cpython-310.pyc",
+    "python/scout/__pycache__/workflow.cpython-310.pyc",
+    "python/scout/workflow.py",
+    "python/scout/workflow/remote.py",
+    "python/test.py",
+    "scout-rig.gemspec",
+    "test/scout/python/test_script.rb",
+    "test/scout/python/test_util.rb",
+    "test/scout/test_python.rb",
+    "test/test_helper.rb"
+  ]
+  s.homepage = "http://github.com/mikisvaz/scout-rig".freeze
+  s.licenses = ["MIT".freeze]
+  s.rubygems_version = "3.6.8".freeze
+  s.summary = "Scouts rigging things together".freeze
+
+  s.specification_version = 4
+
+  s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0".freeze])
+  s.add_runtime_dependency(%q<pycall>.freeze, ["> 0".freeze])
+end
+

data/test/scout/python/test_script.rb

@@ -0,0 +1,61 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+require 'scout/python/paths'
+class TestPythonScript < Test::Unit::TestCase
+  def test_script
+    result = ScoutPython.script <<-EOF, value: 2
+result = value * 3
+    EOF
+    assert_equal 6, result
+  end
+
+  def test_script_tsv
+
+    tsv = TSV.setup({}, "Key~ValueA,ValueB#:type=:list")
+    tsv["k1"] = ["a1", "b1"]
+    tsv["k2"] = ["a2", "b2"]
+
+    TmpFile.with_file(tsv.to_s) do |tsv_file|
+      TmpFile.with_file do |target|
+        result = ScoutPython.script <<-EOF, file: tsv_file, target: target
+import scout
+df = scout.tsv(file)
+result = df.loc["k2", "ValueB"]
+scout.save_tsv(target, df)
+        EOF
+        assert_equal "b2", result
+
+        assert_equal "b2", TSV.open(target, type: :list)["k2"]["ValueB"]
+      end
+
+    end
+  end
+
+  def test_script_tsv_save
+
+    tsv = TSV.setup({}, "Key~ValueA,ValueB#:type=:list")
+    tsv["k1"] = ["a1", "b1"]
+    tsv["k2"] = ["a2", "b2"]
+
+    TmpFile.with_file do |target|
+      result = ScoutPython.script <<-EOF, df: tsv, target: target
+result = df.loc["k2", "ValueB"]
+scout.save_tsv(target, df)
+      EOF
+      assert_equal "b2", result
+
+      assert_equal "b2", TSV.open(target, type: :list)["k2"]["ValueB"]
+    end
+  end
+
+  def test_script_exception
+      assert_raises ConcurrentStreamProcessFailed do
+        result = ScoutPython.script <<-EOF
+afsdfasdf
+        EOF
+      end
+  end
+end
+

data/test/scout/python/test_util.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/python'
+class TestPythonUtil < Test::Unit::TestCase
+
+  def test_tuple
+    tsv = TSV.setup([], :key_field => "Key", :fields => %w(Value1 Value2), :type => :list)
+    tsv["k1"] = %w(V1_1 V2_1)
+    tsv["k2"] = %w(V1_2 V2_2)
+    df = ScoutPython.tsv2df(tsv)
+    new_tsv = ScoutPython.df2tsv(df)
+    assert_equal tsv, new_tsv
+  end
+
+  def test_numpy
+    ra = ScoutPython.run :numpy, :as => :np do
+      na = np.array([[[1,2,3], [4,5,6]]])
+      ScoutPython.numpy2ruby na
+    end
+    assert_equal 6, ra[0][1][2]
+  end
+
+end
+

data/test/scout/test_python.rb

@@ -0,0 +1,158 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestPython < Test::Unit::TestCase
+
+  def _test_python
+    TmpFile.with_file do |tmpdir|
+      code =<<-EOF
+def python_test(a, b):
+	c = a + b
+	return c
+      EOF
+      Open.write(File.join(tmpdir, 'file1.py'), code)
+      Open.write(File.join(tmpdir, 'file2.py'), code)
+      Open.write(File.join(tmpdir, 'file3.py'), code)
+      Open.write(File.join(tmpdir, 'file4.py'), code)
+      ScoutPython.add_path tmpdir
+
+      res = nil
+
+      ScoutPython.run 'file2', :python_test do 
+        res = python_test(1, 3)
+      end
+      assert_equal 4, res
+
+      ScoutPython.run do
+        pyfrom :file3, :import => :python_test
+        res = python_test(1, 4)
+      end
+      assert_equal 5, res
+
+      ScoutPython.run do
+        pyimport :file4
+        res = file4.python_test(1, 4)
+      end
+      assert_equal 5, res
+
+      ScoutPython.run 'file1' do 
+        res = file1.python_test(1, 2)
+      end
+      assert_equal 3, res
+    end
+  end
+
+  def _test_run_log
+    TmpFile.with_file do |tmpdir|
+      code =<<-EOF
+import sys
+def python_print():
+	print("Test STDERR", file=sys.stderr)
+	print("Test STDOUT")
+      EOF
+      Open.write(File.join(tmpdir, 'file_print.py'), code)
+      ScoutPython.add_path tmpdir
+
+      ScoutPython.run_log 'file_print' do 
+        file_print.python_print
+      end
+      ScoutPython.run_log_stderr 'file_print' do 
+        file_print.python_print
+      end
+
+      ScoutPython.run_log 'file_print' do 
+        file_print.python_print
+      end
+    end
+  end
+
+  def _test_keras
+    keyword_test :tensorflow do
+      defined = ScoutPython.run do
+        pyimport "tensorflow.keras.models", as: :km
+        defined?(km.Sequential)
+      end
+      assert defined
+    end
+  end
+
+  def _test_keras_import
+    keyword_test :tensorflow do
+      defined = ScoutPython.run do
+        pyfrom "tensorflow.keras.models", import: :Sequential
+        defined?(self::Sequential)
+      end
+      assert defined
+    end
+  end
+
+  def _test_iterate
+    a2, b2 = nil, nil
+    ScoutPython.run :numpy, as: :np do 
+      a = np.array([1,2])
+      a2 = ScoutPython.collect a do |e|
+        e * 2
+      end
+      b = PyCall.tuple([1,2])
+      b2 = ScoutPython.collect b do |e|
+        e * 2
+      end
+    end
+    assert_equal [2,4], a2
+    assert_equal [2,4], b2
+  end
+
+  def _test_lambda
+    l = PyCall.eval "lambda e: e + 2"
+    assert_equal 5, l.(3)
+  end
+
+  def test_binding
+    raised = false
+    ScoutPython.binding_run do
+      pyimport :torch
+      pyfrom :torch, import: ["nn"]
+      begin
+        torch
+      rescue
+        raised = true
+      end
+    end
+    assert ! raised
+
+    raised = false
+    ScoutPython.binding_run do
+      begin
+        torch
+      rescue
+        raised = true
+      end
+    end
+    assert raised
+  end
+
+  def _test_import_method
+    random = ScoutPython.import_method :torch, :rand, :random
+    assert random.call(1).numpy.to_f > 0
+  end
+
+  def _test_class_new_obj
+    obj = ScoutPython.class_new_obj("torch.nn", "Module")
+    assert_equal "Module()", obj.to_s
+  end
+
+  def _test_single
+    a = ScoutPython.run_direct :numpy do
+      numpy.array([1,2])
+    end
+    assert a.methods.include? :__pyptr__
+  end
+
+  def _test_threaded
+    a = ScoutPython.run_threaded :numpy do
+      numpy.array([1,2])
+    end
+    assert a.methods.include? :__pyptr__
+  end
+end
+