scout-gear 10.7.4 → 10.7.5

data/test/scout/workflow/test_entity.rb

@@ -0,0 +1,58 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflowEntity < Test::Unit::TestCase
+  def get_EWF
+    @m ||= Module.new do
+      extend EntityWorkflow
+
+      self.name = 'TestEWF'
+
+      property :introduction do
+        "Mi name is #{self}"
+      end
+
+      entity_job hi: :string do
+        "Hi. #{entity.introduction}"
+      end
+
+      list_job group_hi: :string do
+        "Here is the group: " + entity_list.hi * "; "
+      end
+
+      list_job bye: :array do
+        entity_list.collect do |e|
+          "Bye from #{e}"
+        end
+      end
+    end
+  end
+
+  def test_property_job
+    ewf = get_EWF
+
+    e = ewf.setup("Miki")
+
+    assert_equal "Mi name is Miki", e.introduction
+    assert_equal "Hi. Mi name is Miki", e.hi
+  end
+
+  def test_list
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.hi.length
+
+    assert_include l.group_hi, "group: "
+  end
+
+  def test_multiple
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.bye.length
+  end
+end
+

data/test/scout/workflow/test_step.rb

@@ -387,7 +387,7 @@ class TestWorkflowStep < Test::Unit::TestCase
       step1.fork(false, sem)
       step2.fork(false, sem)
       sleep 1
-      assert((step1.status == :queue) || (step2.status == :queue))
+      assert((step1.status.to_sym == :queue) || (step2.status.to_sym == :queue))
       step1.join
       step2.join
       assert_equal "done2", step2.run

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.4
+  version: 10.7.5
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-03-10 00:00:00.000000000 Z
+date: 2025-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -65,34 +65,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: juwelier
   requirement: !ruby/object:Gem::Requirement
@@ -107,22 +79,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.0
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Temporary files, logs, path, resources, persistence, workflows, TSV,
-  etc.
+description: 'Scout gear: workflow, TSVs, persistence, entities, associations, and
+  knowledge_bases.'
 email: mikisvaz@gmail.com
 executables:
 - scout
@@ -162,11 +120,6 @@ files:
 - lib/scout/knowledge_base/query.rb
 - lib/scout/knowledge_base/registry.rb
 - lib/scout/knowledge_base/traverse.rb
-- lib/scout/offsite.rb
-- lib/scout/offsite/exceptions.rb
-- lib/scout/offsite/ssh.rb
-- lib/scout/offsite/step.rb
-- lib/scout/offsite/sync.rb
 - lib/scout/persist/engine.rb
 - lib/scout/persist/engine/fix_width_table.rb
 - lib/scout/persist/engine/packed_index.rb
@@ -216,7 +169,9 @@ files:
 - lib/scout/workflow/deployment/orchestrator.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
+- lib/scout/workflow/entity.rb
 - lib/scout/workflow/exceptions.rb
+- lib/scout/workflow/export.rb
 - lib/scout/workflow/path.rb
 - lib/scout/workflow/step.rb
 - lib/scout/workflow/step/archive.rb
@@ -252,7 +207,6 @@ files:
 - scout_commands/kb/show
 - scout_commands/kb/traverse
 - scout_commands/log
-- scout_commands/offsite
 - scout_commands/rbbt
 - scout_commands/resource/produce
 - scout_commands/template
@@ -287,10 +241,6 @@ files:
 - test/scout/knowledge_base/test_query.rb
 - test/scout/knowledge_base/test_registry.rb
 - test/scout/knowledge_base/test_traverse.rb
-- test/scout/offsite/test_ssh.rb
-- test/scout/offsite/test_step.rb
-- test/scout/offsite/test_sync.rb
-- test/scout/offsite/test_task.rb
 - test/scout/persist/engine/test_fix_width_table.rb
 - test/scout/persist/engine/test_packed_index.rb
 - test/scout/persist/engine/test_sharder.rb
@@ -308,7 +258,6 @@ files:
 - test/scout/test_association.rb
 - test/scout/test_entity.rb
 - test/scout/test_knowledge_base.rb
-- test/scout/test_offsite.rb
 - test/scout/test_semaphore.rb
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
@@ -351,6 +300,7 @@ files:
 - test/scout/workflow/task/test_inputs.rb
 - test/scout/workflow/test_definition.rb
 - test/scout/workflow/test_documentation.rb
+- test/scout/workflow/test_entity.rb
 - test/scout/workflow/test_path.rb
 - test/scout/workflow/test_step.rb
 - test/scout/workflow/test_task.rb

data/lib/scout/offsite/exceptions.rb

@@ -1,9 +0,0 @@
-class SSHProcessFailed < StandardError
-  attr_accessor :host, :cmd
-  def initialize(host, cmd)
-    @host = host
-    @cmd = cmd
-    message = "SSH server #{host} failed cmd '#{cmd}'" 
-    super(message)
-  end
-end

data/lib/scout/offsite/ssh.rb

@@ -1,175 +0,0 @@
-require 'net/ssh'
-require_relative 'exceptions'
-
-class SSHLine
-  class << self
-    attr_accessor :default_server
-    def default_server
-      @@default_server ||= begin
-                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
-                           end
-    end
-  end
-
-  def initialize(host = :default, user = nil)
-    host = SSHLine.default_server if host.nil? || host == :default
-    @host = host
-    @user = user
-
-    @ssh = Net::SSH.start(@host, @user)
-
-    @ch = @ssh.open_channel do |ch|
-      ch.exec 'bash -l'
-    end
-
-    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
-    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
-
-    @ch.on_data do |_,data|
-      if m = data.match(/DONECMD: (\d+)\n/)
-        @exit_status = m[1].to_i
-        @output << data.sub(m[0],'')
-        serve_output 
-      else
-        @output << data
-      end
-    end
-
-    @ch.on_extended_data do |_,c,err|
-      STDERR.write err 
-    end
-  end
-
-
-  def self.reach?(server = SSHLine.default_server)
-    Persist.memory(server, :key => "Reach server") do
-      begin
-        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
-        true
-      rescue Exception
-        false
-      end
-    end
-  end
-
-  def send_cmd(command)
-    @output = ""
-    @complete_output = false
-    @ch.send_data(command+"\necho DONECMD: $?\n")
-  end
-
-  def serve_output
-    @complete_output = true
-  end
-
-  def run(command)
-    send_cmd(command)
-    @ssh.loop{ ! @complete_output}
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, command
-    end
-  end
-
-  def ruby(script)
-    @output = ""
-    @complete_output = false
-    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-    Log.debug "Running ruby on #{@host}:\n#{ script }"
-    @ch.send_data(cmd)
-    @ch.send_data("echo DONECMD: $?\n")
-    @ssh.loop{ !@complete_output }
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
-    end
-  end
-
-  def scout(script)
-    scout_script =<<-EOF
-require 'scout'
-SSHLine.run_local do
-#{script.strip}
-end
-    EOF
-
-    m = ruby(scout_script)
-    Marshal.load m
-  end
-
-  def workflow(workflow, script)
-    preamble =<<-EOF
-wf = Workflow.require_workflow('#{workflow}')
-    EOF
-
-    scout(preamble + "\n" + script)
-  end
-
-  class Mock < SSHLine
-    def initialize
-    end
-
-    def run(command)
-      CMD.cmd(command)
-    end
-
-    def ruby(script)
-      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-      CMD.cmd(cmd)
-    end
-  end
-
-  @connections = {}
-  def self.open(host, user = nil)
-    @connections[[host, user]] ||=
-      begin
-        if host == 'localhost'
-          SSHLine::Mock.new
-        else
-          SSHLine.new host, user
-        end
-      end
-  end
-
-  def self.run(server, cmd, options = nil)
-    cmd = cmd * " " if Array === cmd
-    cmd += " " + CMD.process_cmd_options(options) if options
-    open(server).run(cmd)
-  end
-
-  def self.ruby(server, script)
-    open(server).ruby(script)
-  end
-
-  def self.scout(server, script)
-    open(server).scout(script)
-  end
-
-  def self.workflow(server, workflow, script)
-    open(server).workflow(workflow, script)
-  end
-
-  def self.command(server, command, argv = [], options = nil)
-    command = "scout #{command}" unless command && command.include?('scout')
-    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
-    command = "#{command} #{argv_str}"
-    Log.debug "Offsite #{server} running: #{command}"
-    run(server, command, options)
-  end
-
-  def self.mkdir(server, path)
-    self.run server, "mkdir -p '#{path}'"
-  end
-
-  def self.run_local(&block)
-    res = begin
-            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
-            block.call
-          ensure
-            STDOUT.reopen(old_stdout)
-          end
-    puts Marshal.dump(res)
-  end
-end

data/lib/scout/offsite/step.rb

@@ -1,100 +0,0 @@
-require_relative '../workflow/step'
-require_relative 'ssh'
-require_relative 'sync'
-
-module OffsiteStep
-
-  extend Annotation
-  annotation :server, :workflow_name, :clean_id, :slurm
-
-  def inputs_directory
-    @inputs_directory ||= begin
-                            if provided_inputs && provided_inputs.any?
-                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
-                              TmpFile.with_path do |inputs_dir|
-                                save_inputs(inputs_dir)
-                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
-                              end
-                              file
-                            end
-                          end
-  end
-
-  def workflow_name
-    @workflow_name || workflow.to_s
-  end
-
-  def offsite_job_ssh(script)
-    parts = []
-    parts << <<~EOF.strip
-wf = Workflow.require_workflow "#{workflow_name}";
-    EOF
-
-    if inputs_directory
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
-      EOF
-    else
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}");
-      EOF
-    end
-
-    parts << script
-
-
-    SSHLine.scout server, parts * "\n"
-  end
-
-  def offsite_path
-    @path = offsite_job_ssh <<~EOF
-      job.path.identify
-    EOF
-  end
-
-  def info
-    info = @info ||= offsite_job_ssh <<~EOF
-    info = Open.exists?(job.info_file) ? job.info : {}
-    info[:running] = true if job.running?
-    info
-    EOF
-
-    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
-
-    info
-  end
-
-  def done?
-    status == :done
-  end
-
-  def orchestrate_slurm
-    bundle_files = offsite_job_ssh <<~EOF
-    require 'rbbt/hpc'
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, {})
-    job.join
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-
-  def exec
-    bundle_files = offsite_job_ssh <<~EOF
-    job.run
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-  def run
-    if slurm
-      orchestrate_slurm
-    else
-      exec
-    end
-  end
-end

data/lib/scout/offsite/sync.rb

@@ -1,55 +0,0 @@
-class SSHLine
-  def self.locate(server, paths, map: :user)
-    SSHLine.scout server, <<-EOF
-map = :#{map}
-paths = [#{paths.collect{|p| "'" + p + "'" } * ", " }]
-located = paths.collect{|p| Path.setup(p).find(map) }
-identified = paths.collect{|p| Resource.identify(p) }
-[located, identified]
-    EOF
-  end
-
-  def self.rsync(source_path, target_path, directory: false, source: nil, target: nil, dry_run: false, hard_link: false)
-    rsync_args = "-avztHP --copy-unsafe-links --omit-dir-times "
-
-    rsync_args << "--link-dest '#{source_path}' " if hard_link && ! source
-
-    source_path = source_path + "/" if directory && ! source_path.end_with?("/")
-    target_path = target_path + "/" if directory && ! target_path.end_with?("/")
-    if target
-      SSHLine.mkdir target, File.dirname(target_path)
-    else
-      Open.mkdir(File.dirname(target_path))
-    end
-
-    cmd = 'rsync '
-    cmd << rsync_args
-    cmd << '-nv ' if dry_run
-    cmd << (source ? [source, source_path] * ":" : source_path) << " "
-    cmd << (target ? [target, target_path] * ":" : target_path) << " "
-
-    CMD.cmd_log(cmd, :log => Log::HIGH)
-  end
-
-  def self.sync(paths, source: nil, target: nil, map: :user, **kwargs)
-    source = nil if source == 'localhost'
-    target = nil if target == 'localhost'
-
-    if source
-      source_paths, identified_paths = SSHLine.locate(source, paths)
-    else
-      source_paths = paths.collect{|p| Path === p ? p.find : p }
-      identified_paths = paths.collect{|p| Resource.identify(p) }
-    end
-
-    if target
-      target_paths = SSHLine.locate(target, identified_paths, map: map)
-    else
-      target_paths = identified_paths.collect{|p| p.find(map) }
-    end
-
-    source_paths.zip(target_paths).each do |source_path,target_path|
-      rsync(source_path, target_path, source: source, target: target, **kwargs)
-    end
-  end
-end

data/lib/scout/offsite.rb

@@ -1,3 +0,0 @@
-require_relative 'offsite/ssh'
-require_relative 'offsite/step'
-require_relative 'offsite/sync'

data/scout_commands/offsite

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'scout'
-require 'scout/offsite'
-
-$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
-
-options = SOPT.setup <<EOF
-
-Description of the tool
-
-$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
-
-Run a command offsite. Use -- to make sure options reach the offsite
-
--h--help Print this help
-EOF
-if options[:help]
-  if defined? scout_usage
-    scout_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
-end
-
-where = ARGV.shift
-command = ARGV.shift
-
-puts SSHLine.command(where, command, ARGV)

data/test/scout/offsite/test_ssh.rb

@@ -1,15 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestSSH < Test::Unit::TestCase
-  def test_marshal
-    return unless SSHLine.reach?
-
-    assert TrueClass === SSHLine.scout(:default, 'true')
-  end
-
-  def test_localhost
-    assert SSHLine.scout('localhost', 'true')
-  end
-end
-

data/test/scout/offsite/test_step.rb

@@ -1,32 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestOffsiteStep < Test::Unit::TestCase
-  def test_offsite_task
-    workflow_code =<<-EOF
-module TestWF
-  extend Workflow
-
-  input :string, :string, "String", "string"
-  task :string => :string do |string| string end
-end
-
-TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
-    EOF
-
-    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
-      wf = Workflow.require_workflow wffile
-
-      job = wf.job(:string)
-
-      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
-
-      refute off.done?
-      assert_equal 'string', off.run
-
-      assert off.done?
-      assert_equal 'string', off.run
-    end
-  end
-end
-

data/test/scout/offsite/test_sync.rb

@@ -1,36 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestSync < Test::Unit::TestCase
-  def test_sync
-    TmpFile.with_path do |tmpdir|
-      tmpdir = Scout.tmp.tmpdir_sync
-      tmpdir.dir1.foo.write("FOO")
-      tmpdir.dir1.bar.write("BAR")
-
-      TmpFile.with_path do |tmpdir2|
-        Misc.in_dir tmpdir2 do
-          SSHLine.sync([tmpdir.dir1], map: :current)
-
-          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
-        end
-      end
-    end
-  end
-
-  def test_sync_dir_map
-    TmpFile.with_path do |tmpdir|
-      tmpdir = Scout.tmp.tmpdir_sync
-      tmpdir.dir1.foo.write("FOO")
-      tmpdir.dir1.bar.write("BAR")
-
-      TmpFile.with_path do |tmpdir2|
-        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
-        Misc.in_dir tmpdir2 do
-          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
-        end
-      end
-    end
-  end
-end
-