scout-gear 10.7.3 → 10.7.5

data/test/scout/test_association.rb

@@ -61,5 +61,20 @@ class TestAssociation < Test::Unit::TestCase
     database = Association.database(datadir_test.person.brothers, source: "Older=~Older (Alias)=>Name", persist: true)
     assert database.respond_to?(:persistence_path)
   end
+
+  def test_extra_options
+    file=<<-EOF
+#: :extra_option=:test
+#Key,Value
+k,v
+    EOF
+
+    TmpFile.with_path(file.gsub(',', "\t")) do |f|
+      assert_nothing_raised do
+        Association.open(f, target: "Value", source: "Key")
+      end
+    end
+
+  end
 end
 

data/test/scout/test_tsv.rb

@@ -427,4 +427,19 @@ row2    A    B    Id3
       assert_equal %w(a aa aaa), tsv["row1"]
     end
   end
+
+  def test_number_key
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+1    a|aa|aaa    b    Id1|Id2
+2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, field: "ValueB")
+      assert_equal "b", tsv["1"]
+    end
+
+
+  end
 end

data/test/scout/tsv/test_parser.rb

@@ -317,4 +317,8 @@ k4    a|A    b|B
       assert Open.read(tmp_logfile).include?("directly into")
     end
   end
+
+  def test_acceptable_parser_options
+    assert_include TSV.acceptable_parser_options, :namespace
+  end
 end

data/test/scout/tsv/test_transformer.rb

@@ -153,4 +153,17 @@ row2    A2    B2
     tsv = TSV.open(content)
     assert_equal %w(A1), tsv.to_double["row1"]["ValueA"]
   end
+
+  def test_head
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1    B1
+row2    A2    B2
+row3    A3    B3
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal ["row1", "row2"], tsv.head(2).keys
+  end
 end

data/test/scout/workflow/step/test_info.rb

@@ -3,6 +3,17 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 require 'scout/workflow'
 
 class TestStepInfo < Test::Unit::TestCase
+  def test_benchmark
+    i = {a:1, b: [1,2], c: "String"}
+    times = 100000
+    Misc.benchmark(times) do
+      Marshal.dump(i)
+    end
+    Misc.benchmark(times) do
+      i.to_json
+    end
+  end
+
   def test_dependency
     TmpFile.with_file do |tmpdir|
       Path.setup(tmpdir)

data/test/scout/workflow/task/test_dependencies.rb

@@ -120,7 +120,7 @@ class TestTaskDependencies < Test::Unit::TestCase
     job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
     assert_equal 5, job.run
 
-    TmpFile.with_file(4) do |file|
+    TmpFile.with_file("4") do |file|
       job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
       assert_equal 6, job.run
       assert_not_equal Task::DEFAULT_NAME, job.name
@@ -238,7 +238,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1  do |id,options|
+      dep :step1, :input1 => "1"  do |id,options|
         {:inputs => options}
       end
       task :step2 => :string do |i| step(:step1).load end
@@ -271,7 +271,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1  do |id,options|
+      dep :step1, :input1 => "1"  do |id,options|
         [{:inputs => options}]
       end
       input :input2, :string
@@ -289,7 +289,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1
+      dep :step1, :input1 => "1"
       input :input2, :string
       task :step2 => :string do |i| step(:step1).load end
     end
@@ -304,9 +304,9 @@ class TestTaskDependencies < Test::Unit::TestCase
     wf = Workflow.annonymous_workflow "TaskInputs" do
       input :input1, :integer, "", 1
       input :input2, :integer, "", 0
-      task :step1 => :string do |i1,i2| i1 + i2 end
+      task :step1 => :string do |i1,i2| (i1 + i2).to_s end
 
-      dep :step1, :input2 => 1
+      dep :step1, :input2 => "1"
       task :step2 => :string do |i| step(:step1).load end
 
       dep :step2
@@ -334,7 +334,7 @@ class TestTaskDependencies < Test::Unit::TestCase
         if i1 < 0
           raise ScoutException
         else
-          i1
+          i1.to_s
         end
       end
 

data/test/scout/workflow/task/test_info.rb

@@ -0,0 +1,22 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestWorkflowInfo < Test::Unit::TestCase
+  def test_task_info
+    m = Module.new do
+      extend Workflow
+      self.name = "TestWF"
+
+      input :option1
+      task :step1 do end
+
+      dep :step1
+      input :option2
+      task :step2 do end
+    end
+    info = m.task_info :step2
+    assert_equal "TestWF/step2", info[:id]
+  end
+end
+

data/test/scout/workflow/test_definition.rb

@@ -64,7 +64,7 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     refute Open.exist?(dep_path)
     Scout::Config::CACHE.replace old_cache
     assert_include job.archived_info, dep_path
-    assert_equal :done, job.archived_info[dep_path][:status]
+    assert_equal :done, job.archived_info[dep_path][:status].to_sym
   end
 
   def test_task_alias_remove_dep_partial
@@ -95,7 +95,7 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     assert call_name.done?
     Scout::Config::CACHE.replace old_cache
     assert_include job.archived_info, call_name.path
-    assert_equal :done, job.archived_info[call_name.path][:status]
+    assert_equal :done, job.archived_info[call_name.path][:status].to_sym
   end
 end
 

data/test/scout/workflow/test_entity.rb

@@ -0,0 +1,58 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflowEntity < Test::Unit::TestCase
+  def get_EWF
+    @m ||= Module.new do
+      extend EntityWorkflow
+
+      self.name = 'TestEWF'
+
+      property :introduction do
+        "Mi name is #{self}"
+      end
+
+      entity_job hi: :string do
+        "Hi. #{entity.introduction}"
+      end
+
+      list_job group_hi: :string do
+        "Here is the group: " + entity_list.hi * "; "
+      end
+
+      list_job bye: :array do
+        entity_list.collect do |e|
+          "Bye from #{e}"
+        end
+      end
+    end
+  end
+
+  def test_property_job
+    ewf = get_EWF
+
+    e = ewf.setup("Miki")
+
+    assert_equal "Mi name is Miki", e.introduction
+    assert_equal "Hi. Mi name is Miki", e.hi
+  end
+
+  def test_list
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.hi.length
+
+    assert_include l.group_hi, "group: "
+  end
+
+  def test_multiple
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.bye.length
+  end
+end
+

data/test/scout/workflow/test_step.rb

@@ -387,7 +387,7 @@ class TestWorkflowStep < Test::Unit::TestCase
       step1.fork(false, sem)
       step2.fork(false, sem)
       sleep 1
-      assert((step1.status == :queue) || (step2.status == :queue))
+      assert((step1.status.to_sym == :queue) || (step2.status.to_sym == :queue))
       step1.join
       step2.join
       assert_equal "done2", step2.run

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.3
+  version: 10.7.5
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-01-17 00:00:00.000000000 Z
+date: 2025-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -66,34 +65,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: juwelier
   requirement: !ruby/object:Gem::Requirement
@@ -108,22 +79,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.0
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Temporary files, logs, path, resources, persistence, workflows, TSV,
-  etc.
+description: 'Scout gear: workflow, TSVs, persistence, entities, associations, and
+  knowledge_bases.'
 email: mikisvaz@gmail.com
 executables:
 - scout
@@ -163,11 +120,6 @@ files:
 - lib/scout/knowledge_base/query.rb
 - lib/scout/knowledge_base/registry.rb
 - lib/scout/knowledge_base/traverse.rb
-- lib/scout/offsite.rb
-- lib/scout/offsite/exceptions.rb
-- lib/scout/offsite/ssh.rb
-- lib/scout/offsite/step.rb
-- lib/scout/offsite/sync.rb
 - lib/scout/persist/engine.rb
 - lib/scout/persist/engine/fix_width_table.rb
 - lib/scout/persist/engine/packed_index.rb
@@ -217,7 +169,9 @@ files:
 - lib/scout/workflow/deployment/orchestrator.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
+- lib/scout/workflow/entity.rb
 - lib/scout/workflow/exceptions.rb
+- lib/scout/workflow/export.rb
 - lib/scout/workflow/path.rb
 - lib/scout/workflow/step.rb
 - lib/scout/workflow/step/archive.rb
@@ -233,6 +187,7 @@ files:
 - lib/scout/workflow/step/status.rb
 - lib/scout/workflow/task.rb
 - lib/scout/workflow/task/dependencies.rb
+- lib/scout/workflow/task/info.rb
 - lib/scout/workflow/task/inputs.rb
 - lib/scout/workflow/usage.rb
 - lib/scout/workflow/util.rb
@@ -252,7 +207,6 @@ files:
 - scout_commands/kb/show
 - scout_commands/kb/traverse
 - scout_commands/log
-- scout_commands/offsite
 - scout_commands/rbbt
 - scout_commands/resource/produce
 - scout_commands/template
@@ -287,10 +241,6 @@ files:
 - test/scout/knowledge_base/test_query.rb
 - test/scout/knowledge_base/test_registry.rb
 - test/scout/knowledge_base/test_traverse.rb
-- test/scout/offsite/test_ssh.rb
-- test/scout/offsite/test_step.rb
-- test/scout/offsite/test_sync.rb
-- test/scout/offsite/test_task.rb
 - test/scout/persist/engine/test_fix_width_table.rb
 - test/scout/persist/engine/test_packed_index.rb
 - test/scout/persist/engine/test_sharder.rb
@@ -308,7 +258,6 @@ files:
 - test/scout/test_association.rb
 - test/scout/test_entity.rb
 - test/scout/test_knowledge_base.rb
-- test/scout/test_offsite.rb
 - test/scout/test_semaphore.rb
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
@@ -347,9 +296,11 @@ files:
 - test/scout/workflow/step/test_provenance.rb
 - test/scout/workflow/step/test_status.rb
 - test/scout/workflow/task/test_dependencies.rb
+- test/scout/workflow/task/test_info.rb
 - test/scout/workflow/task/test_inputs.rb
 - test/scout/workflow/test_definition.rb
 - test/scout/workflow/test_documentation.rb
+- test/scout/workflow/test_entity.rb
 - test/scout/workflow/test_path.rb
 - test/scout/workflow/test_step.rb
 - test/scout/workflow/test_task.rb
@@ -362,7 +313,6 @@ homepage: http://github.com/mikisvaz/scout-gear
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -377,8 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.23
-signing_key: 
+rubygems_version: 3.6.5
 specification_version: 4
 summary: basic gear for scouts
 test_files: []

data/lib/scout/offsite/exceptions.rb

@@ -1,9 +0,0 @@
-class SSHProcessFailed < StandardError
-  attr_accessor :host, :cmd
-  def initialize(host, cmd)
-    @host = host
-    @cmd = cmd
-    message = "SSH server #{host} failed cmd '#{cmd}'" 
-    super(message)
-  end
-end

data/lib/scout/offsite/ssh.rb

@@ -1,175 +0,0 @@
-require 'net/ssh'
-require_relative 'exceptions'
-
-class SSHLine
-  class << self
-    attr_accessor :default_server
-    def default_server
-      @@default_server ||= begin
-                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
-                           end
-    end
-  end
-
-  def initialize(host = :default, user = nil)
-    host = SSHLine.default_server if host.nil? || host == :default
-    @host = host
-    @user = user
-
-    @ssh = Net::SSH.start(@host, @user)
-
-    @ch = @ssh.open_channel do |ch|
-      ch.exec 'bash -l'
-    end
-
-    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
-    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
-
-    @ch.on_data do |_,data|
-      if m = data.match(/DONECMD: (\d+)\n/)
-        @exit_status = m[1].to_i
-        @output << data.sub(m[0],'')
-        serve_output 
-      else
-        @output << data
-      end
-    end
-
-    @ch.on_extended_data do |_,c,err|
-      STDERR.write err 
-    end
-  end
-
-
-  def self.reach?(server = SSHLine.default_server)
-    Persist.memory(server, :key => "Reach server") do
-      begin
-        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
-        true
-      rescue Exception
-        false
-      end
-    end
-  end
-
-  def send_cmd(command)
-    @output = ""
-    @complete_output = false
-    @ch.send_data(command+"\necho DONECMD: $?\n")
-  end
-
-  def serve_output
-    @complete_output = true
-  end
-
-  def run(command)
-    send_cmd(command)
-    @ssh.loop{ ! @complete_output}
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, command
-    end
-  end
-
-  def ruby(script)
-    @output = ""
-    @complete_output = false
-    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-    Log.debug "Running ruby on #{@host}:\n#{ script }"
-    @ch.send_data(cmd)
-    @ch.send_data("echo DONECMD: $?\n")
-    @ssh.loop{ !@complete_output }
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
-    end
-  end
-
-  def scout(script)
-    scout_script =<<-EOF
-require 'scout'
-SSHLine.run_local do
-#{script.strip}
-end
-    EOF
-
-    m = ruby(scout_script)
-    Marshal.load m
-  end
-
-  def workflow(workflow, script)
-    preamble =<<-EOF
-wf = Workflow.require_workflow('#{workflow}')
-    EOF
-
-    scout(preamble + "\n" + script)
-  end
-
-  class Mock < SSHLine
-    def initialize
-    end
-
-    def run(command)
-      CMD.cmd(command)
-    end
-
-    def ruby(script)
-      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-      CMD.cmd(cmd)
-    end
-  end
-
-  @connections = {}
-  def self.open(host, user = nil)
-    @connections[[host, user]] ||=
-      begin
-        if host == 'localhost'
-          SSHLine::Mock.new
-        else
-          SSHLine.new host, user
-        end
-      end
-  end
-
-  def self.run(server, cmd, options = nil)
-    cmd = cmd * " " if Array === cmd
-    cmd += " " + CMD.process_cmd_options(options) if options
-    open(server).run(cmd)
-  end
-
-  def self.ruby(server, script)
-    open(server).ruby(script)
-  end
-
-  def self.scout(server, script)
-    open(server).scout(script)
-  end
-
-  def self.workflow(server, workflow, script)
-    open(server).workflow(workflow, script)
-  end
-
-  def self.command(server, command, argv = [], options = nil)
-    command = "scout #{command}" unless command && command.include?('scout')
-    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
-    command = "#{command} #{argv_str}"
-    Log.debug "Offsite #{server} running: #{command}"
-    run(server, command, options)
-  end
-
-  def self.mkdir(server, path)
-    self.run server, "mkdir -p '#{path}'"
-  end
-
-  def self.run_local(&block)
-    res = begin
-            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
-            block.call
-          ensure
-            STDOUT.reopen(old_stdout)
-          end
-    puts Marshal.dump(res)
-  end
-end

data/lib/scout/offsite/step.rb

@@ -1,100 +0,0 @@
-require_relative '../workflow/step'
-require_relative 'ssh'
-require_relative 'sync'
-
-module OffsiteStep
-
-  extend Annotation
-  annotation :server, :workflow_name, :clean_id, :slurm
-
-  def inputs_directory
-    @inputs_directory ||= begin
-                            if provided_inputs && provided_inputs.any?
-                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
-                              TmpFile.with_path do |inputs_dir|
-                                save_inputs(inputs_dir)
-                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
-                              end
-                              file
-                            end
-                          end
-  end
-
-  def workflow_name
-    @workflow_name || workflow.to_s
-  end
-
-  def offsite_job_ssh(script)
-    parts = []
-    parts << <<~EOF.strip
-wf = Workflow.require_workflow "#{workflow_name}";
-    EOF
-
-    if inputs_directory
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
-      EOF
-    else
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}");
-      EOF
-    end
-
-    parts << script
-
-
-    SSHLine.scout server, parts * "\n"
-  end
-
-  def offsite_path
-    @path = offsite_job_ssh <<~EOF
-      job.path.identify
-    EOF
-  end
-
-  def info
-    info = @info ||= offsite_job_ssh <<~EOF
-    info = Open.exists?(job.info_file) ? job.info : {}
-    info[:running] = true if job.running?
-    info
-    EOF
-
-    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
-
-    info
-  end
-
-  def done?
-    status == :done
-  end
-
-  def orchestrate_slurm
-    bundle_files = offsite_job_ssh <<~EOF
-    require 'rbbt/hpc'
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, {})
-    job.join
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-
-  def exec
-    bundle_files = offsite_job_ssh <<~EOF
-    job.run
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-  def run
-    if slurm
-      orchestrate_slurm
-    else
-      exec
-    end
-  end
-end