scout-gear 10.1.0 → 10.3.0

data/scout_commands/workflow/prov

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Examine the provenance of a job result
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+-h--help Help
+-p--plot* draw the dependency plot into <file.png>
+-i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
+-t--touch Update modification times to be consistent
+-e--expand_repeats Show all the dependency tree even if reapeated dependencies have already been seen before
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+$inputs = (options[:inputs] || "").split(",")
+$info_fields = (options[:info_fields] || "").split(",")
+
+file = ARGV.shift
+
+$seen = []
+def get_step(file)
+  file = File.expand_path(file) if File.exist?(file)
+  file = file.sub(/\.(info|files)$/,'')
+  $seen << file
+  Step.new file
+end
+
+def touch(step)
+  return unless File.exist?(step.path)
+  step.dependencies.each do |dep|
+    next unless Open.exists?(dep.path)
+    if Open.mtime(dep.path) > Open.mtime(step.path) + 1
+      Log.debug("Updating #{step.path} to #{dep.path}")
+      Open.update_mtime(dep.path, step.path)
+    end
+    touch(dep)
+  end if step.dependencies
+end
+
+step = get_step file
+$main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
+
+def adjacency(step)
+
+  info = step.info || {}
+  path  = step.path
+  status = info[:status] || :missing
+  status = "remote" if Open.remote?(path)
+  if status == 'remote'
+    workflow, task, name = path.sub(/\{.*/,'').split("/")[-3..-1]
+  else
+    workflow, task, name = path.split("/")[-3..-1]
+  end
+  name = name.split(":").last
+  status = :unsync if status == :done and not Open.exist? path
+  shapes = Hash.new "circle"
+  edge_info = {:status =>  status, :workflow => workflow, :task => task, :name => name, :label => task, :shape => shapes[workflow], :color => status == 'remote' ? 'blue' : 'green'}
+  id = Misc.digest(path)
+  edges = []
+  node_info = {}
+  node_info[id] = edge_info
+  if info[:dependencies]
+    info[:dependencies].each do |task,name,path|
+      dep = get_step path
+      _id, _edges, _node_info = adjacency(dep)
+      edges << [_id, id] 
+      edges.concat _edges
+      node_info.merge!(_node_info)
+    end 
+  end
+
+  [id, edges, node_info]
+end
+
+if options[:touch]
+  touch(step)
+end
+
+if options[:plot]
+  id, edges, node_info = adjacency(step)
+  node_info[id][:color] = 'red'
+  TmpFile.with_file do |edge_file|
+    Open.write(edge_file) do |f|
+      f.puts "from,to"
+      edges.uniq.each do |from,to|
+        f.puts [from,to]*","
+      end
+    end
+    TmpFile.with_file do |node_info_file|
+      Open.write(node_info_file) do |f|
+        fields = node_info.first.last.keys
+        f.puts "id," + fields * ","
+        node_info.each do |id,info|
+          f.puts ([id] + info.values_at(*fields)) * ","
+        end
+      end
+
+      require 'rbbt/util/R'
+
+      R.run <<-EOF
+      nodes <- read.csv("#{node_info_file}", header=T, as.is=T)
+      links <- read.csv("#{edge_file}", header=T, as.is=T)
+
+      rbbt.require('igraph')
+
+      net <- graph.data.frame(links, nodes, directed=T)
+      net <- simplify(net, remove.multiple = F, remove.loops = T)
+
+      png("#{options[:plot]}", width=1000, height=1000)
+      plot(net, edge.arrow.size=0.4, vertex.label=net$label, vertex.color=net$color)
+      dev.off()
+      EOF
+    end
+  end
+
+else
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
+end
+

data/scout_commands/workflow/task

@@ -21,7 +21,7 @@ $ #{$0} [<options>] <workflow> <task>
 --deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 -jn--jobname* Name to use as job identifier
 -li--load_inputs* Directory with inputs files to load
--pf--print_filepath Print the file path
+-pf--printpath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
@@ -39,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps,
   :deploy => 'serial'
 
 if help
@@ -102,8 +102,9 @@ else
   when "slurm"
     require 'rbbt-scout'
     require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
+    slurm_options = {}
     HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    HPC::BATCH_MODULE.orchestrate_job(job, slurm_options)
     job.grace
   else 
     if deploy.end_with?('-slurm')
@@ -116,7 +117,7 @@ else
     job.run
   end unless job.done?
 
-  if print_filepath
+  if printpath
     job.join
     path = job.path
     path = path.find if Path === path

data/scout_commands/workflow/trace

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <job-result>
+
+-h--help Help
+-fg--fix_gap Remove execution gaps
+-rk--report_keys* Config keys and info fields to report
+-p--plot* Plot file
+-w--width* Image Width
+-h--height* Image Height
+-s--size* Image Size (Height and Width)
+-pd--plot_data Print plot data
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'rbbt/workflow/util/trace'
+require 'rbbt/util/R'
+
+
+files = ARGV
+plot = options[:plot]
+
+def get_step(file)
+  file = File.expand_path(file)
+  file = file.sub(/\.(info|files)/,'')
+  Step.new file
+end
+
+jobs = []
+jobs = files.collect do |file|
+  get_step file
+end
+
+puts Workflow.trace(jobs, options)

data/scout_commands/workflow/write_info

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the info of a job result
+
+$ #{$0} <job-result> <key> <value>
+
+-h--help Help
+-f--force Write info even if key is already present
+-r--recursive Write info for all dependencies as well
+-p--check_pid Check that recursive jobs where created by the same process
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+file, key, value = ARGV
+
+force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
+
+def get_step(file)
+  file = file.sub(/\.(info|files)/,'')
+  step = Step.new file
+  step
+end
+
+raise ParameterException if key.nil? || value.nil?
+
+if %w(DELETE nil).include? value
+  value = nil
+  force = true
+end
+
+step = get_step file
+
+step.set_info key, value if force || ! step.info.include?(key)
+
+pid = step.info[:pid]
+host = step.info[:pid_hostname]
+
+step.rec_dependencies.each do |dep|
+  begin
+    dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
+  rescue
+    Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+  end
+end if recursive
+
+if recursive && step.info[:archived_info]
+  ad = step.info[:archived_info]
+  ad.each do |d,info|
+    begin
+      info[key] = value if (force || ! info.include?(key)) && (!check_pid || info[:pid].to_s == pid and info[:pid_hostname] == host)
+    rescue
+      Log.warn "Could no set info #{key} for archived_dep #{info[:path]}: #{$!.message}"
+    end 
+  end
+  step.set_info :archived_info, ad
+end

data/test/scout/offsite/test_ssh.rb

@@ -5,7 +5,7 @@ class TestSSH < Test::Unit::TestCase
   def test_marshal
     return unless SSHLine.reach?
 
-    assert TrueClass === SSHLine.rbbt(:default, 'true')
+    assert TrueClass === SSHLine.scout(:default, 'true')
   end
 
   def test_localhost

data/test/scout/test_tsv.rb

@@ -85,6 +85,27 @@ row2    a    a    id3
     assert_equal %w(A a), tsv["row2"][0]
   end
 
+  def test_open_persist_path
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    TmpFile.with_file do |persist_path|
+      orig = TmpFile.with_file(content) do |filename|
+        TSV.open(filename, :persist => true, :merge => true, :persist_path => persist_path)
+      end
+
+      tsv = ScoutCabinet.open persist_path, false
+      assert_equal tsv.persistence_path, persist_path
+      assert_equal orig, tsv
+    end
+
+  end
+
   def test_headerless_fields
     content =<<-EOF
 row1    a|aa|aaa    b    Id1|Id2

data/test/scout/tsv/persist/test_tkrzw.rb

@@ -0,0 +1,123 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestScoutTKRZW < Test::Unit::TestCase
+  def _test_open
+    TmpFile.with_file nil do |tmp|
+      db = ScoutTKRZW.open(tmp, true)
+      1000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+      assert_include db, 'foo1'
+      assert_equal 1000, db.keys.length
+
+      db.close
+      TmpFile.with_file nil do |tmp_alt|
+        Open.cp tmp, tmp_alt
+        db2 = ScoutTKRZW.open(tmp_alt, false)
+        assert_include db2, 'foo1'
+        assert_equal 1000, db2.keys.length
+      end
+    end
+  end
+
+  def _test_tkrzw
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      Persist.persist(__method__, 'tkh') do
+        TSV.open(filename)
+      end
+    end
+
+
+    assert_equal %w(a aa aaa), tsv["row1"][0]
+
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+
+    tsv.close
+    tsv_loaded = assert_nothing_raised do
+      TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, 'tkh') do
+          raise
+        end
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+  end
+
+  def __test_benchmark1
+    TmpFile.with_file nil do |tmp|
+
+      Misc.benchmark(1000) do
+        db = ScoutTKRZW.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        10.times do
+          db.keys
+        end
+        10.times do |i|
+          db["foo#{i}"]
+        end
+        Open.rm tmp
+      end
+
+      Misc.benchmark(1000) do
+        db = ScoutCabinet.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        10.times do
+          db.keys
+        end
+        10.times do |i|
+          db["foo#{i}"]
+        end
+        db.close
+        Open.rm tmp
+      end
+    end
+  end
+
+  def test_benchmark2
+    TmpFile.with_file nil do |tmp|
+
+      db = ScoutTKRZW.open(tmp, true)
+      10000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+
+      Misc.benchmark(1000) do
+        100.times do |i|
+          db["foo#{i}"]
+        end
+      end
+
+      Open.rm tmp
+      db = ScoutCabinet.open(tmp, true)
+      10000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+      Misc.benchmark(1000) do
+        100.times do |i|
+          db["foo#{i}"]
+        end
+      end
+    end
+  end
+
+end
+

data/test/scout/tsv/util/test_melt.rb

@@ -0,0 +1,28 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestClass < Test::Unit::TestCase
+  def test_melt
+    txt =<<-EOF
+#: :sep=,#:type=:list
+#Size,System1,System2
+10,20,30
+20,40,60
+    EOF
+    
+    target =<<-EOF
+#: :sep=,#:type=:list
+#ID,Size,Time,System
+10:0,10,20,System1
+10:1,10,30,System2
+20:0,20,40,System1
+20:1,20,60,System2
+    EOF
+
+    tsv = TSV.open(txt)
+    assert_equal tsv.melt_columns("Time", "System"), TSV.open(target)
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.1.0
+  version: 10.3.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-07 00:00:00.000000000 Z
+date: 2023-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -163,12 +163,14 @@ files:
 - lib/scout/tsv/persist/adapter.rb
 - lib/scout/tsv/persist/fix_width_table.rb
 - lib/scout/tsv/persist/serialize.rb
+- lib/scout/tsv/persist/tkrzw.rb
 - lib/scout/tsv/persist/tokyocabinet.rb
 - lib/scout/tsv/stream.rb
 - lib/scout/tsv/transformer.rb
 - lib/scout/tsv/traverse.rb
 - lib/scout/tsv/util.rb
 - lib/scout/tsv/util/filter.rb
+- lib/scout/tsv/util/melt.rb
 - lib/scout/tsv/util/process.rb
 - lib/scout/tsv/util/reorder.rb
 - lib/scout/tsv/util/select.rb
@@ -201,9 +203,12 @@ files:
 - lib/workflow-scout.rb
 - scout-gear.gemspec
 - scout_commands/alias
+- scout_commands/batch/clean
+- scout_commands/batch/list
 - scout_commands/doc
 - scout_commands/find
 - scout_commands/glob
+- scout_commands/log
 - scout_commands/offsite
 - scout_commands/rbbt
 - scout_commands/resource/produce
@@ -212,7 +217,10 @@ files:
 - scout_commands/workflow/info
 - scout_commands/workflow/install
 - scout_commands/workflow/list
+- scout_commands/workflow/prov
 - scout_commands/workflow/task
+- scout_commands/workflow/trace
+- scout_commands/workflow/write_info
 - share/color/color_names
 - share/color/diverging_colors.hex
 - share/software/install_helpers
@@ -229,6 +237,7 @@ files:
 - test/scout/test_workflow.rb
 - test/scout/tsv/persist/test_adapter.rb
 - test/scout/tsv/persist/test_fix_width_table.rb
+- test/scout/tsv/persist/test_tkrzw.rb
 - test/scout/tsv/persist/test_tokyocabinet.rb
 - test/scout/tsv/test_attach.rb
 - test/scout/tsv/test_change_id.rb
@@ -242,6 +251,7 @@ files:
 - test/scout/tsv/test_traverse.rb
 - test/scout/tsv/test_util.rb
 - test/scout/tsv/util/test_filter.rb
+- test/scout/tsv/util/test_melt.rb
 - test/scout/tsv/util/test_process.rb
 - test/scout/tsv/util/test_reorder.rb
 - test/scout/tsv/util/test_select.rb