scout-gear 10.1.0 → 10.2.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,133 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Examine the provenance of a job result
10
+
11
+ $ #{$0} [<options>] <filename> [<other|->]*
12
+
13
+ -h--help Help
14
+ -p--plot* draw the dependency plot into <file.png>
15
+ -i--inputs* List of inputs to print
16
+ -if--info_fields* List of info fields to print
17
+ -t--touch Update modification times to be consistent
18
+ -e--expand_repeats Show all the dependency tree even if reapeated dependencies have already been seen before
19
+ EOF
20
+ if options[:help]
21
+ if defined? scout_usage
22
+ scout_usage
23
+ else
24
+ puts SOPT.doc
25
+ end
26
+ exit 0
27
+ end
28
+
29
+ $inputs = (options[:inputs] || "").split(",")
30
+ $info_fields = (options[:info_fields] || "").split(",")
31
+
32
+ file = ARGV.shift
33
+
34
+ $seen = []
35
+ def get_step(file)
36
+ file = File.expand_path(file) if File.exist?(file)
37
+ file = file.sub(/\.(info|files)$/,'')
38
+ $seen << file
39
+ Step.new file
40
+ end
41
+
42
+ def touch(step)
43
+ return unless File.exist?(step.path)
44
+ step.dependencies.each do |dep|
45
+ next unless Open.exists?(dep.path)
46
+ if Open.mtime(dep.path) > Open.mtime(step.path) + 1
47
+ Log.debug("Updating #{step.path} to #{dep.path}")
48
+ Open.update_mtime(dep.path, step.path)
49
+ end
50
+ touch(dep)
51
+ end if step.dependencies
52
+ end
53
+
54
+ step = get_step file
55
+ $main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
56
+
57
+ def adjacency(step)
58
+
59
+ info = step.info || {}
60
+ path = step.path
61
+ status = info[:status] || :missing
62
+ status = "remote" if Open.remote?(path)
63
+ if status == 'remote'
64
+ workflow, task, name = path.sub(/\{.*/,'').split("/")[-3..-1]
65
+ else
66
+ workflow, task, name = path.split("/")[-3..-1]
67
+ end
68
+ name = name.split(":").last
69
+ status = :unsync if status == :done and not Open.exist? path
70
+ shapes = Hash.new "circle"
71
+ edge_info = {:status => status, :workflow => workflow, :task => task, :name => name, :label => task, :shape => shapes[workflow], :color => status == 'remote' ? 'blue' : 'green'}
72
+ id = Misc.digest(path)
73
+ edges = []
74
+ node_info = {}
75
+ node_info[id] = edge_info
76
+ if info[:dependencies]
77
+ info[:dependencies].each do |task,name,path|
78
+ dep = get_step path
79
+ _id, _edges, _node_info = adjacency(dep)
80
+ edges << [_id, id]
81
+ edges.concat _edges
82
+ node_info.merge!(_node_info)
83
+ end
84
+ end
85
+
86
+ [id, edges, node_info]
87
+ end
88
+
89
+ if options[:touch]
90
+ touch(step)
91
+ end
92
+
93
+ if options[:plot]
94
+ id, edges, node_info = adjacency(step)
95
+ node_info[id][:color] = 'red'
96
+ TmpFile.with_file do |edge_file|
97
+ Open.write(edge_file) do |f|
98
+ f.puts "from,to"
99
+ edges.uniq.each do |from,to|
100
+ f.puts [from,to]*","
101
+ end
102
+ end
103
+ TmpFile.with_file do |node_info_file|
104
+ Open.write(node_info_file) do |f|
105
+ fields = node_info.first.last.keys
106
+ f.puts "id," + fields * ","
107
+ node_info.each do |id,info|
108
+ f.puts ([id] + info.values_at(*fields)) * ","
109
+ end
110
+ end
111
+
112
+ require 'rbbt/util/R'
113
+
114
+ R.run <<-EOF
115
+ nodes <- read.csv("#{node_info_file}", header=T, as.is=T)
116
+ links <- read.csv("#{edge_file}", header=T, as.is=T)
117
+
118
+ rbbt.require('igraph')
119
+
120
+ net <- graph.data.frame(links, nodes, directed=T)
121
+ net <- simplify(net, remove.multiple = F, remove.loops = T)
122
+
123
+ png("#{options[:plot]}", width=1000, height=1000)
124
+ plot(net, edge.arrow.size=0.4, vertex.label=net$label, vertex.color=net$color)
125
+ dev.off()
126
+ EOF
127
+ end
128
+ end
129
+
130
+ else
131
+ puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
132
+ end
133
+
@@ -21,7 +21,7 @@ $ #{$0} [<options>] <workflow> <task>
21
21
  --deploy* Deploy mode: serial, local, or SLURM (default 'serial')
22
22
  -jn--jobname* Name to use as job identifier
23
23
  -li--load_inputs* Directory with inputs files to load
24
- -pf--print_filepath Print the file path
24
+ -pf--printpath Print the file path
25
25
  -prov--provenance Print the step provenance
26
26
  -cl--clean Clean the last step
27
27
  -rcl--recursive_clean Clean all steps
@@ -39,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
39
39
 
40
40
  options[:help] = true if task.nil?
41
41
 
42
- help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options,
43
- :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
42
+ help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps = IndiferentHash.process_options options,
43
+ :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps,
44
44
  :deploy => 'serial'
45
45
 
46
46
  if help
@@ -102,8 +102,9 @@ else
102
102
  when "slurm"
103
103
  require 'rbbt-scout'
104
104
  require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
105
+ slurm_options = {}
105
106
  HPC::BATCH_MODULE = HPC.batch_system "SLURM"
106
- HPC::BATCH_MODULE.orchestrate_job(job, {})
107
+ HPC::BATCH_MODULE.orchestrate_job(job, slurm_options)
107
108
  job.grace
108
109
  else
109
110
  if deploy.end_with?('-slurm')
@@ -116,7 +117,7 @@ else
116
117
  job.run
117
118
  end unless job.done?
118
119
 
119
- if print_filepath
120
+ if printpath
120
121
  job.join
121
122
  path = job.path
122
123
  path = path.find if Path === path
@@ -0,0 +1,49 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Description of the tool
10
+
11
+ $ #{$0} [<options>] <job-result>
12
+
13
+ -h--help Help
14
+ -fg--fix_gap Remove execution gaps
15
+ -rk--report_keys* Config keys and info fields to report
16
+ -p--plot* Plot file
17
+ -w--width* Image Width
18
+ -h--height* Image Height
19
+ -s--size* Image Size (Height and Width)
20
+ -pd--plot_data Print plot data
21
+ EOF
22
+ if options[:help]
23
+ if defined? scout_usage
24
+ scout_usage
25
+ else
26
+ puts SOPT.doc
27
+ end
28
+ exit 0
29
+ end
30
+
31
+ require 'rbbt/workflow/util/trace'
32
+ require 'rbbt/util/R'
33
+
34
+
35
+ files = ARGV
36
+ plot = options[:plot]
37
+
38
+ def get_step(file)
39
+ file = File.expand_path(file)
40
+ file = file.sub(/\.(info|files)/,'')
41
+ Step.new file
42
+ end
43
+
44
+ jobs = []
45
+ jobs = files.collect do |file|
46
+ get_step file
47
+ end
48
+
49
+ puts Workflow.trace(jobs, options)
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+ Examine the info of a job result
9
+
10
+ $ #{$0} <job-result> <key> <value>
11
+
12
+ -h--help Help
13
+ -f--force Write info even if key is already present
14
+ -r--recursive Write info for all dependencies as well
15
+ -p--check_pid Check that recursive jobs where created by the same process
16
+ EOF
17
+ if options[:help]
18
+ if defined? scout_usage
19
+ scout_usage
20
+ else
21
+ puts SOPT.doc
22
+ end
23
+ exit 0
24
+ end
25
+
26
+
27
+ file, key, value = ARGV
28
+
29
+ force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
30
+
31
+ def get_step(file)
32
+ file = file.sub(/\.(info|files)/,'')
33
+ step = Step.new file
34
+ step
35
+ end
36
+
37
+ raise ParameterException if key.nil? || value.nil?
38
+
39
+ if %w(DELETE nil).include? value
40
+ value = nil
41
+ force = true
42
+ end
43
+
44
+ step = get_step file
45
+
46
+ step.set_info key, value if force || ! step.info.include?(key)
47
+
48
+ pid = step.info[:pid]
49
+ host = step.info[:pid_hostname]
50
+
51
+ step.rec_dependencies.each do |dep|
52
+ begin
53
+ dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
54
+ rescue
55
+ Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
56
+ end
57
+ end if recursive
58
+
59
+ if recursive && step.info[:archived_info]
60
+ ad = step.info[:archived_info]
61
+ ad.each do |d,info|
62
+ begin
63
+ info[key] = value if (force || ! info.include?(key)) && (!check_pid || info[:pid].to_s == pid and info[:pid_hostname] == host)
64
+ rescue
65
+ Log.warn "Could no set info #{key} for archived_dep #{info[:path]}: #{$!.message}"
66
+ end
67
+ end
68
+ step.set_info :archived_info, ad
69
+ end
@@ -5,7 +5,7 @@ class TestSSH < Test::Unit::TestCase
5
5
  def test_marshal
6
6
  return unless SSHLine.reach?
7
7
 
8
- assert TrueClass === SSHLine.rbbt(:default, 'true')
8
+ assert TrueClass === SSHLine.scout(:default, 'true')
9
9
  end
10
10
 
11
11
  def test_localhost
@@ -85,6 +85,27 @@ row2 a a id3
85
85
  assert_equal %w(A a), tsv["row2"][0]
86
86
  end
87
87
 
88
+ def test_open_persist_path
89
+ content =<<-'EOF'
90
+ #: :sep=/\s+/#:type=:double#:merge=:concat
91
+ #Id ValueA ValueB OtherID
92
+ row1 a|aa|aaa b Id1|Id2
93
+ row2 A B Id3
94
+ row2 a a id3
95
+ EOF
96
+
97
+ TmpFile.with_file do |persist_path|
98
+ orig = TmpFile.with_file(content) do |filename|
99
+ TSV.open(filename, :persist => true, :merge => true, :persist_path => persist_path)
100
+ end
101
+
102
+ tsv = ScoutCabinet.open persist_path, false
103
+ assert_equal tsv.persistence_path, persist_path
104
+ assert_equal orig, tsv
105
+ end
106
+
107
+ end
108
+
88
109
  def test_headerless_fields
89
110
  content =<<-EOF
90
111
  row1 a|aa|aaa b Id1|Id2
@@ -0,0 +1,28 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+
6
+ class TestClass < Test::Unit::TestCase
7
+ def test_melt
8
+ txt =<<-EOF
9
+ #: :sep=,#:type=:list
10
+ #Size,System1,System2
11
+ 10,20,30
12
+ 20,40,60
13
+ EOF
14
+
15
+ target =<<-EOF
16
+ #: :sep=,#:type=:list
17
+ #ID,Size,Time,System
18
+ 10:0,10,20,System1
19
+ 10:1,10,30,System2
20
+ 20:0,20,40,System1
21
+ 20:1,20,60,System2
22
+ EOF
23
+
24
+ tsv = TSV.open(txt)
25
+ assert_equal tsv.melt_columns("Time", "System"), TSV.open(target)
26
+ end
27
+ end
28
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scout-gear
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.1.0
4
+ version: 10.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-07-07 00:00:00.000000000 Z
11
+ date: 2023-07-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: scout-essentials
@@ -169,6 +169,7 @@ files:
169
169
  - lib/scout/tsv/traverse.rb
170
170
  - lib/scout/tsv/util.rb
171
171
  - lib/scout/tsv/util/filter.rb
172
+ - lib/scout/tsv/util/melt.rb
172
173
  - lib/scout/tsv/util/process.rb
173
174
  - lib/scout/tsv/util/reorder.rb
174
175
  - lib/scout/tsv/util/select.rb
@@ -201,9 +202,12 @@ files:
201
202
  - lib/workflow-scout.rb
202
203
  - scout-gear.gemspec
203
204
  - scout_commands/alias
205
+ - scout_commands/batch/clean
206
+ - scout_commands/batch/list
204
207
  - scout_commands/doc
205
208
  - scout_commands/find
206
209
  - scout_commands/glob
210
+ - scout_commands/log
207
211
  - scout_commands/offsite
208
212
  - scout_commands/rbbt
209
213
  - scout_commands/resource/produce
@@ -212,7 +216,10 @@ files:
212
216
  - scout_commands/workflow/info
213
217
  - scout_commands/workflow/install
214
218
  - scout_commands/workflow/list
219
+ - scout_commands/workflow/prov
215
220
  - scout_commands/workflow/task
221
+ - scout_commands/workflow/trace
222
+ - scout_commands/workflow/write_info
216
223
  - share/color/color_names
217
224
  - share/color/diverging_colors.hex
218
225
  - share/software/install_helpers
@@ -242,6 +249,7 @@ files:
242
249
  - test/scout/tsv/test_traverse.rb
243
250
  - test/scout/tsv/test_util.rb
244
251
  - test/scout/tsv/util/test_filter.rb
252
+ - test/scout/tsv/util/test_melt.rb
245
253
  - test/scout/tsv/util/test_process.rb
246
254
  - test/scout/tsv/util/test_reorder.rb
247
255
  - test/scout/tsv/util/test_select.rb
@@ -285,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
285
293
  - !ruby/object:Gem::Version
286
294
  version: '0'
287
295
  requirements: []
288
- rubygems_version: 3.4.13
296
+ rubygems_version: 3.5.0.dev
289
297
  signing_key:
290
298
  specification_version: 4
291
299
  summary: basic gear for scouts