scout-gear 5.1.1 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +10 -4
- data/Rakefile +2 -0
- data/VERSION +1 -1
- data/bin/scout +2 -0
- data/lib/scout/meta_extension.rb +4 -2
- data/lib/scout/misc/format.rb +16 -4
- data/lib/scout/misc/monitor.rb +23 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/open/stream.rb +1 -0
- data/lib/scout/path/find.rb +2 -1
- data/lib/scout/path.rb +1 -1
- data/lib/scout/persist/serialize.rb +15 -4
- data/lib/scout/resource/path.rb +5 -0
- data/lib/scout/resource/util.rb +48 -0
- data/lib/scout/resource.rb +2 -0
- data/lib/scout/simple_opt/doc.rb +26 -2
- data/lib/scout/workflow/definition.rb +8 -2
- data/lib/scout/workflow/documentation.rb +32 -26
- data/lib/scout/workflow/step/info.rb +11 -11
- data/lib/scout/workflow/step/load.rb +18 -0
- data/lib/scout/workflow/step.rb +40 -4
- data/lib/scout/workflow/task/inputs.rb +4 -2
- data/lib/scout/workflow/task.rb +15 -1
- data/lib/scout/workflow/usage.rb +96 -76
- data/lib/scout/workflow.rb +1 -0
- data/scout-gear.gemspec +14 -3
- data/scout_commands/workflow/info +29 -0
- data/scout_commands/workflow/list +27 -0
- data/scout_commands/workflow/task +32 -681
- data/scout_commands/workflow/task_old +706 -0
- data/test/scout/resource/test_util.rb +27 -0
- data/test/scout/simple_opt/test_doc.rb +16 -0
- data/test/scout/test_meta_extension.rb +9 -0
- data/test/scout/workflow/step/test_info.rb +17 -15
- data/test/scout/workflow/step/test_load.rb +65 -0
- data/test/scout/workflow/test_definition.rb +0 -0
- data/test/scout/workflow/test_documentation.rb +30 -0
- data/test/scout/workflow/test_task.rb +1 -0
- data/test/scout/workflow/test_usage.rb +12 -3
- metadata +13 -2
data/lib/scout/workflow/task.rb
CHANGED
@@ -16,6 +16,19 @@ module Task
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
+
def inputs
|
20
|
+
@inputs ||= []
|
21
|
+
end
|
22
|
+
|
23
|
+
def recursive_inputs
|
24
|
+
return inputs if deps.nil?
|
25
|
+
deps.inject(inputs) do |acc,dep|
|
26
|
+
workflow, task = dep
|
27
|
+
next if workflow.nil?
|
28
|
+
acc += workflow.tasks[task].recursive_inputs
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
19
32
|
def directory
|
20
33
|
@directory ||= Task.default_directory
|
21
34
|
end
|
@@ -121,6 +134,7 @@ module Task
|
|
121
134
|
def job(id = DEFAULT_NAME, provided_inputs = nil )
|
122
135
|
provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
123
136
|
provided_inputs = {} if provided_inputs.nil?
|
137
|
+
id = DEFAULT_NAME if id.nil?
|
124
138
|
|
125
139
|
inputs, non_default_inputs, input_hash = process_inputs provided_inputs
|
126
140
|
|
@@ -136,6 +150,6 @@ module Task
|
|
136
150
|
|
137
151
|
path = directory[id]
|
138
152
|
|
139
|
-
Step.new path, inputs, dependencies, &self
|
153
|
+
Step.new path.find, inputs, dependencies, &self
|
140
154
|
end
|
141
155
|
end
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -1,24 +1,40 @@
|
|
1
1
|
require 'scout/simple_opt'
|
2
2
|
|
3
3
|
module Task
|
4
|
-
def
|
5
|
-
|
6
|
-
puts
|
7
|
-
puts
|
4
|
+
def usage(workflow = nil, deps = nil)
|
5
|
+
str = StringIO.new
|
6
|
+
str.puts Log.color(:yellow, name)
|
7
|
+
str.puts Log.color(:yellow, "-" * name.length)
|
8
|
+
str.puts "\n" << Misc.format_paragraph(description.strip) << "\n" if description and not description.empty?
|
9
|
+
str.puts
|
8
10
|
|
9
11
|
selects = []
|
10
12
|
if inputs && inputs.any?
|
11
|
-
|
12
|
-
puts
|
13
|
-
puts
|
13
|
+
str.puts Log.color(:magenta, "Inputs")
|
14
|
+
str.puts
|
15
|
+
str.puts SOPT.input_array_doc(inputs)
|
16
|
+
str.puts
|
14
17
|
end
|
15
18
|
|
16
19
|
if deps and deps.any?
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
task.inputs
|
20
|
+
seen = inputs.collect{|name,_| name }
|
21
|
+
dep_inputs = {}
|
22
|
+
deps.each do |dep_workflow,task_name|
|
23
|
+
task = dep_workflow.tasks[task_name]
|
24
|
+
next if task.inputs.nil?
|
25
|
+
inputs = task.inputs.reject{|name, _| seen.include? name }
|
26
|
+
next unless inputs.any?
|
27
|
+
dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
|
28
|
+
dep_inputs[dep] = inputs
|
29
|
+
end
|
30
|
+
|
31
|
+
str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
|
32
|
+
dep_inputs.each do |dep,inputs|
|
33
|
+
str.puts
|
34
|
+
str.puts Log.color :yellow, dep + ":"
|
35
|
+
str.puts
|
36
|
+
str.puts SOPT.input_array_doc(inputs)
|
37
|
+
str.puts
|
22
38
|
end
|
23
39
|
|
24
40
|
#task_inputs = dep_inputs deps, workflow
|
@@ -44,30 +60,49 @@ module Task
|
|
44
60
|
end
|
45
61
|
|
46
62
|
case
|
47
|
-
when
|
48
|
-
puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
63
|
+
when inputs && inputs.select{|name,type| type == :array }.any?
|
64
|
+
str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
49
65
|
also works in addition to the others. You may use the '--array_separator' option
|
50
66
|
the change this default. Whenever a file is specified it may also accept STDIN using
|
51
67
|
the '-' character."))
|
52
|
-
puts
|
68
|
+
str.puts
|
53
69
|
|
54
|
-
when
|
55
|
-
puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
56
|
-
puts
|
70
|
+
when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
|
71
|
+
str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
72
|
+
str.puts
|
57
73
|
end
|
58
74
|
|
59
|
-
puts Log.color(:magenta, "Returns: ") << Log.color(:blue,
|
60
|
-
puts
|
75
|
+
str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
|
76
|
+
str.puts
|
61
77
|
|
62
78
|
if selects.any?
|
63
|
-
puts Log.color(:magenta, "Input select options")
|
64
|
-
puts
|
79
|
+
str.puts Log.color(:magenta, "Input select options")
|
80
|
+
str.puts
|
65
81
|
selects.collect{|p| p}.uniq.each do |input,options|
|
66
|
-
puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
67
|
-
puts unless Log.compact
|
82
|
+
str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
83
|
+
str.puts unless Log.compact
|
68
84
|
end
|
69
|
-
puts
|
85
|
+
str.puts
|
86
|
+
end
|
87
|
+
str.rewind
|
88
|
+
str.read
|
89
|
+
end
|
90
|
+
|
91
|
+
def SOPT_str
|
92
|
+
sopt_options = []
|
93
|
+
self.recursive_inputs.each do |name,type,desc,default,options|
|
94
|
+
shortcut = (options && options[:shortcut]) || name.to_s.slice(0,1)
|
95
|
+
boolean = type == :boolean
|
96
|
+
|
97
|
+
sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
|
70
98
|
end
|
99
|
+
|
100
|
+
sopt_options * ":"
|
101
|
+
end
|
102
|
+
|
103
|
+
def get_SOPT(task)
|
104
|
+
sopt_option_string = self.SOPT_str
|
105
|
+
SOPT.get sopt_option_string
|
71
106
|
end
|
72
107
|
end
|
73
108
|
|
@@ -167,29 +202,34 @@ module Workflow
|
|
167
202
|
lines * "\n"
|
168
203
|
end
|
169
204
|
|
170
|
-
def
|
205
|
+
def usage(task = nil, abridge = false)
|
171
206
|
|
172
|
-
|
173
|
-
puts Log.color :magenta, self.to_s
|
174
|
-
puts Log.color :magenta, "=" * self.to_s.length
|
207
|
+
str = StringIO.new
|
175
208
|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
209
|
+
if self.documentation[:title] and not self.documentation[:title].empty?
|
210
|
+
title = self.name + " - " + self.documentation[:title]
|
211
|
+
str.puts Log.color :magenta, title
|
212
|
+
str.puts Log.color :magenta, "=" * title.length
|
213
|
+
else
|
214
|
+
str.puts Log.color :magenta, self.name
|
215
|
+
str.puts Log.color :magenta, "=" * self.name.length
|
216
|
+
end
|
180
217
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
218
|
+
if self.documentation[:description] and not self.documentation[:description].empty?
|
219
|
+
str.puts
|
220
|
+
str.puts Misc.format_paragraph self.documentation[:description]
|
221
|
+
str.puts
|
222
|
+
end
|
186
223
|
|
187
|
-
|
224
|
+
|
225
|
+
if task.nil?
|
226
|
+
|
227
|
+
str.puts Log.color :magenta, "## TASKS"
|
188
228
|
if self.documentation[:task_description] and not self.documentation[:task_description].empty?
|
189
|
-
puts
|
190
|
-
puts Misc.format_paragraph self.documentation[:task_description]
|
229
|
+
str.puts
|
230
|
+
str.puts Misc.format_paragraph self.documentation[:task_description]
|
191
231
|
end
|
192
|
-
puts
|
232
|
+
str.puts
|
193
233
|
|
194
234
|
final = Set.new
|
195
235
|
not_final = Set.new
|
@@ -208,10 +248,10 @@ module Workflow
|
|
208
248
|
description = description.split("\n\n").first
|
209
249
|
|
210
250
|
next if abridge && ! final.include?(name)
|
211
|
-
puts Misc.format_definition_list_item(name.to_s, description,
|
251
|
+
str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
|
212
252
|
|
213
253
|
prov_string = prov_string(dep_tree(name))
|
214
|
-
puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
254
|
+
str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
215
255
|
end
|
216
256
|
|
217
257
|
else
|
@@ -224,56 +264,36 @@ module Workflow
|
|
224
264
|
end
|
225
265
|
|
226
266
|
#dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
|
227
|
-
task.
|
267
|
+
str.puts task.usage(self, self.recursive_deps(task_name))
|
228
268
|
|
229
269
|
dep_tree = {[self, task_name] => dep_tree(task_name)}
|
230
270
|
prov_tree = prov_tree(dep_tree)
|
231
271
|
if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
|
232
272
|
|
233
|
-
puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
|
234
|
-
puts
|
273
|
+
str.puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
|
274
|
+
str.puts
|
235
275
|
prov_tree.split("\n").each do |line|
|
236
276
|
next if line.strip.empty?
|
237
277
|
if m = line.match(/^( *)(\w+?)#(\w*)/i)
|
238
278
|
offset, workflow, task_name = m.values_at 1, 2, 3
|
239
|
-
|
279
|
+
str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
|
240
280
|
else
|
241
|
-
puts Log.color :blue, line
|
281
|
+
str.puts Log.color :blue, line
|
242
282
|
end
|
243
283
|
end
|
244
|
-
puts
|
245
|
-
end
|
246
|
-
|
247
|
-
if self.examples.include? task_name
|
248
|
-
self.examples[task_name].each do |example|
|
249
|
-
|
250
|
-
puts Log.color(:magenta, "Example ") << Log.color(:green, example) + " -- " + Log.color(:blue, example_dir[task_name][example])
|
251
|
-
|
252
|
-
inputs = self.example(task_name, example)
|
253
|
-
|
254
|
-
inputs.each do |input, type, file|
|
255
|
-
case type
|
256
|
-
when :tsv, :array, :text, :file
|
257
|
-
lines = file.read.split("\n")
|
258
|
-
head = lines[0..5].compact * "\n\n"
|
259
|
-
head = head[0..500]
|
260
|
-
puts Misc.format_definition_list_item(input, head, 1000, -1, :blue).gsub(/\n\s*\n/,"\n")
|
261
|
-
puts '...' if lines.length > 6
|
262
|
-
else
|
263
|
-
puts Misc.format_definition_list_item(input, file.read, Log.tty_size, 20, :blue)
|
264
|
-
end
|
265
|
-
end
|
266
|
-
puts
|
267
|
-
end
|
284
|
+
str.puts
|
268
285
|
end
|
269
286
|
end
|
287
|
+
|
288
|
+
str.rewind
|
289
|
+
str.read
|
270
290
|
end
|
271
291
|
|
272
292
|
def SOPT_str(task)
|
273
293
|
sopt_options = []
|
274
|
-
self.
|
275
|
-
|
276
|
-
boolean =
|
294
|
+
self.tasks[task].recursive_inputs.each do |name,type,desc,default,options|
|
295
|
+
shortcut = options[:shortcut] || name.to_s.slice(0,1)
|
296
|
+
boolean = type == :boolean
|
277
297
|
|
278
298
|
sopt_options << "-#{short}--#{name}#{boolean ? "" : "*"}"
|
279
299
|
end
|
data/lib/scout/workflow.rb
CHANGED
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear 5.
|
5
|
+
# stub: scout-gear 5.2.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "5.
|
9
|
+
s.version = "5.2.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2023-04-
|
14
|
+
s.date = "2023-04-28"
|
15
15
|
s.description = "Temporary files, logs, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -49,6 +49,7 @@ Gem::Specification.new do |s|
|
|
49
49
|
"lib/scout/misc/filesystem.rb",
|
50
50
|
"lib/scout/misc/format.rb",
|
51
51
|
"lib/scout/misc/insist.rb",
|
52
|
+
"lib/scout/misc/monitor.rb",
|
52
53
|
"lib/scout/open.rb",
|
53
54
|
"lib/scout/open/lock.rb",
|
54
55
|
"lib/scout/open/remote.rb",
|
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
|
|
67
68
|
"lib/scout/resource/produce.rb",
|
68
69
|
"lib/scout/resource/produce/rake.rb",
|
69
70
|
"lib/scout/resource/scout.rb",
|
71
|
+
"lib/scout/resource/util.rb",
|
70
72
|
"lib/scout/simple_opt.rb",
|
71
73
|
"lib/scout/simple_opt/accessor.rb",
|
72
74
|
"lib/scout/simple_opt/doc.rb",
|
@@ -79,6 +81,7 @@ Gem::Specification.new do |s|
|
|
79
81
|
"lib/scout/workflow/documentation.rb",
|
80
82
|
"lib/scout/workflow/step.rb",
|
81
83
|
"lib/scout/workflow/step/info.rb",
|
84
|
+
"lib/scout/workflow/step/load.rb",
|
82
85
|
"lib/scout/workflow/task.rb",
|
83
86
|
"lib/scout/workflow/task/inputs.rb",
|
84
87
|
"lib/scout/workflow/usage.rb",
|
@@ -89,7 +92,10 @@ Gem::Specification.new do |s|
|
|
89
92
|
"scout_commands/find",
|
90
93
|
"scout_commands/glob",
|
91
94
|
"scout_commands/rbbt",
|
95
|
+
"scout_commands/workflow/info",
|
96
|
+
"scout_commands/workflow/list",
|
92
97
|
"scout_commands/workflow/task",
|
98
|
+
"scout_commands/workflow/task_old",
|
93
99
|
"test/scout/indiferent_hash/test_case_insensitive.rb",
|
94
100
|
"test/scout/indiferent_hash/test_options.rb",
|
95
101
|
"test/scout/log/test_progress.rb",
|
@@ -107,6 +113,8 @@ Gem::Specification.new do |s|
|
|
107
113
|
"test/scout/persist/test_serialize.rb",
|
108
114
|
"test/scout/resource/test_path.rb",
|
109
115
|
"test/scout/resource/test_produce.rb",
|
116
|
+
"test/scout/resource/test_util.rb",
|
117
|
+
"test/scout/simple_opt/test_doc.rb",
|
110
118
|
"test/scout/simple_opt/test_get.rb",
|
111
119
|
"test/scout/simple_opt/test_parse.rb",
|
112
120
|
"test/scout/simple_opt/test_setup.rb",
|
@@ -123,7 +131,10 @@ Gem::Specification.new do |s|
|
|
123
131
|
"test/scout/test_tmpfile.rb",
|
124
132
|
"test/scout/test_workflow.rb",
|
125
133
|
"test/scout/workflow/step/test_info.rb",
|
134
|
+
"test/scout/workflow/step/test_load.rb",
|
126
135
|
"test/scout/workflow/task/test_inputs.rb",
|
136
|
+
"test/scout/workflow/test_definition.rb",
|
137
|
+
"test/scout/workflow/test_documentation.rb",
|
127
138
|
"test/scout/workflow/test_step.rb",
|
128
139
|
"test/scout/workflow/test_task.rb",
|
129
140
|
"test/scout/workflow/test_usage.rb",
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Show info from job
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <step_path>
|
12
|
+
|
13
|
+
-h--help Print this help
|
14
|
+
EOF
|
15
|
+
if options[:help]
|
16
|
+
if defined? scout_usage
|
17
|
+
scout_usage
|
18
|
+
else
|
19
|
+
puts SOPT.doc
|
20
|
+
end
|
21
|
+
exit 0
|
22
|
+
end
|
23
|
+
|
24
|
+
path = ARGV.first
|
25
|
+
raise MissingParameterException.new :step_path if path.nil?
|
26
|
+
step = Step.load(path)
|
27
|
+
|
28
|
+
puts step.info.to_json
|
29
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
List all workflows
|
10
|
+
|
11
|
+
$ #{$0} [<options>]
|
12
|
+
|
13
|
+
-h--help Print this help
|
14
|
+
EOF
|
15
|
+
if options[:help]
|
16
|
+
if defined? scout_usage
|
17
|
+
scout_usage
|
18
|
+
else
|
19
|
+
puts SOPT.doc
|
20
|
+
end
|
21
|
+
exit 0
|
22
|
+
end
|
23
|
+
|
24
|
+
Path.setup('workflows').glob_all("*").each do |file|
|
25
|
+
puts File.basename(file)
|
26
|
+
end
|
27
|
+
|