scout-gear 7.3.0 → 8.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +44 -16
- data/Rakefile +6 -1
- data/VERSION +1 -1
- data/bin/scout +21 -7
- data/doc/lib/scout/path.md +35 -0
- data/doc/lib/scout/workflow/task.md +13 -0
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +24 -25
- data/lib/scout/concurrent_stream.rb +59 -39
- data/lib/scout/config.rb +1 -1
- data/lib/scout/exceptions.rb +10 -0
- data/lib/scout/log/color.rb +15 -12
- data/lib/scout/log/progress/report.rb +8 -6
- data/lib/scout/log/progress/util.rb +61 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +115 -52
- data/lib/scout/meta_extension.rb +47 -6
- data/lib/scout/misc/digest.rb +12 -3
- data/lib/scout/misc/format.rb +24 -7
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +22 -0
- data/lib/scout/misc/system.rb +58 -0
- data/lib/scout/named_array.rb +73 -3
- data/lib/scout/offsite/ssh.rb +171 -0
- data/lib/scout/offsite/step.rb +83 -0
- data/lib/scout/offsite/sync.rb +55 -0
- data/lib/scout/offsite.rb +3 -0
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +9 -2
- data/lib/scout/open/remote.rb +16 -1
- data/lib/scout/open/stream.rb +146 -83
- data/lib/scout/open/util.rb +22 -3
- data/lib/scout/open.rb +5 -4
- data/lib/scout/path/find.rb +24 -11
- data/lib/scout/path/util.rb +40 -0
- data/lib/scout/persist/serialize.rb +19 -6
- data/lib/scout/persist.rb +29 -13
- data/lib/scout/resource/path.rb +57 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +12 -5
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +74 -46
- data/lib/scout/tsv/index.rb +85 -87
- data/lib/scout/tsv/open.rb +160 -85
- data/lib/scout/tsv/parser.rb +142 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +6 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/stream.rb +204 -0
- data/lib/scout/tsv/transformer.rb +152 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +34 -4
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +51 -20
- data/lib/scout/workflow/definition.rb +23 -3
- data/lib/scout/workflow/deployment/orchestrator.rb +245 -0
- data/lib/scout/workflow/deployment.rb +1 -0
- data/lib/scout/workflow/step/dependencies.rb +56 -10
- data/lib/scout/workflow/step/file.rb +5 -0
- data/lib/scout/workflow/step/info.rb +40 -7
- data/lib/scout/workflow/step/load.rb +1 -1
- data/lib/scout/workflow/step/provenance.rb +9 -7
- data/lib/scout/workflow/step/status.rb +43 -0
- data/lib/scout/workflow/step.rb +160 -49
- data/lib/scout/workflow/task/dependencies.rb +114 -0
- data/lib/scout/workflow/task/inputs.rb +40 -32
- data/lib/scout/workflow/task.rb +38 -102
- data/lib/scout/workflow/usage.rb +48 -18
- data/lib/scout/workflow.rb +4 -2
- data/lib/scout-gear.rb +2 -0
- data/lib/scout.rb +6 -0
- data/scout-gear.gemspec +52 -23
- data/scout_commands/doc +37 -0
- data/scout_commands/find +1 -0
- data/scout_commands/offsite +30 -0
- data/scout_commands/update +29 -0
- data/scout_commands/workflow/info +15 -3
- data/scout_commands/workflow/install +102 -0
- data/scout_commands/workflow/task +57 -9
- data/test/scout/offsite/test_ssh.rb +15 -0
- data/test/scout/offsite/test_step.rb +33 -0
- data/test/scout/offsite/test_sync.rb +36 -0
- data/test/scout/offsite/test_task.rb +0 -0
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_path.rb +6 -0
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +24 -0
- data/test/scout/test_persist.rb +9 -2
- data/test/scout/test_tsv.rb +229 -2
- data/test/scout/test_work_queue.rb +65 -41
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +49 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +33 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_stream.rb +200 -0
- data/test/scout/tsv/test_transformer.rb +120 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/deployment/test_orchestrator.rb +272 -0
- data/test/scout/workflow/step/test_dependencies.rb +68 -0
- data/test/scout/workflow/step/test_info.rb +18 -0
- data/test/scout/workflow/step/test_status.rb +30 -0
- data/test/scout/workflow/task/test_dependencies.rb +355 -0
- data/test/scout/workflow/task/test_inputs.rb +67 -14
- data/test/scout/workflow/test_definition.rb +18 -0
- data/test/scout/workflow/test_documentation.rb +24 -0
- data/test/scout/workflow/test_step.rb +112 -3
- data/test/scout/workflow/test_task.rb +0 -151
- data/test/scout/workflow/test_usage.rb +33 -6
- data/test/test_scout.rb +9 -0
- metadata +100 -8
- data/scout_commands/workflow/task_old +0 -706
data/lib/scout/workflow/task.rb
CHANGED
@@ -2,6 +2,7 @@ require_relative '../meta_extension'
|
|
2
2
|
require_relative '../named_array'
|
3
3
|
require_relative 'step'
|
4
4
|
require_relative 'task/inputs'
|
5
|
+
require_relative 'task/dependencies'
|
5
6
|
|
6
7
|
module Task
|
7
8
|
extend MetaExtension
|
@@ -38,123 +39,58 @@ module Task
|
|
38
39
|
binding.instance_exec(*inputs, &self)
|
39
40
|
end
|
40
41
|
|
41
|
-
def dependencies(id, provided_inputs, non_default_inputs = [])
|
42
|
-
return [] if deps.nil?
|
43
|
-
dependencies = []
|
44
|
-
|
45
|
-
provided_inputs ||= {}
|
46
|
-
|
47
|
-
load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
|
48
|
-
task = hash_options[:task] if hash_options.include?(:task)
|
49
|
-
workflow = hash_options[:workflow] if hash_options.include?(:workflow)
|
50
|
-
id = hash_options[:id] if hash_options.include? :id
|
51
|
-
|
52
|
-
hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
|
53
|
-
inputs = IndiferentHash.add_defaults hash_inputs, inputs
|
54
|
-
|
55
|
-
resolved_inputs = {}
|
56
|
-
inputs.each do |k,v|
|
57
|
-
if Symbol === v
|
58
|
-
input_dep = dependencies.select{|d| d.task_name == v }.first
|
59
|
-
resolved_inputs[k] = input_dep || inputs[v] || k
|
60
|
-
else
|
61
|
-
resolved_inputs[k] = v
|
62
|
-
end
|
63
|
-
end
|
64
|
-
workflow.job(task, id, resolved_inputs)
|
65
|
-
end
|
66
|
-
|
67
|
-
deps.each do |workflow,task,options,block=nil|
|
68
|
-
if provided_inputs.include?(overriden = [workflow.name, task] * "#")
|
69
|
-
dep = provided_inputs[overriden]
|
70
|
-
dep = Step.new dep unless Step === dep
|
71
|
-
dep.type = workflow.tasks[task].type
|
72
|
-
dependencies << dep
|
73
|
-
non_default_inputs << overriden
|
74
|
-
next
|
75
|
-
end
|
76
|
-
|
77
|
-
options ||= {}
|
78
|
-
if block
|
79
|
-
inputs = IndiferentHash.add_defaults options.dup, provided_inputs
|
80
|
-
|
81
|
-
res = block.call id, inputs, dependencies
|
82
|
-
|
83
|
-
case res
|
84
|
-
when Step
|
85
|
-
dep = res
|
86
|
-
dependencies << dep
|
87
|
-
dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
|
88
|
-
non_default_inputs.concat(dep_non_default_inputs - options.keys)
|
89
|
-
when Hash
|
90
|
-
new_options = res
|
91
|
-
dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
|
92
|
-
dependencies << dep
|
93
|
-
dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
|
94
|
-
dep_non_default_inputs -= options.keys
|
95
|
-
if new_options.include?(:inputs)
|
96
|
-
dep_non_default_inputs -= new_options[:inputs].keys
|
97
|
-
else
|
98
|
-
dep_non_default_inputs -= new_options.keys
|
99
|
-
end
|
100
|
-
non_default_inputs.concat(dep_non_default_inputs)
|
101
|
-
when Array
|
102
|
-
res.each do |_res|
|
103
|
-
if Hash === _res
|
104
|
-
new_options = _res
|
105
|
-
dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
|
106
|
-
dependencies << dep
|
107
|
-
dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
|
108
|
-
dep_non_default_inputs -= options.keys
|
109
|
-
if new_options.include?(:inputs)
|
110
|
-
dep_non_default_inputs -= new_options[:inputs].keys
|
111
|
-
else
|
112
|
-
dep_non_default_inputs -= new_options.keys
|
113
|
-
end
|
114
|
-
non_default_inputs.concat(dep_non_default_inputs)
|
115
|
-
else
|
116
|
-
dep = _res
|
117
|
-
dependencies << dep
|
118
|
-
dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
|
119
|
-
non_default_inputs.concat(dep_non_default_inputs - options.keys)
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
123
|
-
else
|
124
|
-
inputs = IndiferentHash.add_defaults options.dup, provided_inputs
|
125
|
-
dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
|
126
|
-
dependencies << dep
|
127
|
-
dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
|
128
|
-
non_default_inputs.concat(dep_non_default_inputs - options.keys)
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
dependencies
|
133
|
-
end
|
134
|
-
|
135
42
|
def job(id = DEFAULT_NAME, provided_inputs = nil )
|
136
43
|
provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
137
44
|
provided_inputs = {} if provided_inputs.nil?
|
138
45
|
id = DEFAULT_NAME if id.nil?
|
139
46
|
|
140
|
-
|
47
|
+
provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
|
141
48
|
|
142
|
-
|
49
|
+
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
|
50
|
+
|
51
|
+
compute = {}
|
52
|
+
dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
|
143
53
|
|
144
54
|
non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
|
145
55
|
|
56
|
+
non_default_inputs.uniq!
|
57
|
+
|
146
58
|
if non_default_inputs.any?
|
147
|
-
hash = Misc.digest(:inputs =>
|
148
|
-
|
149
|
-
|
59
|
+
hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
|
60
|
+
name = [id, hash] * "_"
|
61
|
+
else
|
62
|
+
name = id
|
150
63
|
end
|
151
64
|
|
152
|
-
|
65
|
+
extension = self.extension
|
66
|
+
if extension == :dep_task
|
67
|
+
extension = nil
|
68
|
+
if dependencies.any?
|
69
|
+
dep_basename = File.basename(dependencies.last.path)
|
70
|
+
if dep_basename.include? "."
|
71
|
+
parts = dep_basename.split(".")
|
72
|
+
extension = [parts.pop]
|
73
|
+
while parts.last.length <= 4
|
74
|
+
extension << parts.pop
|
75
|
+
end
|
76
|
+
extension = extension.reverse * "."
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
|
82
|
+
path = directory[name]
|
83
|
+
|
84
|
+
path = path.set_extension(extension) if extension
|
153
85
|
|
154
86
|
Persist.memory(path) do
|
155
|
-
|
87
|
+
if hash
|
88
|
+
Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
|
89
|
+
else
|
90
|
+
Log.debug "ID #{self.name} #{id} - Clean"
|
91
|
+
end
|
156
92
|
NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
|
157
|
-
Step.new path.find, inputs, dependencies, &self
|
93
|
+
Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
|
158
94
|
end
|
159
95
|
end
|
160
96
|
end
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -3,10 +3,31 @@ require 'scout/simple_opt'
|
|
3
3
|
module Task
|
4
4
|
def usage(workflow = nil, deps = nil)
|
5
5
|
str = StringIO.new
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
|
7
|
+
if description
|
8
|
+
title, paragraph = description.split("\n\n")
|
9
|
+
if title.length < Misc::MAX_TTY_LINE_WIDTH
|
10
|
+
title = self.name.to_s + " - " + title
|
11
|
+
str.puts Log.color :yellow, title
|
12
|
+
str.puts Log.color :yellow, "-" * title.length
|
13
|
+
if paragraph
|
14
|
+
str.puts "\n" << Misc.format_paragraph(paragraph)
|
15
|
+
end
|
16
|
+
str.puts
|
17
|
+
else
|
18
|
+
title = self.name.to_s
|
19
|
+
str.puts Log.color :yellow, title
|
20
|
+
str.puts Log.color :yellow, "-" * title.length
|
21
|
+
str.puts "\n" << Misc.format_paragraph(description)
|
22
|
+
str.puts
|
23
|
+
end
|
24
|
+
else
|
25
|
+
title = self.name.to_s
|
26
|
+
str.puts Log.color :yellow, title
|
27
|
+
str.puts Log.color :yellow, "-" * title.length
|
28
|
+
str.puts
|
29
|
+
end
|
30
|
+
|
10
31
|
|
11
32
|
selects = []
|
12
33
|
if inputs && inputs.any?
|
@@ -16,14 +37,16 @@ module Task
|
|
16
37
|
str.puts
|
17
38
|
end
|
18
39
|
|
40
|
+
deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
|
19
41
|
if deps and deps.any?
|
20
42
|
seen = inputs.collect{|name,_| name }
|
21
43
|
dep_inputs = {}
|
22
|
-
deps.each do |dep_workflow,task_name|
|
44
|
+
deps.each do |dep_workflow,task_name,options|
|
23
45
|
next if task_name.nil?
|
24
46
|
task = dep_workflow.tasks[task_name]
|
25
47
|
next if task.inputs.nil?
|
26
48
|
inputs = task.inputs.reject{|name, _| seen.include? name }
|
49
|
+
inputs = task.inputs.reject{|name, _| options.include? name }
|
27
50
|
next unless inputs.any?
|
28
51
|
dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
|
29
52
|
dep_inputs[dep] = inputs
|
@@ -101,33 +124,40 @@ module Task
|
|
101
124
|
sopt_options * ":"
|
102
125
|
end
|
103
126
|
|
104
|
-
def get_SOPT
|
127
|
+
def get_SOPT
|
105
128
|
sopt_option_string = self.SOPT_str
|
106
|
-
SOPT.get sopt_option_string
|
129
|
+
job_options = SOPT.get sopt_option_string
|
130
|
+
recursive_inputs.uniq.each do |name,type|
|
131
|
+
next unless type.to_s.include?('array')
|
132
|
+
if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
|
133
|
+
job_options[name] = job_options[name].split(",")
|
134
|
+
end
|
135
|
+
end
|
136
|
+
job_options
|
107
137
|
end
|
108
138
|
end
|
109
139
|
|
110
140
|
module Workflow
|
111
141
|
|
112
|
-
def dep_tree(task_name, seen = nil)
|
142
|
+
def dep_tree(task_name, seen = nil, seen_options = nil)
|
113
143
|
@dep_tree ||= {}
|
114
144
|
key = [self, task_name]
|
115
145
|
|
116
146
|
return @dep_tree[key] if @dep_tree.include?(key)
|
117
147
|
save = seen.nil?
|
118
148
|
seen = Set.new if seen.nil?
|
149
|
+
seen_options = {} if seen_options.nil?
|
119
150
|
|
120
151
|
dep_tree = {}
|
121
152
|
task = self.tasks[task_name]
|
122
|
-
task.deps.each do |
|
153
|
+
task.deps.each do |workflow, task, options|
|
123
154
|
next if seen.include? dep
|
124
|
-
seen <<
|
125
|
-
workflow, task, *rest = dep
|
155
|
+
seen << [workflow, task, options.merge(seen_options)]
|
126
156
|
next if task.nil?
|
127
157
|
|
128
158
|
key = [workflow, task]
|
129
159
|
|
130
|
-
dep_tree[key] = workflow.dep_tree(task, seen)
|
160
|
+
dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
|
131
161
|
end if task.deps
|
132
162
|
|
133
163
|
@dep_tree[key] = dep_tree if save
|
@@ -217,15 +247,15 @@ module Workflow
|
|
217
247
|
str.puts Log.color :magenta, "=" * self.name.length
|
218
248
|
end
|
219
249
|
|
220
|
-
|
221
|
-
str.puts
|
222
|
-
str.puts Misc.format_paragraph self.documentation[:description]
|
223
|
-
str.puts
|
224
|
-
end
|
225
|
-
|
250
|
+
str.puts
|
226
251
|
|
227
252
|
if task.nil?
|
228
253
|
|
254
|
+
if self.documentation[:description] and not self.documentation[:description].empty?
|
255
|
+
str.puts Misc.format_paragraph self.documentation[:description]
|
256
|
+
str.puts
|
257
|
+
end
|
258
|
+
|
229
259
|
str.puts Log.color :magenta, "## TASKS"
|
230
260
|
if self.documentation[:task_description] and not self.documentation[:task_description].empty?
|
231
261
|
str.puts
|
data/lib/scout/workflow.rb
CHANGED
@@ -4,13 +4,14 @@ require_relative 'workflow/task'
|
|
4
4
|
require_relative 'workflow/step'
|
5
5
|
require_relative 'workflow/documentation'
|
6
6
|
require_relative 'workflow/usage'
|
7
|
+
require_relative 'workflow/deployment'
|
7
8
|
|
8
9
|
require_relative 'resource'
|
9
10
|
require_relative 'resource/scout'
|
10
11
|
|
11
12
|
module Workflow
|
12
13
|
class << self
|
13
|
-
attr_accessor :workflows
|
14
|
+
attr_accessor :workflows, :main
|
14
15
|
def workflows
|
15
16
|
@workflows ||= []
|
16
17
|
end
|
@@ -30,13 +31,14 @@ module Workflow
|
|
30
31
|
workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
31
32
|
workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
32
33
|
if Open.exists?(workflow)
|
34
|
+
self.main = nil
|
33
35
|
workflow = workflow.find if Path === workflow
|
34
36
|
$LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
|
35
37
|
load workflow
|
36
38
|
else
|
37
39
|
raise "Workflow #{workflow_name} not found"
|
38
40
|
end
|
39
|
-
workflows.last
|
41
|
+
self.main || workflows.last
|
40
42
|
end
|
41
43
|
|
42
44
|
def job(name, *args)
|
data/lib/scout-gear.rb
CHANGED
data/lib/scout.rb
CHANGED
data/scout-gear.gemspec
CHANGED
@@ -2,17 +2,17 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear
|
5
|
+
# stub: scout-gear 8.1.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "
|
9
|
+
s.version = "8.1.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2023-
|
15
|
-
s.description = "Temporary files, logs, etc.".freeze
|
14
|
+
s.date = "2023-06-15"
|
15
|
+
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
18
18
|
s.extra_rdoc_files = [
|
@@ -28,6 +28,8 @@ Gem::Specification.new do |s|
|
|
28
28
|
"Rakefile",
|
29
29
|
"VERSION",
|
30
30
|
"bin/scout",
|
31
|
+
"doc/lib/scout/path.md",
|
32
|
+
"doc/lib/scout/workflow/task.md",
|
31
33
|
"lib/rbbt-scout.rb",
|
32
34
|
"lib/scout-gear.rb",
|
33
35
|
"lib/scout.rb",
|
@@ -45,6 +47,7 @@ Gem::Specification.new do |s|
|
|
45
47
|
"lib/scout/log/progress.rb",
|
46
48
|
"lib/scout/log/progress/report.rb",
|
47
49
|
"lib/scout/log/progress/util.rb",
|
50
|
+
"lib/scout/log/trap.rb",
|
48
51
|
"lib/scout/meta_extension.rb",
|
49
52
|
"lib/scout/misc.rb",
|
50
53
|
"lib/scout/misc/digest.rb",
|
@@ -55,8 +58,13 @@ Gem::Specification.new do |s|
|
|
55
58
|
"lib/scout/misc/monitor.rb",
|
56
59
|
"lib/scout/misc/system.rb",
|
57
60
|
"lib/scout/named_array.rb",
|
61
|
+
"lib/scout/offsite.rb",
|
62
|
+
"lib/scout/offsite/ssh.rb",
|
63
|
+
"lib/scout/offsite/step.rb",
|
64
|
+
"lib/scout/offsite/sync.rb",
|
58
65
|
"lib/scout/open.rb",
|
59
66
|
"lib/scout/open/lock.rb",
|
67
|
+
"lib/scout/open/lock/lockfile.rb",
|
60
68
|
"lib/scout/open/remote.rb",
|
61
69
|
"lib/scout/open/stream.rb",
|
62
70
|
"lib/scout/open/util.rb",
|
@@ -84,6 +92,8 @@ Gem::Specification.new do |s|
|
|
84
92
|
"lib/scout/simple_opt/setup.rb",
|
85
93
|
"lib/scout/tmpfile.rb",
|
86
94
|
"lib/scout/tsv.rb",
|
95
|
+
"lib/scout/tsv/attach.rb",
|
96
|
+
"lib/scout/tsv/change_id.rb",
|
87
97
|
"lib/scout/tsv/dumper.rb",
|
88
98
|
"lib/scout/tsv/index.rb",
|
89
99
|
"lib/scout/tsv/open.rb",
|
@@ -94,16 +104,22 @@ Gem::Specification.new do |s|
|
|
94
104
|
"lib/scout/tsv/persist/fix_width_table.rb",
|
95
105
|
"lib/scout/tsv/persist/serialize.rb",
|
96
106
|
"lib/scout/tsv/persist/tokyocabinet.rb",
|
107
|
+
"lib/scout/tsv/stream.rb",
|
108
|
+
"lib/scout/tsv/transformer.rb",
|
97
109
|
"lib/scout/tsv/traverse.rb",
|
98
110
|
"lib/scout/tsv/util.rb",
|
99
111
|
"lib/scout/tsv/util/filter.rb",
|
100
112
|
"lib/scout/tsv/util/process.rb",
|
113
|
+
"lib/scout/tsv/util/reorder.rb",
|
101
114
|
"lib/scout/tsv/util/select.rb",
|
115
|
+
"lib/scout/tsv/util/unzip.rb",
|
102
116
|
"lib/scout/work_queue.rb",
|
103
117
|
"lib/scout/work_queue/socket.rb",
|
104
118
|
"lib/scout/work_queue/worker.rb",
|
105
119
|
"lib/scout/workflow.rb",
|
106
120
|
"lib/scout/workflow/definition.rb",
|
121
|
+
"lib/scout/workflow/deployment.rb",
|
122
|
+
"lib/scout/workflow/deployment/orchestrator.rb",
|
107
123
|
"lib/scout/workflow/documentation.rb",
|
108
124
|
"lib/scout/workflow/step.rb",
|
109
125
|
"lib/scout/workflow/step/config.rb",
|
@@ -113,20 +129,25 @@ Gem::Specification.new do |s|
|
|
113
129
|
"lib/scout/workflow/step/load.rb",
|
114
130
|
"lib/scout/workflow/step/progress.rb",
|
115
131
|
"lib/scout/workflow/step/provenance.rb",
|
132
|
+
"lib/scout/workflow/step/status.rb",
|
116
133
|
"lib/scout/workflow/task.rb",
|
134
|
+
"lib/scout/workflow/task/dependencies.rb",
|
117
135
|
"lib/scout/workflow/task/inputs.rb",
|
118
136
|
"lib/scout/workflow/usage.rb",
|
119
137
|
"lib/scout/workflow/util.rb",
|
120
138
|
"lib/workflow-scout.rb",
|
121
139
|
"scout-gear.gemspec",
|
122
140
|
"scout_commands/alias",
|
141
|
+
"scout_commands/doc",
|
123
142
|
"scout_commands/find",
|
124
143
|
"scout_commands/glob",
|
144
|
+
"scout_commands/offsite",
|
125
145
|
"scout_commands/rbbt",
|
146
|
+
"scout_commands/update",
|
126
147
|
"scout_commands/workflow/info",
|
148
|
+
"scout_commands/workflow/install",
|
127
149
|
"scout_commands/workflow/list",
|
128
150
|
"scout_commands/workflow/task",
|
129
|
-
"scout_commands/workflow/task_old",
|
130
151
|
"share/color/color_names",
|
131
152
|
"share/color/diverging_colors.hex",
|
132
153
|
"share/software/install_helpers",
|
@@ -138,6 +159,10 @@ Gem::Specification.new do |s|
|
|
138
159
|
"test/scout/misc/test_filesystem.rb",
|
139
160
|
"test/scout/misc/test_insist.rb",
|
140
161
|
"test/scout/misc/test_system.rb",
|
162
|
+
"test/scout/offsite/test_ssh.rb",
|
163
|
+
"test/scout/offsite/test_step.rb",
|
164
|
+
"test/scout/offsite/test_sync.rb",
|
165
|
+
"test/scout/offsite/test_task.rb",
|
141
166
|
"test/scout/open/test_lock.rb",
|
142
167
|
"test/scout/open/test_remote.rb",
|
143
168
|
"test/scout/open/test_stream.rb",
|
@@ -175,22 +200,31 @@ Gem::Specification.new do |s|
|
|
175
200
|
"test/scout/tsv/persist/test_adapter.rb",
|
176
201
|
"test/scout/tsv/persist/test_fix_width_table.rb",
|
177
202
|
"test/scout/tsv/persist/test_tokyocabinet.rb",
|
203
|
+
"test/scout/tsv/test_attach.rb",
|
204
|
+
"test/scout/tsv/test_change_id.rb",
|
178
205
|
"test/scout/tsv/test_dumper.rb",
|
179
206
|
"test/scout/tsv/test_index.rb",
|
180
207
|
"test/scout/tsv/test_open.rb",
|
181
208
|
"test/scout/tsv/test_parser.rb",
|
182
209
|
"test/scout/tsv/test_persist.rb",
|
210
|
+
"test/scout/tsv/test_stream.rb",
|
211
|
+
"test/scout/tsv/test_transformer.rb",
|
183
212
|
"test/scout/tsv/test_traverse.rb",
|
184
213
|
"test/scout/tsv/test_util.rb",
|
185
214
|
"test/scout/tsv/util/test_filter.rb",
|
186
215
|
"test/scout/tsv/util/test_process.rb",
|
216
|
+
"test/scout/tsv/util/test_reorder.rb",
|
187
217
|
"test/scout/tsv/util/test_select.rb",
|
218
|
+
"test/scout/tsv/util/test_unzip.rb",
|
188
219
|
"test/scout/work_queue/test_socket.rb",
|
189
220
|
"test/scout/work_queue/test_worker.rb",
|
221
|
+
"test/scout/workflow/deployment/test_orchestrator.rb",
|
190
222
|
"test/scout/workflow/step/test_dependencies.rb",
|
191
223
|
"test/scout/workflow/step/test_info.rb",
|
192
224
|
"test/scout/workflow/step/test_load.rb",
|
193
225
|
"test/scout/workflow/step/test_provenance.rb",
|
226
|
+
"test/scout/workflow/step/test_status.rb",
|
227
|
+
"test/scout/workflow/task/test_dependencies.rb",
|
194
228
|
"test/scout/workflow/task/test_inputs.rb",
|
195
229
|
"test/scout/workflow/test_definition.rb",
|
196
230
|
"test/scout/workflow/test_documentation.rb",
|
@@ -199,29 +233,24 @@ Gem::Specification.new do |s|
|
|
199
233
|
"test/scout/workflow/test_usage.rb",
|
200
234
|
"test/scout/workflow/test_util.rb",
|
201
235
|
"test/test_helper.rb",
|
202
|
-
"test/test_scout-gear.rb"
|
236
|
+
"test/test_scout-gear.rb",
|
237
|
+
"test/test_scout.rb"
|
203
238
|
]
|
204
239
|
s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
|
205
240
|
s.licenses = ["MIT".freeze]
|
206
|
-
s.rubygems_version = "3.
|
241
|
+
s.rubygems_version = "3.4.13".freeze
|
207
242
|
s.summary = "basic gear for scouts".freeze
|
208
243
|
|
209
|
-
|
210
|
-
s.specification_version = 4
|
211
|
-
end
|
244
|
+
s.specification_version = 4
|
212
245
|
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
|
223
|
-
s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
|
224
|
-
s.add_dependency(%q<simplecov>.freeze, [">= 0"])
|
225
|
-
end
|
246
|
+
s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
|
247
|
+
s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
|
248
|
+
s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
|
249
|
+
s.add_runtime_dependency(%q<sys-proctable>.freeze, [">= 0"])
|
250
|
+
s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
|
251
|
+
s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
|
252
|
+
s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
|
253
|
+
s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
|
254
|
+
s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
|
226
255
|
end
|
227
256
|
|
data/scout_commands/doc
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Print documentation for a module
|
10
|
+
|
11
|
+
$ #{$0} [<options>] [module]
|
12
|
+
|
13
|
+
If you don't specify the module it prints those available
|
14
|
+
|
15
|
+
-h--help Print this help
|
16
|
+
EOF
|
17
|
+
if options[:help]
|
18
|
+
if defined? scout_usage
|
19
|
+
scout_usage
|
20
|
+
else
|
21
|
+
puts SOPT.doc
|
22
|
+
end
|
23
|
+
exit 0
|
24
|
+
end
|
25
|
+
|
26
|
+
|
27
|
+
module_name = ARGV.first
|
28
|
+
if module_name.nil?
|
29
|
+
puts Rbbt.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
|
30
|
+
else
|
31
|
+
begin
|
32
|
+
puts Rbbt.doc.lib.scout[module_name].find_with_extension('md', :lib).read
|
33
|
+
rescue
|
34
|
+
puts Rbbt.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
data/scout_commands/find
CHANGED
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
require 'scout/offsite'
|
5
|
+
|
6
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
7
|
+
|
8
|
+
options = SOPT.setup <<EOF
|
9
|
+
|
10
|
+
Description of the tool
|
11
|
+
|
12
|
+
$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
|
13
|
+
|
14
|
+
Run a command offsite. Use -- to make sure options reach the offsite
|
15
|
+
|
16
|
+
-h--help Print this help
|
17
|
+
EOF
|
18
|
+
if options[:help]
|
19
|
+
if defined? scout_usage
|
20
|
+
scout_usage
|
21
|
+
else
|
22
|
+
puts SOPT.doc
|
23
|
+
end
|
24
|
+
exit 0
|
25
|
+
end
|
26
|
+
|
27
|
+
where = ARGV.shift
|
28
|
+
command = ARGV.shift
|
29
|
+
|
30
|
+
puts SSHLine.command(where, command, ARGV)
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Update scout installation
|
10
|
+
|
11
|
+
$ #{$0} [<options>] [gem]
|
12
|
+
|
13
|
+
If no gem is specified 'scout-gear' is selected.
|
14
|
+
|
15
|
+
-h--help Print this help
|
16
|
+
EOF
|
17
|
+
if options[:help]
|
18
|
+
if defined? scout_usage
|
19
|
+
scout_usage
|
20
|
+
else
|
21
|
+
puts SOPT.doc
|
22
|
+
end
|
23
|
+
exit 0
|
24
|
+
end
|
25
|
+
|
26
|
+
gem_name = ARGV.first
|
27
|
+
Misc.update_git(gem_name)
|
28
|
+
|
29
|
+
|
@@ -23,7 +23,19 @@ end
|
|
23
23
|
|
24
24
|
path = ARGV.first
|
25
25
|
raise MissingParameterException.new :step_path if path.nil?
|
26
|
-
step = Step.load(path)
|
27
|
-
|
28
|
-
|
26
|
+
step = Step.load(path.dup)
|
27
|
+
|
28
|
+
step.info.each do |k,v|
|
29
|
+
case v
|
30
|
+
when nil
|
31
|
+
next
|
32
|
+
when Exception
|
33
|
+
puts Log.color(:title, "Exception")
|
34
|
+
Log.exception v
|
35
|
+
else
|
36
|
+
puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
|
37
|
+
end
|
38
|
+
rescue
|
39
|
+
next
|
40
|
+
end
|
29
41
|
|