scout-gear 7.3.0 → 8.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +44 -16
  3. data/Rakefile +6 -1
  4. data/VERSION +1 -1
  5. data/bin/scout +21 -7
  6. data/doc/lib/scout/path.md +35 -0
  7. data/doc/lib/scout/workflow/task.md +13 -0
  8. data/lib/rbbt-scout.rb +1 -0
  9. data/lib/scout/cmd.rb +24 -25
  10. data/lib/scout/concurrent_stream.rb +59 -39
  11. data/lib/scout/config.rb +1 -1
  12. data/lib/scout/exceptions.rb +10 -0
  13. data/lib/scout/log/color.rb +15 -12
  14. data/lib/scout/log/progress/report.rb +8 -6
  15. data/lib/scout/log/progress/util.rb +61 -54
  16. data/lib/scout/log/progress.rb +1 -1
  17. data/lib/scout/log/trap.rb +107 -0
  18. data/lib/scout/log.rb +115 -52
  19. data/lib/scout/meta_extension.rb +47 -6
  20. data/lib/scout/misc/digest.rb +12 -3
  21. data/lib/scout/misc/format.rb +24 -7
  22. data/lib/scout/misc/insist.rb +1 -1
  23. data/lib/scout/misc/monitor.rb +22 -0
  24. data/lib/scout/misc/system.rb +58 -0
  25. data/lib/scout/named_array.rb +73 -3
  26. data/lib/scout/offsite/ssh.rb +171 -0
  27. data/lib/scout/offsite/step.rb +83 -0
  28. data/lib/scout/offsite/sync.rb +55 -0
  29. data/lib/scout/offsite.rb +3 -0
  30. data/lib/scout/open/lock/lockfile.rb +587 -0
  31. data/lib/scout/open/lock.rb +9 -2
  32. data/lib/scout/open/remote.rb +16 -1
  33. data/lib/scout/open/stream.rb +146 -83
  34. data/lib/scout/open/util.rb +22 -3
  35. data/lib/scout/open.rb +5 -4
  36. data/lib/scout/path/find.rb +24 -11
  37. data/lib/scout/path/util.rb +40 -0
  38. data/lib/scout/persist/serialize.rb +19 -6
  39. data/lib/scout/persist.rb +29 -13
  40. data/lib/scout/resource/path.rb +57 -0
  41. data/lib/scout/resource/produce.rb +0 -8
  42. data/lib/scout/resource/util.rb +12 -5
  43. data/lib/scout/tmpfile.rb +7 -8
  44. data/lib/scout/tsv/attach.rb +177 -0
  45. data/lib/scout/tsv/change_id.rb +40 -0
  46. data/lib/scout/tsv/dumper.rb +74 -46
  47. data/lib/scout/tsv/index.rb +85 -87
  48. data/lib/scout/tsv/open.rb +160 -85
  49. data/lib/scout/tsv/parser.rb +142 -80
  50. data/lib/scout/tsv/path.rb +1 -2
  51. data/lib/scout/tsv/persist/adapter.rb +15 -45
  52. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  53. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -1
  54. data/lib/scout/tsv/persist.rb +4 -0
  55. data/lib/scout/tsv/stream.rb +204 -0
  56. data/lib/scout/tsv/transformer.rb +152 -0
  57. data/lib/scout/tsv/traverse.rb +96 -92
  58. data/lib/scout/tsv/util/filter.rb +9 -0
  59. data/lib/scout/tsv/util/reorder.rb +81 -0
  60. data/lib/scout/tsv/util/select.rb +78 -33
  61. data/lib/scout/tsv/util/unzip.rb +86 -0
  62. data/lib/scout/tsv/util.rb +60 -11
  63. data/lib/scout/tsv.rb +34 -4
  64. data/lib/scout/work_queue/socket.rb +6 -1
  65. data/lib/scout/work_queue/worker.rb +5 -2
  66. data/lib/scout/work_queue.rb +51 -20
  67. data/lib/scout/workflow/definition.rb +23 -3
  68. data/lib/scout/workflow/deployment/orchestrator.rb +245 -0
  69. data/lib/scout/workflow/deployment.rb +1 -0
  70. data/lib/scout/workflow/step/dependencies.rb +56 -10
  71. data/lib/scout/workflow/step/file.rb +5 -0
  72. data/lib/scout/workflow/step/info.rb +40 -7
  73. data/lib/scout/workflow/step/load.rb +1 -1
  74. data/lib/scout/workflow/step/provenance.rb +9 -7
  75. data/lib/scout/workflow/step/status.rb +43 -0
  76. data/lib/scout/workflow/step.rb +160 -49
  77. data/lib/scout/workflow/task/dependencies.rb +114 -0
  78. data/lib/scout/workflow/task/inputs.rb +40 -32
  79. data/lib/scout/workflow/task.rb +38 -102
  80. data/lib/scout/workflow/usage.rb +48 -18
  81. data/lib/scout/workflow.rb +4 -2
  82. data/lib/scout-gear.rb +2 -0
  83. data/lib/scout.rb +6 -0
  84. data/scout-gear.gemspec +52 -23
  85. data/scout_commands/doc +37 -0
  86. data/scout_commands/find +1 -0
  87. data/scout_commands/offsite +30 -0
  88. data/scout_commands/update +29 -0
  89. data/scout_commands/workflow/info +15 -3
  90. data/scout_commands/workflow/install +102 -0
  91. data/scout_commands/workflow/task +57 -9
  92. data/test/scout/offsite/test_ssh.rb +15 -0
  93. data/test/scout/offsite/test_step.rb +33 -0
  94. data/test/scout/offsite/test_sync.rb +36 -0
  95. data/test/scout/offsite/test_task.rb +0 -0
  96. data/test/scout/open/test_stream.rb +60 -58
  97. data/test/scout/path/test_find.rb +10 -1
  98. data/test/scout/resource/test_path.rb +6 -0
  99. data/test/scout/resource/test_produce.rb +15 -0
  100. data/test/scout/test_meta_extension.rb +25 -0
  101. data/test/scout/test_named_array.rb +24 -0
  102. data/test/scout/test_persist.rb +9 -2
  103. data/test/scout/test_tsv.rb +229 -2
  104. data/test/scout/test_work_queue.rb +65 -41
  105. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  106. data/test/scout/tsv/test_attach.rb +227 -0
  107. data/test/scout/tsv/test_change_id.rb +98 -0
  108. data/test/scout/tsv/test_dumper.rb +1 -1
  109. data/test/scout/tsv/test_index.rb +49 -3
  110. data/test/scout/tsv/test_open.rb +160 -2
  111. data/test/scout/tsv/test_parser.rb +33 -2
  112. data/test/scout/tsv/test_persist.rb +2 -0
  113. data/test/scout/tsv/test_stream.rb +200 -0
  114. data/test/scout/tsv/test_transformer.rb +120 -0
  115. data/test/scout/tsv/test_traverse.rb +88 -3
  116. data/test/scout/tsv/test_util.rb +1 -0
  117. data/test/scout/tsv/util/test_reorder.rb +94 -0
  118. data/test/scout/tsv/util/test_select.rb +25 -11
  119. data/test/scout/tsv/util/test_unzip.rb +112 -0
  120. data/test/scout/work_queue/test_socket.rb +0 -1
  121. data/test/scout/workflow/deployment/test_orchestrator.rb +272 -0
  122. data/test/scout/workflow/step/test_dependencies.rb +68 -0
  123. data/test/scout/workflow/step/test_info.rb +18 -0
  124. data/test/scout/workflow/step/test_status.rb +30 -0
  125. data/test/scout/workflow/task/test_dependencies.rb +355 -0
  126. data/test/scout/workflow/task/test_inputs.rb +67 -14
  127. data/test/scout/workflow/test_definition.rb +18 -0
  128. data/test/scout/workflow/test_documentation.rb +24 -0
  129. data/test/scout/workflow/test_step.rb +112 -3
  130. data/test/scout/workflow/test_task.rb +0 -151
  131. data/test/scout/workflow/test_usage.rb +33 -6
  132. data/test/test_scout.rb +9 -0
  133. metadata +100 -8
  134. data/scout_commands/workflow/task_old +0 -706
@@ -2,6 +2,7 @@ require_relative '../meta_extension'
2
2
  require_relative '../named_array'
3
3
  require_relative 'step'
4
4
  require_relative 'task/inputs'
5
+ require_relative 'task/dependencies'
5
6
 
6
7
  module Task
7
8
  extend MetaExtension
@@ -38,123 +39,58 @@ module Task
38
39
  binding.instance_exec(*inputs, &self)
39
40
  end
40
41
 
41
- def dependencies(id, provided_inputs, non_default_inputs = [])
42
- return [] if deps.nil?
43
- dependencies = []
44
-
45
- provided_inputs ||= {}
46
-
47
- load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
48
- task = hash_options[:task] if hash_options.include?(:task)
49
- workflow = hash_options[:workflow] if hash_options.include?(:workflow)
50
- id = hash_options[:id] if hash_options.include? :id
51
-
52
- hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
53
- inputs = IndiferentHash.add_defaults hash_inputs, inputs
54
-
55
- resolved_inputs = {}
56
- inputs.each do |k,v|
57
- if Symbol === v
58
- input_dep = dependencies.select{|d| d.task_name == v }.first
59
- resolved_inputs[k] = input_dep || inputs[v] || k
60
- else
61
- resolved_inputs[k] = v
62
- end
63
- end
64
- workflow.job(task, id, resolved_inputs)
65
- end
66
-
67
- deps.each do |workflow,task,options,block=nil|
68
- if provided_inputs.include?(overriden = [workflow.name, task] * "#")
69
- dep = provided_inputs[overriden]
70
- dep = Step.new dep unless Step === dep
71
- dep.type = workflow.tasks[task].type
72
- dependencies << dep
73
- non_default_inputs << overriden
74
- next
75
- end
76
-
77
- options ||= {}
78
- if block
79
- inputs = IndiferentHash.add_defaults options.dup, provided_inputs
80
-
81
- res = block.call id, inputs, dependencies
82
-
83
- case res
84
- when Step
85
- dep = res
86
- dependencies << dep
87
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
88
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
89
- when Hash
90
- new_options = res
91
- dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
92
- dependencies << dep
93
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
94
- dep_non_default_inputs -= options.keys
95
- if new_options.include?(:inputs)
96
- dep_non_default_inputs -= new_options[:inputs].keys
97
- else
98
- dep_non_default_inputs -= new_options.keys
99
- end
100
- non_default_inputs.concat(dep_non_default_inputs)
101
- when Array
102
- res.each do |_res|
103
- if Hash === _res
104
- new_options = _res
105
- dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
106
- dependencies << dep
107
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
108
- dep_non_default_inputs -= options.keys
109
- if new_options.include?(:inputs)
110
- dep_non_default_inputs -= new_options[:inputs].keys
111
- else
112
- dep_non_default_inputs -= new_options.keys
113
- end
114
- non_default_inputs.concat(dep_non_default_inputs)
115
- else
116
- dep = _res
117
- dependencies << dep
118
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
119
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
120
- end
121
- end
122
- end
123
- else
124
- inputs = IndiferentHash.add_defaults options.dup, provided_inputs
125
- dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
126
- dependencies << dep
127
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
128
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
129
- end
130
- end
131
-
132
- dependencies
133
- end
134
-
135
42
  def job(id = DEFAULT_NAME, provided_inputs = nil )
136
43
  provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
137
44
  provided_inputs = {} if provided_inputs.nil?
138
45
  id = DEFAULT_NAME if id.nil?
139
46
 
140
- inputs, non_default_inputs, input_hash = process_inputs provided_inputs
47
+ provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
141
48
 
142
- dependencies = dependencies(id, provided_inputs, non_default_inputs)
49
+ inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
50
+
51
+ compute = {}
52
+ dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
143
53
 
144
54
  non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
145
55
 
56
+ non_default_inputs.uniq!
57
+
146
58
  if non_default_inputs.any?
147
- hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
148
- Log.debug "Hash #{name} - #{hash}: #{Log.fingerprint(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
149
- id = [id, hash] * "_"
59
+ hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
60
+ name = [id, hash] * "_"
61
+ else
62
+ name = id
150
63
  end
151
64
 
152
- path = directory[id]
65
+ extension = self.extension
66
+ if extension == :dep_task
67
+ extension = nil
68
+ if dependencies.any?
69
+ dep_basename = File.basename(dependencies.last.path)
70
+ if dep_basename.include? "."
71
+ parts = dep_basename.split(".")
72
+ extension = [parts.pop]
73
+ while parts.last.length <= 4
74
+ extension << parts.pop
75
+ end
76
+ extension = extension.reverse * "."
77
+ end
78
+ end
79
+ end
80
+
81
+
82
+ path = directory[name]
83
+
84
+ path = path.set_extension(extension) if extension
153
85
 
154
86
  Persist.memory(path) do
155
- Log.debug "Creating job #{path} #{Log.fingerprint inputs} #{Log.fingerprint dependencies}"
87
+ if hash
88
+ Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
89
+ else
90
+ Log.debug "ID #{self.name} #{id} - Clean"
91
+ end
156
92
  NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
157
- Step.new path.find, inputs, dependencies, &self
93
+ Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
158
94
  end
159
95
  end
160
96
  end
@@ -3,10 +3,31 @@ require 'scout/simple_opt'
3
3
  module Task
4
4
  def usage(workflow = nil, deps = nil)
5
5
  str = StringIO.new
6
- str.puts Log.color(:yellow, name)
7
- str.puts Log.color(:yellow, "-" * name.length)
8
- str.puts "\n" << Misc.format_paragraph(description.strip) << "\n" if description and not description.empty?
9
- str.puts
6
+
7
+ if description
8
+ title, paragraph = description.split("\n\n")
9
+ if title.length < Misc::MAX_TTY_LINE_WIDTH
10
+ title = self.name.to_s + " - " + title
11
+ str.puts Log.color :yellow, title
12
+ str.puts Log.color :yellow, "-" * title.length
13
+ if paragraph
14
+ str.puts "\n" << Misc.format_paragraph(paragraph)
15
+ end
16
+ str.puts
17
+ else
18
+ title = self.name.to_s
19
+ str.puts Log.color :yellow, title
20
+ str.puts Log.color :yellow, "-" * title.length
21
+ str.puts "\n" << Misc.format_paragraph(description)
22
+ str.puts
23
+ end
24
+ else
25
+ title = self.name.to_s
26
+ str.puts Log.color :yellow, title
27
+ str.puts Log.color :yellow, "-" * title.length
28
+ str.puts
29
+ end
30
+
10
31
 
11
32
  selects = []
12
33
  if inputs && inputs.any?
@@ -16,14 +37,16 @@ module Task
16
37
  str.puts
17
38
  end
18
39
 
40
+ deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
19
41
  if deps and deps.any?
20
42
  seen = inputs.collect{|name,_| name }
21
43
  dep_inputs = {}
22
- deps.each do |dep_workflow,task_name|
44
+ deps.each do |dep_workflow,task_name,options|
23
45
  next if task_name.nil?
24
46
  task = dep_workflow.tasks[task_name]
25
47
  next if task.inputs.nil?
26
48
  inputs = task.inputs.reject{|name, _| seen.include? name }
49
+ inputs = task.inputs.reject{|name, _| options.include? name }
27
50
  next unless inputs.any?
28
51
  dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
29
52
  dep_inputs[dep] = inputs
@@ -101,33 +124,40 @@ module Task
101
124
  sopt_options * ":"
102
125
  end
103
126
 
104
- def get_SOPT(task)
127
+ def get_SOPT
105
128
  sopt_option_string = self.SOPT_str
106
- SOPT.get sopt_option_string
129
+ job_options = SOPT.get sopt_option_string
130
+ recursive_inputs.uniq.each do |name,type|
131
+ next unless type.to_s.include?('array')
132
+ if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
133
+ job_options[name] = job_options[name].split(",")
134
+ end
135
+ end
136
+ job_options
107
137
  end
108
138
  end
109
139
 
110
140
  module Workflow
111
141
 
112
- def dep_tree(task_name, seen = nil)
142
+ def dep_tree(task_name, seen = nil, seen_options = nil)
113
143
  @dep_tree ||= {}
114
144
  key = [self, task_name]
115
145
 
116
146
  return @dep_tree[key] if @dep_tree.include?(key)
117
147
  save = seen.nil?
118
148
  seen = Set.new if seen.nil?
149
+ seen_options = {} if seen_options.nil?
119
150
 
120
151
  dep_tree = {}
121
152
  task = self.tasks[task_name]
122
- task.deps.each do |dep|
153
+ task.deps.each do |workflow, task, options|
123
154
  next if seen.include? dep
124
- seen << dep
125
- workflow, task, *rest = dep
155
+ seen << [workflow, task, options.merge(seen_options)]
126
156
  next if task.nil?
127
157
 
128
158
  key = [workflow, task]
129
159
 
130
- dep_tree[key] = workflow.dep_tree(task, seen)
160
+ dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
131
161
  end if task.deps
132
162
 
133
163
  @dep_tree[key] = dep_tree if save
@@ -217,15 +247,15 @@ module Workflow
217
247
  str.puts Log.color :magenta, "=" * self.name.length
218
248
  end
219
249
 
220
- if self.documentation[:description] and not self.documentation[:description].empty?
221
- str.puts
222
- str.puts Misc.format_paragraph self.documentation[:description]
223
- str.puts
224
- end
225
-
250
+ str.puts
226
251
 
227
252
  if task.nil?
228
253
 
254
+ if self.documentation[:description] and not self.documentation[:description].empty?
255
+ str.puts Misc.format_paragraph self.documentation[:description]
256
+ str.puts
257
+ end
258
+
229
259
  str.puts Log.color :magenta, "## TASKS"
230
260
  if self.documentation[:task_description] and not self.documentation[:task_description].empty?
231
261
  str.puts
@@ -4,13 +4,14 @@ require_relative 'workflow/task'
4
4
  require_relative 'workflow/step'
5
5
  require_relative 'workflow/documentation'
6
6
  require_relative 'workflow/usage'
7
+ require_relative 'workflow/deployment'
7
8
 
8
9
  require_relative 'resource'
9
10
  require_relative 'resource/scout'
10
11
 
11
12
  module Workflow
12
13
  class << self
13
- attr_accessor :workflows
14
+ attr_accessor :workflows, :main
14
15
  def workflows
15
16
  @workflows ||= []
16
17
  end
@@ -30,13 +31,14 @@ module Workflow
30
31
  workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
31
32
  workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
32
33
  if Open.exists?(workflow)
34
+ self.main = nil
33
35
  workflow = workflow.find if Path === workflow
34
36
  $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
35
37
  load workflow
36
38
  else
37
39
  raise "Workflow #{workflow_name} not found"
38
40
  end
39
- workflows.last
41
+ self.main || workflows.last
40
42
  end
41
43
 
42
44
  def job(name, *args)
data/lib/scout-gear.rb CHANGED
@@ -8,3 +8,5 @@ require_relative 'scout/resource'
8
8
  require_relative 'scout/resource/scout'
9
9
  require_relative 'scout/persist'
10
10
  require_relative 'scout/tsv'
11
+ require_relative 'scout/config'
12
+ require_relative 'scout/offsite'
data/lib/scout.rb CHANGED
@@ -1,2 +1,8 @@
1
1
  require 'workflow-scout'
2
2
  require 'rbbt-scout'
3
+
4
+ module Scout
5
+ def self.version
6
+ Open.read(File.join(__dir__, '../VERSION'))
7
+ end
8
+ end
data/scout-gear.gemspec CHANGED
@@ -2,17 +2,17 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: scout-gear 7.3.0 ruby lib
5
+ # stub: scout-gear 8.1.0 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "scout-gear".freeze
9
- s.version = "7.3.0"
9
+ s.version = "8.1.0"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib".freeze]
13
13
  s.authors = ["Miguel Vazquez".freeze]
14
- s.date = "2023-05-12"
15
- s.description = "Temporary files, logs, etc.".freeze
14
+ s.date = "2023-06-15"
15
+ s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
16
16
  s.email = "mikisvaz@gmail.com".freeze
17
17
  s.executables = ["scout".freeze]
18
18
  s.extra_rdoc_files = [
@@ -28,6 +28,8 @@ Gem::Specification.new do |s|
28
28
  "Rakefile",
29
29
  "VERSION",
30
30
  "bin/scout",
31
+ "doc/lib/scout/path.md",
32
+ "doc/lib/scout/workflow/task.md",
31
33
  "lib/rbbt-scout.rb",
32
34
  "lib/scout-gear.rb",
33
35
  "lib/scout.rb",
@@ -45,6 +47,7 @@ Gem::Specification.new do |s|
45
47
  "lib/scout/log/progress.rb",
46
48
  "lib/scout/log/progress/report.rb",
47
49
  "lib/scout/log/progress/util.rb",
50
+ "lib/scout/log/trap.rb",
48
51
  "lib/scout/meta_extension.rb",
49
52
  "lib/scout/misc.rb",
50
53
  "lib/scout/misc/digest.rb",
@@ -55,8 +58,13 @@ Gem::Specification.new do |s|
55
58
  "lib/scout/misc/monitor.rb",
56
59
  "lib/scout/misc/system.rb",
57
60
  "lib/scout/named_array.rb",
61
+ "lib/scout/offsite.rb",
62
+ "lib/scout/offsite/ssh.rb",
63
+ "lib/scout/offsite/step.rb",
64
+ "lib/scout/offsite/sync.rb",
58
65
  "lib/scout/open.rb",
59
66
  "lib/scout/open/lock.rb",
67
+ "lib/scout/open/lock/lockfile.rb",
60
68
  "lib/scout/open/remote.rb",
61
69
  "lib/scout/open/stream.rb",
62
70
  "lib/scout/open/util.rb",
@@ -84,6 +92,8 @@ Gem::Specification.new do |s|
84
92
  "lib/scout/simple_opt/setup.rb",
85
93
  "lib/scout/tmpfile.rb",
86
94
  "lib/scout/tsv.rb",
95
+ "lib/scout/tsv/attach.rb",
96
+ "lib/scout/tsv/change_id.rb",
87
97
  "lib/scout/tsv/dumper.rb",
88
98
  "lib/scout/tsv/index.rb",
89
99
  "lib/scout/tsv/open.rb",
@@ -94,16 +104,22 @@ Gem::Specification.new do |s|
94
104
  "lib/scout/tsv/persist/fix_width_table.rb",
95
105
  "lib/scout/tsv/persist/serialize.rb",
96
106
  "lib/scout/tsv/persist/tokyocabinet.rb",
107
+ "lib/scout/tsv/stream.rb",
108
+ "lib/scout/tsv/transformer.rb",
97
109
  "lib/scout/tsv/traverse.rb",
98
110
  "lib/scout/tsv/util.rb",
99
111
  "lib/scout/tsv/util/filter.rb",
100
112
  "lib/scout/tsv/util/process.rb",
113
+ "lib/scout/tsv/util/reorder.rb",
101
114
  "lib/scout/tsv/util/select.rb",
115
+ "lib/scout/tsv/util/unzip.rb",
102
116
  "lib/scout/work_queue.rb",
103
117
  "lib/scout/work_queue/socket.rb",
104
118
  "lib/scout/work_queue/worker.rb",
105
119
  "lib/scout/workflow.rb",
106
120
  "lib/scout/workflow/definition.rb",
121
+ "lib/scout/workflow/deployment.rb",
122
+ "lib/scout/workflow/deployment/orchestrator.rb",
107
123
  "lib/scout/workflow/documentation.rb",
108
124
  "lib/scout/workflow/step.rb",
109
125
  "lib/scout/workflow/step/config.rb",
@@ -113,20 +129,25 @@ Gem::Specification.new do |s|
113
129
  "lib/scout/workflow/step/load.rb",
114
130
  "lib/scout/workflow/step/progress.rb",
115
131
  "lib/scout/workflow/step/provenance.rb",
132
+ "lib/scout/workflow/step/status.rb",
116
133
  "lib/scout/workflow/task.rb",
134
+ "lib/scout/workflow/task/dependencies.rb",
117
135
  "lib/scout/workflow/task/inputs.rb",
118
136
  "lib/scout/workflow/usage.rb",
119
137
  "lib/scout/workflow/util.rb",
120
138
  "lib/workflow-scout.rb",
121
139
  "scout-gear.gemspec",
122
140
  "scout_commands/alias",
141
+ "scout_commands/doc",
123
142
  "scout_commands/find",
124
143
  "scout_commands/glob",
144
+ "scout_commands/offsite",
125
145
  "scout_commands/rbbt",
146
+ "scout_commands/update",
126
147
  "scout_commands/workflow/info",
148
+ "scout_commands/workflow/install",
127
149
  "scout_commands/workflow/list",
128
150
  "scout_commands/workflow/task",
129
- "scout_commands/workflow/task_old",
130
151
  "share/color/color_names",
131
152
  "share/color/diverging_colors.hex",
132
153
  "share/software/install_helpers",
@@ -138,6 +159,10 @@ Gem::Specification.new do |s|
138
159
  "test/scout/misc/test_filesystem.rb",
139
160
  "test/scout/misc/test_insist.rb",
140
161
  "test/scout/misc/test_system.rb",
162
+ "test/scout/offsite/test_ssh.rb",
163
+ "test/scout/offsite/test_step.rb",
164
+ "test/scout/offsite/test_sync.rb",
165
+ "test/scout/offsite/test_task.rb",
141
166
  "test/scout/open/test_lock.rb",
142
167
  "test/scout/open/test_remote.rb",
143
168
  "test/scout/open/test_stream.rb",
@@ -175,22 +200,31 @@ Gem::Specification.new do |s|
175
200
  "test/scout/tsv/persist/test_adapter.rb",
176
201
  "test/scout/tsv/persist/test_fix_width_table.rb",
177
202
  "test/scout/tsv/persist/test_tokyocabinet.rb",
203
+ "test/scout/tsv/test_attach.rb",
204
+ "test/scout/tsv/test_change_id.rb",
178
205
  "test/scout/tsv/test_dumper.rb",
179
206
  "test/scout/tsv/test_index.rb",
180
207
  "test/scout/tsv/test_open.rb",
181
208
  "test/scout/tsv/test_parser.rb",
182
209
  "test/scout/tsv/test_persist.rb",
210
+ "test/scout/tsv/test_stream.rb",
211
+ "test/scout/tsv/test_transformer.rb",
183
212
  "test/scout/tsv/test_traverse.rb",
184
213
  "test/scout/tsv/test_util.rb",
185
214
  "test/scout/tsv/util/test_filter.rb",
186
215
  "test/scout/tsv/util/test_process.rb",
216
+ "test/scout/tsv/util/test_reorder.rb",
187
217
  "test/scout/tsv/util/test_select.rb",
218
+ "test/scout/tsv/util/test_unzip.rb",
188
219
  "test/scout/work_queue/test_socket.rb",
189
220
  "test/scout/work_queue/test_worker.rb",
221
+ "test/scout/workflow/deployment/test_orchestrator.rb",
190
222
  "test/scout/workflow/step/test_dependencies.rb",
191
223
  "test/scout/workflow/step/test_info.rb",
192
224
  "test/scout/workflow/step/test_load.rb",
193
225
  "test/scout/workflow/step/test_provenance.rb",
226
+ "test/scout/workflow/step/test_status.rb",
227
+ "test/scout/workflow/task/test_dependencies.rb",
194
228
  "test/scout/workflow/task/test_inputs.rb",
195
229
  "test/scout/workflow/test_definition.rb",
196
230
  "test/scout/workflow/test_documentation.rb",
@@ -199,29 +233,24 @@ Gem::Specification.new do |s|
199
233
  "test/scout/workflow/test_usage.rb",
200
234
  "test/scout/workflow/test_util.rb",
201
235
  "test/test_helper.rb",
202
- "test/test_scout-gear.rb"
236
+ "test/test_scout-gear.rb",
237
+ "test/test_scout.rb"
203
238
  ]
204
239
  s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
205
240
  s.licenses = ["MIT".freeze]
206
- s.rubygems_version = "3.2.15".freeze
241
+ s.rubygems_version = "3.4.13".freeze
207
242
  s.summary = "basic gear for scouts".freeze
208
243
 
209
- if s.respond_to? :specification_version then
210
- s.specification_version = 4
211
- end
244
+ s.specification_version = 4
212
245
 
213
- if s.respond_to? :add_runtime_dependency then
214
- s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
215
- s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
216
- s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
217
- s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
218
- s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
219
- else
220
- s.add_dependency(%q<term-ansicolor>.freeze, [">= 0"])
221
- s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
222
- s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
223
- s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
224
- s.add_dependency(%q<simplecov>.freeze, [">= 0"])
225
- end
246
+ s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
247
+ s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
248
+ s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
249
+ s.add_runtime_dependency(%q<sys-proctable>.freeze, [">= 0"])
250
+ s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
251
+ s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
252
+ s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
253
+ s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
254
+ s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
226
255
  end
227
256
 
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Print documentation for a module
10
+
11
+ $ #{$0} [<options>] [module]
12
+
13
+ If you don't specify the module it prints those available
14
+
15
+ -h--help Print this help
16
+ EOF
17
+ if options[:help]
18
+ if defined? scout_usage
19
+ scout_usage
20
+ else
21
+ puts SOPT.doc
22
+ end
23
+ exit 0
24
+ end
25
+
26
+
27
+ module_name = ARGV.first
28
+ if module_name.nil?
29
+ puts Rbbt.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
30
+ else
31
+ begin
32
+ puts Rbbt.doc.lib.scout[module_name].find_with_extension('md', :lib).read
33
+ rescue
34
+ puts Rbbt.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
35
+ end
36
+ end
37
+
data/scout_commands/find CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  require 'scout-gear'
4
+ require 'scout/offsite'
4
5
 
5
6
  $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
7
 
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+ require 'scout/offsite'
5
+
6
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
7
+
8
+ options = SOPT.setup <<EOF
9
+
10
+ Description of the tool
11
+
12
+ $ #{$0} [<options>] <where> <command> ... -- [<protected options>]
13
+
14
+ Run a command offsite. Use -- to make sure options reach the offsite
15
+
16
+ -h--help Print this help
17
+ EOF
18
+ if options[:help]
19
+ if defined? scout_usage
20
+ scout_usage
21
+ else
22
+ puts SOPT.doc
23
+ end
24
+ exit 0
25
+ end
26
+
27
+ where = ARGV.shift
28
+ command = ARGV.shift
29
+
30
+ puts SSHLine.command(where, command, ARGV)
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Update scout installation
10
+
11
+ $ #{$0} [<options>] [gem]
12
+
13
+ If no gem is specified 'scout-gear' is selected.
14
+
15
+ -h--help Print this help
16
+ EOF
17
+ if options[:help]
18
+ if defined? scout_usage
19
+ scout_usage
20
+ else
21
+ puts SOPT.doc
22
+ end
23
+ exit 0
24
+ end
25
+
26
+ gem_name = ARGV.first
27
+ Misc.update_git(gem_name)
28
+
29
+
@@ -23,7 +23,19 @@ end
23
23
 
24
24
  path = ARGV.first
25
25
  raise MissingParameterException.new :step_path if path.nil?
26
- step = Step.load(path)
27
-
28
- puts step.info.to_json
26
+ step = Step.load(path.dup)
27
+
28
+ step.info.each do |k,v|
29
+ case v
30
+ when nil
31
+ next
32
+ when Exception
33
+ puts Log.color(:title, "Exception")
34
+ Log.exception v
35
+ else
36
+ puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
37
+ end
38
+ rescue
39
+ next
40
+ end
29
41