scout-gear 10.6.1 → 10.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +2 -1
- data/VERSION +1 -1
- data/lib/scout/entity.rb +3 -0
- data/lib/scout/tsv/annotation/repo.rb +11 -7
- data/lib/scout/tsv/attach.rb +2 -2
- data/lib/scout/tsv/parser.rb +1 -0
- data/lib/scout/tsv/transformer.rb +1 -0
- data/lib/scout/tsv/traverse.rb +2 -1
- data/lib/scout/tsv.rb +1 -4
- data/lib/scout/work_queue.rb +5 -2
- data/lib/scout/workflow/definition.rb +25 -8
- data/lib/scout/workflow/deployment/orchestrator.rb +3 -2
- data/lib/scout/workflow/exceptions.rb +1 -0
- data/lib/scout/workflow/path.rb +40 -0
- data/lib/scout/workflow/step/dependencies.rb +2 -2
- data/lib/scout/workflow/step/info.rb +6 -2
- data/lib/scout/workflow/step/status.rb +2 -0
- data/lib/scout/workflow/step.rb +2 -1
- data/lib/scout/workflow/task/dependencies.rb +1 -1
- data/lib/scout/workflow/task/inputs.rb +3 -1
- data/lib/scout/workflow/task.rb +0 -1
- data/lib/scout/workflow/usage.rb +1 -1
- data/lib/scout/workflow.rb +3 -0
- data/scout-gear.gemspec +6 -5
- data/test/scout/test_work_queue.rb +2 -0
- data/test/scout/test_workflow.rb +16 -0
- data/test/scout/tsv/test_attach.rb +29 -0
- data/test/scout/workflow/test_definition.rb +31 -0
- data/test/scout/workflow/test_path.rb +9 -0
- metadata +5 -4
- data/lib/scout/tsv/entity.rb +0 -5
- data/test/scout/tsv/test_entity.rb +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e72b6d2bef9af0067bf2601e059aa992ffdab98fba772dba1e297f958edb02d
|
4
|
+
data.tar.gz: 2c2cc3591c1645ccf34dbb250f1b2acda1b69040c0b051806c02c97eb07cc253
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1be99dd4127f54c5cf16833197e527a3cfa41014c7db4306ec291e8f01b78f8340cad7ee6e342dee2ed25221f2b02a6f6b665f46e81365ad5a12eea087b300d
|
7
|
+
data.tar.gz: cc3a9b28e47520f6756cba18d6037d7e83b7cd4d0cded1597a7cafc2cc167234090c1014d2b38adcc0a6a69c8d3ba1a72255538bd924e14f55b95769f3eca977
|
data/.vimproject
CHANGED
@@ -7,10 +7,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
7
7
|
scout=scout{
|
8
8
|
workflow.rb
|
9
9
|
workflow=workflow{
|
10
|
+
exceptions.rb
|
10
11
|
definition.rb
|
11
12
|
documentation.rb
|
12
13
|
usage.rb
|
13
14
|
util.rb
|
15
|
+
path.rb
|
14
16
|
step.rb
|
15
17
|
step=step{
|
16
18
|
config.rb
|
@@ -80,7 +82,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
80
82
|
sort.rb
|
81
83
|
unzip.rb
|
82
84
|
}
|
83
|
-
entity.rb
|
84
85
|
parser.rb
|
85
86
|
dumper.rb
|
86
87
|
transformer.rb
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
10.
|
1
|
+
10.7.1
|
data/lib/scout/entity.rb
CHANGED
@@ -3,6 +3,7 @@ require_relative 'entity/format'
|
|
3
3
|
require_relative 'entity/property'
|
4
4
|
require_relative 'entity/object'
|
5
5
|
require_relative 'entity/identifiers'
|
6
|
+
|
6
7
|
module Entity
|
7
8
|
def self.extended(base)
|
8
9
|
base.extend Annotation
|
@@ -33,6 +34,8 @@ module Entity
|
|
33
34
|
entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity)
|
34
35
|
|
35
36
|
entity = mod.setup(entity, params)
|
37
|
+
|
38
|
+
entity.extend AnnotatedArray if Array === entity && ! options[:annotated_array] == FalseClass
|
36
39
|
end
|
37
40
|
|
38
41
|
entity
|
@@ -1,17 +1,21 @@
|
|
1
1
|
module Persist
|
2
|
+
REPO_CACHE = {}
|
2
3
|
def self.annotation_repo_persist(repo, name, &block)
|
3
4
|
|
4
5
|
if String === repo
|
5
|
-
repo = repo
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
repo = REPO_CACHE[repo] ||= begin
|
7
|
+
repo = repo.find if Path === repo
|
8
|
+
repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
|
9
|
+
repo_fields = ["literal", "annotation_types", "JSON"]
|
10
|
+
TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
|
11
|
+
repo.save_annotation_hash
|
12
|
+
repo
|
13
|
+
end
|
10
14
|
repo.close
|
11
|
-
else
|
12
|
-
repo_fields = repo.fields
|
13
15
|
end
|
14
16
|
|
17
|
+
repo_fields = repo.fields
|
18
|
+
|
15
19
|
subkey = name + ":"
|
16
20
|
|
17
21
|
keys = repo.read_and_close do
|
data/lib/scout/tsv/attach.rb
CHANGED
@@ -154,7 +154,7 @@ module TSV
|
|
154
154
|
|
155
155
|
other_values.zip(overlaps).each do |v,overlap|
|
156
156
|
if type == :list
|
157
|
-
current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
|
157
|
+
current_values[overlap] = v if current_values[overlap].nil? || (String === current_values[overlap] && current_values[overlap].empty?)
|
158
158
|
elsif type == :flat
|
159
159
|
next if v.nil?
|
160
160
|
v = [v] unless Array === v
|
@@ -206,7 +206,7 @@ module TSV
|
|
206
206
|
if overlap == :key
|
207
207
|
other_key = Array === v ? v : v.first
|
208
208
|
elsif type == :list
|
209
|
-
new_values[overlap] = v if
|
209
|
+
new_values[overlap] = v if new_values[overlap].nil? || (String === new_values[overlap] && new_values[overlap].empty?)
|
210
210
|
else
|
211
211
|
v = [v] unless Array === v
|
212
212
|
new_values[overlap].concat v
|
data/lib/scout/tsv/parser.rb
CHANGED
@@ -352,6 +352,7 @@ module TSV
|
|
352
352
|
all_field_names ||= [@key_field] + @fields
|
353
353
|
fields = all_field_names if fields == :all
|
354
354
|
positions = NamedArray.identify_name(all_field_names, fields)
|
355
|
+
|
355
356
|
raise "Not all fields (#{Log.fingerprint fields}) identified in #{Log.fingerprint all_field_names}" if positions.include?(nil)
|
356
357
|
kwargs[:positions] = positions
|
357
358
|
field_names = all_field_names.values_at *positions
|
@@ -80,6 +80,7 @@ module TSV
|
|
80
80
|
def each(*args, **kwargs, &block)
|
81
81
|
kwargs[:into] = @dumper
|
82
82
|
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
83
|
+
@dumper.namespace ||= @namespace
|
83
84
|
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
84
85
|
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
85
86
|
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
data/lib/scout/tsv/traverse.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require_relative 'parser'
|
2
2
|
module TSV
|
3
|
-
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
|
3
|
+
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
|
4
4
|
key_field = key_field_pos if key_field.nil?
|
5
5
|
fields = fields_pos.dup if fields.nil?
|
6
6
|
type = @type if type.nil?
|
@@ -74,6 +74,7 @@ module TSV
|
|
74
74
|
values = TSV.cast_value(values, cast) if cast
|
75
75
|
|
76
76
|
if Array === key
|
77
|
+
key = key.uniq if uniq
|
77
78
|
if @type == :double && one2one
|
78
79
|
if one2one == :strict
|
79
80
|
key.each_with_index do |key_i,i|
|
data/lib/scout/tsv.rb
CHANGED
@@ -11,13 +11,12 @@ require_relative 'tsv/open'
|
|
11
11
|
require_relative 'tsv/attach'
|
12
12
|
require_relative 'tsv/change_id'
|
13
13
|
require_relative 'tsv/stream'
|
14
|
-
require_relative 'tsv/entity'
|
15
14
|
require_relative 'tsv/annotation'
|
16
15
|
require_relative 'tsv/csv'
|
17
16
|
|
18
17
|
module TSV
|
19
18
|
extend Annotation
|
20
|
-
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :
|
19
|
+
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
|
21
20
|
|
22
21
|
def self.str2options(str)
|
23
22
|
field_options,_sep, rest = str.partition("#")
|
@@ -122,8 +121,6 @@ module TSV
|
|
122
121
|
end
|
123
122
|
end
|
124
123
|
|
125
|
-
tsv.entity_options = entity_options
|
126
|
-
|
127
124
|
tsv
|
128
125
|
end
|
129
126
|
end
|
data/lib/scout/work_queue.rb
CHANGED
@@ -138,17 +138,19 @@ class WorkQueue
|
|
138
138
|
end
|
139
139
|
|
140
140
|
def abort
|
141
|
+
@aborted = true
|
141
142
|
Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
|
142
143
|
@worker_mutex.synchronize do
|
143
144
|
@workers.each do |w|
|
144
|
-
ScoutSemaphore.post_semaphore(@output.write_sem)
|
145
|
-
ScoutSemaphore.post_semaphore(@input.read_sem)
|
145
|
+
ScoutSemaphore.post_semaphore(@output.write_sem)
|
146
|
+
ScoutSemaphore.post_semaphore(@input.read_sem)
|
146
147
|
w.abort
|
147
148
|
end
|
148
149
|
end
|
149
150
|
end
|
150
151
|
|
151
152
|
def close
|
153
|
+
return if @closed || @aborted
|
152
154
|
@closed = true
|
153
155
|
@worker_mutex.synchronize{ @workers.length }.times do
|
154
156
|
begin
|
@@ -165,6 +167,7 @@ class WorkQueue
|
|
165
167
|
end
|
166
168
|
|
167
169
|
def join(clean = true)
|
170
|
+
close
|
168
171
|
begin
|
169
172
|
@waiter.join if @waiter
|
170
173
|
@reader.join if @reader
|
@@ -21,12 +21,14 @@ module Workflow
|
|
21
21
|
@helpers ||= {}
|
22
22
|
end
|
23
23
|
|
24
|
-
def helper(name, *args, &block)
|
24
|
+
def helper(name, *args, **kwargs, &block)
|
25
25
|
if block_given?
|
26
26
|
helpers[name] = block
|
27
27
|
else
|
28
28
|
raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
|
29
|
-
|
29
|
+
o = Object.new
|
30
|
+
o.extend step_module
|
31
|
+
o.send(name, *args, **kwargs)
|
30
32
|
end
|
31
33
|
end
|
32
34
|
|
@@ -138,8 +140,17 @@ module Workflow
|
|
138
140
|
end
|
139
141
|
end
|
140
142
|
|
141
|
-
|
142
|
-
|
143
|
+
FORGET_TASK_ALIAS = begin
|
144
|
+
%w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
|
145
|
+
ENV[var] == 'true'
|
146
|
+
end.any?
|
147
|
+
end
|
148
|
+
REMOVE_TASK_ALIAS = begin
|
149
|
+
remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
|
150
|
+
ENV.include?(var) && ENV[var] != 'false'
|
151
|
+
end.first
|
152
|
+
remove.nil? ? false : remove
|
153
|
+
end
|
143
154
|
def task_alias(name, workflow, oname, *rest, &block)
|
144
155
|
dep(workflow, oname, *rest, &block)
|
145
156
|
extension :dep_task unless @extension
|
@@ -156,9 +167,15 @@ module Workflow
|
|
156
167
|
raise dep.get_exception if dep.error?
|
157
168
|
raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
|
158
169
|
set_info :type, dep.info[:type]
|
159
|
-
|
170
|
+
|
171
|
+
forget = config :forget_task_alias, "forget_task_alias"
|
172
|
+
forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
|
173
|
+
|
160
174
|
if forget
|
161
|
-
remove = config :
|
175
|
+
remove = config :remove_task_alias, "remove_task_alias"
|
176
|
+
remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_TASK_ALIAS if remove.nil?
|
177
|
+
|
178
|
+
Log.medium "Forget task_alias (remove: #{remove}): #{short_path}"
|
162
179
|
|
163
180
|
self.archive_deps
|
164
181
|
self.copy_linked_files_dir
|
@@ -177,9 +194,9 @@ module Workflow
|
|
177
194
|
when 'recursive'
|
178
195
|
(dep.dependencies + dep.rec_dependencies).uniq.each do |d|
|
179
196
|
next if d.overriden
|
180
|
-
d.clean unless
|
197
|
+
d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
|
181
198
|
end
|
182
|
-
dep.clean unless
|
199
|
+
dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
|
183
200
|
end
|
184
201
|
end
|
185
202
|
else
|
@@ -246,13 +246,14 @@ module Workflow
|
|
246
246
|
workload = new_workload
|
247
247
|
sleep timer
|
248
248
|
end
|
249
|
+
all_jobs.each{|s| s.join }
|
249
250
|
rescue TryAgain
|
250
251
|
retry
|
251
252
|
end
|
252
253
|
end
|
253
254
|
end
|
254
255
|
|
255
|
-
def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
|
256
|
+
def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors, produce_timer = 5)
|
256
257
|
jobs = [jobs] unless Array === jobs
|
257
258
|
produce_list = []
|
258
259
|
jobs.each do |job|
|
@@ -264,7 +265,7 @@ module Workflow
|
|
264
265
|
end
|
265
266
|
end
|
266
267
|
|
267
|
-
orchestrator = Orchestrator.new
|
268
|
+
orchestrator = Orchestrator.new produce_timer, cpus: produce_cpus.to_i
|
268
269
|
orchestrator.process({}, produce_list)
|
269
270
|
produce_list
|
270
271
|
end
|
@@ -0,0 +1 @@
|
|
1
|
+
class TaskNotFound < StandardError; end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Path
|
2
|
+
def self.step_file?(path)
|
3
|
+
return false unless path =~ /\.files(?:\/|$)/
|
4
|
+
parts = path.split("/")
|
5
|
+
job = parts.select{|p| p =~ /\.files$/}.first
|
6
|
+
|
7
|
+
if job
|
8
|
+
i = parts.index job
|
9
|
+
begin
|
10
|
+
workflow, task = parts.values_at i - 2, i - 1
|
11
|
+
_loaded = false
|
12
|
+
begin
|
13
|
+
Kernel.const_get(workflow)
|
14
|
+
rescue
|
15
|
+
if ! _loaded
|
16
|
+
Workflow.require_workflow workflow
|
17
|
+
_loaded = true
|
18
|
+
retry
|
19
|
+
end
|
20
|
+
raise $!
|
21
|
+
end
|
22
|
+
return parts[i-2..-1] * "/"
|
23
|
+
rescue
|
24
|
+
Log.exception $!
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
alias original_digest_str digest_str
|
32
|
+
|
33
|
+
def digest_str
|
34
|
+
if step_file = Path.step_file?(self)
|
35
|
+
"Step file: #{step_file}"
|
36
|
+
else
|
37
|
+
original_digest_str
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -11,7 +11,7 @@ class Step
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def recursive_inputs
|
14
|
-
recursive_inputs = NamedArray ===
|
14
|
+
recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
|
15
15
|
return recursive_inputs if dependencies.nil?
|
16
16
|
dependencies.inject(recursive_inputs) do |acc,dep|
|
17
17
|
acc.merge(dep.recursive_inputs)
|
@@ -36,7 +36,7 @@ class Step
|
|
36
36
|
|
37
37
|
dependencies.each do |dep|
|
38
38
|
if dep.present? && ! dep.updated?
|
39
|
-
Log.
|
39
|
+
Log.medium "Clean outdated #{dep.path}"
|
40
40
|
dep.clean
|
41
41
|
end
|
42
42
|
|
@@ -10,7 +10,11 @@ class Step
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def self.load_info(info_file)
|
13
|
-
info =
|
13
|
+
info = begin
|
14
|
+
Persist.load(info_file, SERIALIZER) || {}
|
15
|
+
rescue
|
16
|
+
{status: :noinfo}
|
17
|
+
end
|
14
18
|
IndiferentHash.setup(info)
|
15
19
|
end
|
16
20
|
|
@@ -152,7 +156,7 @@ class Step
|
|
152
156
|
end
|
153
157
|
|
154
158
|
def running?
|
155
|
-
! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
159
|
+
! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
156
160
|
end
|
157
161
|
|
158
162
|
def overriden?
|
@@ -26,10 +26,12 @@ class Step
|
|
26
26
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
27
27
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
28
28
|
|
29
|
+
Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
|
29
30
|
newer.empty?
|
30
31
|
end
|
31
32
|
|
32
33
|
def clean
|
34
|
+
Log.debug "Cleaning job files: #{path}"
|
33
35
|
@take_stream = nil
|
34
36
|
@result = nil
|
35
37
|
@info = nil
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -101,7 +101,7 @@ module Task
|
|
101
101
|
else
|
102
102
|
dep = _res
|
103
103
|
dependencies << dep
|
104
|
-
dep_non_default_inputs = find_dep_non_default_inputs.call(dep,
|
104
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
|
105
105
|
non_default_inputs.concat(dep_non_default_inputs)
|
106
106
|
end
|
107
107
|
end
|
@@ -47,7 +47,7 @@ module Task
|
|
47
47
|
elsif String === value && Symbol === provided && provided.to_s == value
|
48
48
|
same_as_default = true
|
49
49
|
else
|
50
|
-
|
50
|
+
same_as_default = false
|
51
51
|
end
|
52
52
|
if ! provided.nil? && ! same_as_default
|
53
53
|
non_default_inputs << name.to_sym
|
@@ -91,6 +91,8 @@ module Task
|
|
91
91
|
if Path.is_filename?(value)
|
92
92
|
if type == :path
|
93
93
|
Open.write(input_file + ".as_path", value)
|
94
|
+
elsif Path.step_file?(value)
|
95
|
+
Open.write(input_file + ".as_path", value)
|
94
96
|
else
|
95
97
|
relative_file = save_file_input(value, directory)
|
96
98
|
Open.write(input_file + ".as_file", relative_file)
|
data/lib/scout/workflow/task.rb
CHANGED
data/lib/scout/workflow/usage.rb
CHANGED
@@ -138,7 +138,7 @@ module Workflow
|
|
138
138
|
|
139
139
|
dep_tree = {}
|
140
140
|
task = self.tasks[task_name]
|
141
|
-
raise
|
141
|
+
raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
|
142
142
|
task.deps.each do |workflow, task, options|
|
143
143
|
next if seen.include? dep
|
144
144
|
seen << [workflow, task, options.merge(seen_options)]
|
data/lib/scout/workflow.rb
CHANGED
@@ -5,6 +5,8 @@ require_relative 'workflow/step'
|
|
5
5
|
require_relative 'workflow/documentation'
|
6
6
|
require_relative 'workflow/usage'
|
7
7
|
require_relative 'workflow/deployment'
|
8
|
+
require_relative 'workflow/exceptions'
|
9
|
+
require_relative 'workflow/path'
|
8
10
|
|
9
11
|
require 'scout/resource'
|
10
12
|
require 'scout/resource/scout'
|
@@ -153,6 +155,7 @@ module Workflow
|
|
153
155
|
|
154
156
|
def job(name, *args)
|
155
157
|
task = tasks[name]
|
158
|
+
raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
|
156
159
|
step = task.job(*args)
|
157
160
|
step.extend step_module
|
158
161
|
step
|
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear 10.
|
5
|
+
# stub: scout-gear 10.7.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "10.
|
9
|
+
s.version = "10.7.1".freeze
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2024-
|
14
|
+
s.date = "2024-07-01"
|
15
15
|
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -72,7 +72,6 @@ Gem::Specification.new do |s|
|
|
72
72
|
"lib/scout/tsv/change_id/translate.rb",
|
73
73
|
"lib/scout/tsv/csv.rb",
|
74
74
|
"lib/scout/tsv/dumper.rb",
|
75
|
-
"lib/scout/tsv/entity.rb",
|
76
75
|
"lib/scout/tsv/index.rb",
|
77
76
|
"lib/scout/tsv/open.rb",
|
78
77
|
"lib/scout/tsv/parser.rb",
|
@@ -98,6 +97,8 @@ Gem::Specification.new do |s|
|
|
98
97
|
"lib/scout/workflow/deployment/orchestrator.rb",
|
99
98
|
"lib/scout/workflow/deployment/trace.rb",
|
100
99
|
"lib/scout/workflow/documentation.rb",
|
100
|
+
"lib/scout/workflow/exceptions.rb",
|
101
|
+
"lib/scout/workflow/path.rb",
|
101
102
|
"lib/scout/workflow/step.rb",
|
102
103
|
"lib/scout/workflow/step/archive.rb",
|
103
104
|
"lib/scout/workflow/step/children.rb",
|
@@ -184,7 +185,6 @@ Gem::Specification.new do |s|
|
|
184
185
|
"test/scout/tsv/test_change_id.rb",
|
185
186
|
"test/scout/tsv/test_csv.rb",
|
186
187
|
"test/scout/tsv/test_dumper.rb",
|
187
|
-
"test/scout/tsv/test_entity.rb",
|
188
188
|
"test/scout/tsv/test_index.rb",
|
189
189
|
"test/scout/tsv/test_open.rb",
|
190
190
|
"test/scout/tsv/test_parser.rb",
|
@@ -214,6 +214,7 @@ Gem::Specification.new do |s|
|
|
214
214
|
"test/scout/workflow/task/test_inputs.rb",
|
215
215
|
"test/scout/workflow/test_definition.rb",
|
216
216
|
"test/scout/workflow/test_documentation.rb",
|
217
|
+
"test/scout/workflow/test_path.rb",
|
217
218
|
"test/scout/workflow/test_step.rb",
|
218
219
|
"test/scout/workflow/test_task.rb",
|
219
220
|
"test/scout/workflow/test_usage.rb",
|
@@ -117,6 +117,7 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
117
117
|
|
118
118
|
assert_raise ScoutException do
|
119
119
|
begin
|
120
|
+
t.join
|
120
121
|
q.join(false)
|
121
122
|
rescue
|
122
123
|
t.raise($!)
|
@@ -158,6 +159,7 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
158
159
|
|
159
160
|
assert_raise ScoutException do
|
160
161
|
begin
|
162
|
+
t.join
|
161
163
|
q.join(false)
|
162
164
|
rescue Exception
|
163
165
|
t.raise($!)
|
data/test/scout/test_workflow.rb
CHANGED
@@ -85,4 +85,20 @@ class TestWorkflow < Test::Unit::TestCase
|
|
85
85
|
assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
|
86
86
|
Baking.job(:bake_muffin_tray, "Blueberry muffin", :add_bluberries => true).run
|
87
87
|
end
|
88
|
+
|
89
|
+
def test_call_helper
|
90
|
+
wf = Module.new do
|
91
|
+
extend Workflow
|
92
|
+
|
93
|
+
helper :m1 do
|
94
|
+
"TEST"
|
95
|
+
end
|
96
|
+
|
97
|
+
helper :m2 do
|
98
|
+
m1
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
assert_equal "TEST", wf.helper(:m2)
|
103
|
+
end
|
88
104
|
end
|
@@ -449,4 +449,33 @@ A Id3
|
|
449
449
|
assert res["row2"].include? "Id3"
|
450
450
|
assert ! res["row2"].include?("b")
|
451
451
|
end
|
452
|
+
|
453
|
+
def test_attach_list_to_list_with_complete
|
454
|
+
content1 =<<-EOF
|
455
|
+
#Id ValueA ValueB
|
456
|
+
row1 a b
|
457
|
+
row2 A B
|
458
|
+
EOF
|
459
|
+
|
460
|
+
content2 =<<-EOF
|
461
|
+
#Id ValueC
|
462
|
+
row1 c
|
463
|
+
row2 C
|
464
|
+
row3 CC
|
465
|
+
EOF
|
466
|
+
|
467
|
+
tsv1 = tsv2 = index = nil
|
468
|
+
TmpFile.with_file(content1) do |filename|
|
469
|
+
tsv1 = TSV.open(File.open(filename), type: :list, fields: ["ValueA"], sep: /\s+/)
|
470
|
+
end
|
471
|
+
|
472
|
+
TmpFile.with_file(content2) do |filename|
|
473
|
+
tsv2 = TSV.open(File.open(filename), type: :list, sep: /\s+/)
|
474
|
+
end
|
475
|
+
|
476
|
+
res = tsv1.attach tsv2, :fields => ["ValueC"], complete: true
|
477
|
+
assert res["row2"].include?("C")
|
478
|
+
refute res["row2"].include?("b")
|
479
|
+
assert res["row3"].include?("CC")
|
480
|
+
end
|
452
481
|
end
|
@@ -66,5 +66,36 @@ class TestWorkflowDefinition < Test::Unit::TestCase
|
|
66
66
|
assert_include job.archived_info, dep_path
|
67
67
|
assert_equal :done, job.archived_info[dep_path][:status]
|
68
68
|
end
|
69
|
+
|
70
|
+
def test_task_alias_remove_dep_partial
|
71
|
+
wf = Workflow.annonymous_workflow do
|
72
|
+
self.name = "CallName"
|
73
|
+
|
74
|
+
task :salute => :string do |name|
|
75
|
+
"Hi"
|
76
|
+
end
|
77
|
+
|
78
|
+
dep :salute
|
79
|
+
input :name, :string, "Name to call", nil, :jobname => true
|
80
|
+
task :call_name => :string do |name|
|
81
|
+
"#{step(:salute).load} #{name}"
|
82
|
+
end
|
83
|
+
|
84
|
+
task_alias :call_miguel, self, :call_name, name: "Miguel"
|
85
|
+
end
|
86
|
+
|
87
|
+
old_cache = Scout::Config::CACHE.dup
|
88
|
+
Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => :recursive}, 'task:CallName#call_miguel')
|
89
|
+
Scout::Config.set({:remove_dep => false}, 'task:CallName#call_name')
|
90
|
+
job = wf.job(:call_miguel)
|
91
|
+
call_name = job.step(:call_name)
|
92
|
+
salute = job.step(:salute)
|
93
|
+
assert_equal "Hi Miguel", job.run
|
94
|
+
refute salute.done?
|
95
|
+
assert call_name.done?
|
96
|
+
Scout::Config::CACHE.replace old_cache
|
97
|
+
assert_include job.archived_info, call_name.path
|
98
|
+
assert_equal :done, job.archived_info[call_name.path][:status]
|
99
|
+
end
|
69
100
|
end
|
70
101
|
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestClass < Test::Unit::TestCase
|
5
|
+
def test_path_digest
|
6
|
+
iii Misc.digest_str(Path.setup("test_file"))
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scout-gear
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 10.
|
4
|
+
version: 10.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: scout-essentials
|
@@ -184,7 +184,6 @@ files:
|
|
184
184
|
- lib/scout/tsv/change_id/translate.rb
|
185
185
|
- lib/scout/tsv/csv.rb
|
186
186
|
- lib/scout/tsv/dumper.rb
|
187
|
-
- lib/scout/tsv/entity.rb
|
188
187
|
- lib/scout/tsv/index.rb
|
189
188
|
- lib/scout/tsv/open.rb
|
190
189
|
- lib/scout/tsv/parser.rb
|
@@ -210,6 +209,8 @@ files:
|
|
210
209
|
- lib/scout/workflow/deployment/orchestrator.rb
|
211
210
|
- lib/scout/workflow/deployment/trace.rb
|
212
211
|
- lib/scout/workflow/documentation.rb
|
212
|
+
- lib/scout/workflow/exceptions.rb
|
213
|
+
- lib/scout/workflow/path.rb
|
213
214
|
- lib/scout/workflow/step.rb
|
214
215
|
- lib/scout/workflow/step/archive.rb
|
215
216
|
- lib/scout/workflow/step/children.rb
|
@@ -296,7 +297,6 @@ files:
|
|
296
297
|
- test/scout/tsv/test_change_id.rb
|
297
298
|
- test/scout/tsv/test_csv.rb
|
298
299
|
- test/scout/tsv/test_dumper.rb
|
299
|
-
- test/scout/tsv/test_entity.rb
|
300
300
|
- test/scout/tsv/test_index.rb
|
301
301
|
- test/scout/tsv/test_open.rb
|
302
302
|
- test/scout/tsv/test_parser.rb
|
@@ -326,6 +326,7 @@ files:
|
|
326
326
|
- test/scout/workflow/task/test_inputs.rb
|
327
327
|
- test/scout/workflow/test_definition.rb
|
328
328
|
- test/scout/workflow/test_documentation.rb
|
329
|
+
- test/scout/workflow/test_path.rb
|
329
330
|
- test/scout/workflow/test_step.rb
|
330
331
|
- test/scout/workflow/test_task.rb
|
331
332
|
- test/scout/workflow/test_usage.rb
|
data/lib/scout/tsv/entity.rb
DELETED
File without changes
|