scout-gear 10.6.1 → 10.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +2 -1
- data/VERSION +1 -1
- data/lib/scout/entity.rb +3 -0
- data/lib/scout/tsv/annotation/repo.rb +11 -7
- data/lib/scout/tsv/attach.rb +2 -2
- data/lib/scout/tsv/parser.rb +1 -0
- data/lib/scout/tsv/transformer.rb +1 -0
- data/lib/scout/tsv/traverse.rb +2 -1
- data/lib/scout/tsv.rb +1 -4
- data/lib/scout/work_queue.rb +5 -2
- data/lib/scout/workflow/definition.rb +25 -8
- data/lib/scout/workflow/deployment/orchestrator.rb +3 -2
- data/lib/scout/workflow/exceptions.rb +1 -0
- data/lib/scout/workflow/path.rb +40 -0
- data/lib/scout/workflow/step/dependencies.rb +2 -2
- data/lib/scout/workflow/step/info.rb +6 -2
- data/lib/scout/workflow/step/status.rb +2 -0
- data/lib/scout/workflow/step.rb +2 -1
- data/lib/scout/workflow/task/dependencies.rb +1 -1
- data/lib/scout/workflow/task/inputs.rb +3 -1
- data/lib/scout/workflow/task.rb +0 -1
- data/lib/scout/workflow/usage.rb +1 -1
- data/lib/scout/workflow.rb +3 -0
- data/scout-gear.gemspec +6 -5
- data/test/scout/test_work_queue.rb +2 -0
- data/test/scout/test_workflow.rb +16 -0
- data/test/scout/tsv/test_attach.rb +29 -0
- data/test/scout/workflow/test_definition.rb +31 -0
- data/test/scout/workflow/test_path.rb +9 -0
- metadata +5 -4
- data/lib/scout/tsv/entity.rb +0 -5
- data/test/scout/tsv/test_entity.rb +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e72b6d2bef9af0067bf2601e059aa992ffdab98fba772dba1e297f958edb02d
|
4
|
+
data.tar.gz: 2c2cc3591c1645ccf34dbb250f1b2acda1b69040c0b051806c02c97eb07cc253
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1be99dd4127f54c5cf16833197e527a3cfa41014c7db4306ec291e8f01b78f8340cad7ee6e342dee2ed25221f2b02a6f6b665f46e81365ad5a12eea087b300d
|
7
|
+
data.tar.gz: cc3a9b28e47520f6756cba18d6037d7e83b7cd4d0cded1597a7cafc2cc167234090c1014d2b38adcc0a6a69c8d3ba1a72255538bd924e14f55b95769f3eca977
|
data/.vimproject
CHANGED
@@ -7,10 +7,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
7
7
|
scout=scout{
|
8
8
|
workflow.rb
|
9
9
|
workflow=workflow{
|
10
|
+
exceptions.rb
|
10
11
|
definition.rb
|
11
12
|
documentation.rb
|
12
13
|
usage.rb
|
13
14
|
util.rb
|
15
|
+
path.rb
|
14
16
|
step.rb
|
15
17
|
step=step{
|
16
18
|
config.rb
|
@@ -80,7 +82,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
80
82
|
sort.rb
|
81
83
|
unzip.rb
|
82
84
|
}
|
83
|
-
entity.rb
|
84
85
|
parser.rb
|
85
86
|
dumper.rb
|
86
87
|
transformer.rb
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
10.
|
1
|
+
10.7.1
|
data/lib/scout/entity.rb
CHANGED
@@ -3,6 +3,7 @@ require_relative 'entity/format'
|
|
3
3
|
require_relative 'entity/property'
|
4
4
|
require_relative 'entity/object'
|
5
5
|
require_relative 'entity/identifiers'
|
6
|
+
|
6
7
|
module Entity
|
7
8
|
def self.extended(base)
|
8
9
|
base.extend Annotation
|
@@ -33,6 +34,8 @@ module Entity
|
|
33
34
|
entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity)
|
34
35
|
|
35
36
|
entity = mod.setup(entity, params)
|
37
|
+
|
38
|
+
entity.extend AnnotatedArray if Array === entity && ! options[:annotated_array] == FalseClass
|
36
39
|
end
|
37
40
|
|
38
41
|
entity
|
@@ -1,17 +1,21 @@
|
|
1
1
|
module Persist
|
2
|
+
REPO_CACHE = {}
|
2
3
|
def self.annotation_repo_persist(repo, name, &block)
|
3
4
|
|
4
5
|
if String === repo
|
5
|
-
repo = repo
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
repo = REPO_CACHE[repo] ||= begin
|
7
|
+
repo = repo.find if Path === repo
|
8
|
+
repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
|
9
|
+
repo_fields = ["literal", "annotation_types", "JSON"]
|
10
|
+
TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
|
11
|
+
repo.save_annotation_hash
|
12
|
+
repo
|
13
|
+
end
|
10
14
|
repo.close
|
11
|
-
else
|
12
|
-
repo_fields = repo.fields
|
13
15
|
end
|
14
16
|
|
17
|
+
repo_fields = repo.fields
|
18
|
+
|
15
19
|
subkey = name + ":"
|
16
20
|
|
17
21
|
keys = repo.read_and_close do
|
data/lib/scout/tsv/attach.rb
CHANGED
@@ -154,7 +154,7 @@ module TSV
|
|
154
154
|
|
155
155
|
other_values.zip(overlaps).each do |v,overlap|
|
156
156
|
if type == :list
|
157
|
-
current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
|
157
|
+
current_values[overlap] = v if current_values[overlap].nil? || (String === current_values[overlap] && current_values[overlap].empty?)
|
158
158
|
elsif type == :flat
|
159
159
|
next if v.nil?
|
160
160
|
v = [v] unless Array === v
|
@@ -206,7 +206,7 @@ module TSV
|
|
206
206
|
if overlap == :key
|
207
207
|
other_key = Array === v ? v : v.first
|
208
208
|
elsif type == :list
|
209
|
-
new_values[overlap] = v if
|
209
|
+
new_values[overlap] = v if new_values[overlap].nil? || (String === new_values[overlap] && new_values[overlap].empty?)
|
210
210
|
else
|
211
211
|
v = [v] unless Array === v
|
212
212
|
new_values[overlap].concat v
|
data/lib/scout/tsv/parser.rb
CHANGED
@@ -352,6 +352,7 @@ module TSV
|
|
352
352
|
all_field_names ||= [@key_field] + @fields
|
353
353
|
fields = all_field_names if fields == :all
|
354
354
|
positions = NamedArray.identify_name(all_field_names, fields)
|
355
|
+
|
355
356
|
raise "Not all fields (#{Log.fingerprint fields}) identified in #{Log.fingerprint all_field_names}" if positions.include?(nil)
|
356
357
|
kwargs[:positions] = positions
|
357
358
|
field_names = all_field_names.values_at *positions
|
@@ -80,6 +80,7 @@ module TSV
|
|
80
80
|
def each(*args, **kwargs, &block)
|
81
81
|
kwargs[:into] = @dumper
|
82
82
|
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
83
|
+
@dumper.namespace ||= @namespace
|
83
84
|
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
84
85
|
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
85
86
|
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
data/lib/scout/tsv/traverse.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require_relative 'parser'
|
2
2
|
module TSV
|
3
|
-
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
|
3
|
+
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
|
4
4
|
key_field = key_field_pos if key_field.nil?
|
5
5
|
fields = fields_pos.dup if fields.nil?
|
6
6
|
type = @type if type.nil?
|
@@ -74,6 +74,7 @@ module TSV
|
|
74
74
|
values = TSV.cast_value(values, cast) if cast
|
75
75
|
|
76
76
|
if Array === key
|
77
|
+
key = key.uniq if uniq
|
77
78
|
if @type == :double && one2one
|
78
79
|
if one2one == :strict
|
79
80
|
key.each_with_index do |key_i,i|
|
data/lib/scout/tsv.rb
CHANGED
@@ -11,13 +11,12 @@ require_relative 'tsv/open'
|
|
11
11
|
require_relative 'tsv/attach'
|
12
12
|
require_relative 'tsv/change_id'
|
13
13
|
require_relative 'tsv/stream'
|
14
|
-
require_relative 'tsv/entity'
|
15
14
|
require_relative 'tsv/annotation'
|
16
15
|
require_relative 'tsv/csv'
|
17
16
|
|
18
17
|
module TSV
|
19
18
|
extend Annotation
|
20
|
-
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :
|
19
|
+
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
|
21
20
|
|
22
21
|
def self.str2options(str)
|
23
22
|
field_options,_sep, rest = str.partition("#")
|
@@ -122,8 +121,6 @@ module TSV
|
|
122
121
|
end
|
123
122
|
end
|
124
123
|
|
125
|
-
tsv.entity_options = entity_options
|
126
|
-
|
127
124
|
tsv
|
128
125
|
end
|
129
126
|
end
|
data/lib/scout/work_queue.rb
CHANGED
@@ -138,17 +138,19 @@ class WorkQueue
|
|
138
138
|
end
|
139
139
|
|
140
140
|
def abort
|
141
|
+
@aborted = true
|
141
142
|
Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
|
142
143
|
@worker_mutex.synchronize do
|
143
144
|
@workers.each do |w|
|
144
|
-
ScoutSemaphore.post_semaphore(@output.write_sem)
|
145
|
-
ScoutSemaphore.post_semaphore(@input.read_sem)
|
145
|
+
ScoutSemaphore.post_semaphore(@output.write_sem)
|
146
|
+
ScoutSemaphore.post_semaphore(@input.read_sem)
|
146
147
|
w.abort
|
147
148
|
end
|
148
149
|
end
|
149
150
|
end
|
150
151
|
|
151
152
|
def close
|
153
|
+
return if @closed || @aborted
|
152
154
|
@closed = true
|
153
155
|
@worker_mutex.synchronize{ @workers.length }.times do
|
154
156
|
begin
|
@@ -165,6 +167,7 @@ class WorkQueue
|
|
165
167
|
end
|
166
168
|
|
167
169
|
def join(clean = true)
|
170
|
+
close
|
168
171
|
begin
|
169
172
|
@waiter.join if @waiter
|
170
173
|
@reader.join if @reader
|
@@ -21,12 +21,14 @@ module Workflow
|
|
21
21
|
@helpers ||= {}
|
22
22
|
end
|
23
23
|
|
24
|
-
def helper(name, *args, &block)
|
24
|
+
def helper(name, *args, **kwargs, &block)
|
25
25
|
if block_given?
|
26
26
|
helpers[name] = block
|
27
27
|
else
|
28
28
|
raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
|
29
|
-
|
29
|
+
o = Object.new
|
30
|
+
o.extend step_module
|
31
|
+
o.send(name, *args, **kwargs)
|
30
32
|
end
|
31
33
|
end
|
32
34
|
|
@@ -138,8 +140,17 @@ module Workflow
|
|
138
140
|
end
|
139
141
|
end
|
140
142
|
|
141
|
-
|
142
|
-
|
143
|
+
FORGET_TASK_ALIAS = begin
|
144
|
+
%w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
|
145
|
+
ENV[var] == 'true'
|
146
|
+
end.any?
|
147
|
+
end
|
148
|
+
REMOVE_TASK_ALIAS = begin
|
149
|
+
remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
|
150
|
+
ENV.include?(var) && ENV[var] != 'false'
|
151
|
+
end.first
|
152
|
+
remove.nil? ? false : remove
|
153
|
+
end
|
143
154
|
def task_alias(name, workflow, oname, *rest, &block)
|
144
155
|
dep(workflow, oname, *rest, &block)
|
145
156
|
extension :dep_task unless @extension
|
@@ -156,9 +167,15 @@ module Workflow
|
|
156
167
|
raise dep.get_exception if dep.error?
|
157
168
|
raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
|
158
169
|
set_info :type, dep.info[:type]
|
159
|
-
|
170
|
+
|
171
|
+
forget = config :forget_task_alias, "forget_task_alias"
|
172
|
+
forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
|
173
|
+
|
160
174
|
if forget
|
161
|
-
remove = config :
|
175
|
+
remove = config :remove_task_alias, "remove_task_alias"
|
176
|
+
remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_TASK_ALIAS if remove.nil?
|
177
|
+
|
178
|
+
Log.medium "Forget task_alias (remove: #{remove}): #{short_path}"
|
162
179
|
|
163
180
|
self.archive_deps
|
164
181
|
self.copy_linked_files_dir
|
@@ -177,9 +194,9 @@ module Workflow
|
|
177
194
|
when 'recursive'
|
178
195
|
(dep.dependencies + dep.rec_dependencies).uniq.each do |d|
|
179
196
|
next if d.overriden
|
180
|
-
d.clean unless
|
197
|
+
d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
|
181
198
|
end
|
182
|
-
dep.clean unless
|
199
|
+
dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
|
183
200
|
end
|
184
201
|
end
|
185
202
|
else
|
@@ -246,13 +246,14 @@ module Workflow
|
|
246
246
|
workload = new_workload
|
247
247
|
sleep timer
|
248
248
|
end
|
249
|
+
all_jobs.each{|s| s.join }
|
249
250
|
rescue TryAgain
|
250
251
|
retry
|
251
252
|
end
|
252
253
|
end
|
253
254
|
end
|
254
255
|
|
255
|
-
def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
|
256
|
+
def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors, produce_timer = 5)
|
256
257
|
jobs = [jobs] unless Array === jobs
|
257
258
|
produce_list = []
|
258
259
|
jobs.each do |job|
|
@@ -264,7 +265,7 @@ module Workflow
|
|
264
265
|
end
|
265
266
|
end
|
266
267
|
|
267
|
-
orchestrator = Orchestrator.new
|
268
|
+
orchestrator = Orchestrator.new produce_timer, cpus: produce_cpus.to_i
|
268
269
|
orchestrator.process({}, produce_list)
|
269
270
|
produce_list
|
270
271
|
end
|
@@ -0,0 +1 @@
|
|
1
|
+
class TaskNotFound < StandardError; end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Path
|
2
|
+
def self.step_file?(path)
|
3
|
+
return false unless path =~ /\.files(?:\/|$)/
|
4
|
+
parts = path.split("/")
|
5
|
+
job = parts.select{|p| p =~ /\.files$/}.first
|
6
|
+
|
7
|
+
if job
|
8
|
+
i = parts.index job
|
9
|
+
begin
|
10
|
+
workflow, task = parts.values_at i - 2, i - 1
|
11
|
+
_loaded = false
|
12
|
+
begin
|
13
|
+
Kernel.const_get(workflow)
|
14
|
+
rescue
|
15
|
+
if ! _loaded
|
16
|
+
Workflow.require_workflow workflow
|
17
|
+
_loaded = true
|
18
|
+
retry
|
19
|
+
end
|
20
|
+
raise $!
|
21
|
+
end
|
22
|
+
return parts[i-2..-1] * "/"
|
23
|
+
rescue
|
24
|
+
Log.exception $!
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
alias original_digest_str digest_str
|
32
|
+
|
33
|
+
def digest_str
|
34
|
+
if step_file = Path.step_file?(self)
|
35
|
+
"Step file: #{step_file}"
|
36
|
+
else
|
37
|
+
original_digest_str
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -11,7 +11,7 @@ class Step
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def recursive_inputs
|
14
|
-
recursive_inputs = NamedArray ===
|
14
|
+
recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
|
15
15
|
return recursive_inputs if dependencies.nil?
|
16
16
|
dependencies.inject(recursive_inputs) do |acc,dep|
|
17
17
|
acc.merge(dep.recursive_inputs)
|
@@ -36,7 +36,7 @@ class Step
|
|
36
36
|
|
37
37
|
dependencies.each do |dep|
|
38
38
|
if dep.present? && ! dep.updated?
|
39
|
-
Log.
|
39
|
+
Log.medium "Clean outdated #{dep.path}"
|
40
40
|
dep.clean
|
41
41
|
end
|
42
42
|
|
@@ -10,7 +10,11 @@ class Step
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def self.load_info(info_file)
|
13
|
-
info =
|
13
|
+
info = begin
|
14
|
+
Persist.load(info_file, SERIALIZER) || {}
|
15
|
+
rescue
|
16
|
+
{status: :noinfo}
|
17
|
+
end
|
14
18
|
IndiferentHash.setup(info)
|
15
19
|
end
|
16
20
|
|
@@ -152,7 +156,7 @@ class Step
|
|
152
156
|
end
|
153
157
|
|
154
158
|
def running?
|
155
|
-
! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
159
|
+
! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
156
160
|
end
|
157
161
|
|
158
162
|
def overriden?
|
@@ -26,10 +26,12 @@ class Step
|
|
26
26
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
27
27
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
28
28
|
|
29
|
+
Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
|
29
30
|
newer.empty?
|
30
31
|
end
|
31
32
|
|
32
33
|
def clean
|
34
|
+
Log.debug "Cleaning job files: #{path}"
|
33
35
|
@take_stream = nil
|
34
36
|
@result = nil
|
35
37
|
@info = nil
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -101,7 +101,7 @@ module Task
|
|
101
101
|
else
|
102
102
|
dep = _res
|
103
103
|
dependencies << dep
|
104
|
-
dep_non_default_inputs = find_dep_non_default_inputs.call(dep,
|
104
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
|
105
105
|
non_default_inputs.concat(dep_non_default_inputs)
|
106
106
|
end
|
107
107
|
end
|
@@ -47,7 +47,7 @@ module Task
|
|
47
47
|
elsif String === value && Symbol === provided && provided.to_s == value
|
48
48
|
same_as_default = true
|
49
49
|
else
|
50
|
-
|
50
|
+
same_as_default = false
|
51
51
|
end
|
52
52
|
if ! provided.nil? && ! same_as_default
|
53
53
|
non_default_inputs << name.to_sym
|
@@ -91,6 +91,8 @@ module Task
|
|
91
91
|
if Path.is_filename?(value)
|
92
92
|
if type == :path
|
93
93
|
Open.write(input_file + ".as_path", value)
|
94
|
+
elsif Path.step_file?(value)
|
95
|
+
Open.write(input_file + ".as_path", value)
|
94
96
|
else
|
95
97
|
relative_file = save_file_input(value, directory)
|
96
98
|
Open.write(input_file + ".as_file", relative_file)
|
data/lib/scout/workflow/task.rb
CHANGED
data/lib/scout/workflow/usage.rb
CHANGED
@@ -138,7 +138,7 @@ module Workflow
|
|
138
138
|
|
139
139
|
dep_tree = {}
|
140
140
|
task = self.tasks[task_name]
|
141
|
-
raise
|
141
|
+
raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
|
142
142
|
task.deps.each do |workflow, task, options|
|
143
143
|
next if seen.include? dep
|
144
144
|
seen << [workflow, task, options.merge(seen_options)]
|
data/lib/scout/workflow.rb
CHANGED
@@ -5,6 +5,8 @@ require_relative 'workflow/step'
|
|
5
5
|
require_relative 'workflow/documentation'
|
6
6
|
require_relative 'workflow/usage'
|
7
7
|
require_relative 'workflow/deployment'
|
8
|
+
require_relative 'workflow/exceptions'
|
9
|
+
require_relative 'workflow/path'
|
8
10
|
|
9
11
|
require 'scout/resource'
|
10
12
|
require 'scout/resource/scout'
|
@@ -153,6 +155,7 @@ module Workflow
|
|
153
155
|
|
154
156
|
def job(name, *args)
|
155
157
|
task = tasks[name]
|
158
|
+
raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
|
156
159
|
step = task.job(*args)
|
157
160
|
step.extend step_module
|
158
161
|
step
|
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear 10.
|
5
|
+
# stub: scout-gear 10.7.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "10.
|
9
|
+
s.version = "10.7.1".freeze
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2024-
|
14
|
+
s.date = "2024-07-01"
|
15
15
|
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -72,7 +72,6 @@ Gem::Specification.new do |s|
|
|
72
72
|
"lib/scout/tsv/change_id/translate.rb",
|
73
73
|
"lib/scout/tsv/csv.rb",
|
74
74
|
"lib/scout/tsv/dumper.rb",
|
75
|
-
"lib/scout/tsv/entity.rb",
|
76
75
|
"lib/scout/tsv/index.rb",
|
77
76
|
"lib/scout/tsv/open.rb",
|
78
77
|
"lib/scout/tsv/parser.rb",
|
@@ -98,6 +97,8 @@ Gem::Specification.new do |s|
|
|
98
97
|
"lib/scout/workflow/deployment/orchestrator.rb",
|
99
98
|
"lib/scout/workflow/deployment/trace.rb",
|
100
99
|
"lib/scout/workflow/documentation.rb",
|
100
|
+
"lib/scout/workflow/exceptions.rb",
|
101
|
+
"lib/scout/workflow/path.rb",
|
101
102
|
"lib/scout/workflow/step.rb",
|
102
103
|
"lib/scout/workflow/step/archive.rb",
|
103
104
|
"lib/scout/workflow/step/children.rb",
|
@@ -184,7 +185,6 @@ Gem::Specification.new do |s|
|
|
184
185
|
"test/scout/tsv/test_change_id.rb",
|
185
186
|
"test/scout/tsv/test_csv.rb",
|
186
187
|
"test/scout/tsv/test_dumper.rb",
|
187
|
-
"test/scout/tsv/test_entity.rb",
|
188
188
|
"test/scout/tsv/test_index.rb",
|
189
189
|
"test/scout/tsv/test_open.rb",
|
190
190
|
"test/scout/tsv/test_parser.rb",
|
@@ -214,6 +214,7 @@ Gem::Specification.new do |s|
|
|
214
214
|
"test/scout/workflow/task/test_inputs.rb",
|
215
215
|
"test/scout/workflow/test_definition.rb",
|
216
216
|
"test/scout/workflow/test_documentation.rb",
|
217
|
+
"test/scout/workflow/test_path.rb",
|
217
218
|
"test/scout/workflow/test_step.rb",
|
218
219
|
"test/scout/workflow/test_task.rb",
|
219
220
|
"test/scout/workflow/test_usage.rb",
|
@@ -117,6 +117,7 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
117
117
|
|
118
118
|
assert_raise ScoutException do
|
119
119
|
begin
|
120
|
+
t.join
|
120
121
|
q.join(false)
|
121
122
|
rescue
|
122
123
|
t.raise($!)
|
@@ -158,6 +159,7 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
158
159
|
|
159
160
|
assert_raise ScoutException do
|
160
161
|
begin
|
162
|
+
t.join
|
161
163
|
q.join(false)
|
162
164
|
rescue Exception
|
163
165
|
t.raise($!)
|
data/test/scout/test_workflow.rb
CHANGED
@@ -85,4 +85,20 @@ class TestWorkflow < Test::Unit::TestCase
|
|
85
85
|
assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
|
86
86
|
Baking.job(:bake_muffin_tray, "Blueberry muffin", :add_bluberries => true).run
|
87
87
|
end
|
88
|
+
|
89
|
+
def test_call_helper
|
90
|
+
wf = Module.new do
|
91
|
+
extend Workflow
|
92
|
+
|
93
|
+
helper :m1 do
|
94
|
+
"TEST"
|
95
|
+
end
|
96
|
+
|
97
|
+
helper :m2 do
|
98
|
+
m1
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
assert_equal "TEST", wf.helper(:m2)
|
103
|
+
end
|
88
104
|
end
|
@@ -449,4 +449,33 @@ A Id3
|
|
449
449
|
assert res["row2"].include? "Id3"
|
450
450
|
assert ! res["row2"].include?("b")
|
451
451
|
end
|
452
|
+
|
453
|
+
def test_attach_list_to_list_with_complete
|
454
|
+
content1 =<<-EOF
|
455
|
+
#Id ValueA ValueB
|
456
|
+
row1 a b
|
457
|
+
row2 A B
|
458
|
+
EOF
|
459
|
+
|
460
|
+
content2 =<<-EOF
|
461
|
+
#Id ValueC
|
462
|
+
row1 c
|
463
|
+
row2 C
|
464
|
+
row3 CC
|
465
|
+
EOF
|
466
|
+
|
467
|
+
tsv1 = tsv2 = index = nil
|
468
|
+
TmpFile.with_file(content1) do |filename|
|
469
|
+
tsv1 = TSV.open(File.open(filename), type: :list, fields: ["ValueA"], sep: /\s+/)
|
470
|
+
end
|
471
|
+
|
472
|
+
TmpFile.with_file(content2) do |filename|
|
473
|
+
tsv2 = TSV.open(File.open(filename), type: :list, sep: /\s+/)
|
474
|
+
end
|
475
|
+
|
476
|
+
res = tsv1.attach tsv2, :fields => ["ValueC"], complete: true
|
477
|
+
assert res["row2"].include?("C")
|
478
|
+
refute res["row2"].include?("b")
|
479
|
+
assert res["row3"].include?("CC")
|
480
|
+
end
|
452
481
|
end
|
@@ -66,5 +66,36 @@ class TestWorkflowDefinition < Test::Unit::TestCase
|
|
66
66
|
assert_include job.archived_info, dep_path
|
67
67
|
assert_equal :done, job.archived_info[dep_path][:status]
|
68
68
|
end
|
69
|
+
|
70
|
+
def test_task_alias_remove_dep_partial
|
71
|
+
wf = Workflow.annonymous_workflow do
|
72
|
+
self.name = "CallName"
|
73
|
+
|
74
|
+
task :salute => :string do |name|
|
75
|
+
"Hi"
|
76
|
+
end
|
77
|
+
|
78
|
+
dep :salute
|
79
|
+
input :name, :string, "Name to call", nil, :jobname => true
|
80
|
+
task :call_name => :string do |name|
|
81
|
+
"#{step(:salute).load} #{name}"
|
82
|
+
end
|
83
|
+
|
84
|
+
task_alias :call_miguel, self, :call_name, name: "Miguel"
|
85
|
+
end
|
86
|
+
|
87
|
+
old_cache = Scout::Config::CACHE.dup
|
88
|
+
Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => :recursive}, 'task:CallName#call_miguel')
|
89
|
+
Scout::Config.set({:remove_dep => false}, 'task:CallName#call_name')
|
90
|
+
job = wf.job(:call_miguel)
|
91
|
+
call_name = job.step(:call_name)
|
92
|
+
salute = job.step(:salute)
|
93
|
+
assert_equal "Hi Miguel", job.run
|
94
|
+
refute salute.done?
|
95
|
+
assert call_name.done?
|
96
|
+
Scout::Config::CACHE.replace old_cache
|
97
|
+
assert_include job.archived_info, call_name.path
|
98
|
+
assert_equal :done, job.archived_info[call_name.path][:status]
|
99
|
+
end
|
69
100
|
end
|
70
101
|
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestClass < Test::Unit::TestCase
|
5
|
+
def test_path_digest
|
6
|
+
iii Misc.digest_str(Path.setup("test_file"))
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scout-gear
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 10.
|
4
|
+
version: 10.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: scout-essentials
|
@@ -184,7 +184,6 @@ files:
|
|
184
184
|
- lib/scout/tsv/change_id/translate.rb
|
185
185
|
- lib/scout/tsv/csv.rb
|
186
186
|
- lib/scout/tsv/dumper.rb
|
187
|
-
- lib/scout/tsv/entity.rb
|
188
187
|
- lib/scout/tsv/index.rb
|
189
188
|
- lib/scout/tsv/open.rb
|
190
189
|
- lib/scout/tsv/parser.rb
|
@@ -210,6 +209,8 @@ files:
|
|
210
209
|
- lib/scout/workflow/deployment/orchestrator.rb
|
211
210
|
- lib/scout/workflow/deployment/trace.rb
|
212
211
|
- lib/scout/workflow/documentation.rb
|
212
|
+
- lib/scout/workflow/exceptions.rb
|
213
|
+
- lib/scout/workflow/path.rb
|
213
214
|
- lib/scout/workflow/step.rb
|
214
215
|
- lib/scout/workflow/step/archive.rb
|
215
216
|
- lib/scout/workflow/step/children.rb
|
@@ -296,7 +297,6 @@ files:
|
|
296
297
|
- test/scout/tsv/test_change_id.rb
|
297
298
|
- test/scout/tsv/test_csv.rb
|
298
299
|
- test/scout/tsv/test_dumper.rb
|
299
|
-
- test/scout/tsv/test_entity.rb
|
300
300
|
- test/scout/tsv/test_index.rb
|
301
301
|
- test/scout/tsv/test_open.rb
|
302
302
|
- test/scout/tsv/test_parser.rb
|
@@ -326,6 +326,7 @@ files:
|
|
326
326
|
- test/scout/workflow/task/test_inputs.rb
|
327
327
|
- test/scout/workflow/test_definition.rb
|
328
328
|
- test/scout/workflow/test_documentation.rb
|
329
|
+
- test/scout/workflow/test_path.rb
|
329
330
|
- test/scout/workflow/test_step.rb
|
330
331
|
- test/scout/workflow/test_task.rb
|
331
332
|
- test/scout/workflow/test_usage.rb
|
data/lib/scout/tsv/entity.rb
DELETED
File without changes
|