autoflow 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/AutoFlow +19 -19
- data/bin/flow_logger +12 -12
- data/lib/autoflow/batch.rb +16 -16
- data/lib/autoflow/stack.rb +4 -4
- data/lib/autoflow/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f312d125afdd65f9895264cda8383258bd155fb3
|
4
|
+
data.tar.gz: 005f0219a73e5f38811df45b30d5fdaba7f243ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2106ec11e38495b7338bef9cf94a02f2df9a9ddaea8a464167872d4a5919ace690dfd9624ea68f60d3fde74fda5d4aab606f4905c3241eb434697b14049628f2
|
7
|
+
data.tar.gz: 2f046987dd54bee1c4c256e5475c7a544ecbaf5735b4b01677f841188457fe5eaf644dcbc2d0089a0f6ea30cc0cbe4c310f333d73b6a2b673a3be8bb11755033
|
data/bin/AutoFlow
CHANGED
@@ -83,9 +83,9 @@ optparse = OptionParser.new do |opts|
|
|
83
83
|
options[:add] = add
|
84
84
|
end
|
85
85
|
|
86
|
-
options[:batch] =
|
86
|
+
options[:batch] = false
|
87
87
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do
|
88
|
-
options[:batch] =
|
88
|
+
options[:batch] = true
|
89
89
|
end
|
90
90
|
|
91
91
|
options[:cpus] = 16
|
@@ -93,9 +93,9 @@ optparse = OptionParser.new do |opts|
|
|
93
93
|
options[:cpus] = cpus.to_i
|
94
94
|
end
|
95
95
|
|
96
|
-
options[:comment] =
|
96
|
+
options[:comment] = false
|
97
97
|
opts.on( '-C', '--comment_main_command', 'Comment first line of main command job' ) do
|
98
|
-
options[:comment] =
|
98
|
+
options[:comment] = true
|
99
99
|
end
|
100
100
|
|
101
101
|
options[:external_dependencies] = []
|
@@ -103,9 +103,9 @@ optparse = OptionParser.new do |opts|
|
|
103
103
|
options[:external_dependencies] = external_dependencies.split(',')
|
104
104
|
end
|
105
105
|
|
106
|
-
options[:retry] =
|
106
|
+
options[:retry] = false
|
107
107
|
opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do
|
108
|
-
options[:retry] =
|
108
|
+
options[:retry] = true
|
109
109
|
end
|
110
110
|
|
111
111
|
options[:graph] = nil
|
@@ -118,14 +118,14 @@ optparse = OptionParser.new do |opts|
|
|
118
118
|
options[:get_template_repository] = url
|
119
119
|
end
|
120
120
|
|
121
|
-
options[:identifier] =
|
121
|
+
options[:identifier] = false
|
122
122
|
opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
|
123
123
|
options[:identifier] = identifier
|
124
124
|
end
|
125
125
|
|
126
|
-
options[:key_name] =
|
126
|
+
options[:key_name] = false
|
127
127
|
opts.on( '-k', '--use_key_name', ' Use job names as folder names' ) do
|
128
|
-
options[:key_name] =
|
128
|
+
options[:key_name] = true
|
129
129
|
end
|
130
130
|
|
131
131
|
options[:list] = nil
|
@@ -153,9 +153,9 @@ optparse = OptionParser.new do |opts|
|
|
153
153
|
options[:pull] = pull
|
154
154
|
end
|
155
155
|
|
156
|
-
options[:remote] =
|
156
|
+
options[:remote] = false
|
157
157
|
opts.on( '-r', '--remote', 'Connect with remote machine and launch the workflow' ) do
|
158
|
-
options[:remote] =
|
158
|
+
options[:remote] = true
|
159
159
|
puts 'Host to connect and launch workflow:'
|
160
160
|
host = gets.chomp
|
161
161
|
puts 'User:'
|
@@ -166,14 +166,14 @@ optparse = OptionParser.new do |opts|
|
|
166
166
|
puts options[:ssh].exec!('hostname')
|
167
167
|
end
|
168
168
|
|
169
|
-
options[:use_ntasks] =
|
169
|
+
options[:use_ntasks] = false
|
170
170
|
opts.on( '-s', '--use_ntasks', 'Use several nodes on execution' ) do
|
171
|
-
options[:use_ntasks] =
|
171
|
+
options[:use_ntasks] = true
|
172
172
|
end
|
173
173
|
|
174
|
-
options[:show_submit_command] =
|
174
|
+
options[:show_submit_command] = false
|
175
175
|
opts.on( '-S', '--show_submit_command', 'Show the command line used to submit the job' ) do
|
176
|
-
options[:show_submit_command] =
|
176
|
+
options[:show_submit_command] = true
|
177
177
|
end
|
178
178
|
|
179
179
|
options[:time] = '20:00:00'
|
@@ -187,9 +187,9 @@ optparse = OptionParser.new do |opts|
|
|
187
187
|
end
|
188
188
|
|
189
189
|
|
190
|
-
options[:verbose] =
|
190
|
+
options[:verbose] = false
|
191
191
|
opts.on( '-v', '--verbose', 'Show info without launching jobs' ) do
|
192
|
-
options[:verbose] =
|
192
|
+
options[:verbose] = true
|
193
193
|
end
|
194
194
|
|
195
195
|
options[:Variables] = nil
|
@@ -197,7 +197,7 @@ optparse = OptionParser.new do |opts|
|
|
197
197
|
options[:Variables] = [mem]
|
198
198
|
end
|
199
199
|
|
200
|
-
options[:workflow] =
|
200
|
+
options[:workflow] = false
|
201
201
|
opts.on( '-w', '--workflow FILE', 'Input workflow file' ) do |workflow|
|
202
202
|
options[:workflow] = workflow
|
203
203
|
template_file = workflow
|
@@ -338,7 +338,7 @@ if !options[:graph].nil?
|
|
338
338
|
else
|
339
339
|
stack.inspect if options[:verbose]
|
340
340
|
stack.comment_main_command if options[:comment]
|
341
|
-
options[:write_sh] =
|
341
|
+
options[:write_sh] = true # Set in flow logger to FALSE, it is used for relaunch failed jobs
|
342
342
|
manager = QueueManager.select_queue_manager(stack.exec_folder, options, stack.jobs, stack.persist_variables)
|
343
343
|
manager.exec
|
344
344
|
end
|
data/bin/flow_logger
CHANGED
@@ -267,34 +267,34 @@ OptionParser.new do |opts|
|
|
267
267
|
options[:report] = opt
|
268
268
|
end
|
269
269
|
|
270
|
-
options[:workflow_status] =
|
270
|
+
options[:workflow_status] = false
|
271
271
|
opts.on("-w", "--workflow_finished", "When set, logger assumes that the workflow has ended") do |opt|
|
272
|
-
options[:workflow_status] =
|
272
|
+
options[:workflow_status] = true
|
273
273
|
end
|
274
274
|
|
275
|
-
options[:no_size] =
|
275
|
+
options[:no_size] = false
|
276
276
|
opts.on("-n", "--no_size", "When set, logger don't compute the workflow folder sizes") do |opt|
|
277
|
-
options[:no_size] =
|
277
|
+
options[:no_size] = true
|
278
278
|
end
|
279
279
|
|
280
|
-
options[:batch] =
|
280
|
+
options[:batch] = false
|
281
281
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do |opt|
|
282
|
-
options[:batch] =
|
282
|
+
options[:batch] = true
|
283
283
|
end
|
284
284
|
|
285
|
-
options[:launch_failed_jobs] =
|
285
|
+
options[:launch_failed_jobs] = false
|
286
286
|
opts.on("-l", "--launch_failed_jobs", "Launch jobs tagged as ABORT and NOT. This option only works when the -w flag is enabled") do |opt|
|
287
|
-
options[:launch_failed_jobs] =
|
287
|
+
options[:launch_failed_jobs] = true
|
288
288
|
end
|
289
289
|
|
290
|
-
options[:pending] =
|
290
|
+
options[:pending] = false
|
291
291
|
opts.on("-p", "--pending", "Launch jobs tagged as NOT. This option only works when the -w flag is enabled") do |opt|
|
292
|
-
options[:pending] =
|
292
|
+
options[:pending] = true
|
293
293
|
end
|
294
294
|
|
295
|
-
options[:html] =
|
295
|
+
options[:html] = false
|
296
296
|
opts.on("-H", "--html", "Make a workflow execution full report in html format") do |opt|
|
297
|
-
options[:html] =
|
297
|
+
options[:html] = true
|
298
298
|
end
|
299
299
|
end.parse!
|
300
300
|
|
data/lib/autoflow/batch.rb
CHANGED
@@ -34,9 +34,9 @@ class Batch
|
|
34
34
|
@initialization = init
|
35
35
|
@main_command = main_command
|
36
36
|
@attrib = {
|
37
|
-
:done =>
|
38
|
-
:folder =>
|
39
|
-
:buffer =>
|
37
|
+
:done => false,
|
38
|
+
:folder => true,
|
39
|
+
:buffer => false,
|
40
40
|
:exec_folder => exec_folder,
|
41
41
|
:cpu_asign => nil # number, list or mono
|
42
42
|
}.merge(@@general_computation_attrib)
|
@@ -150,13 +150,13 @@ class Batch
|
|
150
150
|
@@batch_iterator_relations[@name] = @iterator
|
151
151
|
end
|
152
152
|
|
153
|
-
def check_execution_modifiers(name, iter_type =
|
154
|
-
done =
|
155
|
-
folder =
|
156
|
-
buffer =
|
157
|
-
done =
|
158
|
-
folder =
|
159
|
-
buffer =
|
153
|
+
def check_execution_modifiers(name, iter_type = false) #The last paremeter iused to indicate tha name is a iterator not an orignal node name
|
154
|
+
done = false
|
155
|
+
folder = true
|
156
|
+
buffer = false
|
157
|
+
done = true if name.include?('%')
|
158
|
+
folder = false if name.include?('!')
|
159
|
+
buffer = true if name.include?('&')
|
160
160
|
if !iter_type
|
161
161
|
name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
|
162
162
|
else
|
@@ -191,9 +191,9 @@ class Batch
|
|
191
191
|
end
|
192
192
|
end
|
193
193
|
if fields.include?('-s')
|
194
|
-
@attrib[:ntask] =
|
194
|
+
@attrib[:ntask] = true
|
195
195
|
else
|
196
|
-
@attrib[:ntask] =
|
196
|
+
@attrib[:ntask] = false
|
197
197
|
end
|
198
198
|
end
|
199
199
|
end
|
@@ -264,7 +264,7 @@ class Batch
|
|
264
264
|
@iterator.each_with_index do |iter, num|
|
265
265
|
job_attrib = @attrib.dup
|
266
266
|
if !iter.nil?
|
267
|
-
iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter,
|
267
|
+
iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter, true)
|
268
268
|
job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
|
269
269
|
end
|
270
270
|
name = "#{@name}#{iter}"
|
@@ -357,14 +357,14 @@ class Batch
|
|
357
357
|
end
|
358
358
|
|
359
359
|
def string_overlap(matched_regions, substr, string)
|
360
|
-
match =
|
360
|
+
match = false
|
361
361
|
range = get_string_position(substr, string)
|
362
362
|
if !range.empty?
|
363
363
|
matched_regions.each do |start, ending|
|
364
364
|
if (range.first >= start && range.first <= ending) ||
|
365
365
|
(range.last >= start && range.last <= ending) ||
|
366
366
|
(range.first <= start && range.last >= ending)
|
367
|
-
match =
|
367
|
+
match = true
|
368
368
|
break
|
369
369
|
end
|
370
370
|
end
|
@@ -471,4 +471,4 @@ class Batch
|
|
471
471
|
return jobs
|
472
472
|
end
|
473
473
|
|
474
|
-
end
|
474
|
+
end
|
data/lib/autoflow/stack.rb
CHANGED
@@ -44,14 +44,14 @@ class Stack
|
|
44
44
|
persist_variables_lines = []
|
45
45
|
node_lines = []
|
46
46
|
|
47
|
-
node_beg =
|
47
|
+
node_beg = false
|
48
48
|
@workflow.each_line do |line|
|
49
|
-
node_beg =
|
49
|
+
node_beg = true if line.include?('{') # This check the context of a variable
|
50
50
|
if line.include?('}') # if a variable is within a node,
|
51
51
|
if node_beg # we consider tha is a bash variable not a static autoflow variable
|
52
|
-
node_beg =
|
52
|
+
node_beg = false
|
53
53
|
else
|
54
|
-
node_beg =
|
54
|
+
node_beg = true
|
55
55
|
end
|
56
56
|
end
|
57
57
|
if line =~ /^\$/ && !node_beg
|
data/lib/autoflow/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pedro Seoane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: net-ssh
|