autoflow 0.8.4 → 0.8.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/AutoFlow +19 -19
- data/bin/flow_logger +12 -12
- data/lib/autoflow/batch.rb +16 -16
- data/lib/autoflow/stack.rb +4 -4
- data/lib/autoflow/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f312d125afdd65f9895264cda8383258bd155fb3
|
4
|
+
data.tar.gz: 005f0219a73e5f38811df45b30d5fdaba7f243ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2106ec11e38495b7338bef9cf94a02f2df9a9ddaea8a464167872d4a5919ace690dfd9624ea68f60d3fde74fda5d4aab606f4905c3241eb434697b14049628f2
|
7
|
+
data.tar.gz: 2f046987dd54bee1c4c256e5475c7a544ecbaf5735b4b01677f841188457fe5eaf644dcbc2d0089a0f6ea30cc0cbe4c310f333d73b6a2b673a3be8bb11755033
|
data/bin/AutoFlow
CHANGED
@@ -83,9 +83,9 @@ optparse = OptionParser.new do |opts|
|
|
83
83
|
options[:add] = add
|
84
84
|
end
|
85
85
|
|
86
|
-
options[:batch] =
|
86
|
+
options[:batch] = false
|
87
87
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do
|
88
|
-
options[:batch] =
|
88
|
+
options[:batch] = true
|
89
89
|
end
|
90
90
|
|
91
91
|
options[:cpus] = 16
|
@@ -93,9 +93,9 @@ optparse = OptionParser.new do |opts|
|
|
93
93
|
options[:cpus] = cpus.to_i
|
94
94
|
end
|
95
95
|
|
96
|
-
options[:comment] =
|
96
|
+
options[:comment] = false
|
97
97
|
opts.on( '-C', '--comment_main_command', 'Comment first line of main command job' ) do
|
98
|
-
options[:comment] =
|
98
|
+
options[:comment] = true
|
99
99
|
end
|
100
100
|
|
101
101
|
options[:external_dependencies] = []
|
@@ -103,9 +103,9 @@ optparse = OptionParser.new do |opts|
|
|
103
103
|
options[:external_dependencies] = external_dependencies.split(',')
|
104
104
|
end
|
105
105
|
|
106
|
-
options[:retry] =
|
106
|
+
options[:retry] = false
|
107
107
|
opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do
|
108
|
-
options[:retry] =
|
108
|
+
options[:retry] = true
|
109
109
|
end
|
110
110
|
|
111
111
|
options[:graph] = nil
|
@@ -118,14 +118,14 @@ optparse = OptionParser.new do |opts|
|
|
118
118
|
options[:get_template_repository] = url
|
119
119
|
end
|
120
120
|
|
121
|
-
options[:identifier] =
|
121
|
+
options[:identifier] = false
|
122
122
|
opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
|
123
123
|
options[:identifier] = identifier
|
124
124
|
end
|
125
125
|
|
126
|
-
options[:key_name] =
|
126
|
+
options[:key_name] = false
|
127
127
|
opts.on( '-k', '--use_key_name', ' Use job names as folder names' ) do
|
128
|
-
options[:key_name] =
|
128
|
+
options[:key_name] = true
|
129
129
|
end
|
130
130
|
|
131
131
|
options[:list] = nil
|
@@ -153,9 +153,9 @@ optparse = OptionParser.new do |opts|
|
|
153
153
|
options[:pull] = pull
|
154
154
|
end
|
155
155
|
|
156
|
-
options[:remote] =
|
156
|
+
options[:remote] = false
|
157
157
|
opts.on( '-r', '--remote', 'Connect with remote machine and launch the workflow' ) do
|
158
|
-
options[:remote] =
|
158
|
+
options[:remote] = true
|
159
159
|
puts 'Host to connect and launch workflow:'
|
160
160
|
host = gets.chomp
|
161
161
|
puts 'User:'
|
@@ -166,14 +166,14 @@ optparse = OptionParser.new do |opts|
|
|
166
166
|
puts options[:ssh].exec!('hostname')
|
167
167
|
end
|
168
168
|
|
169
|
-
options[:use_ntasks] =
|
169
|
+
options[:use_ntasks] = false
|
170
170
|
opts.on( '-s', '--use_ntasks', 'Use several nodes on execution' ) do
|
171
|
-
options[:use_ntasks] =
|
171
|
+
options[:use_ntasks] = true
|
172
172
|
end
|
173
173
|
|
174
|
-
options[:show_submit_command] =
|
174
|
+
options[:show_submit_command] = false
|
175
175
|
opts.on( '-S', '--show_submit_command', 'Show the command line used to submit the job' ) do
|
176
|
-
options[:show_submit_command] =
|
176
|
+
options[:show_submit_command] = true
|
177
177
|
end
|
178
178
|
|
179
179
|
options[:time] = '20:00:00'
|
@@ -187,9 +187,9 @@ optparse = OptionParser.new do |opts|
|
|
187
187
|
end
|
188
188
|
|
189
189
|
|
190
|
-
options[:verbose] =
|
190
|
+
options[:verbose] = false
|
191
191
|
opts.on( '-v', '--verbose', 'Show info without launching jobs' ) do
|
192
|
-
options[:verbose] =
|
192
|
+
options[:verbose] = true
|
193
193
|
end
|
194
194
|
|
195
195
|
options[:Variables] = nil
|
@@ -197,7 +197,7 @@ optparse = OptionParser.new do |opts|
|
|
197
197
|
options[:Variables] = [mem]
|
198
198
|
end
|
199
199
|
|
200
|
-
options[:workflow] =
|
200
|
+
options[:workflow] = false
|
201
201
|
opts.on( '-w', '--workflow FILE', 'Input workflow file' ) do |workflow|
|
202
202
|
options[:workflow] = workflow
|
203
203
|
template_file = workflow
|
@@ -338,7 +338,7 @@ if !options[:graph].nil?
|
|
338
338
|
else
|
339
339
|
stack.inspect if options[:verbose]
|
340
340
|
stack.comment_main_command if options[:comment]
|
341
|
-
options[:write_sh] =
|
341
|
+
options[:write_sh] = true # Set in flow logger to FALSE, it is used for relaunch failed jobs
|
342
342
|
manager = QueueManager.select_queue_manager(stack.exec_folder, options, stack.jobs, stack.persist_variables)
|
343
343
|
manager.exec
|
344
344
|
end
|
data/bin/flow_logger
CHANGED
@@ -267,34 +267,34 @@ OptionParser.new do |opts|
|
|
267
267
|
options[:report] = opt
|
268
268
|
end
|
269
269
|
|
270
|
-
options[:workflow_status] =
|
270
|
+
options[:workflow_status] = false
|
271
271
|
opts.on("-w", "--workflow_finished", "When set, logger assumes that the workflow has ended") do |opt|
|
272
|
-
options[:workflow_status] =
|
272
|
+
options[:workflow_status] = true
|
273
273
|
end
|
274
274
|
|
275
|
-
options[:no_size] =
|
275
|
+
options[:no_size] = false
|
276
276
|
opts.on("-n", "--no_size", "When set, logger don't compute the workflow folder sizes") do |opt|
|
277
|
-
options[:no_size] =
|
277
|
+
options[:no_size] = true
|
278
278
|
end
|
279
279
|
|
280
|
-
options[:batch] =
|
280
|
+
options[:batch] = false
|
281
281
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do |opt|
|
282
|
-
options[:batch] =
|
282
|
+
options[:batch] = true
|
283
283
|
end
|
284
284
|
|
285
|
-
options[:launch_failed_jobs] =
|
285
|
+
options[:launch_failed_jobs] = false
|
286
286
|
opts.on("-l", "--launch_failed_jobs", "Launch jobs tagged as ABORT and NOT. This option only works when the -w flag is enabled") do |opt|
|
287
|
-
options[:launch_failed_jobs] =
|
287
|
+
options[:launch_failed_jobs] = true
|
288
288
|
end
|
289
289
|
|
290
|
-
options[:pending] =
|
290
|
+
options[:pending] = false
|
291
291
|
opts.on("-p", "--pending", "Launch jobs tagged as NOT. This option only works when the -w flag is enabled") do |opt|
|
292
|
-
options[:pending] =
|
292
|
+
options[:pending] = true
|
293
293
|
end
|
294
294
|
|
295
|
-
options[:html] =
|
295
|
+
options[:html] = false
|
296
296
|
opts.on("-H", "--html", "Make a workflow execution full report in html format") do |opt|
|
297
|
-
options[:html] =
|
297
|
+
options[:html] = true
|
298
298
|
end
|
299
299
|
end.parse!
|
300
300
|
|
data/lib/autoflow/batch.rb
CHANGED
@@ -34,9 +34,9 @@ class Batch
|
|
34
34
|
@initialization = init
|
35
35
|
@main_command = main_command
|
36
36
|
@attrib = {
|
37
|
-
:done =>
|
38
|
-
:folder =>
|
39
|
-
:buffer =>
|
37
|
+
:done => false,
|
38
|
+
:folder => true,
|
39
|
+
:buffer => false,
|
40
40
|
:exec_folder => exec_folder,
|
41
41
|
:cpu_asign => nil # number, list or mono
|
42
42
|
}.merge(@@general_computation_attrib)
|
@@ -150,13 +150,13 @@ class Batch
|
|
150
150
|
@@batch_iterator_relations[@name] = @iterator
|
151
151
|
end
|
152
152
|
|
153
|
-
def check_execution_modifiers(name, iter_type =
|
154
|
-
done =
|
155
|
-
folder =
|
156
|
-
buffer =
|
157
|
-
done =
|
158
|
-
folder =
|
159
|
-
buffer =
|
153
|
+
def check_execution_modifiers(name, iter_type = false) #The last paremeter iused to indicate tha name is a iterator not an orignal node name
|
154
|
+
done = false
|
155
|
+
folder = true
|
156
|
+
buffer = false
|
157
|
+
done = true if name.include?('%')
|
158
|
+
folder = false if name.include?('!')
|
159
|
+
buffer = true if name.include?('&')
|
160
160
|
if !iter_type
|
161
161
|
name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
|
162
162
|
else
|
@@ -191,9 +191,9 @@ class Batch
|
|
191
191
|
end
|
192
192
|
end
|
193
193
|
if fields.include?('-s')
|
194
|
-
@attrib[:ntask] =
|
194
|
+
@attrib[:ntask] = true
|
195
195
|
else
|
196
|
-
@attrib[:ntask] =
|
196
|
+
@attrib[:ntask] = false
|
197
197
|
end
|
198
198
|
end
|
199
199
|
end
|
@@ -264,7 +264,7 @@ class Batch
|
|
264
264
|
@iterator.each_with_index do |iter, num|
|
265
265
|
job_attrib = @attrib.dup
|
266
266
|
if !iter.nil?
|
267
|
-
iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter,
|
267
|
+
iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter, true)
|
268
268
|
job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
|
269
269
|
end
|
270
270
|
name = "#{@name}#{iter}"
|
@@ -357,14 +357,14 @@ class Batch
|
|
357
357
|
end
|
358
358
|
|
359
359
|
def string_overlap(matched_regions, substr, string)
|
360
|
-
match =
|
360
|
+
match = false
|
361
361
|
range = get_string_position(substr, string)
|
362
362
|
if !range.empty?
|
363
363
|
matched_regions.each do |start, ending|
|
364
364
|
if (range.first >= start && range.first <= ending) ||
|
365
365
|
(range.last >= start && range.last <= ending) ||
|
366
366
|
(range.first <= start && range.last >= ending)
|
367
|
-
match =
|
367
|
+
match = true
|
368
368
|
break
|
369
369
|
end
|
370
370
|
end
|
@@ -471,4 +471,4 @@ class Batch
|
|
471
471
|
return jobs
|
472
472
|
end
|
473
473
|
|
474
|
-
end
|
474
|
+
end
|
data/lib/autoflow/stack.rb
CHANGED
@@ -44,14 +44,14 @@ class Stack
|
|
44
44
|
persist_variables_lines = []
|
45
45
|
node_lines = []
|
46
46
|
|
47
|
-
node_beg =
|
47
|
+
node_beg = false
|
48
48
|
@workflow.each_line do |line|
|
49
|
-
node_beg =
|
49
|
+
node_beg = true if line.include?('{') # This check the context of a variable
|
50
50
|
if line.include?('}') # if a variable is within a node,
|
51
51
|
if node_beg # we consider tha is a bash variable not a static autoflow variable
|
52
|
-
node_beg =
|
52
|
+
node_beg = false
|
53
53
|
else
|
54
|
-
node_beg =
|
54
|
+
node_beg = true
|
55
55
|
end
|
56
56
|
end
|
57
57
|
if line =~ /^\$/ && !node_beg
|
data/lib/autoflow/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pedro Seoane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: net-ssh
|