autoflow 0.8.7 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5db665d92bc22dfdf35e0160f45af1510c89563f
4
- data.tar.gz: b60d8c6b6f1e7891c5925e2d1abb8e02ce15724c
3
+ metadata.gz: 03a2fce5cddce218e8903d7b7cff49acd3c6962c
4
+ data.tar.gz: 1c2019a7d65d8710bc37baa649301c43a1b61585
5
5
  SHA512:
6
- metadata.gz: 56247ffdc4228a3256fd259f9899af2c61921a6ef410ef09480591601c44f7b16daa3e0e89249f3e5d7e97285ce5a18bbff2036164345058ec7689aa00e355a8
7
- data.tar.gz: 16e2f4b8255a2ec76213b001a50257a489d14eed148712a31d75b2c170debe82ef9a0f4138fe46e01eaf6c6983f3d745bbe167e4536dc681d8226e64846d8c37
6
+ metadata.gz: a52c86fde55c4854e587710c1969e62c282d7fbdd51f645c9a94ce1454882097cecf8f58566f1711cd6c6f4fb92fc02f6e4aef9a3fa46154453b6c09b8a01c6e
7
+ data.tar.gz: 1d8efc8bd847a2c1fea463497a0a02637582cd744d4654951551759a0e425dd74839622ec23f25a94b938c0a21e35edec129188a04409d9f84079c6171345486
@@ -83,6 +83,11 @@ optparse = OptionParser.new do |opts|
83
83
  options[:add] = add
84
84
  end
85
85
 
86
+ options[:additional_job_options] = nil
87
+ opts.on( '-A', '--additional_job_options STRING', 'Additional option in queue system jobs. Format: "parameter:value"' ) do |opt|
88
+ options[:additional_job_options] = opt.split(':')
89
+ end
90
+
86
91
  options[:batch] = false
87
92
  opts.on( '-b', '--batch', 'Workflow execution using batch' ) do
88
93
  options[:batch] = true
@@ -4,14 +4,15 @@ class Batch
4
4
  @@all_batch = {}
5
5
  @@jobs_names = []
6
6
  @@batch_iterator_relations = {}
7
- @@state_iterations = {}
7
+ @@nested_iteration_relations = {}
8
8
  @@general_computation_attrib = {
9
9
  :cpu => nil,
10
10
  :mem => nil,
11
11
  :time => nil,
12
12
  :node => nil,
13
13
  :multinode => nil,
14
- :ntask => nil
14
+ :ntask => nil,
15
+ :additional_job_options => nil
15
16
  }
16
17
 
17
18
  def self.set_general_attrib(attrib_hash)
@@ -20,6 +21,7 @@ class Batch
20
21
 
21
22
 
22
23
  def initialize(tag, init, main_command, id, exec_folder)
24
+ @regex_deps = nil
23
25
  replace_regexp(tag, init, main_command)
24
26
  @name = nil
25
27
  @id = id
@@ -63,6 +65,7 @@ class Batch
63
65
  data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
64
66
  #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
65
67
  job_names = get_dependencies_by_regexp(data[1], data[2])
68
+ @regex_deps = 'command' if job_names.length > 0
66
69
  new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
67
70
  command.gsub!(data[0], new_string)
68
71
  #puts command.inspect
@@ -72,6 +75,9 @@ class Batch
72
75
  data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
73
76
  #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
74
77
  job_names = get_dependencies_by_regexp(data[1], data[2])
78
+ if job_names.length > 0
79
+ @regex_deps = 'tag'
80
+ end
75
81
  new_string = job_names.map{|jn| jn + ')'}.join(';')
76
82
  tag.gsub!(data[0], new_string)
77
83
  end
@@ -188,6 +194,8 @@ class Batch
188
194
  @attrib[:time] = fields[index+1]
189
195
  elsif field == '-u'
190
196
  @attrib[:multinode] = fields[index+1].to_i
197
+ elsif field == '-A'
198
+ @attrib[:additional_job_options] = fields[index+1].split(':')
191
199
  end
192
200
  end
193
201
  if fields.include?('-s')
@@ -253,6 +261,7 @@ class Batch
253
261
  new_job = duplicate_job(tmp_j, iter)
254
262
  check_dependencies(new_job, iter, temp_jobs)
255
263
  parse_iter(iter, @name, new_job)
264
+ add_nested_iteration_relation(tmp_j, new_job)
256
265
  @@jobs_names << new_job.name
257
266
  jobs << new_job
258
267
  @jobs << new_job
@@ -261,6 +270,7 @@ class Batch
261
270
  end
262
271
  temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
263
272
  else
273
+ check_regex_dependencies
264
274
  @iterator.each_with_index do |iter, num|
265
275
  job_attrib = @attrib.dup
266
276
  if !iter.nil?
@@ -283,6 +293,61 @@ class Batch
283
293
  return jobs
284
294
  end
285
295
 
296
+ def add_nested_iteration_relation(tmp_j, new_job)
297
+ query = @@nested_iteration_relations[tmp_j.name]
298
+ if query.nil?
299
+ @@nested_iteration_relations[tmp_j.name] = [new_job.name]
300
+ else
301
+ query << new_job.name
302
+ end
303
+ end
304
+
305
+ def check_regex_dependencies
306
+ if @regex_deps == 'tag'
307
+ new_job_names = []
308
+ @iterator.each do |iter|
309
+ new_names = find_job_names(iter.gsub(')', ''))
310
+ new_job_names.concat(new_names)
311
+ end
312
+ @iterator = new_job_names.map{|nj| nj + ')'} if !new_job_names.empty?
313
+ elsif @regex_deps == 'command'
314
+ [@initialization, @main_command].each do |command|
315
+ patterns = command.scan(/([^\s)]+)\)([^\s]*)/)
316
+ if !patterns.empty?
317
+ patterns.each do |putative_job, sufix|
318
+ job_names = find_job_names(putative_job)
319
+ if !job_names.empty?
320
+ new_string = job_names.map{|jn| "#{jn})#{sufix}"}.join(' ')
321
+ old_string = "#{putative_job})#{sufix}"
322
+ command.gsub!(old_string, new_string)
323
+ end
324
+ end
325
+ end
326
+ end
327
+ end
328
+ end
329
+
330
+ def find_job_names(name)
331
+ final_names = []
332
+ intermediary_names = @@nested_iteration_relations[name]
333
+ if !intermediary_names.nil?
334
+ while !intermediary_names.empty?
335
+ final_names = intermediary_names
336
+ i_names = []
337
+ intermediary_names.each do |i_n|
338
+ query = @@nested_iteration_relations[i_n]
339
+ i_names.concat(query) if !query.nil?
340
+ end
341
+ if !i_names.empty?
342
+ intermediary_names = i_names
343
+ else
344
+ break
345
+ end
346
+ end
347
+ end
348
+ return final_names
349
+ end
350
+
286
351
  #tmp_j => job to set dependencies in iteration
287
352
  #iter => sufix of current iteration
288
353
  #jobs => array of jobs which has the job dependency
@@ -313,7 +378,7 @@ class Batch
313
378
  def handle_dependencies(dinamic_variables)
314
379
  [@initialization, @main_command].each do |instructions|
315
380
  if instructions.class.to_s == 'String'
316
- scan_dependencies(instructions)
381
+ #scan_dependencies(instructions) # NOT NECESSARY? REMOVED BY COLLISION CON REGEX SYSTEM. THE DINAMYC VARIABLES ARE NO USED
317
382
  dinamic_variables.concat(collect_dinamic_variables(instructions))
318
383
  @dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
319
384
  end
@@ -42,10 +42,10 @@ class BashManager < QueueManager
42
42
  end
43
43
 
44
44
  def self.available?(options)
45
- return TRUE
45
+ return true
46
46
  end
47
47
 
48
48
  def self.priority
49
49
  return 0
50
50
  end
51
- end
51
+ end
@@ -1,20 +1,34 @@
1
1
  require 'queue_manager'
2
2
  class SlurmManager < QueueManager
3
+ def parse_additional_options(string, attribs)
4
+ expresions = %w[%C %T %M %N ]
5
+ values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
6
+ new_string = string.dup
7
+ expresions.each_with_index do |exp, i|
8
+ new_string.gsub!(exp, "#{values[i]}")
9
+ end
10
+ return new_string
11
+ end
12
+
3
13
  def write_header(id, job, sh_name)
4
14
  if !job.attrib[:ntask]
5
15
  write_file(sh_name, "#SBATCH --cpus=#{job.attrib[:cpu]}")
6
16
  else
7
17
  write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
8
18
  write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
9
- write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
10
19
  end
11
20
  write_file(sh_name, "#SBATCH --mem=#{job.attrib[:mem]}")
12
21
  write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
13
22
  write_file(sh_name, "#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
14
23
  write_file(sh_name, '#SBATCH --error=job.%J.err')
15
24
  write_file(sh_name, '#SBATCH --output=job.%J.out')
25
+ write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
26
+ if job.attrib[:ntask]
27
+ write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
28
+ end
16
29
  end
17
30
 
31
+
18
32
  def submit_job(job, ar_dependencies)
19
33
  final_dep = get_all_deps(ar_dependencies)
20
34
  dependencies = nil
@@ -34,13 +48,13 @@ class SlurmManager < QueueManager
34
48
  end
35
49
 
36
50
  def self.available?(options)
37
- available = TRUE
51
+ available = true
38
52
  shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
39
- available = FALSE if shell_output.empty?
53
+ available = false if shell_output.empty?
40
54
  return available
41
55
  end
42
56
 
43
57
  def self.priority
44
58
  return 100
45
59
  end
46
- end
60
+ end
@@ -16,7 +16,8 @@ class Stack
16
16
  :time => options[:time],
17
17
  :node => options[:node_type],
18
18
  :multinode => options[:use_multinode],
19
- :ntask => options[:use_ntasks]
19
+ :ntask => options[:use_ntasks],
20
+ :additional_job_options => options[:additional_job_options]
20
21
  })
21
22
  @@folder_name = :program_name
22
23
  @@folder_name = :job_name if options[:key_name]
@@ -297,7 +298,7 @@ class Stack
297
298
  end
298
299
  file.puts '}'
299
300
  file.close
300
- system('dot -Tpng '+name+representation_type+'.dot -o '+name+representation_type+'.png')
301
+ system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
301
302
  end
302
303
 
303
304
  end
@@ -1,3 +1,3 @@
1
1
  module Autoflow
2
- VERSION = "0.8.7"
2
+ VERSION = "0.9.2"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: autoflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.7
4
+ version: 0.9.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pedro Seoane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-01-04 00:00:00.000000000 Z
11
+ date: 2019-10-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: net-ssh
@@ -161,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
161
161
  version: '0'
162
162
  requirements: []
163
163
  rubyforge_project:
164
- rubygems_version: 2.4.8
164
+ rubygems_version: 2.6.14
165
165
  signing_key:
166
166
  specification_version: 4
167
167
  summary: '"This gem take a pipeline and launch it on a queue system"'