miga-base 0.5.4.0 → 0.5.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/miga/cli/action/daemon.rb +5 -1
- data/lib/miga/cli/action/init.rb +1 -1
- data/lib/miga/cli/action/init/daemon_helper.rb +85 -42
- data/lib/miga/cli/action/run.rb +15 -4
- data/lib/miga/common/format.rb +8 -0
- data/lib/miga/daemon.rb +58 -25
- data/lib/miga/daemon/base.rb +22 -9
- data/lib/miga/version.rb +2 -2
- data/test/daemon_test.rb +3 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59649a6c1b7aaa4e3c0db090a828469e9595eba42d311eff114417741a2816bd
|
4
|
+
data.tar.gz: a765a214e5b64d3f1758ae50942dc7f0649f438e8c2b12312648012d9cceeacc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f76b1691a30d7ce94314a4bd8d22601d86d3101c2db33f262b53548bba817d17bc783f2c93ec0502844adda1f5a17bd0c530e82c82c7fb590f07abf8ffc6788c
|
7
|
+
data.tar.gz: c27cd47089f65633cefaca74cd217967b583df43d60682dce9f755a6610fdc6e728638d8d7b884f4de627de79b48080c8325f7b4d60794253f69511c15f997a4
|
@@ -36,6 +36,10 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
|
|
36
36
|
'--max-jobs INT',
|
37
37
|
'Maximum number of jobs to use simultaneously'
|
38
38
|
) { |v| cli[:maxjobs] = v.to_i }
|
39
|
+
opt.on(
|
40
|
+
'--node-list PATH',
|
41
|
+
'Path to the list of execution hostnames'
|
42
|
+
) { |v| cli[:nodelist] = v }
|
39
43
|
opt.on(
|
40
44
|
'--ppn INT',
|
41
45
|
'Maximum number of cores to use in a single job'
|
@@ -70,7 +74,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
|
|
70
74
|
def perform
|
71
75
|
p = cli.load_project
|
72
76
|
d = MiGA::Daemon.new(p, cli[:json])
|
73
|
-
[:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
|
77
|
+
[:latency, :maxjobs, :nodelist, :ppn, :shutdown_when_done].each do |k|
|
74
78
|
d.runopts(k, cli[k]) unless cli[k].nil?
|
75
79
|
end
|
76
80
|
d.daemon(cli.operation, cli[:daemon_opts])
|
data/lib/miga/cli/action/init.rb
CHANGED
@@ -26,7 +26,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
|
|
26
26
|
){ |v| cli[:mytaxa] = v }
|
27
27
|
opt.on(
|
28
28
|
'--daemon-type STRING',
|
29
|
-
'Type of daemon launcher, one of: bash, qsub, msub, slurm',
|
29
|
+
'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
|
30
30
|
"By default: interactive (#{cli[:dtype]} if --auto)"
|
31
31
|
){ |v| cli[:dtype] = v.to_sym }
|
32
32
|
opt.on(
|
@@ -13,15 +13,18 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
13
13
|
v = {created: Time.now.to_s, updated: Time.now.to_s}
|
14
14
|
v[:type] = cli.ask_user(
|
15
15
|
'Please select the type of daemon you want to setup',
|
16
|
-
cli[:dtype], %w(bash qsub msub slurm))
|
16
|
+
cli[:dtype], %w(bash ssh qsub msub slurm))
|
17
17
|
case v[:type]
|
18
18
|
when 'bash'
|
19
19
|
v = configure_bash_daemon(v)
|
20
|
+
when 'ssh'
|
21
|
+
v = configure_ssh_daemon(v)
|
20
22
|
when 'slurm'
|
21
23
|
v = configure_slurm_daemon(v)
|
22
24
|
else # [qm]sub
|
23
25
|
v = configure_qsub_msub_daemon(v)
|
24
26
|
end
|
27
|
+
v[:format_version] = 1
|
25
28
|
File.open(daemon_f, 'w') { |fh| fh.puts JSON.pretty_generate(v) }
|
26
29
|
end
|
27
30
|
cli.puts ''
|
@@ -34,19 +37,51 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
34
37
|
cli.puts 'Setting up internal daemon defaults.'
|
35
38
|
cli.puts 'If you don\'t understand this just leave default values:'
|
36
39
|
v[:cmd] = cli.ask_user(
|
37
|
-
"How should I launch tasks?\n
|
38
|
-
"
|
39
|
-
"
|
40
|
+
"How should I launch tasks?\n" \
|
41
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga\n ",
|
42
|
+
"{{vars}} {{miga}} run -r '{{script}}' -l '{{log}}' -e")
|
40
43
|
v[:var] = cli.ask_user(
|
41
|
-
"How should I pass variables?\n
|
42
|
-
|
44
|
+
"How should I pass variables?\n" \
|
45
|
+
" {{variables}}: key, value\n ",
|
46
|
+
"{{key}}={{value}}")
|
43
47
|
v[:varsep] = cli.ask_user('What should I use to separate variables?', ' ')
|
44
48
|
v[:alive] = cli.ask_user(
|
45
|
-
"How can I know that a process is still alive?\n
|
46
|
-
"
|
47
|
-
|
49
|
+
"How can I know that a process is still alive?\n" \
|
50
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
51
|
+
" {{variables}}: pid\n ",
|
52
|
+
"ps -p '{{pid}}' | tail -n +2 | wc -l")
|
48
53
|
v[:kill] = cli.ask_user(
|
49
|
-
"How should I terminate tasks?\n
|
54
|
+
"How should I terminate tasks?\n" \
|
55
|
+
" {{variables}}: pid\n ",
|
56
|
+
"kill -9 '{{pid}}'")
|
57
|
+
v
|
58
|
+
end
|
59
|
+
|
60
|
+
def configure_ssh_daemon(v)
|
61
|
+
v[:latency] = cli.ask_user('How long should I sleep? (in secs)', '3').to_i
|
62
|
+
v[:nodelist] = cli.ask_user(
|
63
|
+
'What environmental variable points to node list?', '$MIGA_NODELIST')
|
64
|
+
v[:ppn] = cli.ask_user('How many CPUs can I use per job?', '2').to_i
|
65
|
+
cli.puts 'Setting up internal daemon defaults.'
|
66
|
+
cli.puts 'If you don\'t understand this just leave default values:'
|
67
|
+
v[:cmd] = cli.ask_user(
|
68
|
+
"How should I launch tasks?\n" \
|
69
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga, host\n ",
|
70
|
+
"{{vars}} {{miga}} run -r '{{script}}' -l '{{log}}' -R {{host}} -e")
|
71
|
+
v[:var] = cli.ask_user(
|
72
|
+
"How should I pass variables?\n" \
|
73
|
+
" {{variables}}: key, value\n ",
|
74
|
+
"{{key}}={{value}}")
|
75
|
+
v[:varsep] = cli.ask_user('What should I use to separate variables?', ' ')
|
76
|
+
v[:alive] = cli.ask_user(
|
77
|
+
"How can I know that a process is still alive?\n" \
|
78
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
79
|
+
" {{variables}}: pid\n ",
|
80
|
+
"ps -p '{{pid}}' | tail -n +2 | wc -l")
|
81
|
+
v[:kill] = cli.ask_user(
|
82
|
+
"How should I terminate tasks?\n" \
|
83
|
+
" {{variables}}: pid\n ",
|
84
|
+
"kill -9 '{{pid}}'")
|
50
85
|
v
|
51
86
|
end
|
52
87
|
|
@@ -58,24 +93,28 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
58
93
|
cli.puts 'Setting up internal daemon defaults'
|
59
94
|
cli.puts 'If you don\'t understand this just leave default values:'
|
60
95
|
v[:cmd] = cli.ask_user(
|
61
|
-
"How should I launch tasks?\n
|
62
|
-
"
|
63
|
-
"
|
64
|
-
"--nodes=1 --ntasks-per-node
|
65
|
-
"--job-name='
|
96
|
+
"How should I launch tasks?\n" \
|
97
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga\n ",
|
98
|
+
"{{vars}} sbatch --partition='#{queue}' --export=ALL " \
|
99
|
+
"--nodes=1 --ntasks-per-node={{cpus}} --output='{{log}}' " \
|
100
|
+
"--job-name='{{task_name}}' --mem=9G --time=12:00:00 {{script}} " \
|
66
101
|
"| perl -pe 's/.* //'")
|
67
102
|
v[:var] = cli.ask_user(
|
68
|
-
"How should I pass variables?\n
|
69
|
-
|
103
|
+
"How should I pass variables?\n" \
|
104
|
+
" {{variables}}: key, value\n ",
|
105
|
+
"{{key}}={{value}}")
|
70
106
|
v[:varsep] = cli.ask_user(
|
71
107
|
'What should I use to separate variables?', ' ')
|
72
108
|
v[:alive] = cli.ask_user(
|
73
|
-
"How can I know that a process is still alive?\n
|
74
|
-
"
|
75
|
-
|
109
|
+
"How can I know that a process is still alive?\n" \
|
110
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
111
|
+
" {{variables}}: pid\n ",
|
112
|
+
"squeue -h -o %t -j '{{pid}}' | grep '^PD\\|R\\|CF\\|CG$' " \
|
76
113
|
"| tail -n 1 | wc -l")
|
77
114
|
v[:kill] = cli.ask_user(
|
78
|
-
"How should I terminate tasks?\n
|
115
|
+
"How should I terminate tasks?\n" \
|
116
|
+
" {{variables}}: pid\n ",
|
117
|
+
"scancel '{{pid}}'")
|
79
118
|
v
|
80
119
|
end
|
81
120
|
|
@@ -87,36 +126,40 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
87
126
|
cli.puts 'Setting up internal daemon defaults.'
|
88
127
|
cli.puts 'If you don\'t understand this just leave default values:'
|
89
128
|
v[:cmd] = cli.ask_user(
|
90
|
-
"How should I launch tasks?\n
|
91
|
-
"
|
92
|
-
"#{v[:type]} -q '#{queue}' -v '
|
93
|
-
"-j oe -o '
|
94
|
-
"| grep .")
|
129
|
+
"How should I launch tasks?\n" \
|
130
|
+
" {{variables}}: script, vars, cpus, log, task_name\n ",
|
131
|
+
"#{v[:type]} -q '#{queue}' -v '{{vars}}' -l nodes=1:ppn={{cpus}} " \
|
132
|
+
"{{script}} -j oe -o '{{log}}' -N '{{task_name}}' -l mem=9g " \
|
133
|
+
"-l walltime=12:00:00 | grep .")
|
95
134
|
v[:var] = cli.ask_user(
|
96
|
-
"How should I pass variables?\n
|
97
|
-
|
135
|
+
"How should I pass variables?\n" \
|
136
|
+
" {{variables}}: key, value\n ",
|
137
|
+
"{{key}}={{value}}")
|
98
138
|
v[:varsep] = cli.ask_user(
|
99
139
|
'What should I use to separate variables?', ',')
|
100
140
|
if v[:type] == 'qsub'
|
101
141
|
v[:alive] = cli.ask_user(
|
102
|
-
"How can I know that a process is still alive?\n
|
103
|
-
"
|
104
|
-
"
|
105
|
-
"qstat -f '
|
106
|
-
"|grep '[^C]'|tail -
|
142
|
+
"How can I know that a process is still alive?\n" \
|
143
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
144
|
+
" {{variables}}: pid\n ",
|
145
|
+
"qstat -f '{{pid}}' | grep ' job_state =' | perl -pe 's/.*= //' " \
|
146
|
+
"| grep '[^C]' | tail -n 1 | wc -l | awk '{print $1}'")
|
107
147
|
v[:kill] = cli.ask_user(
|
108
|
-
"How should I terminate tasks?\n
|
148
|
+
"How should I terminate tasks?\n" \
|
149
|
+
" {{variables}}: pid\n ",
|
150
|
+
"qdel '{{pid}}'")
|
109
151
|
else # msub
|
110
152
|
v[:alive] = cli.ask_user(
|
111
|
-
"How can I know that a process is still alive?\n
|
112
|
-
"
|
113
|
-
"
|
114
|
-
"checkjob '
|
115
|
-
"|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'" \
|
116
|
-
"|tail -
|
153
|
+
"How can I know that a process is still alive?\n" \
|
154
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
155
|
+
" {{variables}}: pid\n ",
|
156
|
+
"checkjob '{{pid}}'|grep '^State:' | perl -pe 's/.*: //' " \
|
157
|
+
"| grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'" \
|
158
|
+
"| tail -n 1 | wc -l | awk '{print $1}'")
|
117
159
|
v[:kill] = cli.ask_user(
|
118
|
-
"How should I terminate tasks?\n
|
119
|
-
|
160
|
+
"How should I terminate tasks?\n" \
|
161
|
+
" {{variables}}: pid\n ",
|
162
|
+
"canceljob '{{pid}}'")
|
120
163
|
end
|
121
164
|
v
|
122
165
|
end
|
data/lib/miga/cli/action/run.rb
CHANGED
@@ -7,7 +7,7 @@ require 'shellwords'
|
|
7
7
|
class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
8
8
|
|
9
9
|
def parse_cli
|
10
|
-
cli.defaults = {try_load: false, thr: 1}
|
10
|
+
cli.defaults = { try_load: false, thr: 1, env: false }
|
11
11
|
cli.parse do |opt|
|
12
12
|
cli.opt_object(opt, [:project, :dataset_opt, :result])
|
13
13
|
opt.on(
|
@@ -23,10 +23,20 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
|
23
23
|
'-l', '--log PATH',
|
24
24
|
'Path to the output log file to be created. If not set, STDOUT'
|
25
25
|
) { |v| cli[:log] = v }
|
26
|
+
opt.on(
|
27
|
+
'-e', '--environment',
|
28
|
+
'Load PROJECT, DATASET, and CORES from the environment'
|
29
|
+
) { |v| cli[:env] = v }
|
26
30
|
end
|
27
31
|
end
|
28
32
|
|
29
33
|
def perform
|
34
|
+
if cli[:env]
|
35
|
+
cli[:project] ||= ENV['PROJECT']
|
36
|
+
cli[:dataset] ||= ENV['DATASET']
|
37
|
+
cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
|
38
|
+
cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
|
39
|
+
end
|
30
40
|
virtual_task = false
|
31
41
|
miga = MiGA.root_path
|
32
42
|
p = cli.load_project
|
@@ -43,10 +53,11 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
|
43
53
|
end
|
44
54
|
cmd << MiGA.script_path(cli[:result], miga: miga, project: p).shellescape
|
45
55
|
if cli[:remote]
|
46
|
-
cmd.unshift '.', '/etc/profile', ';'
|
47
|
-
cmd = ['ssh', cli[:remote].shellescape,
|
56
|
+
#cmd.unshift '.', '/etc/profile', ';'
|
57
|
+
cmd = ['ssh', '-t', '-t', cli[:remote].shellescape,
|
58
|
+
cmd.join(' ').shellescape]
|
48
59
|
end
|
49
|
-
cmd << ['
|
60
|
+
cmd << ['>', cli[:log].shellescape, '2>&1'] if cli[:log]
|
50
61
|
pid = spawn cmd.join(' ')
|
51
62
|
Process.wait pid
|
52
63
|
end
|
data/lib/miga/common/format.rb
CHANGED
@@ -136,4 +136,12 @@ class String
|
|
136
136
|
def wrap_width(width)
|
137
137
|
gsub(/([^\n\r]{1,#{width}})/, "\\1\n")
|
138
138
|
end
|
139
|
+
|
140
|
+
##
|
141
|
+
# Replace {{variables}} using the +vars+ hash
|
142
|
+
def miga_variables(vars)
|
143
|
+
o = "#{self}"
|
144
|
+
vars.each { |k, v| o.gsub!("{{#{k}}}", v.to_s) }
|
145
|
+
o
|
146
|
+
end
|
139
147
|
end
|
data/lib/miga/daemon.rb
CHANGED
@@ -42,9 +42,12 @@ class MiGA::Daemon < MiGA::MiGA
|
|
42
42
|
def initialize(project, json = nil)
|
43
43
|
$_MIGA_DAEMON_LAIR << self
|
44
44
|
@project = project
|
45
|
+
@runopts = {}
|
45
46
|
json ||= File.expand_path('daemon/daemon.json', project.path)
|
46
|
-
|
47
|
-
json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
47
|
+
MiGA::Json.parse(
|
48
|
+
json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
49
|
+
).each { |k,v| runopts(k, v) }
|
50
|
+
update_format_0 unless runopts(:format_version)
|
48
51
|
@jobs_to_run = []
|
49
52
|
@jobs_running = []
|
50
53
|
@loop_i = -1
|
@@ -145,8 +148,8 @@ class MiGA::Daemon < MiGA::MiGA
|
|
145
148
|
##
|
146
149
|
# Add the task to the internal queue with symbol key +job+. If the task is
|
147
150
|
# dataset-specific, +ds+ specifies the dataset. To submit jobs to the
|
148
|
-
# scheduler (or to bash) see #flush
|
149
|
-
def queue_job(job, ds=nil)
|
151
|
+
# scheduler (or to bash or ssh) see #flush!
|
152
|
+
def queue_job(job, ds = nil)
|
150
153
|
return nil unless get_job(job, ds).nil?
|
151
154
|
ds_name = (ds.nil? ? 'miga-project' : ds.name)
|
152
155
|
say 'Queueing %s:%s' % [ds_name, job]
|
@@ -160,19 +163,16 @@ class MiGA::Daemon < MiGA::MiGA
|
|
160
163
|
log_dir = File.expand_path("daemon/#{job}", project.path)
|
161
164
|
Dir.mkdir(log_dir) unless Dir.exist? log_dir
|
162
165
|
task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
|
163
|
-
to_run = {ds: ds, ds_name: ds_name, job: job, task_name: task_name
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
File.expand_path("#{ds_name}.log", log_dir),
|
174
|
-
# 5: task name
|
175
|
-
task_name)}
|
166
|
+
to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
|
167
|
+
to_run[:cmd] = runopts(:cmd).miga_variables(
|
168
|
+
script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
|
169
|
+
vars: vars.map { |k, v|
|
170
|
+
runopts(:var).miga_variables(key: k, value: v) }.join(runopts(:varsep)),
|
171
|
+
cpus: ppn,
|
172
|
+
log: File.expand_path("#{ds_name}.log", log_dir),
|
173
|
+
task_name: task_name,
|
174
|
+
miga: File.expand_path('bin/miga', MiGA::MiGA.root_path).shellescape
|
175
|
+
)
|
176
176
|
@jobs_to_run << to_run
|
177
177
|
end
|
178
178
|
|
@@ -191,7 +191,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
191
191
|
|
192
192
|
##
|
193
193
|
# Remove finished jobs from the internal queue and launch as many as
|
194
|
-
# possible respecting #maxjobs.
|
194
|
+
# possible respecting #maxjobs or #nodelist (if set).
|
195
195
|
def flush!
|
196
196
|
# Check for finished jobs
|
197
197
|
@jobs_running.select! do |job|
|
@@ -209,17 +209,30 @@ class MiGA::Daemon < MiGA::MiGA
|
|
209
209
|
# Avoid single datasets hogging resources
|
210
210
|
@jobs_to_run.rotate! rand(jobs_to_run.size)
|
211
211
|
# Launch as many +jobs_to_run+ as possible
|
212
|
-
while
|
212
|
+
while hostk = next_host
|
213
213
|
break if jobs_to_run.empty?
|
214
|
-
launch_job
|
214
|
+
launch_job(@jobs_to_run.shift, hostk)
|
215
215
|
end
|
216
216
|
end
|
217
217
|
|
218
|
+
##
|
219
|
+
# Retrieves the host index of an available node (if any), nil otherwise. If
|
220
|
+
# #nodelist is not set, returns true as long as #maxjobs is not reached
|
221
|
+
def next_host
|
222
|
+
if nodelist.nil?
|
223
|
+
return jobs_running.size < maxjobs
|
224
|
+
end
|
225
|
+
allk = (0 .. nodelist.size-1).to_a
|
226
|
+
busyk = jobs_running.map { |k| k[:hostk] }
|
227
|
+
availk = allk - busyk
|
228
|
+
availk.empty? ? nil : availk.first
|
229
|
+
end
|
230
|
+
|
218
231
|
##
|
219
232
|
# Remove dead jobs.
|
220
233
|
def purge!
|
221
234
|
@jobs_running.select! do |job|
|
222
|
-
`#{
|
235
|
+
`#{runopts(:alive).miga_variables(pid: job[:pid])}`.chomp.to_i == 1
|
223
236
|
end
|
224
237
|
end
|
225
238
|
|
@@ -270,14 +283,21 @@ class MiGA::Daemon < MiGA::MiGA
|
|
270
283
|
|
271
284
|
private
|
272
285
|
|
273
|
-
def launch_job(job)
|
286
|
+
def launch_job(job, hostk = nil)
|
274
287
|
# Execute job
|
275
|
-
|
288
|
+
case runopts(:type)
|
289
|
+
when 'ssh'
|
290
|
+
# Remote job
|
291
|
+
job[:hostk] = hostk
|
292
|
+
job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
|
293
|
+
job[:pid] = spawn job[:cmd]
|
294
|
+
Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
|
295
|
+
when 'bash'
|
276
296
|
# Local job
|
277
297
|
job[:pid] = spawn job[:cmd]
|
278
298
|
Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
|
279
299
|
else
|
280
|
-
# Schedule cluster job
|
300
|
+
# Schedule cluster job (qsub, msub, slurm)
|
281
301
|
job[:pid] = `#{job[:cmd]}`.chomp
|
282
302
|
end
|
283
303
|
|
@@ -288,7 +308,20 @@ class MiGA::Daemon < MiGA::MiGA
|
|
288
308
|
say "Unsuccessful #{job[:task_name]}, rescheduling"
|
289
309
|
else
|
290
310
|
@jobs_running << job
|
291
|
-
say "Spawned pid:#{job[:pid]}
|
311
|
+
say "Spawned pid:#{job[:pid]}#{
|
312
|
+
" to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
|
313
|
+
} for #{job[:task_name]}"
|
292
314
|
end
|
293
315
|
end
|
316
|
+
|
317
|
+
def update_format_0
|
318
|
+
say 'Outdated daemon.json format, updating from version 0'
|
319
|
+
{
|
320
|
+
cmd: %w[script vars cpus log task_name],
|
321
|
+
var: %w[key value],
|
322
|
+
alive: %w[pid],
|
323
|
+
kill: %w[pid]
|
324
|
+
}.each { |k,v| runopts(k, sprintf(runopts(k), *v.map{ |i| "{{#{i}}}" })) }
|
325
|
+
runopts(:format_version, 1)
|
326
|
+
end
|
294
327
|
end
|
data/lib/miga/daemon/base.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
|
2
2
|
require 'daemons'
|
3
3
|
require 'date'
|
4
|
+
require 'shellwords'
|
4
5
|
|
5
6
|
class MiGA::Daemon < MiGA::MiGA
|
6
7
|
end
|
@@ -14,10 +15,18 @@ module MiGA::Daemon::Base
|
|
14
15
|
def runopts(k, v = nil, force = false)
|
15
16
|
k = k.to_sym
|
16
17
|
unless v.nil?
|
17
|
-
|
18
|
+
case k
|
19
|
+
when :latency, :maxjobs, :ppn, :format_version
|
18
20
|
v = v.to_i
|
19
|
-
|
21
|
+
when :shutdown_when_done
|
20
22
|
v = !!v
|
23
|
+
when :nodelist
|
24
|
+
if v =~ /^\$/
|
25
|
+
vv = ENV[v.sub('$','')] or raise "Unset environment variable: #{v}"
|
26
|
+
v = vv
|
27
|
+
end
|
28
|
+
say "Reading node list: #{v}"
|
29
|
+
v = File.readlines(v).map(&:chomp)
|
21
30
|
end
|
22
31
|
raise "Daemon's #{k} cannot be set to zero." if !force and v == 0
|
23
32
|
@runopts[k] = v
|
@@ -26,32 +35,36 @@ module MiGA::Daemon::Base
|
|
26
35
|
end
|
27
36
|
|
28
37
|
##
|
29
|
-
# Returns Integer indicating the number of seconds to sleep between checks
|
38
|
+
# Returns Integer indicating the number of seconds to sleep between checks
|
30
39
|
def latency() runopts(:latency); end
|
31
40
|
|
32
41
|
##
|
33
|
-
# Returns Integer indicating the maximum number of concurrent jobs to run
|
42
|
+
# Returns Integer indicating the maximum number of concurrent jobs to run
|
34
43
|
def maxjobs() runopts(:maxjobs); end
|
35
44
|
|
36
45
|
##
|
37
|
-
# Returns
|
46
|
+
# Returns the path to the list of execution hostnames
|
47
|
+
def nodelist() runopts(:nodelist); end
|
48
|
+
|
49
|
+
##
|
50
|
+
# Returns Integer indicating the number of CPUs per job
|
38
51
|
def ppn() runopts(:ppn); end
|
39
52
|
|
40
53
|
##
|
41
54
|
# Returns Boolean indicating if the daemon should shutdown when processing is
|
42
|
-
# complete
|
55
|
+
# complete
|
43
56
|
def shutdown_when_done?() !!runopts(:shutdown_when_done); end
|
44
57
|
|
45
58
|
##
|
46
|
-
# Initializes the daemon with +opts
|
59
|
+
# Initializes the daemon with +opts+
|
47
60
|
def start(opts = []) daemon('start', opts); end
|
48
61
|
|
49
62
|
##
|
50
|
-
# Stops the daemon with +opts
|
63
|
+
# Stops the daemon with +opts+
|
51
64
|
def stop(opts = []) daemon('stop', opts); end
|
52
65
|
|
53
66
|
##
|
54
|
-
# Restarts the daemon with +opts
|
67
|
+
# Restarts the daemon with +opts+
|
55
68
|
def restart(opts = []) daemon('restart', opts); end
|
56
69
|
|
57
70
|
##
|
data/lib/miga/version.rb
CHANGED
@@ -10,7 +10,7 @@ module MiGA
|
|
10
10
|
# - Float representing the major.minor version.
|
11
11
|
# - Integer representing gem releases of the current version.
|
12
12
|
# - Integer representing minor changes that require new version number.
|
13
|
-
VERSION = [0.5,
|
13
|
+
VERSION = [0.5, 5, 0]
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nickname for the current major.minor version.
|
@@ -18,7 +18,7 @@ module MiGA
|
|
18
18
|
|
19
19
|
##
|
20
20
|
# Date of the current gem release.
|
21
|
-
VERSION_DATE = Date.new(2020,
|
21
|
+
VERSION_DATE = Date.new(2020, 2, 4)
|
22
22
|
|
23
23
|
##
|
24
24
|
# Reference of MiGA.
|
data/test/daemon_test.rb
CHANGED
@@ -9,8 +9,9 @@ class DaemonTest < Test::Unit::TestCase
|
|
9
9
|
ENV["MIGA_HOME"] = $tmp
|
10
10
|
FileUtils.touch("#{ENV["MIGA_HOME"]}/.miga_rc")
|
11
11
|
File.open("#{ENV["MIGA_HOME"]}/.miga_daemon.json", "w") do |fh|
|
12
|
-
fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
|
13
|
-
"
|
12
|
+
fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
|
13
|
+
"var":"{{key}}={{value}}","cmd":"{{task_name}}",
|
14
|
+
"alive":"echo 1 # {{pid}}","type":"bash","format_version":1}'
|
14
15
|
end
|
15
16
|
$p1 = MiGA::Project.new(File.expand_path("project1", $tmp))
|
16
17
|
$d1 = MiGA::Daemon.new($p1)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|