miga-base 0.5.4.0 → 0.5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli/action/daemon.rb +5 -1
- data/lib/miga/cli/action/init.rb +1 -1
- data/lib/miga/cli/action/init/daemon_helper.rb +85 -42
- data/lib/miga/cli/action/run.rb +15 -4
- data/lib/miga/common/format.rb +8 -0
- data/lib/miga/daemon.rb +58 -25
- data/lib/miga/daemon/base.rb +22 -9
- data/lib/miga/version.rb +2 -2
- data/test/daemon_test.rb +3 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59649a6c1b7aaa4e3c0db090a828469e9595eba42d311eff114417741a2816bd
|
4
|
+
data.tar.gz: a765a214e5b64d3f1758ae50942dc7f0649f438e8c2b12312648012d9cceeacc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f76b1691a30d7ce94314a4bd8d22601d86d3101c2db33f262b53548bba817d17bc783f2c93ec0502844adda1f5a17bd0c530e82c82c7fb590f07abf8ffc6788c
|
7
|
+
data.tar.gz: c27cd47089f65633cefaca74cd217967b583df43d60682dce9f755a6610fdc6e728638d8d7b884f4de627de79b48080c8325f7b4d60794253f69511c15f997a4
|
@@ -36,6 +36,10 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
|
|
36
36
|
'--max-jobs INT',
|
37
37
|
'Maximum number of jobs to use simultaneously'
|
38
38
|
) { |v| cli[:maxjobs] = v.to_i }
|
39
|
+
opt.on(
|
40
|
+
'--node-list PATH',
|
41
|
+
'Path to the list of execution hostnames'
|
42
|
+
) { |v| cli[:nodelist] = v }
|
39
43
|
opt.on(
|
40
44
|
'--ppn INT',
|
41
45
|
'Maximum number of cores to use in a single job'
|
@@ -70,7 +74,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
|
|
70
74
|
def perform
|
71
75
|
p = cli.load_project
|
72
76
|
d = MiGA::Daemon.new(p, cli[:json])
|
73
|
-
[:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
|
77
|
+
[:latency, :maxjobs, :nodelist, :ppn, :shutdown_when_done].each do |k|
|
74
78
|
d.runopts(k, cli[k]) unless cli[k].nil?
|
75
79
|
end
|
76
80
|
d.daemon(cli.operation, cli[:daemon_opts])
|
data/lib/miga/cli/action/init.rb
CHANGED
@@ -26,7 +26,7 @@ class MiGA::Cli::Action::Init < MiGA::Cli::Action
|
|
26
26
|
){ |v| cli[:mytaxa] = v }
|
27
27
|
opt.on(
|
28
28
|
'--daemon-type STRING',
|
29
|
-
'Type of daemon launcher, one of: bash, qsub, msub, slurm',
|
29
|
+
'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
|
30
30
|
"By default: interactive (#{cli[:dtype]} if --auto)"
|
31
31
|
){ |v| cli[:dtype] = v.to_sym }
|
32
32
|
opt.on(
|
@@ -13,15 +13,18 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
13
13
|
v = {created: Time.now.to_s, updated: Time.now.to_s}
|
14
14
|
v[:type] = cli.ask_user(
|
15
15
|
'Please select the type of daemon you want to setup',
|
16
|
-
cli[:dtype], %w(bash qsub msub slurm))
|
16
|
+
cli[:dtype], %w(bash ssh qsub msub slurm))
|
17
17
|
case v[:type]
|
18
18
|
when 'bash'
|
19
19
|
v = configure_bash_daemon(v)
|
20
|
+
when 'ssh'
|
21
|
+
v = configure_ssh_daemon(v)
|
20
22
|
when 'slurm'
|
21
23
|
v = configure_slurm_daemon(v)
|
22
24
|
else # [qm]sub
|
23
25
|
v = configure_qsub_msub_daemon(v)
|
24
26
|
end
|
27
|
+
v[:format_version] = 1
|
25
28
|
File.open(daemon_f, 'w') { |fh| fh.puts JSON.pretty_generate(v) }
|
26
29
|
end
|
27
30
|
cli.puts ''
|
@@ -34,19 +37,51 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
34
37
|
cli.puts 'Setting up internal daemon defaults.'
|
35
38
|
cli.puts 'If you don\'t understand this just leave default values:'
|
36
39
|
v[:cmd] = cli.ask_user(
|
37
|
-
"How should I launch tasks?\n
|
38
|
-
"
|
39
|
-
"
|
40
|
+
"How should I launch tasks?\n" \
|
41
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga\n ",
|
42
|
+
"{{vars}} {{miga}} run -r '{{script}}' -l '{{log}}' -e")
|
40
43
|
v[:var] = cli.ask_user(
|
41
|
-
"How should I pass variables?\n
|
42
|
-
|
44
|
+
"How should I pass variables?\n" \
|
45
|
+
" {{variables}}: key, value\n ",
|
46
|
+
"{{key}}={{value}}")
|
43
47
|
v[:varsep] = cli.ask_user('What should I use to separate variables?', ' ')
|
44
48
|
v[:alive] = cli.ask_user(
|
45
|
-
"How can I know that a process is still alive?\n
|
46
|
-
"
|
47
|
-
|
49
|
+
"How can I know that a process is still alive?\n" \
|
50
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
51
|
+
" {{variables}}: pid\n ",
|
52
|
+
"ps -p '{{pid}}' | tail -n +2 | wc -l")
|
48
53
|
v[:kill] = cli.ask_user(
|
49
|
-
"How should I terminate tasks?\n
|
54
|
+
"How should I terminate tasks?\n" \
|
55
|
+
" {{variables}}: pid\n ",
|
56
|
+
"kill -9 '{{pid}}'")
|
57
|
+
v
|
58
|
+
end
|
59
|
+
|
60
|
+
def configure_ssh_daemon(v)
|
61
|
+
v[:latency] = cli.ask_user('How long should I sleep? (in secs)', '3').to_i
|
62
|
+
v[:nodelist] = cli.ask_user(
|
63
|
+
'What environmental variable points to node list?', '$MIGA_NODELIST')
|
64
|
+
v[:ppn] = cli.ask_user('How many CPUs can I use per job?', '2').to_i
|
65
|
+
cli.puts 'Setting up internal daemon defaults.'
|
66
|
+
cli.puts 'If you don\'t understand this just leave default values:'
|
67
|
+
v[:cmd] = cli.ask_user(
|
68
|
+
"How should I launch tasks?\n" \
|
69
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga, host\n ",
|
70
|
+
"{{vars}} {{miga}} run -r '{{script}}' -l '{{log}}' -R {{host}} -e")
|
71
|
+
v[:var] = cli.ask_user(
|
72
|
+
"How should I pass variables?\n" \
|
73
|
+
" {{variables}}: key, value\n ",
|
74
|
+
"{{key}}={{value}}")
|
75
|
+
v[:varsep] = cli.ask_user('What should I use to separate variables?', ' ')
|
76
|
+
v[:alive] = cli.ask_user(
|
77
|
+
"How can I know that a process is still alive?\n" \
|
78
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
79
|
+
" {{variables}}: pid\n ",
|
80
|
+
"ps -p '{{pid}}' | tail -n +2 | wc -l")
|
81
|
+
v[:kill] = cli.ask_user(
|
82
|
+
"How should I terminate tasks?\n" \
|
83
|
+
" {{variables}}: pid\n ",
|
84
|
+
"kill -9 '{{pid}}'")
|
50
85
|
v
|
51
86
|
end
|
52
87
|
|
@@ -58,24 +93,28 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
58
93
|
cli.puts 'Setting up internal daemon defaults'
|
59
94
|
cli.puts 'If you don\'t understand this just leave default values:'
|
60
95
|
v[:cmd] = cli.ask_user(
|
61
|
-
"How should I launch tasks?\n
|
62
|
-
"
|
63
|
-
"
|
64
|
-
"--nodes=1 --ntasks-per-node
|
65
|
-
"--job-name='
|
96
|
+
"How should I launch tasks?\n" \
|
97
|
+
" {{variables}}: script, vars, cpus, log, task_name, miga\n ",
|
98
|
+
"{{vars}} sbatch --partition='#{queue}' --export=ALL " \
|
99
|
+
"--nodes=1 --ntasks-per-node={{cpus}} --output='{{log}}' " \
|
100
|
+
"--job-name='{{task_name}}' --mem=9G --time=12:00:00 {{script}} " \
|
66
101
|
"| perl -pe 's/.* //'")
|
67
102
|
v[:var] = cli.ask_user(
|
68
|
-
"How should I pass variables?\n
|
69
|
-
|
103
|
+
"How should I pass variables?\n" \
|
104
|
+
" {{variables}}: key, value\n ",
|
105
|
+
"{{key}}={{value}}")
|
70
106
|
v[:varsep] = cli.ask_user(
|
71
107
|
'What should I use to separate variables?', ' ')
|
72
108
|
v[:alive] = cli.ask_user(
|
73
|
-
"How can I know that a process is still alive?\n
|
74
|
-
"
|
75
|
-
|
109
|
+
"How can I know that a process is still alive?\n" \
|
110
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
111
|
+
" {{variables}}: pid\n ",
|
112
|
+
"squeue -h -o %t -j '{{pid}}' | grep '^PD\\|R\\|CF\\|CG$' " \
|
76
113
|
"| tail -n 1 | wc -l")
|
77
114
|
v[:kill] = cli.ask_user(
|
78
|
-
"How should I terminate tasks?\n
|
115
|
+
"How should I terminate tasks?\n" \
|
116
|
+
" {{variables}}: pid\n ",
|
117
|
+
"scancel '{{pid}}'")
|
79
118
|
v
|
80
119
|
end
|
81
120
|
|
@@ -87,36 +126,40 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
87
126
|
cli.puts 'Setting up internal daemon defaults.'
|
88
127
|
cli.puts 'If you don\'t understand this just leave default values:'
|
89
128
|
v[:cmd] = cli.ask_user(
|
90
|
-
"How should I launch tasks?\n
|
91
|
-
"
|
92
|
-
"#{v[:type]} -q '#{queue}' -v '
|
93
|
-
"-j oe -o '
|
94
|
-
"| grep .")
|
129
|
+
"How should I launch tasks?\n" \
|
130
|
+
" {{variables}}: script, vars, cpus, log, task_name\n ",
|
131
|
+
"#{v[:type]} -q '#{queue}' -v '{{vars}}' -l nodes=1:ppn={{cpus}} " \
|
132
|
+
"{{script}} -j oe -o '{{log}}' -N '{{task_name}}' -l mem=9g " \
|
133
|
+
"-l walltime=12:00:00 | grep .")
|
95
134
|
v[:var] = cli.ask_user(
|
96
|
-
"How should I pass variables?\n
|
97
|
-
|
135
|
+
"How should I pass variables?\n" \
|
136
|
+
" {{variables}}: key, value\n ",
|
137
|
+
"{{key}}={{value}}")
|
98
138
|
v[:varsep] = cli.ask_user(
|
99
139
|
'What should I use to separate variables?', ',')
|
100
140
|
if v[:type] == 'qsub'
|
101
141
|
v[:alive] = cli.ask_user(
|
102
|
-
"How can I know that a process is still alive?\n
|
103
|
-
"
|
104
|
-
"
|
105
|
-
"qstat -f '
|
106
|
-
"|grep '[^C]'|tail -
|
142
|
+
"How can I know that a process is still alive?\n" \
|
143
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
144
|
+
" {{variables}}: pid\n ",
|
145
|
+
"qstat -f '{{pid}}' | grep ' job_state =' | perl -pe 's/.*= //' " \
|
146
|
+
"| grep '[^C]' | tail -n 1 | wc -l | awk '{print $1}'")
|
107
147
|
v[:kill] = cli.ask_user(
|
108
|
-
"How should I terminate tasks?\n
|
148
|
+
"How should I terminate tasks?\n" \
|
149
|
+
" {{variables}}: pid\n ",
|
150
|
+
"qdel '{{pid}}'")
|
109
151
|
else # msub
|
110
152
|
v[:alive] = cli.ask_user(
|
111
|
-
"How can I know that a process is still alive?\n
|
112
|
-
"
|
113
|
-
"
|
114
|
-
"checkjob '
|
115
|
-
"|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'" \
|
116
|
-
"|tail -
|
153
|
+
"How can I know that a process is still alive?\n" \
|
154
|
+
" Output should be 1 for running and 0 for non-running\n" \
|
155
|
+
" {{variables}}: pid\n ",
|
156
|
+
"checkjob '{{pid}}'|grep '^State:' | perl -pe 's/.*: //' " \
|
157
|
+
"| grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'" \
|
158
|
+
"| tail -n 1 | wc -l | awk '{print $1}'")
|
117
159
|
v[:kill] = cli.ask_user(
|
118
|
-
"How should I terminate tasks?\n
|
119
|
-
|
160
|
+
"How should I terminate tasks?\n" \
|
161
|
+
" {{variables}}: pid\n ",
|
162
|
+
"canceljob '{{pid}}'")
|
120
163
|
end
|
121
164
|
v
|
122
165
|
end
|
data/lib/miga/cli/action/run.rb
CHANGED
@@ -7,7 +7,7 @@ require 'shellwords'
|
|
7
7
|
class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
8
8
|
|
9
9
|
def parse_cli
|
10
|
-
cli.defaults = {try_load: false, thr: 1}
|
10
|
+
cli.defaults = { try_load: false, thr: 1, env: false }
|
11
11
|
cli.parse do |opt|
|
12
12
|
cli.opt_object(opt, [:project, :dataset_opt, :result])
|
13
13
|
opt.on(
|
@@ -23,10 +23,20 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
|
23
23
|
'-l', '--log PATH',
|
24
24
|
'Path to the output log file to be created. If not set, STDOUT'
|
25
25
|
) { |v| cli[:log] = v }
|
26
|
+
opt.on(
|
27
|
+
'-e', '--environment',
|
28
|
+
'Load PROJECT, DATASET, and CORES from the environment'
|
29
|
+
) { |v| cli[:env] = v }
|
26
30
|
end
|
27
31
|
end
|
28
32
|
|
29
33
|
def perform
|
34
|
+
if cli[:env]
|
35
|
+
cli[:project] ||= ENV['PROJECT']
|
36
|
+
cli[:dataset] ||= ENV['DATASET']
|
37
|
+
cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
|
38
|
+
cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
|
39
|
+
end
|
30
40
|
virtual_task = false
|
31
41
|
miga = MiGA.root_path
|
32
42
|
p = cli.load_project
|
@@ -43,10 +53,11 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
|
|
43
53
|
end
|
44
54
|
cmd << MiGA.script_path(cli[:result], miga: miga, project: p).shellescape
|
45
55
|
if cli[:remote]
|
46
|
-
cmd.unshift '.', '/etc/profile', ';'
|
47
|
-
cmd = ['ssh', cli[:remote].shellescape,
|
56
|
+
#cmd.unshift '.', '/etc/profile', ';'
|
57
|
+
cmd = ['ssh', '-t', '-t', cli[:remote].shellescape,
|
58
|
+
cmd.join(' ').shellescape]
|
48
59
|
end
|
49
|
-
cmd << ['
|
60
|
+
cmd << ['>', cli[:log].shellescape, '2>&1'] if cli[:log]
|
50
61
|
pid = spawn cmd.join(' ')
|
51
62
|
Process.wait pid
|
52
63
|
end
|
data/lib/miga/common/format.rb
CHANGED
@@ -136,4 +136,12 @@ class String
|
|
136
136
|
def wrap_width(width)
|
137
137
|
gsub(/([^\n\r]{1,#{width}})/, "\\1\n")
|
138
138
|
end
|
139
|
+
|
140
|
+
##
|
141
|
+
# Replace {{variables}} using the +vars+ hash
|
142
|
+
def miga_variables(vars)
|
143
|
+
o = "#{self}"
|
144
|
+
vars.each { |k, v| o.gsub!("{{#{k}}}", v.to_s) }
|
145
|
+
o
|
146
|
+
end
|
139
147
|
end
|
data/lib/miga/daemon.rb
CHANGED
@@ -42,9 +42,12 @@ class MiGA::Daemon < MiGA::MiGA
|
|
42
42
|
def initialize(project, json = nil)
|
43
43
|
$_MIGA_DAEMON_LAIR << self
|
44
44
|
@project = project
|
45
|
+
@runopts = {}
|
45
46
|
json ||= File.expand_path('daemon/daemon.json', project.path)
|
46
|
-
|
47
|
-
json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
47
|
+
MiGA::Json.parse(
|
48
|
+
json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
49
|
+
).each { |k,v| runopts(k, v) }
|
50
|
+
update_format_0 unless runopts(:format_version)
|
48
51
|
@jobs_to_run = []
|
49
52
|
@jobs_running = []
|
50
53
|
@loop_i = -1
|
@@ -145,8 +148,8 @@ class MiGA::Daemon < MiGA::MiGA
|
|
145
148
|
##
|
146
149
|
# Add the task to the internal queue with symbol key +job+. If the task is
|
147
150
|
# dataset-specific, +ds+ specifies the dataset. To submit jobs to the
|
148
|
-
# scheduler (or to bash) see #flush
|
149
|
-
def queue_job(job, ds=nil)
|
151
|
+
# scheduler (or to bash or ssh) see #flush!
|
152
|
+
def queue_job(job, ds = nil)
|
150
153
|
return nil unless get_job(job, ds).nil?
|
151
154
|
ds_name = (ds.nil? ? 'miga-project' : ds.name)
|
152
155
|
say 'Queueing %s:%s' % [ds_name, job]
|
@@ -160,19 +163,16 @@ class MiGA::Daemon < MiGA::MiGA
|
|
160
163
|
log_dir = File.expand_path("daemon/#{job}", project.path)
|
161
164
|
Dir.mkdir(log_dir) unless Dir.exist? log_dir
|
162
165
|
task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
|
163
|
-
to_run = {ds: ds, ds_name: ds_name, job: job, task_name: task_name
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
File.expand_path("#{ds_name}.log", log_dir),
|
174
|
-
# 5: task name
|
175
|
-
task_name)}
|
166
|
+
to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
|
167
|
+
to_run[:cmd] = runopts(:cmd).miga_variables(
|
168
|
+
script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
|
169
|
+
vars: vars.map { |k, v|
|
170
|
+
runopts(:var).miga_variables(key: k, value: v) }.join(runopts(:varsep)),
|
171
|
+
cpus: ppn,
|
172
|
+
log: File.expand_path("#{ds_name}.log", log_dir),
|
173
|
+
task_name: task_name,
|
174
|
+
miga: File.expand_path('bin/miga', MiGA::MiGA.root_path).shellescape
|
175
|
+
)
|
176
176
|
@jobs_to_run << to_run
|
177
177
|
end
|
178
178
|
|
@@ -191,7 +191,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
191
191
|
|
192
192
|
##
|
193
193
|
# Remove finished jobs from the internal queue and launch as many as
|
194
|
-
# possible respecting #maxjobs.
|
194
|
+
# possible respecting #maxjobs or #nodelist (if set).
|
195
195
|
def flush!
|
196
196
|
# Check for finished jobs
|
197
197
|
@jobs_running.select! do |job|
|
@@ -209,17 +209,30 @@ class MiGA::Daemon < MiGA::MiGA
|
|
209
209
|
# Avoid single datasets hogging resources
|
210
210
|
@jobs_to_run.rotate! rand(jobs_to_run.size)
|
211
211
|
# Launch as many +jobs_to_run+ as possible
|
212
|
-
while
|
212
|
+
while hostk = next_host
|
213
213
|
break if jobs_to_run.empty?
|
214
|
-
launch_job
|
214
|
+
launch_job(@jobs_to_run.shift, hostk)
|
215
215
|
end
|
216
216
|
end
|
217
217
|
|
218
|
+
##
|
219
|
+
# Retrieves the host index of an available node (if any), nil otherwise. If
|
220
|
+
# #nodelist is not set, returns true as long as #maxjobs is not reached
|
221
|
+
def next_host
|
222
|
+
if nodelist.nil?
|
223
|
+
return jobs_running.size < maxjobs
|
224
|
+
end
|
225
|
+
allk = (0 .. nodelist.size-1).to_a
|
226
|
+
busyk = jobs_running.map { |k| k[:hostk] }
|
227
|
+
availk = allk - busyk
|
228
|
+
availk.empty? ? nil : availk.first
|
229
|
+
end
|
230
|
+
|
218
231
|
##
|
219
232
|
# Remove dead jobs.
|
220
233
|
def purge!
|
221
234
|
@jobs_running.select! do |job|
|
222
|
-
`#{
|
235
|
+
`#{runopts(:alive).miga_variables(pid: job[:pid])}`.chomp.to_i == 1
|
223
236
|
end
|
224
237
|
end
|
225
238
|
|
@@ -270,14 +283,21 @@ class MiGA::Daemon < MiGA::MiGA
|
|
270
283
|
|
271
284
|
private
|
272
285
|
|
273
|
-
def launch_job(job)
|
286
|
+
def launch_job(job, hostk = nil)
|
274
287
|
# Execute job
|
275
|
-
|
288
|
+
case runopts(:type)
|
289
|
+
when 'ssh'
|
290
|
+
# Remote job
|
291
|
+
job[:hostk] = hostk
|
292
|
+
job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
|
293
|
+
job[:pid] = spawn job[:cmd]
|
294
|
+
Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
|
295
|
+
when 'bash'
|
276
296
|
# Local job
|
277
297
|
job[:pid] = spawn job[:cmd]
|
278
298
|
Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
|
279
299
|
else
|
280
|
-
# Schedule cluster job
|
300
|
+
# Schedule cluster job (qsub, msub, slurm)
|
281
301
|
job[:pid] = `#{job[:cmd]}`.chomp
|
282
302
|
end
|
283
303
|
|
@@ -288,7 +308,20 @@ class MiGA::Daemon < MiGA::MiGA
|
|
288
308
|
say "Unsuccessful #{job[:task_name]}, rescheduling"
|
289
309
|
else
|
290
310
|
@jobs_running << job
|
291
|
-
say "Spawned pid:#{job[:pid]}
|
311
|
+
say "Spawned pid:#{job[:pid]}#{
|
312
|
+
" to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
|
313
|
+
} for #{job[:task_name]}"
|
292
314
|
end
|
293
315
|
end
|
316
|
+
|
317
|
+
def update_format_0
|
318
|
+
say 'Outdated daemon.json format, updating from version 0'
|
319
|
+
{
|
320
|
+
cmd: %w[script vars cpus log task_name],
|
321
|
+
var: %w[key value],
|
322
|
+
alive: %w[pid],
|
323
|
+
kill: %w[pid]
|
324
|
+
}.each { |k,v| runopts(k, sprintf(runopts(k), *v.map{ |i| "{{#{i}}}" })) }
|
325
|
+
runopts(:format_version, 1)
|
326
|
+
end
|
294
327
|
end
|
data/lib/miga/daemon/base.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
|
2
2
|
require 'daemons'
|
3
3
|
require 'date'
|
4
|
+
require 'shellwords'
|
4
5
|
|
5
6
|
class MiGA::Daemon < MiGA::MiGA
|
6
7
|
end
|
@@ -14,10 +15,18 @@ module MiGA::Daemon::Base
|
|
14
15
|
def runopts(k, v = nil, force = false)
|
15
16
|
k = k.to_sym
|
16
17
|
unless v.nil?
|
17
|
-
|
18
|
+
case k
|
19
|
+
when :latency, :maxjobs, :ppn, :format_version
|
18
20
|
v = v.to_i
|
19
|
-
|
21
|
+
when :shutdown_when_done
|
20
22
|
v = !!v
|
23
|
+
when :nodelist
|
24
|
+
if v =~ /^\$/
|
25
|
+
vv = ENV[v.sub('$','')] or raise "Unset environment variable: #{v}"
|
26
|
+
v = vv
|
27
|
+
end
|
28
|
+
say "Reading node list: #{v}"
|
29
|
+
v = File.readlines(v).map(&:chomp)
|
21
30
|
end
|
22
31
|
raise "Daemon's #{k} cannot be set to zero." if !force and v == 0
|
23
32
|
@runopts[k] = v
|
@@ -26,32 +35,36 @@ module MiGA::Daemon::Base
|
|
26
35
|
end
|
27
36
|
|
28
37
|
##
|
29
|
-
# Returns Integer indicating the number of seconds to sleep between checks
|
38
|
+
# Returns Integer indicating the number of seconds to sleep between checks
|
30
39
|
def latency() runopts(:latency); end
|
31
40
|
|
32
41
|
##
|
33
|
-
# Returns Integer indicating the maximum number of concurrent jobs to run
|
42
|
+
# Returns Integer indicating the maximum number of concurrent jobs to run
|
34
43
|
def maxjobs() runopts(:maxjobs); end
|
35
44
|
|
36
45
|
##
|
37
|
-
# Returns
|
46
|
+
# Returns the path to the list of execution hostnames
|
47
|
+
def nodelist() runopts(:nodelist); end
|
48
|
+
|
49
|
+
##
|
50
|
+
# Returns Integer indicating the number of CPUs per job
|
38
51
|
def ppn() runopts(:ppn); end
|
39
52
|
|
40
53
|
##
|
41
54
|
# Returns Boolean indicating if the daemon should shutdown when processing is
|
42
|
-
# complete
|
55
|
+
# complete
|
43
56
|
def shutdown_when_done?() !!runopts(:shutdown_when_done); end
|
44
57
|
|
45
58
|
##
|
46
|
-
# Initializes the daemon with +opts
|
59
|
+
# Initializes the daemon with +opts+
|
47
60
|
def start(opts = []) daemon('start', opts); end
|
48
61
|
|
49
62
|
##
|
50
|
-
# Stops the daemon with +opts
|
63
|
+
# Stops the daemon with +opts+
|
51
64
|
def stop(opts = []) daemon('stop', opts); end
|
52
65
|
|
53
66
|
##
|
54
|
-
# Restarts the daemon with +opts
|
67
|
+
# Restarts the daemon with +opts+
|
55
68
|
def restart(opts = []) daemon('restart', opts); end
|
56
69
|
|
57
70
|
##
|
data/lib/miga/version.rb
CHANGED
@@ -10,7 +10,7 @@ module MiGA
|
|
10
10
|
# - Float representing the major.minor version.
|
11
11
|
# - Integer representing gem releases of the current version.
|
12
12
|
# - Integer representing minor changes that require new version number.
|
13
|
-
VERSION = [0.5,
|
13
|
+
VERSION = [0.5, 5, 0]
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nickname for the current major.minor version.
|
@@ -18,7 +18,7 @@ module MiGA
|
|
18
18
|
|
19
19
|
##
|
20
20
|
# Date of the current gem release.
|
21
|
-
VERSION_DATE = Date.new(2020,
|
21
|
+
VERSION_DATE = Date.new(2020, 2, 4)
|
22
22
|
|
23
23
|
##
|
24
24
|
# Reference of MiGA.
|
data/test/daemon_test.rb
CHANGED
@@ -9,8 +9,9 @@ class DaemonTest < Test::Unit::TestCase
|
|
9
9
|
ENV["MIGA_HOME"] = $tmp
|
10
10
|
FileUtils.touch("#{ENV["MIGA_HOME"]}/.miga_rc")
|
11
11
|
File.open("#{ENV["MIGA_HOME"]}/.miga_daemon.json", "w") do |fh|
|
12
|
-
fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
|
13
|
-
"
|
12
|
+
fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
|
13
|
+
"var":"{{key}}={{value}}","cmd":"{{task_name}}",
|
14
|
+
"alive":"echo 1 # {{pid}}","type":"bash","format_version":1}'
|
14
15
|
end
|
15
16
|
$p1 = MiGA::Project.new(File.expand_path("project1", $tmp))
|
16
17
|
$d1 = MiGA::Daemon.new($p1)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|