miga-base 0.6.4.2 → 0.7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4b861cfbdaa0d9cd28403345bd4dc61c4814a07c09a59709f09415dd05007a60
4
- data.tar.gz: 4738c33442f61be5a6fe0c6cf36c05cabb4fcd5b08f481099651bdacbc2d97fc
3
+ metadata.gz: 209e314b880d88c662aa9fbb04d62df7b4676e5d6af885e0cd23cea354accdec
4
+ data.tar.gz: d7fd5f25b2132880374693773e4cd1d58a331aa34d06ba31be53a5926e47f0e0
5
5
  SHA512:
6
- metadata.gz: ad69cb41e41f11d5aa0ad38af53a007eac213ab5901df9432d140d510fda2d8b8a7e6f4fda5302709e47c0150030cfdf601c54d9ce308d99aa82deeeb05ed0e8
7
- data.tar.gz: aba5401cef4cafba7afdf48765ab3690313d768140229fcb297b1d02eba441816fc0f8f69ecfa7e477de6ac0d5387c324e900fb6b20151bd736bfe7814737155
6
+ metadata.gz: d2e34541cd0191e36524aa1f0811fe3933837cdbc2fb550ced6692423ca873bc900b188ddd852a18839f3bb47a13a19cf9e9de60e65ed1b11c112f0bc217996d
7
+ data.tar.gz: 1dd8e5a74546e47271fb0f35a983234364ce1e855d0ab6194c3782dabc5a82920504c99ca513409b7e6c60244fc9320c2ff88c0f041f72cbc83733332e605f39
@@ -11,13 +11,10 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
11
11
  cli.expect_operation = true
12
12
  cli.parse do |opt|
13
13
  opt.separator 'Available operations:'
14
- { start: 'Start an instance of the application.',
15
- stop: 'Start an instance of the application.',
16
- restart: 'Stop all instances and restart them afterwards.',
17
- reload: 'Send a SIGHUP to all instances of the application.',
18
- run: 'Start the application and stay on top.',
19
- zap: 'Set the application to a stopped state.',
20
- status: 'Show status (PID) of application instances.'
14
+ { start: 'Start an instance of the application',
15
+ stop: 'Start an instance of the application',
16
+ run: 'Start the application and stay on top',
17
+ status: 'Show status (PID) of application instances'
21
18
  }.each { |k,v| opt.separator sprintf ' %*s%s', -33, k, v }
22
19
  opt.separator ''
23
20
 
@@ -11,13 +11,11 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
11
11
  cli.expect_operation = true
12
12
  cli.parse do |opt|
13
13
  opt.separator 'Available operations:'
14
- { start: 'Start an instance of the application.',
15
- stop: 'Start an instance of the application.',
16
- restart: 'Stop all instances and restart them afterwards.',
17
- reload: 'Send a SIGHUP to all instances of the application.',
18
- run: 'Start the application and stay on top.',
19
- zap: 'Set the application to a stopped state.',
20
- status: 'Show status (PID) of application instances.'
14
+ { start: 'Start an instance of the application',
15
+ stop: 'Start an instance of the application',
16
+ run: 'Start the application and stay on top',
17
+ status: 'Show status (PID) of application instances',
18
+ terminate: 'Terminate all daemons in the lair and exit'
21
19
  }.each { |k,v| opt.separator sprintf ' %*s%s', -33, k, v }
22
20
  opt.separator ''
23
21
 
@@ -26,6 +24,10 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
26
24
  '-p', '--path PATH',
27
25
  '(Mandatory) Path to the directory where the MiGA projects are located'
28
26
  ) { |v| cli[:path] = v }
27
+ opt.on(
28
+ '--json PATH',
29
+ 'Path to a custom daemon definition in json format'
30
+ ) { |v| cli[:json] = v }
29
31
  opt.on(
30
32
  '--latency INT', Integer,
31
33
  'Time to wait between iterations in seconds, by default: 120'
@@ -33,21 +35,25 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
33
35
  opt.on(
34
36
  '--wait-for INT', Integer,
35
37
  'Time to wait for a daemon to report being alive in seconds',
36
- 'by default: 900'
38
+ 'by default: 30'
37
39
  ) { |v| cli[:wait_for] = v }
38
40
  opt.on(
39
41
  '--keep-inactive',
40
42
  'If set, daemons are kept alive even when inactive;',
41
43
  'i.e., when all tasks are complete'
42
44
  ) { |v| cli[:keep_inactive] = v }
45
+ opt.on(
46
+ '--no-trust-timestamp',
47
+ 'Check all results instead of trusting project timestamps'
48
+ ) { |v| cli[:trust_timestamp] = v }
43
49
  opt.on(
44
50
  '--name STRING',
45
51
  'A name for the chief daemon process'
46
52
  ) { |v| cli[:name] = v }
47
53
  opt.on(
48
- '--json PATH',
49
- 'Path to a custom daemon definition in json format'
50
- ) { |v| cli[:json] = v }
54
+ '--dry',
55
+ 'Report when daemons would be launched, but don\'t actually launch them'
56
+ ) { |v| cli[:dry] = v }
51
57
  cli.opt_common(opt)
52
58
 
53
59
  opt.separator 'Daemon options:'
@@ -73,9 +79,13 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
73
79
 
74
80
  def perform
75
81
  cli.ensure_par(path: '-p')
76
- k_opts = %i[latency wait_for keep_inactive name json]
77
- opts = Hash[k_opts.map { |k| [k, cli[k]] }]
78
- lair = MiGA::Lair.new(cli[:path], opts)
79
- lair.daemon(cli.operation, cli[:daemon_opts])
82
+ if cli.operation.to_sym == :terminate
83
+ MiGA::Lair.new(cli[:path]).terminate_daemons
84
+ else
85
+ k_opts = %i[json latency wait_for keep_inactive trust_timestamp name dry]
86
+ opts = Hash[k_opts.map { |k| [k, cli[k]] }]
87
+ lair = MiGA::Lair.new(cli[:path], opts)
88
+ lair.daemon(cli.operation, cli[:daemon_opts])
89
+ end
80
90
  end
81
91
  end
@@ -0,0 +1,203 @@
1
+
2
+ require 'daemons'
3
+ require 'miga/common/with_daemon_class'
4
+
5
+ ##
6
+ # Helper module with specific functions to handle objects that have daemons.
7
+ # The class including it must +extend MiGA::Common::WithDaemonClass+ and define:
8
+ # - +#daemon_home+ Path to the daemon's home
9
+ # - +#daemon_name+ Name of the daemon
10
+ # - +#daemon_loop+ One loop of the daemon to be repeatedly called
11
+ # - +#daemon_first_loop+ To be executed before the first call to +#daemon_loop+
12
+ module MiGA::Common::WithDaemon
13
+ # Process ID of the forked process declaring the daemon alive
14
+ attr :declare_alive_pid
15
+
16
+ # Loop counter
17
+ attr :loop_i
18
+
19
+ def pid_file
20
+ File.join(daemon_home, "#{daemon_name}.pid")
21
+ end
22
+
23
+ def output_file
24
+ File.join(daemon_home, "#{daemon_name}.output")
25
+ end
26
+
27
+ def terminate_file
28
+ File.join(daemon_home, 'terminate-daemon')
29
+ end
30
+
31
+ def alive_file
32
+ self.class.alive_file(daemon_home)
33
+ end
34
+
35
+ def terminated_file
36
+ self.class.terminated_file(daemon_home)
37
+ end
38
+
39
+ ##
40
+ # When was the daemon last seen active?
41
+ def last_alive
42
+ self.class.last_alive(daemon_home)
43
+ end
44
+
45
+ ##
46
+ # Is the daemon active?
47
+ def active?
48
+ return false unless File.exist? alive_file
49
+ last_alive > Time.now - 60
50
+ end
51
+
52
+ ##
53
+ # Tell the world that you're alive.
54
+ def declare_alive
55
+ if active?
56
+ raise "Trying to declare alive an active daemon, if you think this is a" \
57
+ " mistake please remove #{alive_file} or try again in 1 minute"
58
+ end
59
+ @declare_alive_pid = fork { declare_alive_loop }
60
+ sleep(1) # <- to wait for the process check
61
+ end
62
+
63
+ ##
64
+ # Loop checking if the process with PID +pid+ is still alive.
65
+ # By default, the parent process.
66
+ # Do not use directly, use +declare_alive+ instead.
67
+ # Returns a symbol indicating the reason to stop:
68
+ # - +:no_home+ Daemon's home does not exist
69
+ # - +:no_process_alive+ Process is not currently running
70
+ # - +:termination_file+ Found termination file
71
+ def declare_alive_loop(pid = Process.ppid)
72
+ i = -1
73
+ loop do
74
+ i += 1
75
+ return :no_home unless Dir.exist? daemon_home
76
+ return :no_process_alive unless process_alive? pid
77
+ write_alive_file if i % 30 == 0
78
+ return :termination_file if termination_file? pid
79
+ sleep(1)
80
+ end
81
+ end
82
+
83
+ def write_alive_file
84
+ File.open(alive_file, 'w') { |fh| fh.print Time.now.to_s }
85
+ end
86
+
87
+ ##
88
+ # Check if the process with PID +pid+ is still alive,
89
+ # call +terminate+ otherwise.
90
+ def process_alive?(pid)
91
+ Process.kill(0, pid)
92
+ true
93
+ rescue Errno::ESRCH, Errno::EPERM, Errno::ENOENT
94
+ terminate
95
+ false
96
+ end
97
+
98
+ ##
99
+ # Check if a termination file exists and terminate process with PID +pid+
100
+ # if it does. Do not kill any process if +pid+ is +nil+
101
+ def termination_file?(pid)
102
+ return false unless File.exist? terminate_file
103
+ say 'Found termination file, terminating'
104
+ File.unlink(terminate_file)
105
+ terminate
106
+ Process.kill(9, pid) unless pid.nil?
107
+ true
108
+ end
109
+
110
+ ##
111
+ # Returns Hash containing the default options for the daemon.
112
+ def default_options
113
+ {
114
+ dir_mode: :normal, dir: daemon_home, multiple: false, log_output: true,
115
+ stop_proc: :terminate
116
+ }
117
+ end
118
+
119
+ ##
120
+ # Launches the +task+ with options +opts+ (as command-line arguments) and
121
+ # returns the process ID as an Integer. If +wait+ it waits for the process to
122
+ # complete, immediately returns otherwise.
123
+ # Supported tasks: start, run, stop, status.
124
+ def daemon(task, opts = [], wait = true)
125
+ MiGA::MiGA.DEBUG "#{self.class}#daemon #{task} #{opts}"
126
+ task = task.to_sym
127
+ raise "Unsupported task: #{task}" unless respond_to? task
128
+ return send(task, opts, wait) unless %i[start run].include? task
129
+
130
+ # start & run:
131
+ options = default_options
132
+ opts.unshift(task.to_s)
133
+ options[:ARGV] = opts
134
+ # This additional degree of separation below was introduced so the Daemons
135
+ # package doesn't kill the parent process in workflows.
136
+ pid = fork { launch_daemon_proc(options) }
137
+ Process.wait(pid) if wait
138
+ pid
139
+ end
140
+
141
+ ##
142
+ # Stops the daemon with +opts+
143
+ def stop(opts = [], wait = true)
144
+ if active?
145
+ say 'Sending termination message'
146
+ FileUtils.touch(terminate_file)
147
+ sleep(0.5) while active? if wait
148
+ File.unlink(pid_file) if File.exist?(pid_file)
149
+ else
150
+ say 'No running instances'
151
+ end
152
+ end
153
+
154
+ ##
155
+ # Returns the status of the daemon with +opts+
156
+ def status(opts = [], wait = true)
157
+ if active?
158
+ say "Running with pid #{File.read(pid_file)}"
159
+ else
160
+ say 'Not running'
161
+ end
162
+ end
163
+
164
+ ##
165
+ # Pass daemon options to +Daemons+. Do not use directly, use +daemon+ instead.
166
+ def launch_daemon_proc(options)
167
+ Daemons.run_proc("#{daemon_name}", options) { while in_loop; end }
168
+ end
169
+
170
+ ##
171
+ # Initializes the daemon with +opts+
172
+ def start(opts = [], wait = true)
173
+ daemon(:start, opts, wait)
174
+ end
175
+
176
+ ##
177
+ # Initializes the daemon on top with +opts+
178
+ def run(opts = [], wait = true)
179
+ daemon(:run, opts, wait)
180
+ end
181
+
182
+ ##
183
+ # One loop, returns a boolean indicating if the execution should continue
184
+ def in_loop
185
+ if loop_i.nil?
186
+ declare_alive
187
+ daemon_first_loop
188
+ @loop_i = -1
189
+ end
190
+ @loop_i += 1
191
+ daemon_loop
192
+ end
193
+
194
+ ##
195
+ # Declares a daemon termination. Do not use, directly, use #stop instead.
196
+ def terminate
197
+ unless declare_alive_pid.nil?
198
+ Process.kill(9, declare_alive_pid)
199
+ @declare_alive_pid = nil
200
+ end
201
+ File.rename(alive_file, terminated_file) if File.exist? alive_file
202
+ end
203
+ end
@@ -0,0 +1,32 @@
1
+
2
+ ##
3
+ # Helper module with specific class-level functions to be used with
4
+ # +include MiGA::Common::WithDaemon+.
5
+ module MiGA::Common::WithDaemonClass
6
+ ##
7
+ # Path to the daemon home from the parent's +path+
8
+ def daemon_home(path)
9
+ path
10
+ end
11
+
12
+ ##
13
+ # Path to the alive file
14
+ def alive_file(path)
15
+ File.join(daemon_home(path), '.daemon-alive')
16
+ end
17
+
18
+ ##
19
+ # Path to the terminated file
20
+ def terminated_file(path)
21
+ File.join(daemon_home(path), '.daemon-terminated')
22
+ end
23
+
24
+ ##
25
+ # When was a daemon last seen at +path+?
26
+ def last_alive(path)
27
+ f = alive_file(path)
28
+ f = terminated_file(path) unless File.exist? f
29
+ return nil unless File.exist? f
30
+ Time.parse(File.read(f))
31
+ end
32
+ end
data/lib/miga/daemon.rb CHANGED
@@ -2,6 +2,7 @@
2
2
  # @license Artistic-2.0
3
3
 
4
4
  require 'miga/project'
5
+ require 'miga/common/with_daemon'
5
6
  require 'miga/daemon/base'
6
7
 
7
8
  ##
@@ -9,29 +10,30 @@ require 'miga/daemon/base'
9
10
  class MiGA::Daemon < MiGA::MiGA
10
11
 
11
12
  include MiGA::Daemon::Base
13
+ include MiGA::Common::WithDaemon
14
+ extend MiGA::Common::WithDaemonClass
12
15
 
13
- ##
14
- # When was the last time a daemon for the MiGA::Project +project+ was seen
15
- # active? Returns Time.
16
- def self.last_alive(project)
17
- f = File.expand_path('daemon/alive', project.path)
18
- return nil unless File.exist? f
19
- Time.parse(File.read(f))
16
+ class << self
17
+ ##
18
+ # Daemon's home inside the MiGA::Project +project+ or a String with the
19
+ # full path to the project's 'daemon' folder
20
+ def daemon_home(project)
21
+ return project if project.is_a? String
22
+ File.join(project.path, 'daemon')
23
+ end
20
24
  end
21
25
 
22
- # Array of all spawned daemons.
23
- $_MIGA_DAEMON_LAIR = []
24
-
25
26
  # MiGA::Project in which the daemon is running
26
27
  attr_reader :project
28
+
27
29
  # Options used to setup the daemon
28
30
  attr_reader :options
31
+
29
32
  # Array of jobs next to be executed
30
33
  attr_reader :jobs_to_run
34
+
31
35
  # Array of jobs currently running
32
36
  attr_reader :jobs_running
33
- # Integer indicating the current iteration
34
- attr_reader :loop_i
35
37
 
36
38
  ##
37
39
  # Initialize an unactive daemon for the MiGA::Project +project+. See #daemon
@@ -40,72 +42,74 @@ class MiGA::Daemon < MiGA::MiGA
40
42
  # is used. In either case, missing variables are used as defined in
41
43
  # ~/.miga_daemon.json.
42
44
  def initialize(project, json = nil)
43
- $_MIGA_DAEMON_LAIR << self
44
45
  @project = project
45
46
  @runopts = {}
46
- json ||= File.expand_path('daemon/daemon.json', project.path)
47
+ json ||= File.join(project.path, 'daemon/daemon.json')
47
48
  MiGA::Json.parse(
48
49
  json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
49
50
  ).each { |k,v| runopts(k, v) }
50
51
  update_format_0
51
52
  @jobs_to_run = []
52
53
  @jobs_running = []
53
- @loop_i = -1
54
54
  end
55
55
 
56
56
  ##
57
- # When was the last time a daemon for the current project was seen active?
58
- # Returns Time.
59
- def last_alive
60
- MiGA::Daemon.last_alive project
57
+ # Path to the daemon home
58
+ def daemon_home
59
+ self.class.daemon_home(project)
61
60
  end
62
61
 
63
62
  ##
64
- # Returns Hash containing the default options for the daemon.
65
- def default_options
66
- { dir_mode: :normal, dir: File.expand_path('daemon', project.path),
67
- multiple: false, log_output: true }
63
+ # Name of the daemon
64
+ def daemon_name
65
+ "MiGA:#{project.name}"
68
66
  end
69
67
 
70
68
  ##
71
- # Launches the +task+ with options +opts+ (as command-line arguments) and
72
- # returns the process ID as an Integer. If +wait+ it waits for the process to
73
- # complete, immediately returns otherwise.
74
- # Supported tasks: start, stop, restart, status.
75
- def daemon(task, opts = [], wait = true)
76
- MiGA.DEBUG "Daemon.daemon #{task} #{opts}"
77
- options = default_options
78
- opts.unshift(task.to_s)
79
- options[:ARGV] = opts
80
- # This additional degree of separation below was introduced so the Daemons
81
- # package doesn't kill the parent process in workflows.
82
- pid = fork do
83
- Daemons.run_proc("MiGA:#{project.name}", options) { while in_loop; end }
84
- end
85
- Process.wait(pid) if wait
86
- pid
69
+ # Run only in the first loop
70
+ def daemon_first_loop
71
+ say '-----------------------------------'
72
+ say 'MiGA:%s launched' % project.name
73
+ say '-----------------------------------'
74
+ load_status
75
+ say 'Configuration options:'
76
+ say @runopts.to_s
87
77
  end
88
78
 
89
79
  ##
90
- # Tell the world that you're alive.
91
- def declare_alive
92
- f = File.open(File.expand_path('daemon/alive', project.path), 'w')
93
- f.print Time.now.to_s
94
- f.close
80
+ # Run one loop step. Returns a Boolean indicating if the loop should continue.
81
+ def daemon_loop
82
+ project.load
83
+ check_datasets
84
+ check_project
85
+ if shutdown_when_done? and jobs_running.size + jobs_to_run.size == 0
86
+ say 'Nothing else to do, shutting down.'
87
+ return false
88
+ end
89
+ flush!
90
+ if loop_i >= 12
91
+ say 'Probing running jobs'
92
+ @loop_i = 0
93
+ purge!
94
+ end
95
+ report_status
96
+ sleep(latency)
97
+ true
95
98
  end
96
99
 
97
100
  ##
98
101
  # Report status in a JSON file.
99
102
  def report_status
100
103
  MiGA::Json.generate(
101
- {jobs_running: @jobs_running, jobs_to_run: @jobs_to_run},
102
- File.expand_path('daemon/status.json', project.path))
104
+ { jobs_running: @jobs_running, jobs_to_run: @jobs_to_run },
105
+ File.join(daemon_home, 'status.json')
106
+ )
103
107
  end
104
108
 
105
109
  ##
106
110
  # Load the status of a previous instance.
107
111
  def load_status
108
- f_path = File.expand_path('daemon/status.json', project.path)
112
+ f_path = File.join(daemon_home, 'status.json')
109
113
  return unless File.size? f_path
110
114
  say 'Loading previous status in daemon/status.json:'
111
115
  status = MiGA::Json.parse(f_path)
@@ -234,53 +238,6 @@ class MiGA::Daemon < MiGA::MiGA
234
238
  end
235
239
  end
236
240
 
237
- ##
238
- # Run one loop step. Returns a Boolean indicating if the loop should continue.
239
- def in_loop
240
- declare_alive
241
- project.load
242
- if loop_i == -1
243
- say '-----------------------------------'
244
- say 'MiGA:%s launched' % project.name
245
- say '-----------------------------------'
246
- load_status
247
- say 'Configuration options:'
248
- say @runopts.to_s
249
- @loop_i = 0
250
- end
251
- @loop_i += 1
252
- check_datasets
253
- check_project
254
- if shutdown_when_done? and jobs_running.size + jobs_to_run.size == 0
255
- say 'Nothing else to do, shutting down.'
256
- return false
257
- end
258
- flush!
259
- if loop_i == 12
260
- say 'Probing running jobs'
261
- @loop_i = 0
262
- purge!
263
- end
264
- report_status
265
- sleep(latency)
266
- true
267
- end
268
-
269
- ##
270
- # Send a datestamped message to the log.
271
- def say(*opts)
272
- print "[#{Time.new.inspect}] ", *opts, "\n"
273
- end
274
-
275
- ##
276
- # Terminates a daemon.
277
- def terminate
278
- say 'Terminating daemon...'
279
- report_status
280
- f = File.expand_path('daemon/alive', project.path)
281
- File.unlink(f) if File.exist? f
282
- end
283
-
284
241
  ##
285
242
  # Launch the job described by Hash +job+ to +hostk+-th host
286
243
  def launch_job(job, hostk = nil)