locd 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +108 -0
  3. data/.gitmodules +9 -0
  4. data/.qb-options.yml +4 -0
  5. data/.rspec +3 -0
  6. data/.travis.yml +5 -0
  7. data/.yardopts +7 -0
  8. data/Gemfile +10 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +72 -0
  11. data/Rakefile +6 -0
  12. data/VERSION +1 -0
  13. data/config/default.yml +24 -0
  14. data/doc/files/design/domains_and_labels.md +117 -0
  15. data/doc/files/topics/agents.md +16 -0
  16. data/doc/files/topics/launchd.md +15 -0
  17. data/doc/files/topics/plists.md +39 -0
  18. data/doc/include/plist.md +3 -0
  19. data/exe/locd +24 -0
  20. data/lib/locd.rb +75 -0
  21. data/lib/locd/agent.rb +1186 -0
  22. data/lib/locd/agent/job.rb +142 -0
  23. data/lib/locd/agent/proxy.rb +111 -0
  24. data/lib/locd/agent/rotate_logs.rb +82 -0
  25. data/lib/locd/agent/site.rb +174 -0
  26. data/lib/locd/agent/system.rb +270 -0
  27. data/lib/locd/cli.rb +4 -0
  28. data/lib/locd/cli/command.rb +9 -0
  29. data/lib/locd/cli/command/agent.rb +310 -0
  30. data/lib/locd/cli/command/base.rb +243 -0
  31. data/lib/locd/cli/command/job.rb +110 -0
  32. data/lib/locd/cli/command/main.rb +201 -0
  33. data/lib/locd/cli/command/proxy.rb +177 -0
  34. data/lib/locd/cli/command/rotate_logs.rb +152 -0
  35. data/lib/locd/cli/command/site.rb +47 -0
  36. data/lib/locd/cli/table.rb +157 -0
  37. data/lib/locd/config.rb +237 -0
  38. data/lib/locd/config/base.rb +93 -0
  39. data/lib/locd/errors.rb +65 -0
  40. data/lib/locd/label.rb +61 -0
  41. data/lib/locd/launchctl.rb +209 -0
  42. data/lib/locd/logging.rb +360 -0
  43. data/lib/locd/newsyslog.rb +402 -0
  44. data/lib/locd/pattern.rb +193 -0
  45. data/lib/locd/proxy.rb +272 -0
  46. data/lib/locd/proxymachine.rb +34 -0
  47. data/lib/locd/util.rb +49 -0
  48. data/lib/locd/version.rb +26 -0
  49. data/locd.gemspec +66 -0
  50. metadata +262 -0
@@ -0,0 +1,402 @@
1
+ # encoding: UTF-8
2
+ # frozen_string_literal: true
3
+
4
+ # Requirements
5
+ # =======================================================================
6
+
7
+ # Stdlib
8
+ # -----------------------------------------------------------------------
9
+
10
+ # Deps
11
+ # -----------------------------------------------------------------------
12
+ require 'cmds'
13
+
14
+ # Project / Package
15
+ # -----------------------------------------------------------------------
16
+
17
+
18
+ # Refinements
19
+ # =======================================================================
20
+
21
+ using NRSER
22
+ using NRSER::Types
23
+
24
+
25
+ # Definitions
26
+ # =======================================================================
27
+
28
+ # Use `newsyslog` to rotate {Locd::Agent} log files.
29
+ #
30
+ module Locd::Newsyslog
31
+
32
+ include SemanticLogger::Loggable
33
+
34
+
35
+ # Default place to work out of.
36
+ #
37
+ # @return [Pathname]
38
+ #
39
+ DEFAULT_WORKDIR = Pathname.new( '~/.locd/tmp/newsyslog' ).expand_path
40
+
41
+
42
+ # Lil' class that holds values for a `newsyslog` conf file entry.
43
+ #
44
+ # @see https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man5/newsyslog.conf.5.html
45
+ # @see https://archive.is/nIiP9
46
+ #
47
+ class Entry
48
+
49
+ # Constants
50
+ # ======================================================================
51
+
52
+ # Value for the `when` field when we want to never rotate based on time.
53
+ #
54
+ # @return [String]
55
+ #
56
+ WHEN_NEVER = '*'
57
+
58
+ # Conf file value for `flags` where there are none
59
+ #
60
+ # @return [String]
61
+ #
62
+ NO_FLAGS = '-'
63
+
64
+
65
+ FIELD_SEP = " "
66
+
67
+
68
+ # Class Methods
69
+ # ======================================================================
70
+
71
+
72
+ # @todo Document normalize_mode method.
73
+ #
74
+ # @param [type] arg_name
75
+ # @todo Add name param description.
76
+ #
77
+ # @return [return_type]
78
+ # @todo Document return value.
79
+ #
80
+ def self.mode_string_for mode
81
+ # t.match mode,
82
+ # 0000..0777, mode.to_s( 8 ),
83
+ # /\A[0-7]{3}\z/, mode
84
+
85
+ case mode
86
+ when 0000..0777
87
+ mode.to_s 8
88
+ when /\A[0-7]{3}\z/
89
+ mode
90
+ else
91
+ raise ArgumentError,
92
+ "Bad mode: #{ mode.inspect }, need String or Fixnum: '644'; 0644"
93
+ end
94
+ end # .normalize_mode
95
+
96
+
97
+ def self.sig_num_for signal
98
+ case signal
99
+ when String
100
+ Signal.list.fetch signal
101
+ when Fixnum
102
+ signal
103
+ else
104
+ raise ArgumentError,
105
+ "Bad signal: #{ signal.inspect }, need String or Fixnum: 'HUP'; 1"
106
+ end
107
+ end
108
+
109
+
110
+ # Attributes
111
+ # ======================================================================
112
+
113
+ attr_reader :log_path,
114
+ :pid_path,
115
+ :owner,
116
+ :group,
117
+ :mode,
118
+ :count,
119
+ :size,
120
+ :when_,
121
+ :flags,
122
+ :sig_num
123
+
124
+ # Constructor
125
+ # ======================================================================
126
+
127
+ # Instantiate a new `Entry`.
128
+ def initialize log_path:,
129
+ pid_path:,
130
+ owner: nil,
131
+ group: nil,
132
+ mode: '644',
133
+ count: 7,
134
+ size: 100, # 100 KB max size
135
+ when_: '*', # '$D0', # rotate every day at midnight
136
+ flags: [],
137
+ signal: 'HUP'
138
+ @log_path = log_path
139
+ @pid_path = pid_path
140
+ @owner = owner
141
+ @group = group
142
+ @mode = self.class.mode_string_for mode
143
+ @count = t.pos_int.check count
144
+ @size = t.pos_int.check size
145
+ @when_ = when_
146
+ @flags = flags
147
+ @sig_num = self.class.sig_num_for signal
148
+
149
+ render
150
+ end # #initialize
151
+
152
+
153
+ # Instance Methods
154
+ # ======================================================================
155
+
156
+ def sig_name
157
+ Signal.signame @sig_num
158
+ end
159
+
160
+
161
+ def render_flags
162
+ if flags.empty?
163
+ NO_FLAGS
164
+ else
165
+ flags.join
166
+ end
167
+ end
168
+
169
+
170
+ def render
171
+ @render ||= begin
172
+ fields = [
173
+ log_path,
174
+ "#{ owner }:#{ group }",
175
+ mode,
176
+ count,
177
+ size,
178
+ when_,
179
+ render_flags,
180
+ ]
181
+
182
+ if pid_path
183
+ fields << pid_path
184
+ fields << sig_num
185
+ end
186
+
187
+ fields.map( &:to_s ).join FIELD_SEP
188
+ end
189
+ end
190
+
191
+ end # class Entry
192
+
193
+
194
+ # Module Methods
195
+ # ============================================================================
196
+
197
+
198
+ # Run `newsyslog` for an agent to rotate it's log files (if present and
199
+ # needed).
200
+ #
201
+ # @param [Locd::Agent] agent:
202
+ # Agent to run for.
203
+ #
204
+ # @param [Pathname | String] workdir:
205
+ # Directory to write working files to, which are removed after successful
206
+ # runs.
207
+ #
208
+ # @return [Cmds::Result]
209
+ # Result of running the `newsyslog` command.
210
+ #
211
+ # @return [nil]
212
+ # If we didn't run the command 'cause the agent doesn't have any logs
213
+ # (that we know/care about).
214
+ #
215
+ def self.run agent, workdir: DEFAULT_WORKDIR
216
+ logger.debug "Calling {.run_for} agent #{ agent.label }...",
217
+ agent: agent,
218
+ workdir: workdir
219
+
220
+ # Make sure `workdir` is a {Pathname}
221
+ workdir = workdir.to_pn
222
+
223
+ # Collect the unique log paths
224
+ log_paths = [
225
+ agent.out_path,
226
+ agent.err_path,
227
+ ].compact.uniq
228
+
229
+ if log_paths.empty?
230
+ logger.info "Agent #{ agent.label } has no log files."
231
+ return nil
232
+ end
233
+
234
+ logger.info "Setting up to run `newsyslog` for agent `#{ agent.label }`",
235
+ log_paths: log_paths.map( &:to_s )
236
+
237
+ # NOTE Total race condition since agent may be started after this and
238
+ # before we rotate... f-it for now.
239
+ pid_path = nil
240
+ if pid = agent.pid( refresh: true )
241
+ logger.debug "Agent is running", pid: pid
242
+
243
+ pid_path = workdir / 'pids' / "#{ agent.label }.pid"
244
+ FileUtils.mkdir_p( pid_path.dirname ) unless pid_path.dirname.exist?
245
+ pid_path.write pid
246
+
247
+ logger.debug "Wrote PID #{ pid } to file", pid_path: pid_path
248
+ end
249
+
250
+ entries = log_paths.map { |log_path|
251
+ Entry.new log_path: log_path, pid_path: pid_path
252
+ }
253
+ conf_contents = entries.map( &:render ).join( "\n" ) + "\n"
254
+
255
+ logger.debug "Generated conf entries",
256
+ entries.map { |entry|
257
+ [
258
+ entry.log_path.to_s,
259
+ entry.instance_variables.map_values { |name|
260
+ entry.instance_variable_get name
261
+ }
262
+ ]
263
+ }.to_h
264
+
265
+ conf_path = workdir / 'confs' / "#{ agent.label }.conf"
266
+ FileUtils.mkdir_p( conf_path.dirname ) unless conf_path.dirname.exist?
267
+ conf_path.write conf_contents
268
+
269
+ logger.debug "Wrote entries to conf file",
270
+ conf_path: conf_path.to_s,
271
+ conf_contents: conf_contents
272
+
273
+ cmd = Cmds.new "newsyslog <%= opts %>", kwds: {
274
+ opts: {
275
+ # Turn on verbose output
276
+ v: true,
277
+ # Point to the conf file
278
+ f: conf_path,
279
+ # Don't run as root
280
+ r: true,
281
+ }
282
+ }
283
+
284
+ logger.info "Executing `#{ cmd.prepare }`"
285
+
286
+ result = cmd.capture
287
+
288
+ if result.ok?
289
+ logger.info \
290
+ "`newsyslog` command succeeded for agent `#{ agent.label }`" +
291
+ ( result.out.empty? ?
292
+ nil :
293
+ ", output:\n" + result.out.indent(1, indent_string: '> ') )
294
+
295
+ FileUtils.rm( pid_path ) if pid_path
296
+ FileUtils.rm conf_path
297
+
298
+ logger.debug "Files cleaned up."
299
+
300
+ else
301
+ logger.error "`newsyslog` command failed for agent #{ agent.label }",
302
+ result: result.to_h
303
+ end
304
+
305
+ logger.debug "Returning",
306
+ result: result.to_h
307
+
308
+ result
309
+ end # .run
310
+
311
+
312
+ # Call {.run} for each agent.
313
+ #
314
+ # @param workdir (see .run)
315
+ #
316
+ # @return [Hash<Locd::Agent, Cmds::Result?>]
317
+ # Hash mapping each agent to it's {.run} result (which may be `nil`).
318
+ #
319
+ def self.run_all workdir: DEFAULT_WORKDIR, trim_logs: true
320
+ log_to_file do
321
+ Locd::Agent.all.values.
322
+ reject { |agent|
323
+ agent.label == Locd::ROTATE_LOGS_LABEL
324
+ }.
325
+ map { |agent|
326
+ [agent, run( agent, workdir: workdir )]
327
+ }.
328
+ to_h.
329
+ tap { |_|
330
+ self.trim_logs if trim_logs
331
+ }
332
+ end
333
+ end # .run_all
334
+
335
+
336
+ def self.log_dir
337
+ @log_dir ||= Locd.config.log_dir / Locd::ROTATE_LOGS_LABEL
338
+ end
339
+
340
+
341
+ def self.log_to_file &block
342
+ time = Time.now.iso8601
343
+ date = time.split( 'T', 2 )[0]
344
+
345
+ # Like
346
+ #
347
+ # ~/.locd/log/com.nrser.locd.rotate-logs/2018-02-14/2018-02-14T03:46:57+08:00.log
348
+ #
349
+ path = self.log_dir / date / "#{ time }.log"
350
+
351
+ FileUtils.mkdir_p( path.dirname ) unless path.dirname.exist?
352
+
353
+ appender = SemanticLogger.add_appender \
354
+ file_name: path.to_s
355
+
356
+ begin
357
+ result = block.call
358
+ ensure
359
+ SemanticLogger.remove_appender appender
360
+ end
361
+ end
362
+
363
+
364
+ def self.trim_logs keep_days: 7
365
+ logger.info "Removing old self run log directories...",
366
+ log_dir: self.log_dir.to_s,
367
+ keep_days: keep_days
368
+
369
+ unless self.log_dir.directory?
370
+ logger.warn "{Locd::Newsyslog.log_dir} does not exist!",
371
+ log_dir: self.log_dir
372
+ return nil
373
+ end
374
+
375
+ day_dirs = self.log_dir.entries.select { |dir_name|
376
+ dir_name.to_s =~ /\d{4}\-\d{2}\-\d{2}/ &&
377
+ (self.log_dir / dir_name).directory?
378
+ }
379
+
380
+ to_remove = day_dirs.sort[0...(-1 * keep_days)]
381
+
382
+ if to_remove.empty?
383
+ logger.info "No old self run log directories to remove."
384
+ else
385
+ to_remove.each { |dir_name|
386
+ path = self.log_dir / dir_name
387
+
388
+ logger.info "Removing old day directory",
389
+ path: path
390
+
391
+ FileUtils.rm_rf path
392
+ }
393
+
394
+ logger.info "Done.",
395
+ log_dir: self.log_dir.to_s,
396
+ keep_days: keep_days
397
+ end
398
+
399
+ to_remove
400
+ end
401
+
402
+ end # module Locd::Newsyslog
@@ -0,0 +1,193 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Refinements
4
+ # =======================================================================
5
+
6
+ using NRSER
7
+
8
+
9
+ # Definitions
10
+ # =======================================================================
11
+
12
+ # Abstract base class for patterns used to match {Locd::Agent} instances.
13
+ #
14
+ class Locd::Pattern
15
+
16
+ # Factory method to construct the correct concrete subclass.
17
+ #
18
+ # @param [String | Locd::Pattern] object
19
+ # 1. {String} - a new pattern will be constructed.
20
+ # 2. {Locd::Pattern} - will just be returned.
21
+ #
22
+ # @return [Locd::Pattern::Label | Locd::Pattern::Workdir]
23
+ # Pattern instance.
24
+ #
25
+ def self.from object, **options
26
+ case object
27
+ when String
28
+ string = object
29
+ case string
30
+ when ''
31
+ raise ArgumentError, "Empty string is not a valid pattern"
32
+ when /\A[\.\~\/]/
33
+ Locd::Pattern::Workdir.new string, **options
34
+ else
35
+ Locd::Pattern::Label.new string, **options
36
+ end
37
+ when Locd::Pattern
38
+ object
39
+ else
40
+ raise TypeError.new binding.erb <<-END
41
+ Expected `object` to be {String} or {Locd::Pattern}, found <%= object.class %>
42
+
43
+ `object` (first argument):
44
+
45
+ <%= object.pretty_inspect %>
46
+
47
+ Options:
48
+
49
+ <%= options.pretty_inspect %>
50
+
51
+ END
52
+ end
53
+ end # .from
54
+
55
+
56
+ # Raw source string provided at initialization.
57
+ #
58
+ # @return [String]
59
+ #
60
+ attr_reader :source
61
+
62
+
63
+ # Construct a new pattern. Should only be called via `super`.
64
+ #
65
+ # @param [String] source
66
+ # Raw source string the pattern is built off.
67
+ #
68
+ def initialize source
69
+ @source = source
70
+ end # #initialize
71
+
72
+
73
+ # @raise [NRSER::AbstractMethodError]
74
+ # Abstract method, concrete subclasses must implement.
75
+ #
76
+ def match? agent
77
+ raise NRSER::AbstractMethodError.new( self, __method__ )
78
+ end # #match?
79
+
80
+ end # class Locd::Pattern
81
+
82
+
83
+ # A {Locd::Pattern} that matches against {Locd::Agent} labels.
84
+ #
85
+ class Locd::Pattern::Label < Locd::Pattern
86
+
87
+
88
+ # TODO document `string` attribute.
89
+ #
90
+ # @return [attr_type]
91
+ #
92
+ attr_reader :string
93
+
94
+
95
+ # Label {Regexp} used by {#match?}
96
+ #
97
+ # @return [Regexp]
98
+ #
99
+ attr_reader :regexp
100
+
101
+
102
+ # Instantiate a new `Locd::Pattern::Label`.
103
+ #
104
+ # @param (see Locd::Label.regexp_for_glob)
105
+ #
106
+ def initialize source, exact: false, ignore_case: false
107
+ super source
108
+ @regexp = Locd::Label.regexp_for_glob source,
109
+ exact: exact,
110
+ ignore_case: ignore_case
111
+ end # #initialize
112
+
113
+
114
+ # See if this patten matches an agent.
115
+ #
116
+ # @param [Locd::Agent] agent
117
+ # Agent to test against.
118
+ #
119
+ # @return [Boolean]
120
+ # `true` if this pattern matches the `agent` {Locd::Agent#label}.
121
+ #
122
+ def match? agent
123
+ agent.label =~ regexp
124
+ end # #match?
125
+
126
+ end # class Locd::Pattern::Label
127
+
128
+
129
+ # A {Locd::Pattern} that matches against {Locd::Agent} workdir.
130
+ #
131
+ class Locd::Pattern::Workdir < Locd::Pattern
132
+
133
+ # "Current" absolute directory path that a relative {#raw_path} would have
134
+ # been expanded against.
135
+ #
136
+ # @return [Pathname]
137
+ #
138
+ attr_reader :cwd
139
+
140
+
141
+ # When `true`, pattern will additionally match any agents who's
142
+ # {Locd::Agent#workdir} is a subdirectory of the {#path}.
143
+ #
144
+ # @return [Boolean]
145
+ #
146
+ attr_reader :recursive
147
+
148
+
149
+ # Expanded absolute path to test {Locd::Agent#workdir} against.
150
+ #
151
+ # @return [Pathname]
152
+ #
153
+ attr_reader :path
154
+
155
+
156
+ # Instantiate a new `Locd::Pattern::Workdir`.
157
+ #
158
+ # @param [String] raw_path
159
+ # Path to construct for, which may be relative to `cwd`.
160
+ #
161
+ # @param [Boolean] recursive:
162
+ # Additionally match agents with `workdir` in subdirectories.
163
+ #
164
+ # See {#recursive}.
165
+ #
166
+ # @param [Pathname] cwd:
167
+ # Directory to expand relative paths against.
168
+ #
169
+ def initialize source, recursive: false, cwd: Pathname.getwd
170
+ super source
171
+ @cwd = cwd.to_pn
172
+ @recursive = recursive
173
+ @path = Pathname.new( source ).expand_path @cwd
174
+ end # #initialize
175
+
176
+
177
+ # See if this patten matches an agent.
178
+ #
179
+ # @param [Locd::Agent] agent
180
+ # Agent to test against.
181
+ #
182
+ # @return [Boolean]
183
+ # `true` if this pattern matches the `agent` {Locd::Agent#label}.
184
+ #
185
+ def match? agent
186
+ if recursive
187
+ agent.workdir.to_s.start_with?( path.to_s + '/' )
188
+ else
189
+ agent.workdir == path
190
+ end
191
+ end # #match?
192
+
193
+ end # class Locd::Pattern::Label