locd 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +108 -0
  3. data/.gitmodules +9 -0
  4. data/.qb-options.yml +4 -0
  5. data/.rspec +3 -0
  6. data/.travis.yml +5 -0
  7. data/.yardopts +7 -0
  8. data/Gemfile +10 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +72 -0
  11. data/Rakefile +6 -0
  12. data/VERSION +1 -0
  13. data/config/default.yml +24 -0
  14. data/doc/files/design/domains_and_labels.md +117 -0
  15. data/doc/files/topics/agents.md +16 -0
  16. data/doc/files/topics/launchd.md +15 -0
  17. data/doc/files/topics/plists.md +39 -0
  18. data/doc/include/plist.md +3 -0
  19. data/exe/locd +24 -0
  20. data/lib/locd.rb +75 -0
  21. data/lib/locd/agent.rb +1186 -0
  22. data/lib/locd/agent/job.rb +142 -0
  23. data/lib/locd/agent/proxy.rb +111 -0
  24. data/lib/locd/agent/rotate_logs.rb +82 -0
  25. data/lib/locd/agent/site.rb +174 -0
  26. data/lib/locd/agent/system.rb +270 -0
  27. data/lib/locd/cli.rb +4 -0
  28. data/lib/locd/cli/command.rb +9 -0
  29. data/lib/locd/cli/command/agent.rb +310 -0
  30. data/lib/locd/cli/command/base.rb +243 -0
  31. data/lib/locd/cli/command/job.rb +110 -0
  32. data/lib/locd/cli/command/main.rb +201 -0
  33. data/lib/locd/cli/command/proxy.rb +177 -0
  34. data/lib/locd/cli/command/rotate_logs.rb +152 -0
  35. data/lib/locd/cli/command/site.rb +47 -0
  36. data/lib/locd/cli/table.rb +157 -0
  37. data/lib/locd/config.rb +237 -0
  38. data/lib/locd/config/base.rb +93 -0
  39. data/lib/locd/errors.rb +65 -0
  40. data/lib/locd/label.rb +61 -0
  41. data/lib/locd/launchctl.rb +209 -0
  42. data/lib/locd/logging.rb +360 -0
  43. data/lib/locd/newsyslog.rb +402 -0
  44. data/lib/locd/pattern.rb +193 -0
  45. data/lib/locd/proxy.rb +272 -0
  46. data/lib/locd/proxymachine.rb +34 -0
  47. data/lib/locd/util.rb +49 -0
  48. data/lib/locd/version.rb +26 -0
  49. data/locd.gemspec +66 -0
  50. metadata +262 -0
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Requirements
4
+ # =======================================================================
5
+
6
+ # Stdlib
7
+ # -----------------------------------------------------------------------
8
+
9
+ # Deps
10
+ # -----------------------------------------------------------------------
11
+ require 'cmds'
12
+
13
+ # Project / Package
14
+ # -----------------------------------------------------------------------
15
+
16
+
17
+ # Refinements
18
+ # =======================================================================
19
+
20
+ using NRSER
21
+ using NRSER::Types
22
+
23
+
24
+ # Definitions
25
+ # =======================================================================
26
+
27
+ # Thin wrapper to run `launchctl` commands and capture outputs via {Cmds}.
28
+ #
29
+ # Basically just centralizes this stuff all in one place and presents a
30
+ # friendly API, does some common debug logging, etc.
31
+ #
32
+ module Locd::Launchctl
33
+
34
+ # Mappings between any option keyword names we use to the ones `launchctl`
35
+ # expects.
36
+ #
37
+ # @return [Hash<Symbol, Symbol>]
38
+ #
39
+ OPTS_MAP = {
40
+ force: :F,
41
+ write: :w,
42
+ }
43
+
44
+
45
+ # Regexp to parse `launchd list` output for {#status} method.
46
+ #
47
+ # @return [Regexp]
48
+ #
49
+ STATUS_RE = /^(?<pid>\S+)\s+(?<status>\S+)\s+(?<label>\S+)/.freeze
50
+
51
+
52
+ # Max number of seconds to
53
+ STATUS_CACHE_TIMEOUT_SEC = 5
54
+
55
+
56
+ # Add {.logger} method
57
+ include SemanticLogger::Loggable
58
+
59
+
60
+ # `launchctl` executable to use. You shouldn't need to touch this, but might
61
+ # be useful for testing or if you have path weirdness.
62
+ #
63
+ # @return [String]
64
+ #
65
+ @@bin = 'launchctl'
66
+
67
+
68
+ # Swap any keys in {OPTS_MAP}.
69
+ #
70
+ # @param [Hash<Symbol, V>] **opts
71
+ # Options as passed to {.execute}, etc.
72
+ #
73
+ # @return [Hash<Symbol, V>]
74
+ # Options ready for `launchctl`.
75
+ #
76
+ def self.map_opts **opts
77
+ opts.map { |key, value|
78
+ if OPTS_MAP[key]
79
+ [OPTS_MAP[key], value]
80
+ else
81
+ [key, value]
82
+ end
83
+ }.to_h
84
+ end
85
+
86
+
87
+ # Run a `launchctl` subcommand with positional args and options.
88
+ #
89
+ # @param [Symbol | String] subcmd
90
+ # Subcommand to run, like `:load`.
91
+ #
92
+ # @param [Array<String>] *args
93
+ # Positional args, will be added on end of command.
94
+ #
95
+ # @param [Hash<Symbol, V>] **opts
96
+ # Options, will be added between subcommand and positional args.
97
+ #
98
+ # Keys are mapped via {.map_opts}.
99
+ #
100
+ # @return [Cmds::Result]
101
+ #
102
+ def self.execute subcmd, *args, **opts
103
+ logger.debug "Executing `#{ @@bin } #{ subcmd }` command",
104
+ args: args,
105
+ opts: opts
106
+
107
+ result = Cmds "#{ @@bin } #{ subcmd } <%= opts %> <%= *args %>",
108
+ *args,
109
+ opts: map_opts( **opts )
110
+
111
+ logger.debug "`#{ @@bin }` command result",
112
+ cmd: result.cmd,
113
+ status: result.status,
114
+ out: NRSER.ellipsis( result.out.lines, 20 ),
115
+ err: NRSER.ellipsis( result.err.lines, 20 )
116
+
117
+ result
118
+ end
119
+
120
+
121
+ # Just like {#execute} but will raise if the exit status is not `0`.
122
+ #
123
+ # @param (see .execute)
124
+ # @return (see .execute)
125
+ # @raise (see Cmds::Result#assert)
126
+ #
127
+ def self.execute! *args
128
+ execute( *args ).assert
129
+ end
130
+
131
+
132
+ %w{load unload start stop}.each do |subcmd|
133
+ define_singleton_method subcmd do |*args, **opts|
134
+ execute subcmd, *args, **opts
135
+ end
136
+
137
+ define_singleton_method "#{ subcmd }!" do |*args, **opts|
138
+ execute! subcmd, *args, **opts
139
+ end
140
+ end
141
+
142
+
143
+ def self.disabled
144
+ execute :'print-disabled', Cmds.expr( "user/$(id -u)" )
145
+ end
146
+
147
+
148
+ def self.refresh_status?
149
+ @last_status_time.nil? ||
150
+ (Time.now - @last_status_time) > STATUS_CACHE_TIMEOUT_SEC
151
+ end
152
+
153
+
154
+ #
155
+ # @param [Boolean] refresh:
156
+ #
157
+ #
158
+ # @return [Hash<String, {pid: Fixnum?, status: Fixnum?}>]
159
+ # Map of agent labels to hash with process ID (if any) and last exit
160
+ # code (if any).
161
+ #
162
+ def self.status refresh: refresh_status?
163
+ if refresh || @status.nil?
164
+ @status = execute!( :list ).out.lines.rest.
165
+ map { |line|
166
+ if match = STATUS_RE.match( line )
167
+ pid = case match[:pid]
168
+ when '-'
169
+ nil
170
+ when /^\d+$/
171
+ match[:pid].to_i
172
+ else
173
+ logger.error "Unexpected `pid` parse in {#status}",
174
+ captures: match.captures,
175
+ line: line
176
+ nil
177
+ end
178
+
179
+ status = case match[:status]
180
+ when /^\-?\d+$/
181
+ match[:status].to_i
182
+ else
183
+ logger.error "Unexpected `status` parse in {#status}",
184
+ captures: match.captures,
185
+ line: line
186
+ nil
187
+ end
188
+
189
+ label = match[:label].freeze
190
+
191
+ [label, {pid: pid, status: status}]
192
+
193
+ else
194
+ logger.error "FAILED to parse status line", line: line
195
+ nil
196
+
197
+ end
198
+ }.
199
+ reject( &:nil? ).
200
+ to_h
201
+
202
+ # Set the time so we can compare next call
203
+ @last_status_time = Time.now
204
+ end
205
+
206
+ @status
207
+ end # .status
208
+
209
+ end # module Locd::Launchctl
@@ -0,0 +1,360 @@
1
+ # Requirements
2
+ # =======================================================================
3
+
4
+ # Stdlib
5
+ # -----------------------------------------------------------------------
6
+ require 'forwardable'
7
+
8
+ # Deps
9
+ # -----------------------------------------------------------------------
10
+ require 'awesome_print'
11
+ require 'semantic_logger'
12
+
13
+ # Project / Package
14
+ # -----------------------------------------------------------------------
15
+
16
+
17
+ # Refinements
18
+ # =======================================================================
19
+
20
+ using NRSER
21
+ using NRSER::Types
22
+
23
+
24
+ # Definitions
25
+ # =======================================================================
26
+
27
+ # Utility methods to setup logging with [semantic_logger][].
28
+ #
29
+ # [semantic_logger]: http://rocketjob.github.io/semantic_logger/
30
+ #
31
+ module Locd::Logging
32
+ include SemanticLogger::Loggable
33
+
34
+
35
+ # @todo document Formatters module.
36
+ module Formatters
37
+
38
+ # Custom tweaked color formatter (for CLI output).
39
+ #
40
+ # - Turns on multiline output in Awesome Print by default.
41
+ #
42
+ class Color < SemanticLogger::Formatters::Color
43
+
44
+ # Constants
45
+ # ======================================================================
46
+
47
+
48
+ # Class Methods
49
+ # ======================================================================
50
+
51
+
52
+ # Attributes
53
+ # ======================================================================
54
+
55
+
56
+ # Constructor
57
+ # ======================================================================
58
+
59
+ # Instantiate a new `ColorFormatter`.
60
+ def initialize **options
61
+ super ap: { multiline: true },
62
+ color_map: SemanticLogger::Formatters::Color::ColorMap.new(
63
+ debug: SemanticLogger::AnsiColors::MAGENTA,
64
+ trace: "\e[1;30m", # "Dark Gray"
65
+ ),
66
+ **options
67
+ end # #initialize
68
+
69
+
70
+ # Instance Methods
71
+ # ======================================================================
72
+
73
+
74
+ # Upcase the log level.
75
+ #
76
+ # @return [String]
77
+ #
78
+ def level
79
+ "#{ color }#{ log.level.upcase }#{ color_map.clear }"
80
+ end
81
+
82
+
83
+ # Create the log entry text. Overridden to customize appearance -
84
+ # generally reduce amount of info and put payload on it's own line.
85
+ #
86
+ # We need to replace *two* super functions, the first being
87
+ # [SemanticLogger::Formatters::Color#call][]:
88
+ #
89
+ # def call(log, logger)
90
+ # self.color = color_map[log.level]
91
+ # super(log, logger)
92
+ # end
93
+ #
94
+ # [SemanticLogger::Formatters::Color#call]: https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/formatters/color.rb#L98
95
+ #
96
+ # which doesn't do all too much, and the next being it's super-method,
97
+ # [SemanticLogger::Formatters::Default#call][]:
98
+ #
99
+ # # Default text log format
100
+ # # Generates logs of the form:
101
+ # # 2011-07-19 14:36:15.660235 D [1149:ScriptThreadProcess] Rails -- Hello World
102
+ # def call(log, logger)
103
+ # self.log = log
104
+ # self.logger = logger
105
+ #
106
+ # [time, level, process_info, tags, named_tags, duration, name, message, payload, exception].compact.join(' ')
107
+ # end
108
+ #
109
+ # [SemanticLogger::Formatters::Default#call]: https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/formatters/default.rb#L64
110
+ #
111
+ # which does most the real assembly.
112
+ #
113
+ # @param [SemanticLogger::Log] log
114
+ # The log entry to format.
115
+ #
116
+ # See [SemanticLogger::Log](https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/log.rb)
117
+ #
118
+ # @param [SemanticLogger::Logger] logger
119
+ # The logger doing the logging (pretty sure, haven't checked).
120
+ #
121
+ # See [SemanticLogger::Logger](https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/logger.rb)
122
+ #
123
+ # @return [String]
124
+ # The full log string.
125
+ #
126
+ def call log, logger
127
+ # SemanticLogger::Formatters::Color code
128
+ self.color = color_map[log.level]
129
+
130
+ # SemanticLogger::Formatters::Default code
131
+ self.log = log
132
+ self.logger = logger
133
+
134
+ is_info = log.level == :info
135
+
136
+ [
137
+ level,
138
+ tags,
139
+ named_tags,
140
+ duration,
141
+ (is_info ? nil : name),
142
+ message,
143
+ payload,
144
+ exception,
145
+ ].compact.join(' ')
146
+
147
+ end # #call
148
+
149
+
150
+ end # class Color
151
+
152
+ end # module Formatters
153
+
154
+
155
+ module Appender
156
+ # Replacement for {SemanticLogger::Appender::Async} that implements the
157
+ # same interface but just logs synchronously in the current thread.
158
+ #
159
+ class Sync
160
+ extend Forwardable
161
+
162
+ # The appender we forward to, which is a {SemanticLogger::Processor}
163
+ # in practice, since it wouldn't make any sense to wrap a regular
164
+ # appender in a Sync.
165
+ #
166
+ # @return [SemanticLogger::Processor]
167
+ #
168
+ attr_accessor :appender
169
+
170
+ # Forward methods that can be called directly
171
+ def_delegator :@appender, :name
172
+ def_delegator :@appender, :should_log?
173
+ def_delegator :@appender, :filter
174
+ def_delegator :@appender, :host
175
+ def_delegator :@appender, :application
176
+ def_delegator :@appender, :level
177
+ def_delegator :@appender, :level=
178
+ def_delegator :@appender, :logger
179
+ # Added for sync
180
+ def_delegator :@appender, :log
181
+ def_delegator :@appender, :on_log
182
+ def_delegator :@appender, :flush
183
+ def_delegator :@appender, :close
184
+
185
+ class FakeQueue
186
+ def self.size
187
+ 0
188
+ end
189
+ end
190
+
191
+ # Appender proxy to allow an existing appender to run asynchronously in a separate thread.
192
+ #
193
+ # Parameters:
194
+ # name: [String]
195
+ # Name to use for the log thread and the log name when logging any errors from this appender.
196
+ #
197
+ # lag_threshold_s [Float]
198
+ # Log a warning when a log message has been on the queue for longer than this period in seconds.
199
+ # Default: 30
200
+ #
201
+ # lag_check_interval: [Integer]
202
+ # Number of messages to process before checking for slow logging.
203
+ # Default: 1,000
204
+ def initialize(appender:,
205
+ name: appender.class.name)
206
+
207
+ @appender = appender
208
+ end
209
+
210
+ # Needs to be there to support {SemanticLogger::Processor.queue_size},
211
+ # which gets the queue and returns it's size (which will always be zero
212
+ # for us).
213
+ #
214
+ # We return {FakeQueue}, which only implements a `size` method that
215
+ # returns zero.
216
+ #
217
+ # @return [#size]
218
+ #
219
+ def queue; FakeQueue; end
220
+
221
+ def lag_check_interval; -1; end
222
+
223
+ def lag_check_interval= value
224
+ raise "Can't set `lag_check_interval` on Sync appender"
225
+ end
226
+
227
+ def lag_threshold_s; -1; end
228
+
229
+ def lag_threshold_s= value
230
+ raise "Can't set `lag_threshold_s` on Sync appender"
231
+ end
232
+
233
+ # @return [false] Sync appender is of course not size-capped.
234
+ def capped?; false; end
235
+
236
+ # The {SemanticLogger::Appender::Async} worker thread is exposed via
237
+ # this method, which creates it if it doesn't exist and returns it, but
238
+ # it doesn't seem like the returned value is ever used; the method
239
+ # call is just invoked to start the thread.
240
+ #
241
+ # Hence it seems to make most sense to just return `nil` since we don't
242
+ # have a thread, and figure out what to do if that causes errors (so far
243
+ # it seems fine).
244
+ #
245
+ # @return [nil]
246
+ #
247
+ def thread; end
248
+
249
+ # @return [true] Sync appender is always active
250
+ def active?; true; end
251
+
252
+ end # class Sync
253
+ end # module Appenders
254
+
255
+
256
+ # Module (Class) Methods
257
+ # =====================================================================
258
+
259
+
260
+ def self.level
261
+ SemanticLogger.default_level
262
+ end
263
+
264
+
265
+ def self.level= level
266
+ SemanticLogger.default_level = level
267
+ end
268
+
269
+
270
+ def self.setup?
271
+ !!@setup
272
+ end
273
+
274
+
275
+ def self.get_env_level
276
+ if ENV['LOCD_TRACE'].truthy?
277
+ return :trace
278
+ elsif ENV['LOCD_DEBUG'].truthy?
279
+ return :debug
280
+ elsif ENV['LOCD_LOG_LEVEL']
281
+ return ENV['LOCD_LOG_LEVEL'].to_sym
282
+ end
283
+
284
+ nil
285
+ end
286
+
287
+
288
+ # Setup logging.
289
+ #
290
+ # @param [type] arg_name
291
+ # @todo Add name param description.
292
+ #
293
+ # @return [return_type]
294
+ # @todo Document return value.
295
+ #
296
+ def self.setup level: nil, sync: false, dest: nil
297
+ if setup?
298
+ logger.warn "Logging is already setup!"
299
+ return false
300
+ end
301
+
302
+ SemanticLogger.application = 'locd'
303
+
304
+ level = get_env_level if level.nil?
305
+ self.level = level if level
306
+ self.appender = dest if dest
307
+
308
+ if sync
309
+ # Hack up SemanticLogger to do sync logging in the main thread
310
+
311
+ # Create a {Locd::Logging::Appender::Sync}, which implements the
312
+ # {SemanticLogger::Appender::Async} interface but just forwards directly
313
+ # to it's appender in the same thread, and point it where
314
+ # {SemanticLogger::Processor.instance} (which is an Async) points.
315
+ #
316
+ sync_appender = Appender::Sync.new \
317
+ appender: SemanticLogger::Processor.instance.appender
318
+
319
+ # Swap our sync in for the async
320
+ SemanticLogger::Processor.instance_variable_set \
321
+ :@processor,
322
+ sync_appender
323
+ end
324
+
325
+ @setup = true
326
+
327
+ true
328
+ end # .setup
329
+
330
+
331
+ def self.appender
332
+ @appender
333
+ end
334
+
335
+
336
+ def self.appender= value
337
+ # Save ref to current appender (if any) so we can remove it after adding
338
+ # the new one.
339
+ old_appender = @appender
340
+
341
+ @appender = case value
342
+ when Hash
343
+ SemanticLogger.add_appender value
344
+ when String
345
+ SemanticLogger.add_appender file_name: value
346
+ else
347
+ SemanticLogger.add_appender \
348
+ io: value,
349
+ formatter: Formatters::Color.new
350
+ end
351
+
352
+ # Remove the old appender (if there was one). This is done after adding
353
+ # the new one so that failing won't result with no appenders.
354
+ SemanticLogger.remove_appender( old_appender ) if old_appender
355
+
356
+ @appender
357
+ end
358
+
359
+
360
+ end # module Locd::Logging