locd 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +108 -0
  3. data/.gitmodules +9 -0
  4. data/.qb-options.yml +4 -0
  5. data/.rspec +3 -0
  6. data/.travis.yml +5 -0
  7. data/.yardopts +7 -0
  8. data/Gemfile +10 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +72 -0
  11. data/Rakefile +6 -0
  12. data/VERSION +1 -0
  13. data/config/default.yml +24 -0
  14. data/doc/files/design/domains_and_labels.md +117 -0
  15. data/doc/files/topics/agents.md +16 -0
  16. data/doc/files/topics/launchd.md +15 -0
  17. data/doc/files/topics/plists.md +39 -0
  18. data/doc/include/plist.md +3 -0
  19. data/exe/locd +24 -0
  20. data/lib/locd.rb +75 -0
  21. data/lib/locd/agent.rb +1186 -0
  22. data/lib/locd/agent/job.rb +142 -0
  23. data/lib/locd/agent/proxy.rb +111 -0
  24. data/lib/locd/agent/rotate_logs.rb +82 -0
  25. data/lib/locd/agent/site.rb +174 -0
  26. data/lib/locd/agent/system.rb +270 -0
  27. data/lib/locd/cli.rb +4 -0
  28. data/lib/locd/cli/command.rb +9 -0
  29. data/lib/locd/cli/command/agent.rb +310 -0
  30. data/lib/locd/cli/command/base.rb +243 -0
  31. data/lib/locd/cli/command/job.rb +110 -0
  32. data/lib/locd/cli/command/main.rb +201 -0
  33. data/lib/locd/cli/command/proxy.rb +177 -0
  34. data/lib/locd/cli/command/rotate_logs.rb +152 -0
  35. data/lib/locd/cli/command/site.rb +47 -0
  36. data/lib/locd/cli/table.rb +157 -0
  37. data/lib/locd/config.rb +237 -0
  38. data/lib/locd/config/base.rb +93 -0
  39. data/lib/locd/errors.rb +65 -0
  40. data/lib/locd/label.rb +61 -0
  41. data/lib/locd/launchctl.rb +209 -0
  42. data/lib/locd/logging.rb +360 -0
  43. data/lib/locd/newsyslog.rb +402 -0
  44. data/lib/locd/pattern.rb +193 -0
  45. data/lib/locd/proxy.rb +272 -0
  46. data/lib/locd/proxymachine.rb +34 -0
  47. data/lib/locd/util.rb +49 -0
  48. data/lib/locd/version.rb +26 -0
  49. data/locd.gemspec +66 -0
  50. metadata +262 -0
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Requirements
4
+ # =======================================================================
5
+
6
+ # Stdlib
7
+ # -----------------------------------------------------------------------
8
+
9
+ # Deps
10
+ # -----------------------------------------------------------------------
11
+ require 'cmds'
12
+
13
+ # Project / Package
14
+ # -----------------------------------------------------------------------
15
+
16
+
17
+ # Refinements
18
+ # =======================================================================
19
+
20
+ using NRSER
21
+ using NRSER::Types
22
+
23
+
24
+ # Definitions
25
+ # =======================================================================
26
+
27
+ # Thin wrapper to run `launchctl` commands and capture outputs via {Cmds}.
28
+ #
29
+ # Basically just centralizes this stuff all in one place and presents a
30
+ # friendly API, does some common debug logging, etc.
31
+ #
32
+ module Locd::Launchctl
33
+
34
+ # Mappings between any option keyword names we use to the ones `launchctl`
35
+ # expects.
36
+ #
37
+ # @return [Hash<Symbol, Symbol>]
38
+ #
39
+ OPTS_MAP = {
40
+ force: :F,
41
+ write: :w,
42
+ }
43
+
44
+
45
+ # Regexp to parse `launchd list` output for {#status} method.
46
+ #
47
+ # @return [Regexp]
48
+ #
49
+ STATUS_RE = /^(?<pid>\S+)\s+(?<status>\S+)\s+(?<label>\S+)/.freeze
50
+
51
+
52
+ # Max number of seconds to
53
+ STATUS_CACHE_TIMEOUT_SEC = 5
54
+
55
+
56
+ # Add {.logger} method
57
+ include SemanticLogger::Loggable
58
+
59
+
60
+ # `launchctl` executable to use. You shouldn't need to touch this, but might
61
+ # be useful for testing or if you have path weirdness.
62
+ #
63
+ # @return [String]
64
+ #
65
+ @@bin = 'launchctl'
66
+
67
+
68
+ # Swap any keys in {OPTS_MAP}.
69
+ #
70
+ # @param [Hash<Symbol, V>] **opts
71
+ # Options as passed to {.execute}, etc.
72
+ #
73
+ # @return [Hash<Symbol, V>]
74
+ # Options ready for `launchctl`.
75
+ #
76
+ def self.map_opts **opts
77
+ opts.map { |key, value|
78
+ if OPTS_MAP[key]
79
+ [OPTS_MAP[key], value]
80
+ else
81
+ [key, value]
82
+ end
83
+ }.to_h
84
+ end
85
+
86
+
87
+ # Run a `launchctl` subcommand with positional args and options.
88
+ #
89
+ # @param [Symbol | String] subcmd
90
+ # Subcommand to run, like `:load`.
91
+ #
92
+ # @param [Array<String>] *args
93
+ # Positional args, will be added on end of command.
94
+ #
95
+ # @param [Hash<Symbol, V>] **opts
96
+ # Options, will be added between subcommand and positional args.
97
+ #
98
+ # Keys are mapped via {.map_opts}.
99
+ #
100
+ # @return [Cmds::Result]
101
+ #
102
+ def self.execute subcmd, *args, **opts
103
+ logger.debug "Executing `#{ @@bin } #{ subcmd }` command",
104
+ args: args,
105
+ opts: opts
106
+
107
+ result = Cmds "#{ @@bin } #{ subcmd } <%= opts %> <%= *args %>",
108
+ *args,
109
+ opts: map_opts( **opts )
110
+
111
+ logger.debug "`#{ @@bin }` command result",
112
+ cmd: result.cmd,
113
+ status: result.status,
114
+ out: NRSER.ellipsis( result.out.lines, 20 ),
115
+ err: NRSER.ellipsis( result.err.lines, 20 )
116
+
117
+ result
118
+ end
119
+
120
+
121
+ # Just like {#execute} but will raise if the exit status is not `0`.
122
+ #
123
+ # @param (see .execute)
124
+ # @return (see .execute)
125
+ # @raise (see Cmds::Result#assert)
126
+ #
127
+ def self.execute! *args
128
+ execute( *args ).assert
129
+ end
130
+
131
+
132
+ %w{load unload start stop}.each do |subcmd|
133
+ define_singleton_method subcmd do |*args, **opts|
134
+ execute subcmd, *args, **opts
135
+ end
136
+
137
+ define_singleton_method "#{ subcmd }!" do |*args, **opts|
138
+ execute! subcmd, *args, **opts
139
+ end
140
+ end
141
+
142
+
143
+ def self.disabled
144
+ execute :'print-disabled', Cmds.expr( "user/$(id -u)" )
145
+ end
146
+
147
+
148
+ def self.refresh_status?
149
+ @last_status_time.nil? ||
150
+ (Time.now - @last_status_time) > STATUS_CACHE_TIMEOUT_SEC
151
+ end
152
+
153
+
154
+ #
155
+ # @param [Boolean] refresh:
156
+ #
157
+ #
158
+ # @return [Hash<String, {pid: Fixnum?, status: Fixnum?}>]
159
+ # Map of agent labels to hash with process ID (if any) and last exit
160
+ # code (if any).
161
+ #
162
+ def self.status refresh: refresh_status?
163
+ if refresh || @status.nil?
164
+ @status = execute!( :list ).out.lines.rest.
165
+ map { |line|
166
+ if match = STATUS_RE.match( line )
167
+ pid = case match[:pid]
168
+ when '-'
169
+ nil
170
+ when /^\d+$/
171
+ match[:pid].to_i
172
+ else
173
+ logger.error "Unexpected `pid` parse in {#status}",
174
+ captures: match.captures,
175
+ line: line
176
+ nil
177
+ end
178
+
179
+ status = case match[:status]
180
+ when /^\-?\d+$/
181
+ match[:status].to_i
182
+ else
183
+ logger.error "Unexpected `status` parse in {#status}",
184
+ captures: match.captures,
185
+ line: line
186
+ nil
187
+ end
188
+
189
+ label = match[:label].freeze
190
+
191
+ [label, {pid: pid, status: status}]
192
+
193
+ else
194
+ logger.error "FAILED to parse status line", line: line
195
+ nil
196
+
197
+ end
198
+ }.
199
+ reject( &:nil? ).
200
+ to_h
201
+
202
+ # Set the time so we can compare next call
203
+ @last_status_time = Time.now
204
+ end
205
+
206
+ @status
207
+ end # .status
208
+
209
+ end # module Locd::Launchctl
@@ -0,0 +1,360 @@
1
+ # Requirements
2
+ # =======================================================================
3
+
4
+ # Stdlib
5
+ # -----------------------------------------------------------------------
6
+ require 'forwardable'
7
+
8
+ # Deps
9
+ # -----------------------------------------------------------------------
10
+ require 'awesome_print'
11
+ require 'semantic_logger'
12
+
13
+ # Project / Package
14
+ # -----------------------------------------------------------------------
15
+
16
+
17
+ # Refinements
18
+ # =======================================================================
19
+
20
+ using NRSER
21
+ using NRSER::Types
22
+
23
+
24
+ # Definitions
25
+ # =======================================================================
26
+
27
+ # Utility methods to setup logging with [semantic_logger][].
28
+ #
29
+ # [semantic_logger]: http://rocketjob.github.io/semantic_logger/
30
+ #
31
+ module Locd::Logging
32
+ include SemanticLogger::Loggable
33
+
34
+
35
+ # @todo document Formatters module.
36
+ module Formatters
37
+
38
+ # Custom tweaked color formatter (for CLI output).
39
+ #
40
+ # - Turns on multiline output in Awesome Print by default.
41
+ #
42
+ class Color < SemanticLogger::Formatters::Color
43
+
44
+ # Constants
45
+ # ======================================================================
46
+
47
+
48
+ # Class Methods
49
+ # ======================================================================
50
+
51
+
52
+ # Attributes
53
+ # ======================================================================
54
+
55
+
56
+ # Constructor
57
+ # ======================================================================
58
+
59
+ # Instantiate a new `ColorFormatter`.
60
+ def initialize **options
61
+ super ap: { multiline: true },
62
+ color_map: SemanticLogger::Formatters::Color::ColorMap.new(
63
+ debug: SemanticLogger::AnsiColors::MAGENTA,
64
+ trace: "\e[1;30m", # "Dark Gray"
65
+ ),
66
+ **options
67
+ end # #initialize
68
+
69
+
70
+ # Instance Methods
71
+ # ======================================================================
72
+
73
+
74
+ # Upcase the log level.
75
+ #
76
+ # @return [String]
77
+ #
78
+ def level
79
+ "#{ color }#{ log.level.upcase }#{ color_map.clear }"
80
+ end
81
+
82
+
83
+ # Create the log entry text. Overridden to customize appearance -
84
+ # generally reduce amount of info and put payload on it's own line.
85
+ #
86
+ # We need to replace *two* super functions, the first being
87
+ # [SemanticLogger::Formatters::Color#call][]:
88
+ #
89
+ # def call(log, logger)
90
+ # self.color = color_map[log.level]
91
+ # super(log, logger)
92
+ # end
93
+ #
94
+ # [SemanticLogger::Formatters::Color#call]: https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/formatters/color.rb#L98
95
+ #
96
+ # which doesn't do all too much, and the next being it's super-method,
97
+ # [SemanticLogger::Formatters::Default#call][]:
98
+ #
99
+ # # Default text log format
100
+ # # Generates logs of the form:
101
+ # # 2011-07-19 14:36:15.660235 D [1149:ScriptThreadProcess] Rails -- Hello World
102
+ # def call(log, logger)
103
+ # self.log = log
104
+ # self.logger = logger
105
+ #
106
+ # [time, level, process_info, tags, named_tags, duration, name, message, payload, exception].compact.join(' ')
107
+ # end
108
+ #
109
+ # [SemanticLogger::Formatters::Default#call]: https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/formatters/default.rb#L64
110
+ #
111
+ # which does most the real assembly.
112
+ #
113
+ # @param [SemanticLogger::Log] log
114
+ # The log entry to format.
115
+ #
116
+ # See [SemanticLogger::Log](https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/log.rb)
117
+ #
118
+ # @param [SemanticLogger::Logger] logger
119
+ # The logger doing the logging (pretty sure, haven't checked).
120
+ #
121
+ # See [SemanticLogger::Logger](https://github.com/rocketjob/semantic_logger/blob/v4.2.0/lib/semantic_logger/logger.rb)
122
+ #
123
+ # @return [String]
124
+ # The full log string.
125
+ #
126
+ def call log, logger
127
+ # SemanticLogger::Formatters::Color code
128
+ self.color = color_map[log.level]
129
+
130
+ # SemanticLogger::Formatters::Default code
131
+ self.log = log
132
+ self.logger = logger
133
+
134
+ is_info = log.level == :info
135
+
136
+ [
137
+ level,
138
+ tags,
139
+ named_tags,
140
+ duration,
141
+ (is_info ? nil : name),
142
+ message,
143
+ payload,
144
+ exception,
145
+ ].compact.join(' ')
146
+
147
+ end # #call
148
+
149
+
150
+ end # class Color
151
+
152
+ end # module Formatters
153
+
154
+
155
+ module Appender
156
+ # Replacement for {SemanticLogger::Appender::Async} that implements the
157
+ # same interface but just logs synchronously in the current thread.
158
+ #
159
+ class Sync
160
+ extend Forwardable
161
+
162
+ # The appender we forward to, which is a {SemanticLogger::Processor}
163
+ # in practice, since it wouldn't make any sense to wrap a regular
164
+ # appender in a Sync.
165
+ #
166
+ # @return [SemanticLogger::Processor]
167
+ #
168
+ attr_accessor :appender
169
+
170
+ # Forward methods that can be called directly
171
+ def_delegator :@appender, :name
172
+ def_delegator :@appender, :should_log?
173
+ def_delegator :@appender, :filter
174
+ def_delegator :@appender, :host
175
+ def_delegator :@appender, :application
176
+ def_delegator :@appender, :level
177
+ def_delegator :@appender, :level=
178
+ def_delegator :@appender, :logger
179
+ # Added for sync
180
+ def_delegator :@appender, :log
181
+ def_delegator :@appender, :on_log
182
+ def_delegator :@appender, :flush
183
+ def_delegator :@appender, :close
184
+
185
+ class FakeQueue
186
+ def self.size
187
+ 0
188
+ end
189
+ end
190
+
191
+ # Appender proxy to allow an existing appender to run asynchronously in a separate thread.
192
+ #
193
+ # Parameters:
194
+ # name: [String]
195
+ # Name to use for the log thread and the log name when logging any errors from this appender.
196
+ #
197
+ # lag_threshold_s [Float]
198
+ # Log a warning when a log message has been on the queue for longer than this period in seconds.
199
+ # Default: 30
200
+ #
201
+ # lag_check_interval: [Integer]
202
+ # Number of messages to process before checking for slow logging.
203
+ # Default: 1,000
204
+ def initialize(appender:,
205
+ name: appender.class.name)
206
+
207
+ @appender = appender
208
+ end
209
+
210
+ # Needs to be there to support {SemanticLogger::Processor.queue_size},
211
+ # which gets the queue and returns it's size (which will always be zero
212
+ # for us).
213
+ #
214
+ # We return {FakeQueue}, which only implements a `size` method that
215
+ # returns zero.
216
+ #
217
+ # @return [#size]
218
+ #
219
+ def queue; FakeQueue; end
220
+
221
+ def lag_check_interval; -1; end
222
+
223
+ def lag_check_interval= value
224
+ raise "Can't set `lag_check_interval` on Sync appender"
225
+ end
226
+
227
+ def lag_threshold_s; -1; end
228
+
229
+ def lag_threshold_s= value
230
+ raise "Can't set `lag_threshold_s` on Sync appender"
231
+ end
232
+
233
+ # @return [false] Sync appender is of course not size-capped.
234
+ def capped?; false; end
235
+
236
+ # The {SemanticLogger::Appender::Async} worker thread is exposed via
237
+ # this method, which creates it if it doesn't exist and returns it, but
238
+ # it doesn't seem like the returned value is ever used; the method
239
+ # call is just invoked to start the thread.
240
+ #
241
+ # Hence it seems to make most sense to just return `nil` since we don't
242
+ # have a thread, and figure out what to do if that causes errors (so far
243
+ # it seems fine).
244
+ #
245
+ # @return [nil]
246
+ #
247
+ def thread; end
248
+
249
+ # @return [true] Sync appender is always active
250
+ def active?; true; end
251
+
252
+ end # class Sync
253
+ end # module Appenders
254
+
255
+
256
+ # Module (Class) Methods
257
+ # =====================================================================
258
+
259
+
260
+ def self.level
261
+ SemanticLogger.default_level
262
+ end
263
+
264
+
265
+ def self.level= level
266
+ SemanticLogger.default_level = level
267
+ end
268
+
269
+
270
+ def self.setup?
271
+ !!@setup
272
+ end
273
+
274
+
275
+ def self.get_env_level
276
+ if ENV['LOCD_TRACE'].truthy?
277
+ return :trace
278
+ elsif ENV['LOCD_DEBUG'].truthy?
279
+ return :debug
280
+ elsif ENV['LOCD_LOG_LEVEL']
281
+ return ENV['LOCD_LOG_LEVEL'].to_sym
282
+ end
283
+
284
+ nil
285
+ end
286
+
287
+
288
+ # Setup logging.
289
+ #
290
+ # @param [type] arg_name
291
+ # @todo Add name param description.
292
+ #
293
+ # @return [return_type]
294
+ # @todo Document return value.
295
+ #
296
+ def self.setup level: nil, sync: false, dest: nil
297
+ if setup?
298
+ logger.warn "Logging is already setup!"
299
+ return false
300
+ end
301
+
302
+ SemanticLogger.application = 'locd'
303
+
304
+ level = get_env_level if level.nil?
305
+ self.level = level if level
306
+ self.appender = dest if dest
307
+
308
+ if sync
309
+ # Hack up SemanticLogger to do sync logging in the main thread
310
+
311
+ # Create a {Locd::Logging::Appender::Sync}, which implements the
312
+ # {SemanticLogger::Appender::Async} interface but just forwards directly
313
+ # to it's appender in the same thread, and point it where
314
+ # {SemanticLogger::Processor.instance} (which is an Async) points.
315
+ #
316
+ sync_appender = Appender::Sync.new \
317
+ appender: SemanticLogger::Processor.instance.appender
318
+
319
+ # Swap our sync in for the async
320
+ SemanticLogger::Processor.instance_variable_set \
321
+ :@processor,
322
+ sync_appender
323
+ end
324
+
325
+ @setup = true
326
+
327
+ true
328
+ end # .setup
329
+
330
+
331
+ def self.appender
332
+ @appender
333
+ end
334
+
335
+
336
+ def self.appender= value
337
+ # Save ref to current appender (if any) so we can remove it after adding
338
+ # the new one.
339
+ old_appender = @appender
340
+
341
+ @appender = case value
342
+ when Hash
343
+ SemanticLogger.add_appender value
344
+ when String
345
+ SemanticLogger.add_appender file_name: value
346
+ else
347
+ SemanticLogger.add_appender \
348
+ io: value,
349
+ formatter: Formatters::Color.new
350
+ end
351
+
352
+ # Remove the old appender (if there was one). This is done after adding
353
+ # the new one so that failing won't result with no appenders.
354
+ SemanticLogger.remove_appender( old_appender ) if old_appender
355
+
356
+ @appender
357
+ end
358
+
359
+
360
+ end # module Locd::Logging