mcproc 2016.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/Announce.txt +135 -0
  3. data/Gemfile +9 -0
  4. data/History.txt +469 -0
  5. data/LICENSE +22 -0
  6. data/README.md +37 -0
  7. data/Rakefile +185 -0
  8. data/TODO.md +37 -0
  9. data/bin/mcproc +134 -0
  10. data/doc/intro.asciidoc +20 -0
  11. data/doc/mcproc.asciidoc +1592 -0
  12. data/ext/god/.gitignore +5 -0
  13. data/ext/god/extconf.rb +56 -0
  14. data/ext/god/kqueue_handler.c +133 -0
  15. data/ext/god/netlink_handler.c +182 -0
  16. data/lib/god.rb +780 -0
  17. data/lib/god/behavior.rb +52 -0
  18. data/lib/god/behaviors/clean_pid_file.rb +21 -0
  19. data/lib/god/behaviors/clean_unix_socket.rb +21 -0
  20. data/lib/god/behaviors/notify_when_flapping.rb +51 -0
  21. data/lib/god/cli/command.rb +268 -0
  22. data/lib/god/cli/run.rb +170 -0
  23. data/lib/god/cli/version.rb +23 -0
  24. data/lib/god/compat19.rb +33 -0
  25. data/lib/god/condition.rb +96 -0
  26. data/lib/god/conditions/always.rb +36 -0
  27. data/lib/god/conditions/complex.rb +86 -0
  28. data/lib/god/conditions/cpu_usage.rb +80 -0
  29. data/lib/god/conditions/degrading_lambda.rb +52 -0
  30. data/lib/god/conditions/disk_usage.rb +32 -0
  31. data/lib/god/conditions/file_mtime.rb +28 -0
  32. data/lib/god/conditions/file_touched.rb +44 -0
  33. data/lib/god/conditions/flapping.rb +128 -0
  34. data/lib/god/conditions/http_response_code.rb +184 -0
  35. data/lib/god/conditions/lambda.rb +25 -0
  36. data/lib/god/conditions/memory_usage.rb +82 -0
  37. data/lib/god/conditions/process_exits.rb +66 -0
  38. data/lib/god/conditions/process_running.rb +63 -0
  39. data/lib/god/conditions/socket_responding.rb +142 -0
  40. data/lib/god/conditions/tries.rb +44 -0
  41. data/lib/god/configurable.rb +57 -0
  42. data/lib/god/contact.rb +114 -0
  43. data/lib/god/contacts/airbrake.rb +44 -0
  44. data/lib/god/contacts/campfire.rb +121 -0
  45. data/lib/god/contacts/email.rb +130 -0
  46. data/lib/god/contacts/hipchat.rb +117 -0
  47. data/lib/god/contacts/jabber.rb +75 -0
  48. data/lib/god/contacts/prowl.rb +57 -0
  49. data/lib/god/contacts/scout.rb +55 -0
  50. data/lib/god/contacts/sensu.rb +59 -0
  51. data/lib/god/contacts/slack.rb +98 -0
  52. data/lib/god/contacts/statsd.rb +46 -0
  53. data/lib/god/contacts/twitter.rb +51 -0
  54. data/lib/god/contacts/webhook.rb +74 -0
  55. data/lib/god/driver.rb +238 -0
  56. data/lib/god/errors.rb +24 -0
  57. data/lib/god/event_handler.rb +112 -0
  58. data/lib/god/event_handlers/dummy_handler.rb +13 -0
  59. data/lib/god/event_handlers/kqueue_handler.rb +17 -0
  60. data/lib/god/event_handlers/netlink_handler.rb +13 -0
  61. data/lib/god/logger.rb +109 -0
  62. data/lib/god/metric.rb +87 -0
  63. data/lib/god/process.rb +381 -0
  64. data/lib/god/registry.rb +32 -0
  65. data/lib/god/simple_logger.rb +59 -0
  66. data/lib/god/socket.rb +113 -0
  67. data/lib/god/sugar.rb +62 -0
  68. data/lib/god/sys_logger.rb +45 -0
  69. data/lib/god/system/portable_poller.rb +42 -0
  70. data/lib/god/system/process.rb +50 -0
  71. data/lib/god/system/slash_proc_poller.rb +92 -0
  72. data/lib/god/task.rb +552 -0
  73. data/lib/god/timeline.rb +25 -0
  74. data/lib/god/trigger.rb +43 -0
  75. data/lib/god/watch.rb +340 -0
  76. data/mcproc.gemspec +192 -0
  77. data/test/configs/child_events/child_events.god +44 -0
  78. data/test/configs/child_events/simple_server.rb +3 -0
  79. data/test/configs/child_polls/child_polls.god +37 -0
  80. data/test/configs/child_polls/simple_server.rb +12 -0
  81. data/test/configs/complex/complex.god +59 -0
  82. data/test/configs/complex/simple_server.rb +3 -0
  83. data/test/configs/contact/contact.god +118 -0
  84. data/test/configs/contact/simple_server.rb +3 -0
  85. data/test/configs/daemon_events/daemon_events.god +37 -0
  86. data/test/configs/daemon_events/simple_server.rb +8 -0
  87. data/test/configs/daemon_events/simple_server_stop.rb +11 -0
  88. data/test/configs/daemon_polls/daemon_polls.god +17 -0
  89. data/test/configs/daemon_polls/simple_server.rb +6 -0
  90. data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
  91. data/test/configs/degrading_lambda/tcp_server.rb +15 -0
  92. data/test/configs/keepalive/keepalive.god +9 -0
  93. data/test/configs/keepalive/keepalive.rb +12 -0
  94. data/test/configs/lifecycle/lifecycle.god +25 -0
  95. data/test/configs/matias/matias.god +50 -0
  96. data/test/configs/real.rb +59 -0
  97. data/test/configs/running_load/running_load.god +16 -0
  98. data/test/configs/stop_options/simple_server.rb +12 -0
  99. data/test/configs/stop_options/stop_options.god +39 -0
  100. data/test/configs/stress/simple_server.rb +3 -0
  101. data/test/configs/stress/stress.god +15 -0
  102. data/test/configs/task/logs/.placeholder +0 -0
  103. data/test/configs/task/task.god +26 -0
  104. data/test/configs/test.rb +61 -0
  105. data/test/configs/usr1_trapper.rb +10 -0
  106. data/test/helper.rb +172 -0
  107. data/test/suite.rb +6 -0
  108. data/test/test_airbrake.rb +14 -0
  109. data/test/test_behavior.rb +18 -0
  110. data/test/test_campfire.rb +22 -0
  111. data/test/test_condition.rb +52 -0
  112. data/test/test_conditions_disk_usage.rb +50 -0
  113. data/test/test_conditions_http_response_code.rb +109 -0
  114. data/test/test_conditions_process_running.rb +40 -0
  115. data/test/test_conditions_socket_responding.rb +176 -0
  116. data/test/test_conditions_tries.rb +67 -0
  117. data/test/test_contact.rb +109 -0
  118. data/test/test_driver.rb +26 -0
  119. data/test/test_email.rb +34 -0
  120. data/test/test_event_handler.rb +82 -0
  121. data/test/test_god.rb +710 -0
  122. data/test/test_god_system.rb +201 -0
  123. data/test/test_handlers_kqueue_handler.rb +16 -0
  124. data/test/test_hipchat.rb +23 -0
  125. data/test/test_jabber.rb +29 -0
  126. data/test/test_logger.rb +55 -0
  127. data/test/test_metric.rb +74 -0
  128. data/test/test_process.rb +263 -0
  129. data/test/test_prowl.rb +15 -0
  130. data/test/test_registry.rb +15 -0
  131. data/test/test_sensu.rb +11 -0
  132. data/test/test_slack.rb +57 -0
  133. data/test/test_socket.rb +34 -0
  134. data/test/test_statsd.rb +22 -0
  135. data/test/test_sugar.rb +42 -0
  136. data/test/test_system_portable_poller.rb +17 -0
  137. data/test/test_system_process.rb +30 -0
  138. data/test/test_task.rb +246 -0
  139. data/test/test_timeline.rb +37 -0
  140. data/test/test_trigger.rb +63 -0
  141. data/test/test_watch.rb +286 -0
  142. data/test/test_webhook.rb +22 -0
  143. metadata +475 -0
data/lib/god/driver.rb ADDED
@@ -0,0 +1,238 @@
1
+ require 'monitor'
2
+
3
+ # Ruby 1.9.1 specific fixes.
4
+ if RUBY_VERSION.between?('1.9', '1.9.1')
5
+ require 'god/compat19'
6
+ end
7
+
8
+ module God
9
+ # The TimedEvent class represents an event in the future. This class is used
10
+ # by the drivers to schedule upcoming conditional tests and other scheduled
11
+ # events.
12
+ class TimedEvent
13
+ include Comparable
14
+
15
+ # The Time at which this event is due.
16
+ attr_accessor :at
17
+
18
+ # Instantiate a new TimedEvent that will be triggered after the specified
19
+ # delay.
20
+ #
21
+ # delay - The optional Numeric number of seconds from now at which to
22
+ # trigger (default: 0).
23
+ def initialize(delay = 0)
24
+ self.at = Time.now + delay
25
+ end
26
+
27
+ # Is the current event due (current time >= event time)?
28
+ #
29
+ # Returns true if the event is due, false if not.
30
+ def due?
31
+ Time.now >= self.at
32
+ end
33
+
34
+ # Compare this event to another.
35
+ #
36
+ # other - The other TimedEvent.
37
+ #
38
+ # Returns -1 if this event is before the other, 0 if the two events are
39
+ # due at the same time, 1 if the other event is later.
40
+ def <=>(other)
41
+ self.at <=> other.at
42
+ end
43
+ end
44
+
45
+ # A DriverEvent is a TimedEvent with an associated Task and Condition. This
46
+ # is the primary mechanism for poll conditions to be scheduled.
47
+ class DriverEvent < TimedEvent
48
+ # Initialize a new DriverEvent.
49
+ #
50
+ # delay - The Numeric delay for this event.
51
+ # task - The Task associated with this event.
52
+ # condition - The Condition associated with this event.
53
+ def initialize(delay, task, condition)
54
+ super(delay)
55
+ @task = task
56
+ @condition = condition
57
+ end
58
+
59
+ # Handle this event by invoking the underlying condition on the associated
60
+ # task.
61
+ #
62
+ # Returns nothing.
63
+ def handle_event
64
+ @task.handle_poll(@condition)
65
+ end
66
+ end
67
+
68
+ # A DriverOperation is a TimedEvent that is due as soon as possible. It is
69
+ # used to execute an arbitrary method on the associated Task.
70
+ class DriverOperation < TimedEvent
71
+ # Initialize a new DriverOperation.
72
+ #
73
+ # task - The Task upon which to operate.
74
+ # name - The Symbol name of the method to call.
75
+ # args - The Array of arguments to send to the method.
76
+ def initialize(task, name, args)
77
+ super(0)
78
+ @task = task
79
+ @name = name
80
+ @args = args
81
+ end
82
+
83
+ # Handle the operation that was issued asynchronously.
84
+ #
85
+ # Returns nothing.
86
+ def handle_event
87
+ @task.send(@name, *@args)
88
+ end
89
+ end
90
+
91
+ # The DriverEventQueue is a simple queue that holds TimedEvent instances in
92
+ # order to maintain the schedule of upcoming events.
93
+ class DriverEventQueue
94
+ # Initialize a DriverEventQueue.
95
+ def initialize
96
+ @shutdown = false
97
+ @events = []
98
+ @monitor = Monitor.new
99
+ @resource = @monitor.new_cond
100
+ end
101
+
102
+ # Wake any sleeping threads after setting the sentinel.
103
+ #
104
+ # Returns nothing.
105
+ def shutdown
106
+ @shutdown = true
107
+ @monitor.synchronize do
108
+ @resource.broadcast
109
+ end
110
+ end
111
+
112
+ # Wait until the queue has something due, pop it off the queue, and return
113
+ # it.
114
+ #
115
+ # Returns the popped event.
116
+ def pop
117
+ @monitor.synchronize do
118
+ if @events.empty?
119
+ raise ThreadError, "queue empty" if @shutdown
120
+ @resource.wait
121
+ else
122
+ delay = @events.first.at - Time.now
123
+ @resource.wait(delay) if delay > 0
124
+ end
125
+
126
+ @events.shift
127
+ end
128
+ end
129
+
130
+ # Add an event to the queue, wake any waiters if what we added needs to
131
+ # happen sooner than the next pending event.
132
+ #
133
+ # Returns nothing.
134
+ def push(event)
135
+ @monitor.synchronize do
136
+ @events << event
137
+ @events.sort!
138
+
139
+ # If we've sorted the events and found the one we're adding is at
140
+ # the front, it will likely need to run before the next due date.
141
+ @resource.signal if @events.first == event
142
+ end
143
+ end
144
+
145
+ # Returns true if the queue is empty, false if not.
146
+ def empty?
147
+ @events.empty?
148
+ end
149
+
150
+ # Clear the queue.
151
+ #
152
+ # Returns nothing.
153
+ def clear
154
+ @events.clear
155
+ end
156
+
157
+ # Returns the Integer length of the queue.
158
+ def length
159
+ @events.length
160
+ end
161
+
162
+ alias size length
163
+ end
164
+
165
+ # The Driver class is responsible for scheduling all of the events for a
166
+ # given Task.
167
+ class Driver
168
+ # The Thread running the driver loop.
169
+ attr_reader :thread
170
+
171
+ # Instantiate a new Driver and start the scheduler loop to handle events.
172
+ #
173
+ # task - The Task this Driver belongs to.
174
+ def initialize(task)
175
+ @task = task
176
+ @events = God::DriverEventQueue.new
177
+
178
+ @thread = Thread.new do
179
+ loop do
180
+ begin
181
+ @events.pop.handle_event
182
+ rescue ThreadError => e
183
+ # queue is empty
184
+ break
185
+ rescue Object => e
186
+ message = format("Unhandled exception in driver loop - (%s): %s\n%s",
187
+ e.class, e.message, e.backtrace.join("\n"))
188
+ applog(nil, :fatal, message)
189
+ end
190
+ end
191
+ end
192
+ end
193
+
194
+ # Check if we're in the driver context.
195
+ #
196
+ # Returns true if in driver thread, false if not.
197
+ def in_driver_context?
198
+ Thread.current == @thread
199
+ end
200
+
201
+ # Clear all events for this Driver.
202
+ #
203
+ # Returns nothing.
204
+ def clear_events
205
+ @events.clear
206
+ end
207
+
208
+ # Shutdown the DriverEventQueue threads.
209
+ #
210
+ # Returns nothing.
211
+ def shutdown
212
+ @events.shutdown
213
+ end
214
+
215
+ # Queue an asynchronous message.
216
+ #
217
+ # name - The Symbol name of the operation.
218
+ # args - An optional Array of arguments.
219
+ #
220
+ # Returns nothing.
221
+ def message(name, args = [])
222
+ @events.push(DriverOperation.new(@task, name, args))
223
+ end
224
+
225
+ # Create and schedule a new DriverEvent.
226
+ #
227
+ # condition - The Condition.
228
+ # delay - The Numeric number of seconds to delay (default: interval
229
+ # defined in condition).
230
+ #
231
+ # Returns nothing.
232
+ def schedule(condition, delay = condition.interval)
233
+ applog(nil, :debug, "driver schedule #{condition} in #{delay} seconds")
234
+
235
+ @events.push(DriverEvent.new(delay, @task, condition))
236
+ end
237
+ end
238
+ end
data/lib/god/errors.rb ADDED
@@ -0,0 +1,24 @@
1
+ module God
2
+
3
+ class AbstractMethodNotOverriddenError < StandardError
4
+ end
5
+
6
+ class NoSuchWatchError < StandardError
7
+ end
8
+
9
+ class NoSuchConditionError < StandardError
10
+ end
11
+
12
+ class NoSuchBehaviorError < StandardError
13
+ end
14
+
15
+ class NoSuchContactError < StandardError
16
+ end
17
+
18
+ class InvalidCommandError < StandardError
19
+ end
20
+
21
+ class EventRegistrationFailedError < StandardError
22
+ end
23
+
24
+ end
@@ -0,0 +1,112 @@
1
+ module God
2
+ class EventHandler
3
+ @@actions = {}
4
+ @@handler = nil
5
+ @@loaded = false
6
+
7
+ def self.loaded?
8
+ @@loaded
9
+ end
10
+
11
+ def self.event_system
12
+ @@handler::EVENT_SYSTEM
13
+ end
14
+
15
+ def self.load
16
+ begin
17
+ case RUBY_PLATFORM
18
+ when /darwin/i, /bsd/i
19
+ require 'god/event_handlers/kqueue_handler'
20
+ @@handler = KQueueHandler
21
+ when /linux/i
22
+ require 'god/event_handlers/netlink_handler'
23
+ @@handler = NetlinkHandler
24
+ else
25
+ raise NotImplementedError, "Platform not supported for EventHandler"
26
+ end
27
+ @@loaded = true
28
+ rescue Exception
29
+ require 'god/event_handlers/dummy_handler'
30
+ @@handler = DummyHandler
31
+ @@loaded = false
32
+ end
33
+ end
34
+
35
+ def self.register(pid, event, &block)
36
+ @@actions[pid] ||= {}
37
+ @@actions[pid][event] = block
38
+ @@handler.register_process(pid, @@actions[pid].keys)
39
+ end
40
+
41
+ def self.deregister(pid, event)
42
+ if watching_pid? pid
43
+ running = ::Process.kill(0, pid.to_i) rescue false
44
+ @@actions[pid].delete(event)
45
+ @@handler.register_process(pid, @@actions[pid].keys) if running
46
+ @@actions.delete(pid) if @@actions[pid].empty?
47
+ end
48
+ end
49
+
50
+ def self.call(pid, event, extra_data = {})
51
+ @@actions[pid][event].call(extra_data) if watching_pid?(pid) && @@actions[pid][event]
52
+ end
53
+
54
+ def self.watching_pid?(pid)
55
+ @@actions[pid]
56
+ end
57
+
58
+ def self.start
59
+ @@thread = Thread.new do
60
+ loop do
61
+ begin
62
+ @@handler.handle_events
63
+ rescue Exception => e
64
+ message = format("Unhandled exception (%s): %s\n%s",
65
+ e.class, e.message, e.backtrace.join("\n"))
66
+ applog(nil, :fatal, message)
67
+ end
68
+ end
69
+ end
70
+
71
+ # do a real test to make sure events are working properly
72
+ @@loaded = self.operational?
73
+ end
74
+
75
+ def self.stop
76
+ @@thread.kill if @@thread
77
+ end
78
+
79
+ def self.operational?
80
+ com = [false]
81
+
82
+ Thread.new do
83
+ begin
84
+ event_system = God::EventHandler.event_system
85
+
86
+ pid = fork do
87
+ loop { sleep(1) }
88
+ end
89
+
90
+ self.register(pid, :proc_exit) do
91
+ com[0] = true
92
+ end
93
+
94
+ ::Process.kill('KILL', pid)
95
+ ::Process.waitpid(pid)
96
+
97
+ sleep(0.1)
98
+
99
+ self.deregister(pid, :proc_exit) rescue nil
100
+ rescue => e
101
+ puts e.message
102
+ puts e.backtrace.join("\n")
103
+ end
104
+ end.join
105
+
106
+ sleep(0.1)
107
+
108
+ com.first
109
+ end
110
+
111
+ end
112
+ end
@@ -0,0 +1,13 @@
1
+ module God
2
+ class DummyHandler
3
+ EVENT_SYSTEM = "none"
4
+
5
+ def self.register_process(pid, events)
6
+ raise NotImplementedError
7
+ end
8
+
9
+ def self.handle_events
10
+ raise NotImplementedError
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,17 @@
1
+ require 'kqueue_handler_ext'
2
+
3
+ module God
4
+ class KQueueHandler
5
+ EVENT_SYSTEM = "kqueue"
6
+
7
+ def self.register_process(pid, events)
8
+ monitor_process(pid, events_mask(events))
9
+ end
10
+
11
+ def self.events_mask(events)
12
+ events.inject(0) do |mask, event|
13
+ mask |= event_mask(event)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,13 @@
1
+ require 'netlink_handler_ext'
2
+
3
+ module God
4
+ class NetlinkHandler
5
+ EVENT_SYSTEM = "netlink"
6
+
7
+ def self.register_process(pid, events)
8
+ # netlink doesn't need to do this
9
+ # it just reads from the eventhandler actions to see if the pid
10
+ # matches the list we're looking for -- Kev
11
+ end
12
+ end
13
+ end
data/lib/god/logger.rb ADDED
@@ -0,0 +1,109 @@
1
+ module God
2
+
3
+ class Logger < SimpleLogger
4
+
5
+ attr_accessor :logs
6
+
7
+ class << self
8
+ attr_accessor :syslog
9
+ end
10
+
11
+ self.syslog = defined?(Syslog)
12
+
13
+ # Instantiate a new Logger object
14
+ def initialize(io = $stdout)
15
+ super(io)
16
+ self.logs = {}
17
+ @mutex = Mutex.new
18
+ @capture = nil
19
+ @spool = Time.now - 10
20
+ @templogio = StringIO.new
21
+ @templog = SimpleLogger.new(@templogio)
22
+ @templog.level = Logger::INFO
23
+ end
24
+
25
+
26
+ def level=(lev)
27
+ SysLogger.level = SimpleLogger::CONSTANT_TO_SYMBOL[lev] if Logger.syslog
28
+ super(lev)
29
+ end
30
+
31
+ # Log a message
32
+ # +watch+ is the String name of the Watch (may be nil if not Watch is applicable)
33
+ # +level+ is the log level [:debug|:info|:warn|:error|:fatal]
34
+ # +text+ is the String message
35
+ #
36
+ # Returns nothing
37
+ def log(watch, level, text)
38
+ # initialize watch log if necessary
39
+ self.logs[watch.name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT) if watch
40
+
41
+ # push onto capture and timeline for the given watch
42
+ if @capture || (watch && (Time.now - @spool < 2))
43
+ @mutex.synchronize do
44
+ @templogio.truncate(0)
45
+ @templogio.rewind
46
+ @templog.send(level, text)
47
+
48
+ message = @templogio.string.dup
49
+
50
+ if @capture
51
+ @capture.puts(message)
52
+ else
53
+ self.logs[watch.name] << [Time.now, message]
54
+ end
55
+ end
56
+ end
57
+
58
+ # send to regular logger
59
+ self.send(level, text)
60
+
61
+ # send to syslog
62
+ SysLogger.log(level, text) if Logger.syslog
63
+ end
64
+
65
+ # Get all log output for a given Watch since a certain Time.
66
+ # +watch_name+ is the String name of the Watch
67
+ # +since+ is the Time since which to fetch log lines
68
+ #
69
+ # Returns String
70
+ def watch_log_since(watch_name, since)
71
+ # initialize watch log if necessary
72
+ self.logs[watch_name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT)
73
+
74
+ # get and join lines since given time
75
+ @mutex.synchronize do
76
+ @spool = Time.now
77
+ self.logs[watch_name].select do |x|
78
+ x.first > since
79
+ end.map do |x|
80
+ x[1]
81
+ end.join
82
+ end
83
+ end
84
+
85
+ # private
86
+
87
+ # Enable capturing of log
88
+ #
89
+ # Returns nothing
90
+ def start_capture
91
+ @mutex.synchronize do
92
+ @capture = StringIO.new
93
+ end
94
+ end
95
+
96
+ # Disable capturing of log and return what was captured since
97
+ # capturing was enabled with Logger#start_capture
98
+ #
99
+ # Returns String
100
+ def finish_capture
101
+ @mutex.synchronize do
102
+ cap = @capture.string if @capture
103
+ @capture = nil
104
+ cap
105
+ end
106
+ end
107
+ end
108
+
109
+ end