mcproc 2016.2.20

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/Announce.txt +135 -0
  3. data/Gemfile +9 -0
  4. data/History.txt +469 -0
  5. data/LICENSE +22 -0
  6. data/README.md +37 -0
  7. data/Rakefile +185 -0
  8. data/TODO.md +37 -0
  9. data/bin/mcproc +134 -0
  10. data/doc/intro.asciidoc +20 -0
  11. data/doc/mcproc.asciidoc +1592 -0
  12. data/ext/god/.gitignore +5 -0
  13. data/ext/god/extconf.rb +56 -0
  14. data/ext/god/kqueue_handler.c +133 -0
  15. data/ext/god/netlink_handler.c +182 -0
  16. data/lib/god.rb +780 -0
  17. data/lib/god/behavior.rb +52 -0
  18. data/lib/god/behaviors/clean_pid_file.rb +21 -0
  19. data/lib/god/behaviors/clean_unix_socket.rb +21 -0
  20. data/lib/god/behaviors/notify_when_flapping.rb +51 -0
  21. data/lib/god/cli/command.rb +268 -0
  22. data/lib/god/cli/run.rb +170 -0
  23. data/lib/god/cli/version.rb +23 -0
  24. data/lib/god/compat19.rb +33 -0
  25. data/lib/god/condition.rb +96 -0
  26. data/lib/god/conditions/always.rb +36 -0
  27. data/lib/god/conditions/complex.rb +86 -0
  28. data/lib/god/conditions/cpu_usage.rb +80 -0
  29. data/lib/god/conditions/degrading_lambda.rb +52 -0
  30. data/lib/god/conditions/disk_usage.rb +32 -0
  31. data/lib/god/conditions/file_mtime.rb +28 -0
  32. data/lib/god/conditions/file_touched.rb +44 -0
  33. data/lib/god/conditions/flapping.rb +128 -0
  34. data/lib/god/conditions/http_response_code.rb +184 -0
  35. data/lib/god/conditions/lambda.rb +25 -0
  36. data/lib/god/conditions/memory_usage.rb +82 -0
  37. data/lib/god/conditions/process_exits.rb +66 -0
  38. data/lib/god/conditions/process_running.rb +63 -0
  39. data/lib/god/conditions/socket_responding.rb +142 -0
  40. data/lib/god/conditions/tries.rb +44 -0
  41. data/lib/god/configurable.rb +57 -0
  42. data/lib/god/contact.rb +114 -0
  43. data/lib/god/contacts/airbrake.rb +44 -0
  44. data/lib/god/contacts/campfire.rb +121 -0
  45. data/lib/god/contacts/email.rb +130 -0
  46. data/lib/god/contacts/hipchat.rb +117 -0
  47. data/lib/god/contacts/jabber.rb +75 -0
  48. data/lib/god/contacts/prowl.rb +57 -0
  49. data/lib/god/contacts/scout.rb +55 -0
  50. data/lib/god/contacts/sensu.rb +59 -0
  51. data/lib/god/contacts/slack.rb +98 -0
  52. data/lib/god/contacts/statsd.rb +46 -0
  53. data/lib/god/contacts/twitter.rb +51 -0
  54. data/lib/god/contacts/webhook.rb +74 -0
  55. data/lib/god/driver.rb +238 -0
  56. data/lib/god/errors.rb +24 -0
  57. data/lib/god/event_handler.rb +112 -0
  58. data/lib/god/event_handlers/dummy_handler.rb +13 -0
  59. data/lib/god/event_handlers/kqueue_handler.rb +17 -0
  60. data/lib/god/event_handlers/netlink_handler.rb +13 -0
  61. data/lib/god/logger.rb +109 -0
  62. data/lib/god/metric.rb +87 -0
  63. data/lib/god/process.rb +381 -0
  64. data/lib/god/registry.rb +32 -0
  65. data/lib/god/simple_logger.rb +59 -0
  66. data/lib/god/socket.rb +113 -0
  67. data/lib/god/sugar.rb +62 -0
  68. data/lib/god/sys_logger.rb +45 -0
  69. data/lib/god/system/portable_poller.rb +42 -0
  70. data/lib/god/system/process.rb +50 -0
  71. data/lib/god/system/slash_proc_poller.rb +92 -0
  72. data/lib/god/task.rb +552 -0
  73. data/lib/god/timeline.rb +25 -0
  74. data/lib/god/trigger.rb +43 -0
  75. data/lib/god/watch.rb +340 -0
  76. data/mcproc.gemspec +192 -0
  77. data/test/configs/child_events/child_events.god +44 -0
  78. data/test/configs/child_events/simple_server.rb +3 -0
  79. data/test/configs/child_polls/child_polls.god +37 -0
  80. data/test/configs/child_polls/simple_server.rb +12 -0
  81. data/test/configs/complex/complex.god +59 -0
  82. data/test/configs/complex/simple_server.rb +3 -0
  83. data/test/configs/contact/contact.god +118 -0
  84. data/test/configs/contact/simple_server.rb +3 -0
  85. data/test/configs/daemon_events/daemon_events.god +37 -0
  86. data/test/configs/daemon_events/simple_server.rb +8 -0
  87. data/test/configs/daemon_events/simple_server_stop.rb +11 -0
  88. data/test/configs/daemon_polls/daemon_polls.god +17 -0
  89. data/test/configs/daemon_polls/simple_server.rb +6 -0
  90. data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
  91. data/test/configs/degrading_lambda/tcp_server.rb +15 -0
  92. data/test/configs/keepalive/keepalive.god +9 -0
  93. data/test/configs/keepalive/keepalive.rb +12 -0
  94. data/test/configs/lifecycle/lifecycle.god +25 -0
  95. data/test/configs/matias/matias.god +50 -0
  96. data/test/configs/real.rb +59 -0
  97. data/test/configs/running_load/running_load.god +16 -0
  98. data/test/configs/stop_options/simple_server.rb +12 -0
  99. data/test/configs/stop_options/stop_options.god +39 -0
  100. data/test/configs/stress/simple_server.rb +3 -0
  101. data/test/configs/stress/stress.god +15 -0
  102. data/test/configs/task/logs/.placeholder +0 -0
  103. data/test/configs/task/task.god +26 -0
  104. data/test/configs/test.rb +61 -0
  105. data/test/configs/usr1_trapper.rb +10 -0
  106. data/test/helper.rb +172 -0
  107. data/test/suite.rb +6 -0
  108. data/test/test_airbrake.rb +14 -0
  109. data/test/test_behavior.rb +18 -0
  110. data/test/test_campfire.rb +22 -0
  111. data/test/test_condition.rb +52 -0
  112. data/test/test_conditions_disk_usage.rb +50 -0
  113. data/test/test_conditions_http_response_code.rb +109 -0
  114. data/test/test_conditions_process_running.rb +40 -0
  115. data/test/test_conditions_socket_responding.rb +176 -0
  116. data/test/test_conditions_tries.rb +67 -0
  117. data/test/test_contact.rb +109 -0
  118. data/test/test_driver.rb +26 -0
  119. data/test/test_email.rb +34 -0
  120. data/test/test_event_handler.rb +82 -0
  121. data/test/test_god.rb +710 -0
  122. data/test/test_god_system.rb +201 -0
  123. data/test/test_handlers_kqueue_handler.rb +16 -0
  124. data/test/test_hipchat.rb +23 -0
  125. data/test/test_jabber.rb +29 -0
  126. data/test/test_logger.rb +55 -0
  127. data/test/test_metric.rb +74 -0
  128. data/test/test_process.rb +263 -0
  129. data/test/test_prowl.rb +15 -0
  130. data/test/test_registry.rb +15 -0
  131. data/test/test_sensu.rb +11 -0
  132. data/test/test_slack.rb +57 -0
  133. data/test/test_socket.rb +34 -0
  134. data/test/test_statsd.rb +22 -0
  135. data/test/test_sugar.rb +42 -0
  136. data/test/test_system_portable_poller.rb +17 -0
  137. data/test/test_system_process.rb +30 -0
  138. data/test/test_task.rb +246 -0
  139. data/test/test_timeline.rb +37 -0
  140. data/test/test_trigger.rb +63 -0
  141. data/test/test_watch.rb +286 -0
  142. data/test/test_webhook.rb +22 -0
  143. metadata +475 -0
data/lib/god/driver.rb ADDED
@@ -0,0 +1,238 @@
1
+ require 'monitor'
2
+
3
+ # Ruby 1.9.1 specific fixes.
4
+ if RUBY_VERSION.between?('1.9', '1.9.1')
5
+ require 'god/compat19'
6
+ end
7
+
8
+ module God
9
+ # The TimedEvent class represents an event in the future. This class is used
10
+ # by the drivers to schedule upcoming conditional tests and other scheduled
11
+ # events.
12
+ class TimedEvent
13
+ include Comparable
14
+
15
+ # The Time at which this event is due.
16
+ attr_accessor :at
17
+
18
+ # Instantiate a new TimedEvent that will be triggered after the specified
19
+ # delay.
20
+ #
21
+ # delay - The optional Numeric number of seconds from now at which to
22
+ # trigger (default: 0).
23
+ def initialize(delay = 0)
24
+ self.at = Time.now + delay
25
+ end
26
+
27
+ # Is the current event due (current time >= event time)?
28
+ #
29
+ # Returns true if the event is due, false if not.
30
+ def due?
31
+ Time.now >= self.at
32
+ end
33
+
34
+ # Compare this event to another.
35
+ #
36
+ # other - The other TimedEvent.
37
+ #
38
+ # Returns -1 if this event is before the other, 0 if the two events are
39
+ # due at the same time, 1 if the other event is later.
40
+ def <=>(other)
41
+ self.at <=> other.at
42
+ end
43
+ end
44
+
45
+ # A DriverEvent is a TimedEvent with an associated Task and Condition. This
46
+ # is the primary mechanism for poll conditions to be scheduled.
47
+ class DriverEvent < TimedEvent
48
+ # Initialize a new DriverEvent.
49
+ #
50
+ # delay - The Numeric delay for this event.
51
+ # task - The Task associated with this event.
52
+ # condition - The Condition associated with this event.
53
+ def initialize(delay, task, condition)
54
+ super(delay)
55
+ @task = task
56
+ @condition = condition
57
+ end
58
+
59
+ # Handle this event by invoking the underlying condition on the associated
60
+ # task.
61
+ #
62
+ # Returns nothing.
63
+ def handle_event
64
+ @task.handle_poll(@condition)
65
+ end
66
+ end
67
+
68
+ # A DriverOperation is a TimedEvent that is due as soon as possible. It is
69
+ # used to execute an arbitrary method on the associated Task.
70
+ class DriverOperation < TimedEvent
71
+ # Initialize a new DriverOperation.
72
+ #
73
+ # task - The Task upon which to operate.
74
+ # name - The Symbol name of the method to call.
75
+ # args - The Array of arguments to send to the method.
76
+ def initialize(task, name, args)
77
+ super(0)
78
+ @task = task
79
+ @name = name
80
+ @args = args
81
+ end
82
+
83
+ # Handle the operation that was issued asynchronously.
84
+ #
85
+ # Returns nothing.
86
+ def handle_event
87
+ @task.send(@name, *@args)
88
+ end
89
+ end
90
+
91
+ # The DriverEventQueue is a simple queue that holds TimedEvent instances in
92
+ # order to maintain the schedule of upcoming events.
93
+ class DriverEventQueue
94
+ # Initialize a DriverEventQueue.
95
+ def initialize
96
+ @shutdown = false
97
+ @events = []
98
+ @monitor = Monitor.new
99
+ @resource = @monitor.new_cond
100
+ end
101
+
102
+ # Wake any sleeping threads after setting the sentinel.
103
+ #
104
+ # Returns nothing.
105
+ def shutdown
106
+ @shutdown = true
107
+ @monitor.synchronize do
108
+ @resource.broadcast
109
+ end
110
+ end
111
+
112
+ # Wait until the queue has something due, pop it off the queue, and return
113
+ # it.
114
+ #
115
+ # Returns the popped event.
116
+ def pop
117
+ @monitor.synchronize do
118
+ if @events.empty?
119
+ raise ThreadError, "queue empty" if @shutdown
120
+ @resource.wait
121
+ else
122
+ delay = @events.first.at - Time.now
123
+ @resource.wait(delay) if delay > 0
124
+ end
125
+
126
+ @events.shift
127
+ end
128
+ end
129
+
130
+ # Add an event to the queue, wake any waiters if what we added needs to
131
+ # happen sooner than the next pending event.
132
+ #
133
+ # Returns nothing.
134
+ def push(event)
135
+ @monitor.synchronize do
136
+ @events << event
137
+ @events.sort!
138
+
139
+ # If we've sorted the events and found the one we're adding is at
140
+ # the front, it will likely need to run before the next due date.
141
+ @resource.signal if @events.first == event
142
+ end
143
+ end
144
+
145
+ # Returns true if the queue is empty, false if not.
146
+ def empty?
147
+ @events.empty?
148
+ end
149
+
150
+ # Clear the queue.
151
+ #
152
+ # Returns nothing.
153
+ def clear
154
+ @events.clear
155
+ end
156
+
157
+ # Returns the Integer length of the queue.
158
+ def length
159
+ @events.length
160
+ end
161
+
162
+ alias size length
163
+ end
164
+
165
+ # The Driver class is responsible for scheduling all of the events for a
166
+ # given Task.
167
+ class Driver
168
+ # The Thread running the driver loop.
169
+ attr_reader :thread
170
+
171
+ # Instantiate a new Driver and start the scheduler loop to handle events.
172
+ #
173
+ # task - The Task this Driver belongs to.
174
+ def initialize(task)
175
+ @task = task
176
+ @events = God::DriverEventQueue.new
177
+
178
+ @thread = Thread.new do
179
+ loop do
180
+ begin
181
+ @events.pop.handle_event
182
+ rescue ThreadError => e
183
+ # queue is empty
184
+ break
185
+ rescue Object => e
186
+ message = format("Unhandled exception in driver loop - (%s): %s\n%s",
187
+ e.class, e.message, e.backtrace.join("\n"))
188
+ applog(nil, :fatal, message)
189
+ end
190
+ end
191
+ end
192
+ end
193
+
194
+ # Check if we're in the driver context.
195
+ #
196
+ # Returns true if in driver thread, false if not.
197
+ def in_driver_context?
198
+ Thread.current == @thread
199
+ end
200
+
201
+ # Clear all events for this Driver.
202
+ #
203
+ # Returns nothing.
204
+ def clear_events
205
+ @events.clear
206
+ end
207
+
208
+ # Shutdown the DriverEventQueue threads.
209
+ #
210
+ # Returns nothing.
211
+ def shutdown
212
+ @events.shutdown
213
+ end
214
+
215
+ # Queue an asynchronous message.
216
+ #
217
+ # name - The Symbol name of the operation.
218
+ # args - An optional Array of arguments.
219
+ #
220
+ # Returns nothing.
221
+ def message(name, args = [])
222
+ @events.push(DriverOperation.new(@task, name, args))
223
+ end
224
+
225
+ # Create and schedule a new DriverEvent.
226
+ #
227
+ # condition - The Condition.
228
+ # delay - The Numeric number of seconds to delay (default: interval
229
+ # defined in condition).
230
+ #
231
+ # Returns nothing.
232
+ def schedule(condition, delay = condition.interval)
233
+ applog(nil, :debug, "driver schedule #{condition} in #{delay} seconds")
234
+
235
+ @events.push(DriverEvent.new(delay, @task, condition))
236
+ end
237
+ end
238
+ end
data/lib/god/errors.rb ADDED
@@ -0,0 +1,24 @@
1
+ module God
2
+
3
+ class AbstractMethodNotOverriddenError < StandardError
4
+ end
5
+
6
+ class NoSuchWatchError < StandardError
7
+ end
8
+
9
+ class NoSuchConditionError < StandardError
10
+ end
11
+
12
+ class NoSuchBehaviorError < StandardError
13
+ end
14
+
15
+ class NoSuchContactError < StandardError
16
+ end
17
+
18
+ class InvalidCommandError < StandardError
19
+ end
20
+
21
+ class EventRegistrationFailedError < StandardError
22
+ end
23
+
24
+ end
@@ -0,0 +1,112 @@
1
+ module God
2
+ class EventHandler
3
+ @@actions = {}
4
+ @@handler = nil
5
+ @@loaded = false
6
+
7
+ def self.loaded?
8
+ @@loaded
9
+ end
10
+
11
+ def self.event_system
12
+ @@handler::EVENT_SYSTEM
13
+ end
14
+
15
+ def self.load
16
+ begin
17
+ case RUBY_PLATFORM
18
+ when /darwin/i, /bsd/i
19
+ require 'god/event_handlers/kqueue_handler'
20
+ @@handler = KQueueHandler
21
+ when /linux/i
22
+ require 'god/event_handlers/netlink_handler'
23
+ @@handler = NetlinkHandler
24
+ else
25
+ raise NotImplementedError, "Platform not supported for EventHandler"
26
+ end
27
+ @@loaded = true
28
+ rescue Exception
29
+ require 'god/event_handlers/dummy_handler'
30
+ @@handler = DummyHandler
31
+ @@loaded = false
32
+ end
33
+ end
34
+
35
+ def self.register(pid, event, &block)
36
+ @@actions[pid] ||= {}
37
+ @@actions[pid][event] = block
38
+ @@handler.register_process(pid, @@actions[pid].keys)
39
+ end
40
+
41
+ def self.deregister(pid, event)
42
+ if watching_pid? pid
43
+ running = ::Process.kill(0, pid.to_i) rescue false
44
+ @@actions[pid].delete(event)
45
+ @@handler.register_process(pid, @@actions[pid].keys) if running
46
+ @@actions.delete(pid) if @@actions[pid].empty?
47
+ end
48
+ end
49
+
50
+ def self.call(pid, event, extra_data = {})
51
+ @@actions[pid][event].call(extra_data) if watching_pid?(pid) && @@actions[pid][event]
52
+ end
53
+
54
+ def self.watching_pid?(pid)
55
+ @@actions[pid]
56
+ end
57
+
58
+ def self.start
59
+ @@thread = Thread.new do
60
+ loop do
61
+ begin
62
+ @@handler.handle_events
63
+ rescue Exception => e
64
+ message = format("Unhandled exception (%s): %s\n%s",
65
+ e.class, e.message, e.backtrace.join("\n"))
66
+ applog(nil, :fatal, message)
67
+ end
68
+ end
69
+ end
70
+
71
+ # do a real test to make sure events are working properly
72
+ @@loaded = self.operational?
73
+ end
74
+
75
+ def self.stop
76
+ @@thread.kill if @@thread
77
+ end
78
+
79
+ def self.operational?
80
+ com = [false]
81
+
82
+ Thread.new do
83
+ begin
84
+ event_system = God::EventHandler.event_system
85
+
86
+ pid = fork do
87
+ loop { sleep(1) }
88
+ end
89
+
90
+ self.register(pid, :proc_exit) do
91
+ com[0] = true
92
+ end
93
+
94
+ ::Process.kill('KILL', pid)
95
+ ::Process.waitpid(pid)
96
+
97
+ sleep(0.1)
98
+
99
+ self.deregister(pid, :proc_exit) rescue nil
100
+ rescue => e
101
+ puts e.message
102
+ puts e.backtrace.join("\n")
103
+ end
104
+ end.join
105
+
106
+ sleep(0.1)
107
+
108
+ com.first
109
+ end
110
+
111
+ end
112
+ end
@@ -0,0 +1,13 @@
1
+ module God
2
+ class DummyHandler
3
+ EVENT_SYSTEM = "none"
4
+
5
+ def self.register_process(pid, events)
6
+ raise NotImplementedError
7
+ end
8
+
9
+ def self.handle_events
10
+ raise NotImplementedError
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,17 @@
1
+ require 'kqueue_handler_ext'
2
+
3
+ module God
4
+ class KQueueHandler
5
+ EVENT_SYSTEM = "kqueue"
6
+
7
+ def self.register_process(pid, events)
8
+ monitor_process(pid, events_mask(events))
9
+ end
10
+
11
+ def self.events_mask(events)
12
+ events.inject(0) do |mask, event|
13
+ mask |= event_mask(event)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,13 @@
1
+ require 'netlink_handler_ext'
2
+
3
+ module God
4
+ class NetlinkHandler
5
+ EVENT_SYSTEM = "netlink"
6
+
7
+ def self.register_process(pid, events)
8
+ # netlink doesn't need to do this
9
+ # it just reads from the eventhandler actions to see if the pid
10
+ # matches the list we're looking for -- Kev
11
+ end
12
+ end
13
+ end
data/lib/god/logger.rb ADDED
@@ -0,0 +1,109 @@
1
+ module God
2
+
3
+ class Logger < SimpleLogger
4
+
5
+ attr_accessor :logs
6
+
7
+ class << self
8
+ attr_accessor :syslog
9
+ end
10
+
11
+ self.syslog = defined?(Syslog)
12
+
13
+ # Instantiate a new Logger object
14
+ def initialize(io = $stdout)
15
+ super(io)
16
+ self.logs = {}
17
+ @mutex = Mutex.new
18
+ @capture = nil
19
+ @spool = Time.now - 10
20
+ @templogio = StringIO.new
21
+ @templog = SimpleLogger.new(@templogio)
22
+ @templog.level = Logger::INFO
23
+ end
24
+
25
+
26
+ def level=(lev)
27
+ SysLogger.level = SimpleLogger::CONSTANT_TO_SYMBOL[lev] if Logger.syslog
28
+ super(lev)
29
+ end
30
+
31
+ # Log a message
32
+ # +watch+ is the String name of the Watch (may be nil if not Watch is applicable)
33
+ # +level+ is the log level [:debug|:info|:warn|:error|:fatal]
34
+ # +text+ is the String message
35
+ #
36
+ # Returns nothing
37
+ def log(watch, level, text)
38
+ # initialize watch log if necessary
39
+ self.logs[watch.name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT) if watch
40
+
41
+ # push onto capture and timeline for the given watch
42
+ if @capture || (watch && (Time.now - @spool < 2))
43
+ @mutex.synchronize do
44
+ @templogio.truncate(0)
45
+ @templogio.rewind
46
+ @templog.send(level, text)
47
+
48
+ message = @templogio.string.dup
49
+
50
+ if @capture
51
+ @capture.puts(message)
52
+ else
53
+ self.logs[watch.name] << [Time.now, message]
54
+ end
55
+ end
56
+ end
57
+
58
+ # send to regular logger
59
+ self.send(level, text)
60
+
61
+ # send to syslog
62
+ SysLogger.log(level, text) if Logger.syslog
63
+ end
64
+
65
+ # Get all log output for a given Watch since a certain Time.
66
+ # +watch_name+ is the String name of the Watch
67
+ # +since+ is the Time since which to fetch log lines
68
+ #
69
+ # Returns String
70
+ def watch_log_since(watch_name, since)
71
+ # initialize watch log if necessary
72
+ self.logs[watch_name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT)
73
+
74
+ # get and join lines since given time
75
+ @mutex.synchronize do
76
+ @spool = Time.now
77
+ self.logs[watch_name].select do |x|
78
+ x.first > since
79
+ end.map do |x|
80
+ x[1]
81
+ end.join
82
+ end
83
+ end
84
+
85
+ # private
86
+
87
+ # Enable capturing of log
88
+ #
89
+ # Returns nothing
90
+ def start_capture
91
+ @mutex.synchronize do
92
+ @capture = StringIO.new
93
+ end
94
+ end
95
+
96
+ # Disable capturing of log and return what was captured since
97
+ # capturing was enabled with Logger#start_capture
98
+ #
99
+ # Returns String
100
+ def finish_capture
101
+ @mutex.synchronize do
102
+ cap = @capture.string if @capture
103
+ @capture = nil
104
+ cap
105
+ end
106
+ end
107
+ end
108
+
109
+ end