cognizant 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.travis.yml +17 -0
- data/Gemfile +4 -1
- data/{LICENSE → License.md} +4 -2
- data/Rakefile +5 -0
- data/Readme.md +95 -0
- data/bin/cognizant +76 -122
- data/bin/cognizantd +28 -61
- data/cognizant.gemspec +8 -4
- data/examples/apps/redis-server.cz +42 -0
- data/examples/apps/redis-server.yml +29 -0
- data/examples/apps/redis-server_dsl.cz +54 -0
- data/examples/apps/resque.cz +17 -0
- data/examples/apps/thin.cz +32 -0
- data/examples/apps/thin.yml +48 -0
- data/examples/cognizantd.yml +18 -47
- data/features/child_process.feature +62 -0
- data/features/commands.feature +65 -0
- data/features/cpu_usage.feature +45 -0
- data/features/daemon.feature +12 -0
- data/features/flapping.feature +39 -0
- data/features/memory_usage.feature +45 -0
- data/features/shell.feature +30 -0
- data/features/step_definitions/common_steps.rb +14 -0
- data/features/step_definitions/daemon_steps.rb +25 -0
- data/features/step_definitions/shell_steps.rb +96 -0
- data/features/support/env.rb +54 -0
- data/lib/cognizant.rb +1 -5
- data/lib/cognizant/application.rb +122 -0
- data/lib/cognizant/application/dsl_proxy.rb +23 -0
- data/lib/cognizant/client.rb +61 -0
- data/lib/cognizant/commands.rb +164 -0
- data/lib/cognizant/commands/actions.rb +30 -0
- data/lib/cognizant/commands/help.rb +10 -0
- data/lib/cognizant/commands/load.rb +10 -0
- data/lib/cognizant/commands/shutdown.rb +7 -0
- data/lib/cognizant/commands/status.rb +11 -0
- data/lib/cognizant/commands/use.rb +15 -0
- data/lib/cognizant/controller.rb +17 -0
- data/lib/cognizant/daemon.rb +279 -0
- data/lib/cognizant/interface.rb +17 -0
- data/lib/cognizant/log.rb +25 -0
- data/lib/cognizant/process.rb +138 -94
- data/lib/cognizant/process/actions.rb +30 -41
- data/lib/cognizant/process/actions/restart.rb +73 -17
- data/lib/cognizant/process/actions/start.rb +35 -12
- data/lib/cognizant/process/actions/stop.rb +38 -17
- data/lib/cognizant/process/attributes.rb +41 -10
- data/lib/cognizant/process/children.rb +36 -0
- data/lib/cognizant/process/{condition_check.rb → condition_delegate.rb} +11 -13
- data/lib/cognizant/process/conditions.rb +7 -4
- data/lib/cognizant/process/conditions/cpu_usage.rb +5 -6
- data/lib/cognizant/process/conditions/memory_usage.rb +2 -6
- data/lib/cognizant/process/dsl_proxy.rb +23 -0
- data/lib/cognizant/process/execution.rb +16 -9
- data/lib/cognizant/process/pid.rb +16 -6
- data/lib/cognizant/process/status.rb +14 -2
- data/lib/cognizant/process/trigger_delegate.rb +57 -0
- data/lib/cognizant/process/triggers.rb +19 -0
- data/lib/cognizant/process/triggers/flapping.rb +68 -0
- data/lib/cognizant/process/triggers/transition.rb +22 -0
- data/lib/cognizant/process/triggers/trigger.rb +15 -0
- data/lib/cognizant/shell.rb +142 -0
- data/lib/cognizant/system.rb +16 -0
- data/lib/cognizant/system/ps.rb +1 -1
- data/lib/cognizant/system/signal.rb +2 -2
- data/lib/cognizant/util/dsl_proxy_methods_handler.rb +25 -0
- data/lib/cognizant/util/fixnum_percent.rb +5 -0
- data/lib/cognizant/util/transform_hash_keys.rb +33 -0
- data/lib/cognizant/validations.rb +142 -142
- data/lib/cognizant/version.rb +1 -1
- metadata +131 -71
- data/README.md +0 -221
- data/examples/redis-server.rb +0 -28
- data/examples/resque.rb +0 -10
- data/images/logo-small.png +0 -0
- data/images/logo.png +0 -0
- data/images/logo.pxm +0 -0
- data/lib/cognizant/logging.rb +0 -33
- data/lib/cognizant/process/conditions/flapping.rb +0 -57
- data/lib/cognizant/process/conditions/trigger_condition.rb +0 -52
- data/lib/cognizant/server.rb +0 -14
- data/lib/cognizant/server/commands.rb +0 -80
- data/lib/cognizant/server/daemon.rb +0 -277
- data/lib/cognizant/server/interface.rb +0 -86
- data/lib/cognizant/util/symbolize_hash_keys.rb +0 -19
@@ -1,24 +1,34 @@
|
|
1
1
|
module Cognizant
|
2
2
|
class Process
|
3
3
|
module PID
|
4
|
+
def cached_pid
|
5
|
+
if not @process_pid or @process_pid == 0
|
6
|
+
read_pid
|
7
|
+
else
|
8
|
+
@process_pid
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
4
12
|
def read_pid
|
5
13
|
if self.pid_command
|
6
14
|
str = execute(self.pid_command).stdout.to_i
|
7
|
-
|
15
|
+
process_pid = str unless not str or str.zero?
|
8
16
|
# TODO: Write pid to pidfile, since our source was pid_command instead.
|
9
17
|
elsif self.pidfile and File.exists?(self.pidfile)
|
10
18
|
str = File.read(self.pidfile).to_i
|
11
|
-
|
19
|
+
process_pid = str unless not str or str.zero?
|
12
20
|
end
|
13
|
-
|
14
|
-
@process_pid
|
21
|
+
process_pid = 0 unless Cognizant::System.pid_running?(process_pid) # If the newly fetched pid is not running, reset it.
|
22
|
+
@process_pid = process_pid
|
15
23
|
end
|
16
24
|
|
25
|
+
# Note: Expected that this method is not called to overwrite pidfile if the process daemonizes itself (and hence manages the pidfile by itself).
|
17
26
|
def write_pid(pid = nil)
|
18
|
-
@process_pid = pid
|
19
|
-
File.open(self.pidfile, "w") { |f| f.write(@process_pid) } if self.pidfile and @process_pid
|
27
|
+
@process_pid = pid
|
28
|
+
File.open(self.pidfile, "w") { |f| f.write(@process_pid) } if self.pidfile and @process_pid and @process_pid != 0
|
20
29
|
end
|
21
30
|
|
31
|
+
# Note: Expected that this method is not called to unlink pidfile if the process daemonizes itself (and hence manages the pidfile by itself).
|
22
32
|
def unlink_pid
|
23
33
|
File.unlink(self.pidfile) if self.pidfile
|
24
34
|
rescue Errno::ENOENT
|
@@ -3,12 +3,24 @@ require "cognizant/system"
|
|
3
3
|
module Cognizant
|
4
4
|
class Process
|
5
5
|
module Status
|
6
|
+
def process_running?
|
7
|
+
@process_running = begin
|
8
|
+
if @ping_command and run(@ping_command).succeeded?
|
9
|
+
true
|
10
|
+
elsif pid_running?
|
11
|
+
true
|
12
|
+
else
|
13
|
+
false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
6
18
|
def pid_running?
|
7
|
-
Cognizant::System.pid_running?(
|
19
|
+
Cognizant::System.pid_running?(cached_pid)
|
8
20
|
end
|
9
21
|
|
10
22
|
def signal(signal, pid = nil)
|
11
|
-
Cognizant::System.signal(signal, (pid ||
|
23
|
+
Cognizant::System.signal(signal, (pid || cached_pid))
|
12
24
|
end
|
13
25
|
end
|
14
26
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "cognizant/process/triggers"
|
2
|
+
|
3
|
+
module Cognizant
|
4
|
+
class Process
|
5
|
+
class TriggerDelegate
|
6
|
+
attr_accessor :name, :process, :mutex, :scheduled_events
|
7
|
+
|
8
|
+
def initialize(name, process, options = {}, &block)
|
9
|
+
@name, @process = name, process
|
10
|
+
@mutex = Mutex.new
|
11
|
+
@scheduled_events = []
|
12
|
+
|
13
|
+
@trigger = Cognizant::Process::Triggers[@name].new(options, &block)
|
14
|
+
# TODO: This is hackish even though it keeps trigger implementations simple.
|
15
|
+
@trigger.instance_variable_set(:@delegate, self)
|
16
|
+
end
|
17
|
+
|
18
|
+
def notify(transition)
|
19
|
+
@trigger.notify(transition)
|
20
|
+
end
|
21
|
+
|
22
|
+
def reset!
|
23
|
+
@trigger.reset!
|
24
|
+
self.cancel_all_events
|
25
|
+
end
|
26
|
+
|
27
|
+
def dispatch!(event)
|
28
|
+
@process.dispatch!(event, @name)
|
29
|
+
end
|
30
|
+
|
31
|
+
def schedule_event(event, delay)
|
32
|
+
# TODO: Maybe wrap this in a ScheduledEvent class with methods like cancel.
|
33
|
+
thread = Thread.new(self) do |trigger|
|
34
|
+
begin
|
35
|
+
sleep(delay)
|
36
|
+
trigger.dispatch!(event)
|
37
|
+
trigger.mutex.synchronize do
|
38
|
+
trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
|
39
|
+
end
|
40
|
+
rescue StandardError => e
|
41
|
+
puts(e)
|
42
|
+
puts(e.backtrace.join("\n"))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
@scheduled_events.push([event, thread])
|
47
|
+
end
|
48
|
+
|
49
|
+
def cancel_all_events
|
50
|
+
Log[self].debug "Canceling all scheduled events"
|
51
|
+
@mutex.synchronize do
|
52
|
+
@scheduled_events.each {|_, thread| thread.kill}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require "cognizant/process/triggers/trigger"
|
2
|
+
|
3
|
+
Dir["#{File.dirname(__FILE__)}/triggers/*.rb"].each do |trigger|
|
4
|
+
require trigger
|
5
|
+
end
|
6
|
+
|
7
|
+
module Cognizant
|
8
|
+
class Process
|
9
|
+
module Triggers
|
10
|
+
def self.[](name)
|
11
|
+
begin
|
12
|
+
const_get(name.to_s.camelcase)
|
13
|
+
rescue NameError
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require "cognizant/util/rotational_array"
|
2
|
+
|
3
|
+
module Cognizant
|
4
|
+
class Process
|
5
|
+
module Triggers
|
6
|
+
class Flapping < Trigger
|
7
|
+
TRIGGER_STATES = [:starting, :restarting]
|
8
|
+
|
9
|
+
attr_accessor :times, :within, :retry_after, :retries
|
10
|
+
attr_reader :timeline
|
11
|
+
|
12
|
+
def initialize(options = {})
|
13
|
+
@times = options[:times] || 5
|
14
|
+
@within = options[:within] || 1
|
15
|
+
@retry_after = options[:retry_after] || 5
|
16
|
+
@retries = options[:retries] || 0
|
17
|
+
|
18
|
+
@timeline = Util::RotationalArray.new(@times)
|
19
|
+
@num_of_tries = 0
|
20
|
+
end
|
21
|
+
|
22
|
+
def notify(transition)
|
23
|
+
if TRIGGER_STATES.include?(transition.to_name)
|
24
|
+
self.timeline << Time.now.to_i
|
25
|
+
self.check_flapping
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def reset!
|
30
|
+
@timeline.clear
|
31
|
+
@num_of_tries = 0
|
32
|
+
end
|
33
|
+
|
34
|
+
def within_duration?
|
35
|
+
(@timeline.last - @timeline.first) <= self.within
|
36
|
+
end
|
37
|
+
|
38
|
+
def can_retry?
|
39
|
+
# retry_after = 0 means do not retry.
|
40
|
+
self.retry_after > 0 and
|
41
|
+
# retries = 0 means always retry.
|
42
|
+
(self.retries == 0 or (self.retries > 0 and @num_of_tries <= self.retries))
|
43
|
+
end
|
44
|
+
|
45
|
+
def check_flapping
|
46
|
+
# The process has not flapped if we haven't encountered enough incidents.
|
47
|
+
return unless (@timeline.compact.length == self.times)
|
48
|
+
|
49
|
+
# Check if the incident happend within the timeframe.
|
50
|
+
if within_duration?
|
51
|
+
@num_of_tries += 1
|
52
|
+
|
53
|
+
Log[self].debug "Flapping detected (##{@num_of_tries}) for #{@delegate.process.name}(pid:#{@delegate.process.cached_pid})."
|
54
|
+
|
55
|
+
# 0.1 to ensure the state isn't randomly caught in throw :halt below.
|
56
|
+
@delegate.schedule_event(:unmonitor, [0.1, self.retry_after].min)
|
57
|
+
@delegate.schedule_event(:start, self.retry_after) if can_retry?
|
58
|
+
|
59
|
+
@timeline.clear
|
60
|
+
|
61
|
+
# This will prevent a transition from happening in the process state_machine.
|
62
|
+
throw :halt
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Cognizant
|
2
|
+
class Process
|
3
|
+
module Triggers
|
4
|
+
class Transition < Trigger
|
5
|
+
def initialize(options = {}, &block)
|
6
|
+
@from = [*options[:from]]
|
7
|
+
@to = [*options[:to]]
|
8
|
+
@do = block
|
9
|
+
end
|
10
|
+
|
11
|
+
def notify(transition)
|
12
|
+
if @from.include?(transition.from_name) and @to.include?(transition.to_name)
|
13
|
+
@do.call(@delegate.process) if @do and @do.respond_to?(:call)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def reset!
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "optparse"
|
3
|
+
|
4
|
+
require "readline"
|
5
|
+
require "shellwords"
|
6
|
+
|
7
|
+
require "cognizant/client"
|
8
|
+
|
9
|
+
module Cognizant
|
10
|
+
class Shell
|
11
|
+
def initialize(options = {})
|
12
|
+
@app = ""
|
13
|
+
@app = options[:app] if options.has_key?(:app) and options[:app].to_s.size > 0
|
14
|
+
|
15
|
+
@path_to_socket = "/var/run/cognizant/cognizantd.sock"
|
16
|
+
@path_to_socket = options[:socket] if options.has_key?(:socket) and options[:socket].to_s.size > 0
|
17
|
+
|
18
|
+
@@is_shell = true
|
19
|
+
@@is_shell = options[:shell] if options.has_key?(:shell)
|
20
|
+
|
21
|
+
@autocomplete_keywords = []
|
22
|
+
connect
|
23
|
+
end
|
24
|
+
|
25
|
+
def run(&block)
|
26
|
+
Signal.trap("INT") do
|
27
|
+
Cognizant::Shell.emit("\nGoodbye!")
|
28
|
+
exit(0)
|
29
|
+
end
|
30
|
+
|
31
|
+
emit("Enter 'help' if you're not sure what to do.")
|
32
|
+
emit
|
33
|
+
emit("Type 'quit' or 'exit' to quit at any time.")
|
34
|
+
|
35
|
+
setup_readline(&block)
|
36
|
+
end
|
37
|
+
|
38
|
+
def setup_readline(&block)
|
39
|
+
Readline.completion_proc = Proc.new do |input|
|
40
|
+
case input
|
41
|
+
when /^\//
|
42
|
+
# Handle file and directory name autocompletion.
|
43
|
+
Readline.completion_append_character = "/"
|
44
|
+
Dir[input + '*'].grep(/^#{Regexp.escape(input)}/)
|
45
|
+
else
|
46
|
+
# Handle commands and process name autocompletion.
|
47
|
+
Readline.completion_append_character = " "
|
48
|
+
(@autocomplete_keywords + ['quit', 'exit']).grep(/^#{Regexp.escape(input)}/)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
while line = Readline.readline(prompt, true).to_s.strip
|
53
|
+
if line.size > 0
|
54
|
+
command, args = parse_command(line)
|
55
|
+
return emit("Goodbye!") if ['quit', 'exit'].include?(command)
|
56
|
+
run_command(command, args, &block)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def prompt
|
62
|
+
@app.to_s.size > 0 ? "(#{@app})> " : "> "
|
63
|
+
end
|
64
|
+
|
65
|
+
def run_command(command, args, &block)
|
66
|
+
command = command.to_s
|
67
|
+
|
68
|
+
begin
|
69
|
+
response = @client.command({'command' => command, 'args' => args, 'app' => @app})
|
70
|
+
rescue Errno::EPIPE => e
|
71
|
+
emit("cognizant: Error communicating with cognizantd: #{e} (#{e.class})")
|
72
|
+
exit(1)
|
73
|
+
end
|
74
|
+
|
75
|
+
@app = response["use"] if response.is_a?(Hash) and response.has_key?("use")
|
76
|
+
|
77
|
+
if block
|
78
|
+
block.call(response, command)
|
79
|
+
elsif response.kind_of?(Hash)
|
80
|
+
puts response['message']
|
81
|
+
else
|
82
|
+
puts "Invalid response type #{response.class}: #{response.inspect}"
|
83
|
+
end
|
84
|
+
|
85
|
+
fetch_autocomplete_keywords
|
86
|
+
end
|
87
|
+
|
88
|
+
def parse_command(line)
|
89
|
+
command, *args = Shellwords.shellsplit(line)
|
90
|
+
[command, args]
|
91
|
+
end
|
92
|
+
|
93
|
+
def connect
|
94
|
+
begin
|
95
|
+
@client = Cognizant::Client.for_path(@path_to_socket)
|
96
|
+
rescue Errno::ENOENT => e
|
97
|
+
# TODO: The exit here is a biit of a layering violation.
|
98
|
+
Cognizant::Shell.emit(<<EOF, true)
|
99
|
+
Could not connect to Cognizant daemon process:
|
100
|
+
|
101
|
+
#{e}
|
102
|
+
|
103
|
+
HINT: Are you sure you are running the Cognizant daemon? If so, you
|
104
|
+
should pass cognizant the socket argument provided to cognizantd.
|
105
|
+
EOF
|
106
|
+
exit(1)
|
107
|
+
end
|
108
|
+
ehlo if interactive?
|
109
|
+
fetch_autocomplete_keywords
|
110
|
+
end
|
111
|
+
|
112
|
+
def ehlo
|
113
|
+
response = @client.command('command' => '_ehlo', 'user' => ENV['USER'], 'app' => @app)
|
114
|
+
@app = response["use"] if response.is_a?(Hash) and response.has_key?("use")
|
115
|
+
|
116
|
+
emit(response['message'])
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_autocomplete_keywords
|
120
|
+
return unless @@is_shell
|
121
|
+
@autocomplete_keywords = @client.command('command' => '_autocomplete_keywords', 'app' => @app)
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.emit(message = nil, force = false)
|
125
|
+
$stdout.puts(message || '') if interactive? || force
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.interactive?
|
129
|
+
# TODO: It is not a tty during tests.
|
130
|
+
# $stdin.isatty and @@is_shell
|
131
|
+
@@is_shell
|
132
|
+
end
|
133
|
+
|
134
|
+
def emit(*args)
|
135
|
+
self.class.emit(*args)
|
136
|
+
end
|
137
|
+
|
138
|
+
def interactive?
|
139
|
+
self.class.interactive?
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
data/lib/cognizant/system.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require "fileutils"
|
2
|
+
|
1
3
|
require "cognizant/system/signal"
|
2
4
|
require "cognizant/system/ps"
|
3
5
|
|
@@ -24,5 +26,19 @@ module Cognizant
|
|
24
26
|
# Possibly running.
|
25
27
|
true
|
26
28
|
end
|
29
|
+
|
30
|
+
def unlink_file(path)
|
31
|
+
begin
|
32
|
+
File.unlink(path) if path
|
33
|
+
rescue Errno::ENOENT
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def mkdir(*directories)
|
39
|
+
[*directories].each do |directory|
|
40
|
+
FileUtils.mkdir_p(File.expand_path(directory))
|
41
|
+
end
|
42
|
+
end
|
27
43
|
end
|
28
44
|
end
|
data/lib/cognizant/system/ps.rb
CHANGED
@@ -14,8 +14,8 @@ module Cognizant
|
|
14
14
|
# Return if the process is not running.
|
15
15
|
return true unless pid_running?(pid)
|
16
16
|
|
17
|
-
signals = options[:signals] || ["TERM", "INT"
|
18
|
-
timeout = options[:timeout] ||
|
17
|
+
signals = options[:signals] || ["TERM", "INT"]
|
18
|
+
timeout = options[:timeout] || 30
|
19
19
|
|
20
20
|
catch :stopped do
|
21
21
|
signals.each do |stop_signal|
|