cognizant 0.0.1 → 0.0.2

data/lib/cognizant/process/actions.rb

@@ -1,6 +1,7 @@
 require "cognizant/process/actions/start"
 require "cognizant/process/actions/stop"
 require "cognizant/process/actions/restart"
+require "cognizant/system"
 
 module Cognizant
   class Process
@@ -55,7 +56,7 @@ module Cognizant
             end
             sleep 1
           end
-        
+
           # Kill the nested thread.
           thread.kill
 
@@ -73,22 +74,7 @@ module Cognizant
       def send_signals(options = {})
         # Return if the process is already stopped.
         return true unless pid_running?
-
-        signals = options[:signals] || ["TERM", "INT", "KILL"]
-        timeout = options[:timeout] || 10
-
-        catch :stopped do
-          signals.each do |stop_signal|
-            # Send the stop signal and wait for it to stop.
-            signal(stop_signal, @process_pid)
-
-            # Poll to see if it's stopped yet. Minimum 2 so that we check at least once again.
-            ([timeout / signals.size, 2].max).times do
-              throw :stopped unless pid_running?
-              sleep 1
-            end
-          end
-        end
+        Cognizant::System.send_signals(@process_pid, options)
         not pid_running?
       end
     end

data/lib/cognizant/process/condition_check.rb

@@ -0,0 +1,59 @@
+require "cognizant/process/conditions"
+require "cognizant/util/rotational_array"
+
+module Cognizant
+  class Process
+    class ConditionCheck
+      class HistoryValue < Struct.new(:value, :critical); end
+
+      # No need to recreate one every tick.
+      EMPTY_ARRAY = [].freeze
+
+      attr_accessor :condition_name
+      def initialize(condition_name, options = {}, &block)
+        @condition_name = condition_name
+
+        if block
+          @do = Array(block)
+        else
+          @do = options.has_key?(:do) ? Array(options.delete(:do)) : [:restart]
+        end
+
+        @every = options.delete(:every).to_i
+        @times = options.delete(:times) || [1, 1]
+        @times = [@times, @times] unless @times.is_a?(Array) # handles :times => 5
+        @times.map(&:to_i)
+
+        clear_history!
+
+        @condition = Cognizant::Process::Conditions[@condition_name].new(options)
+      end
+
+      def run(pid, tick_number = Time.now.to_i)
+        if @last_ran_at.nil? || (@last_ran_at + @every) <= tick_number
+          @last_ran_at = tick_number
+
+          value = @condition.run(pid)
+          @history << HistoryValue.new(@condition.format_value(value), @condition.check(value))
+          # puts self.to_s
+
+          return @do if failed_check?
+        end
+        EMPTY_ARRAY
+      end
+
+      def clear_history!
+        @history = Util::RotationalArray.new(@times.last)
+      end
+
+      def failed_check?
+        @history.count { |v| v and v.critical } >= @times.first
+      end
+
+      def to_s
+        data = @history.collect { |v| v and "#{v.value}#{'*' unless v.critical}" }.join(", ")
+        "#{@condition_name}: [#{data}]\n"
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/always_true.rb

@@ -0,0 +1,15 @@
+module Cognizant
+  class Process
+    module Conditions
+      class AlwaysTrue < PollCondition
+        def run(pid)
+          1
+        end
+
+        def check(value)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/cpu_usage.rb

@@ -0,0 +1,21 @@
+require "cognizant/system"
+
+module Cognizant
+  class Process
+    module Conditions
+      class CpuUsage < PollCondition
+        def initialize(options = {})
+          @above = options[:above].to_f
+        end
+
+        def run(pid)
+          System.cpu_usage(pid).to_f
+        end
+
+        def check(value)
+          value > @above
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/flapping.rb

@@ -0,0 +1,57 @@
+require "cognizant/util/rotational_array"
+
+module Cognizant
+  class Process
+    module Conditions
+      class Flapping < TriggerCondition
+        TRIGGER_STATES = [:starting, :restarting]
+
+        attr_accessor :times, :within, :retry_after
+        attr_reader :timeline
+
+        def initialize(process, options = {})
+          options = { :times => 5, :within => 1, :retry_after => 5 }.merge(options)
+
+          options.each_pair do |name, value|
+            self.send("#{name}=", value) if self.respond_to?("#{name}=")
+          end
+
+          @timeline = Util::RotationalArray.new(@times)
+          super
+        end
+
+        def notify(transition)
+          if TRIGGER_STATES.include?(transition.to_name)
+            self.timeline << Time.now.to_i
+            self.check_flapping
+          end
+        end
+
+        def reset!
+          @timeline.clear
+          super
+        end
+
+        def check_flapping
+          # The process has not flapped if we haven't encountered enough incidents.
+          return unless (@timeline.compact.length == self.times)
+
+          # Check if the incident happend within the timeframe.
+          duration = (@timeline.last - @timeline.first) <= self.within
+
+          if duration
+            puts "Flapping detected: retrying in #{self.retry_after} seconds"
+
+            self.schedule_event(:start, self.retry_after) unless self.retry_after == 0 # retry_after zero means "do not retry, ever".
+            self.schedule_event(:unmonitor, 0)
+
+            @timeline.clear
+
+            # This will prevent a transition from happening in the process state_machine.
+            throw :halt
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/memory_usage.rb

@@ -0,0 +1,32 @@
+module Cognizant
+  class Process
+    module Conditions
+      class MemoryUsage < PollCondition
+        MB = 1024 ** 2
+        FORMAT_STR = "%d%s"
+        MB_LABEL = "MB"
+        KB_LABEL = "KB"
+
+        def initialize(options = {})
+          @above = options[:above].to_f
+        end
+
+        def run(pid)
+          System.memory_usage(pid).to_f
+        end
+
+        def check(value)
+          value.kilobytes > @above
+        end
+
+        def format_value(value)
+          if value.kilobytes >= MB
+            FORMAT_STR % [(value / 1024).round, MB_LABEL]
+          else
+            FORMAT_STR % [value, KB_LABEL]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/poll_condition.rb

@@ -0,0 +1,23 @@
+module Cognizant
+  class Process
+    module Conditions
+      class PollCondition
+        def initialize(options = {})
+          @options = options
+        end
+
+        def run(pid)
+          raise "Implement in subclass!"
+        end
+
+        def check(value)
+          raise "Implement in subclass!"
+        end
+
+        def format_value(value)
+          value
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions/trigger_condition.rb

@@ -0,0 +1,52 @@
+module Cognizant
+  class Process
+    module Conditions
+      class TriggerCondition
+        attr_accessor :process, :mutex, :scheduled_events
+
+        def initialize(process, options = {})
+          self.process = process
+          self.mutex = Mutex.new
+          self.scheduled_events = []
+        end
+
+        def reset!
+          self.cancel_all_events
+        end
+
+        def notify(transition)
+          raise "Implement in subclass"
+        end
+
+        def dispatch!(event)
+          self.process.dispatch!(event, self.class.name.split("::").last)
+        end
+
+        def schedule_event(event, delay)
+          # TODO: Maybe wrap this in a ScheduledEvent class with methods like cancel.
+          thread = Thread.new(self) do |trigger|
+            begin
+              sleep delay.to_f
+              trigger.dispatch!(event)
+              trigger.mutex.synchronize do
+                trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+              end
+            rescue StandardError => e
+              puts(e)
+              puts(e.backtrace.join("\n"))
+            end
+          end
+
+          self.scheduled_events.push([event, thread])
+        end
+
+        def cancel_all_events
+          puts "Canceling all scheduled events"
+          self.mutex.synchronize do
+            self.scheduled_events.each {|_, thread| thread.kill}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cognizant/process/conditions.rb

@@ -0,0 +1,16 @@
+require "cognizant/process/conditions/poll_condition"
+require "cognizant/process/conditions/trigger_condition"
+
+Dir["#{File.dirname(__FILE__)}/conditions/*.rb"].each do |c|
+  require c
+end
+
+module Cognizant
+  class Process
+    module Conditions
+      def self.[](name)
+        const_get(name.to_s.camelcase)
+      end
+    end
+  end
+end

data/lib/cognizant/process/execution.rb

@@ -65,11 +65,11 @@ module Cognizant
           end
 
           # Merge spawn options.
-        	spawn_options = construct_spawn_options(options, {
-        		:in  => stdin,
-        		:out => stdout,
-        		:err => stderr
-        	})
+          spawn_options = construct_spawn_options(options, {
+            :in  => stdin,
+            :out => stdout,
+            :err => stderr
+          })
 
           # Spawn a process to execute the command.
           process_pid = ::Process.spawn(options[:env], command, spawn_options)

data/lib/cognizant/process/pid.rb

@@ -4,11 +4,13 @@ module Cognizant
       def read_pid
         if self.pid_command
           str = execute(self.pid_command).stdout.to_i
-          @process_pid = str unless not str or str.zero? # TODO: Also check if the pid is alive.
+          @process_pid = str unless not str or str.zero?
+          # TODO: Write pid to pidfile, since our source was pid_command instead.
         elsif self.pidfile and File.exists?(self.pidfile)
           str = File.read(self.pidfile).to_i
-          @process_pid = str unless not str or str.zero? # TODO: Also check if the pid is alive.
+          @process_pid = str unless not str or str.zero?
         end
+        @process_pid = 0 unless System.pid_running?(@process_pid)
         @process_pid
       end
 

data/lib/cognizant/process/status.rb

@@ -1,26 +1,14 @@
+require "cognizant/system"
+
 module Cognizant
   class Process
     module Status
       def pid_running?
-        pid = read_pid
-        return false unless pid and pid != 0
-        signal(0, pid)
-        # It's running since no exception was raised.
-        true
-      rescue Errno::ESRCH
-        # No such process.
-        false
-      rescue Errno::EPERM
-        # Probably running, but we're not allowed to pass signals.
-        # TODO: Is this a sign of problems ahead?
-        true
-      else
-        # Possibly running.
-        true
+        Cognizant::System.pid_running?(read_pid)
       end
 
       def signal(signal, pid = nil)
-        ::Process.kill(signal, (pid || read_pid))
+        Cognizant::System.signal(signal, (pid || read_pid))
       end
     end
   end

data/lib/cognizant/process.rb

@@ -1,10 +1,15 @@
-require 'state_machine'
+require "monitor"
+require "thread"
+
+require "state_machine"
 
 require "cognizant/process/pid"
 require "cognizant/process/status"
 require "cognizant/process/execution"
 require "cognizant/process/attributes"
 require "cognizant/process/actions"
+require "cognizant/process/condition_check"
+require "cognizant/util/symbolize_hash_keys"
 
 module Cognizant
   class Process
@@ -66,21 +71,30 @@ module Cognizant
       before_transition any      => :restarting, :do => lambda { |p| p.autostart = true }
       after_transition  any      => :restarting, :do => :restart_process
 
-      before_transition any => any, :do => :record_transition_start
-      after_transition  any => any, :do => :record_transition_end
+      before_transition any => any, :do => :notify_triggers
+      after_transition  any => any, :do => :record_transition
     end
 
     def initialize(process_name = nil, options = {})
-      # Default.
-      self.autostart = true
-      self.name = process_name if process_name
+      @ticks_to_skip = 0
+      @checks = []
+      @triggers = []
+      @action_mutex = Monitor.new
+      
+      self.name = process_name.to_s if process_name
+      self.autostart = true # Default.
+
+      if options.has_key?(:checks) and options[:checks].kind_of?(Hash)
+        options[:checks].each do |condition_name, args|
+          self.check(condition_name, args)
+        end
+      end
+      options.delete(:checks)
 
       options.each do |attribute_name, value|
         self.send("#{attribute_name}=", value) if self.respond_to?("#{attribute_name}=")
       end
 
-      @ticks_to_skip = 0
-
       yield(self) if block_given?
 
       # Let state_machine initialize as well.
@@ -93,14 +107,80 @@ module Cognizant
 
       # Invoke the state_machine event.
       super
+
+      self.run_checks if self.running?
+    end
+
+    def record_transition(transition)
+      unless transition.loopback?
+        @transitioned = true
+        @last_transition_time = Time.now.to_i
+
+        # When a process changes state, we should clear the memory of all the checks.
+        @checks.each { |check| check.clear_history! }
+        puts "#{name} changing from #{transition.from_name} => #{transition.to_name}"
+      end
+    end
+
+    def last_transition_time
+      @last_transition_time || 0
+    end
+
+    def handle_user_command(command)
+      if command == :unmonitor
+        # When the user issues an unmonitor command, reset any
+        # triggers so that scheduled events gets cleared.
+        @triggers.each { |trigger| trigger.reset! }
+      end
+      dispatch!(command, "user initiated")
+    end
+
+    def dispatch!(action, reason = nil)
+      @action_mutex.synchronize do
+        if action.respond_to?(:call)
+          action.call(self)
+        else
+          self.send("#{action}")
+        end
+      end
     end
 
-    def record_transition_start
-      print "#{name}: changing state from `#{state}`"
+    def check(condition_name, options, &block)
+      klass = Cognizant::Process::Conditions[condition_name]
+      case klass.superclass.name.split("::").last
+      when "TriggerCondition"
+        @triggers << klass.new(self, options.deep_symbolize_keys!)
+      when "PollCondition"
+        @checks << ConditionCheck.new(condition_name, options.deep_symbolize_keys!, &block)
+      end
     end
 
-    def record_transition_end
-      puts " to `#{state}`"
+    def notify_triggers(transition)
+      @triggers.each { |trigger| trigger.notify(transition) }
+    end
+
+    def run_checks
+      now = Time.now.to_i
+
+      threads = @checks.collect do |check|
+        [check, Thread.new { Thread.current[:actions] = check.run(read_pid, now) }]
+      end
+
+      @transitioned = false
+
+      threads.inject([]) do |actions, (check, thread)|
+        thread.join
+        if thread[:actions].size > 0
+          puts "#{check.condition_name} dispatched: #{thread[:actions].join(',')}"
+          thread[:actions].each do |action|
+            actions << [action, check.to_s]
+          end
+        end
+        actions
+      end.each do |(action, reason)|
+        break if @transitioned
+        self.dispatch!(action, reason)
+      end
     end
 
     def process_running?

data/lib/cognizant/server/commands.rb

@@ -1,52 +1,79 @@
+require "json"
+require "cognizant/system"
+
 module Cognizant
   module Server
     module Commands
       def self.load(config_file)
         Cognizant::Server.daemon.load(config_file)
-        yield("OK")
+        # yield("OK")
       end
 
       def self.status(*args)
-        if process_name = args.shift
-          Cognizant::Server.daemon.processes.each do |name, process|
-            if process.name.eql?(process_name)
-              yield("#{process.name}: #{process.state}")
-              return yield("OK")
-            end
-          end  
-          yield("ERR: No such process")
-          return yield("OK")
+        output_processes = []
+        if args.size > 0
+          Cognizant::Server.daemon.processes.values.each do |process|
+            output_processes << process if args.include?(process.name) or args.include?(process.group)
+          end
+          if output_processes.size == 0
+            raise("ERROR: No such process")
+            # yield("OK")
+          end
+        else
+          output_processes = Cognizant::Server.daemon.processes.values
         end
-        yield("OK")
+        
+        output = []
+        output_processes.each do |process|
+          pid = process.read_pid
+          output << {
+            "Process" => process.name,
+            "PID"     => pid,
+            "Group"   => process.group,
+            "State"   => process.state,
+            "Since"   => process.last_transition_time,
+            "% CPU"   => System.cpu_usage(pid).to_f,
+            "Memory"  => System.memory_usage(pid).to_f # in KBs.
+          }
+        end
+        yield(output.to_json)
+        # yield("OK")
       end
 
       %w(monitor start stop restart unmonitor).each do |action|
         class_eval <<-END
           def self.#{action}(*args)
-            unless process_name = args.shift
-              yield("ERR: Missing process name")
-              return yield("OK")
+            unless args.size > 0
+              raise("ERROR: Missing process name")
+              return # yield("OK")
+            end
+            output_processes = []
+            Cognizant::Server.daemon.processes.values.each do |process|
+              if args.include?(process.name) or args.include?(process.group)
+                output_processes << process
+              end
             end
-            Cognizant::Server.daemon.processes.each do |name, process|
-              if process.name.eql?(process_name)
-                process.#{action}
-                return yield("OK")
+
+            if output_processes.size == 0
+              raise("ERROR: No such process")
+              # yield("OK")
+            else
+              output_processes.each do |process|
+                process.handle_user_command(:#{action})
               end
             end
-            yield("ERR: No such process")
-            yield("OK")
           end
         END
       end
 
       def self.shutdown
         Cognizant::Server.daemon.shutdown
-        yield("OK")
+        # yield("OK")
       end
 
       def self.method_missing(command, *args)
-        yield("ERR: Unknown command '#{command}'")
-        yield("OK")
+        raise("ERROR: Unknown command '#{command}'")
+        # yield("OK")
       end
     end
   end