pazuzu 0.1.2

data/lib/pazuzu/control/socket_client.rb

@@ -0,0 +1,30 @@
+module Pazuzu
+  module Control
+
+    class SocketClient
+
+      def initialize(path)
+        @path = path
+        @socket = UNIXSocket.new(@path)
+      end
+
+      def execute(command, *arguments)
+        command_line = [command, *arguments].join(' ')
+        command_line << "\n"
+        
+        @socket.write(command_line)
+        
+        num_bytes = @socket.read(10).to_i(16)
+        if num_bytes > 0
+          data = @socket.read(num_bytes)
+          return JSON.parse(data)
+        end
+      end
+
+      attr_reader :path
+
+    end
+    
+  end
+
+end

data/lib/pazuzu/control/socket_server.rb

@@ -0,0 +1,75 @@
+module Pazuzu
+  module Control
+
+    class SocketServer
+
+      include Utility::Runnable
+
+      def initialize(supervisor, path)
+        @supervisor = supervisor
+        @path = path
+        @logger = Utility::AnnotatedLogger.new(
+          supervisor.logger, 'Command socket server')
+        super()
+      end
+
+      attr_reader :path
+      attr_reader :supervisor
+
+      protected
+
+        def listen
+          unless @socket
+            File.unlink(@path) if File.exist?(@path) and File.socket?(@path)
+            @server = UNIXServer.new(@path)
+          end
+        end
+
+        def run_protocol
+          while @server
+            begin
+              socket = @server.accept
+              if socket
+                Thread.start {
+                  begin
+                    protocol = Protocol.new(socket, @supervisor)
+                    protocol.run!
+                  rescue Errno::EPIPE, Errno::ECONNRESET
+                    # Ignore
+                  rescue Exception => e
+                    @logger.error "Exception during command protocol exchange: #{e.class}: #{e}"
+                  ensure
+                    socket.close
+                  end
+                }
+              end
+            rescue Exception => e
+              @logger.error "Socket server exception: #{e.class}: #{e}"
+              sleep(0.1)
+            end
+          end
+        end
+
+        def on_starting
+          listen
+          @thread = Thread.start { run_protocol }
+          runnable_state.started!
+        end
+
+        def on_stopping
+          if @socket
+            @socket.close rescue nil
+            @socket = nil
+          end
+          if @thread
+            @thread.terminate
+            @thread = nil
+          end
+          @server = nil
+          runnable_state.stopped!
+        end
+
+    end
+
+  end
+end

data/lib/pazuzu/instance.rb

@@ -0,0 +1,218 @@
+module Pazuzu
+
+  class Instance
+
+    # Time that we give processes to shut down normally on SIGTERM.
+    SHUTDOWN_GRACE_TIME = 10.seconds
+
+    include Utility::Runnable
+
+    def initialize(worker, index, root_path, command_line)
+      @worker = worker
+      @index = index
+      @logger = Utility::AnnotatedLogger.new(
+        @worker.application.supervisor.logger, qname)
+      @root_path = root_path
+      @command_line = command_line
+      @tailer = Utility::OutputTailer.new(
+        :limit => 5000,
+        :on_line => proc { |timestamp, line| @logger.info("[output] #{line}") })
+      @flap_limiter = Utility::RateLimiter.new(0.1)
+      @recovery_count = 0
+      @cgroup = @worker.application.supervisor.cgroup_for_instance(self)
+      super()
+    end
+
+    # Does this instance have any unattached PIDs running?
+    def attachable?
+      run_state == :stopped && @cgroup.pids.any?
+    end
+
+    def qname
+      [@worker.qname, @index].join('.')
+    end
+
+    def log_entries
+      source = qname
+      return @tailer.entries.map { |(time, message)| [source, time, message] }
+    end
+
+    attr_reader :command_line
+    attr_reader :worker
+    attr_reader :index
+    attr_reader :root_path
+    attr_reader :pid
+    attr_reader :recovery_count
+
+    private
+
+      def on_starting
+        start_process!
+      end
+
+      def on_started
+        @logger.info 'Started'
+      end
+
+      def on_stopping
+        pids = @cgroup.pids
+        if pids.any?
+          if signal!('TERM')
+            @termination_deadline = Time.now + SHUTDOWN_GRACE_TIME
+          else
+            process_lost!
+          end
+        else
+          runnable_state.stopped!
+        end
+      end
+
+      def on_stopped
+        @recovery_count = 0
+        @logger.info "Stopped"
+      end
+
+      def on_failed
+      end
+
+      def signal!(signal_name)
+        any_signaled = false
+        @cgroup.pids.each do |pid|
+          begin
+            Process.kill(signal_name, pid)
+          rescue Errno::ESRCH
+          else
+            @logger.info "Signaled instance PID=#{pid} with SIG#{signal_name}"
+            any_signaled = true
+          end
+        end
+        any_signaled
+      end
+
+      def start_process!
+        if @cgroup.pids.empty?
+          unless @flap_limiter.count!
+            return unless [:starting, :running].include?(run_state)
+            @logger.warn "Restarting too fast, process may be flapping"
+          end
+
+          reset_process!
+
+          @logger.info "Spawning process with command (in #{@root_path}): #{@command_line}"
+
+          stream = @tailer.open
+          pid = Process.fork do
+            exec!(stream)
+          end
+          stream.close
+
+          @logger.info "Spawned process with PID=#{pid}"
+        end
+
+        @spawned_at = Time.now
+        @monitor_thread ||= Thread.start { monitor_process }
+      end
+
+      def exec!(io_out)
+        @worker.setup_spawned_process!
+        Process.setsid
+
+        # Double fork to detach and avoid zombies
+        Process.fork do
+          Process.setsid
+
+          Dir.chdir(@root_path)
+
+          # TODO: Log to file so we can tail
+          $stderr.reopen("/tmp/#{qname}")
+          $stdout.reopen("/tmp/#{qname}")
+          #$stderr.reopen(io_out)
+          #$stdout.reopen(io_out)
+
+          begin
+            @cgroup.exec(@command_line)
+          rescue Errno::ENOENT => e
+            abort e.message
+          end
+          Process.exit(0)
+        end
+
+        # Never gets here, exit purely for aesthetic and/or superstitious reasons
+        Process.exit(1)
+      end
+
+      def process_lost!
+        @logger.info "Child process lost, assuming it stopped on its own"
+        process_stopped!
+      end
+
+      def process_stopped!
+        reset_process!
+        case run_state
+          when :running, :starting
+            @recovery_count += 1
+            @logger.info "Respawning after failure"
+            if run_state == :running
+              runnable_state.starting!
+            else
+              start_process!
+            end
+          when :stopping
+            runnable_state.stopped!
+        end
+      end
+
+      def process_confirmed_running!
+        if @termination_deadline and @termination_deadline <= Time.now
+          begin
+            Process.kill('KILL', @pid)
+            @logger.info "Child did not terminate in time, forcibly killing with SIGKILL"
+          rescue Errno::ESRCH
+            @logger.info "Child process lost, assuming it stopped on its own"
+            process_stopped!
+          else
+            @termination_deadline = Time.now + SHUTDOWN_GRACE_TIME
+          end
+        else
+          if run_state == :starting
+            @logger.info "Child seems to be running"
+            runnable_state.started!
+          end
+        end
+      end
+
+      def reset_process!
+        @spawned_at = nil
+        @tailer.close
+        @termination_deadline = nil
+      end
+
+      def monitor_process
+        while [:starting, :running, :stopping].include?(run_state)
+          begin
+            case run_state
+              when :starting
+                if @cgroup.pids.any?
+                  process_confirmed_running!
+                end
+              when :running
+                if @cgroup.pids.empty?
+                  process_lost!
+                end
+              when :stopping
+                if @cgroup.pids.empty?
+                  runnable_state.stopped!
+                end
+            end
+            sleep(2)
+          rescue Exception => e
+            @logger.error(e.message)
+          end
+        end
+      ensure
+        @monitor_thread = nil
+      end
+
+  end
+
+end

data/lib/pazuzu/procfiles.rb

@@ -0,0 +1,16 @@
+module Pazuzu
+
+  module Procfiles
+
+    def self.normalize_procfile_path(path)
+      path = File.expand_path(path)
+      if File.directory?(path)
+        automatic_path = File.join(path, 'Procfile')
+        path = automatic_path if File.exist?(automatic_path)
+      end
+      path
+    end
+
+  end
+
+end

data/lib/pazuzu/supervisor.rb

@@ -0,0 +1,201 @@
+module Pazuzu
+
+  class ConfigurationError < StandardError; end
+
+  # The supervisor controls the different applications registered with it.
+  class Supervisor
+
+    DEFAULT_SOCKET_PATH = '/var/run/pazuzud.socket'
+
+    include Utility::Runnable
+
+    def initialize(options)
+      options.assert_valid_keys(:config_path)
+      @logger = Logger.new($stderr)
+      @logger.level = Logger::WARN
+      @applications = Utility::RunnablePool.new
+      @config_path = options[:config_path]
+      Thread.start {
+        # Wait for PIDs, otherwise we will get zombies
+        loop do
+          begin
+            Process.waitpid
+          rescue Errno::ECHILD
+            sleep(1)
+          rescue
+            # Ignore
+          end
+        end
+      }
+      super()
+    end
+
+    def run!
+      load_configuration!
+      start!
+      while [:starting, :running].include?(run_state)
+        sleep(1)
+      end
+    end
+
+    def load_configuration!
+      configure!(load_config_from_yaml(@config_path))
+    end
+
+    def configure!(configuration)
+      case configuration['log_path']
+        when 'syslog'
+          @logger = SyslogLogger('pazuzu')
+        when nil
+          @logger = Logger.new($stderr)
+        else
+          @logger = Logger.new(configuration['log_path'])
+      end
+      @logger.level = Logger::DEBUG
+
+      new_socket_path = configuration['socket_path']
+      new_socket_path ||= DEFAULT_SOCKET_PATH
+      if @socket_path and new_socket_path != @socket_path
+        @logger.warn("Cannot change socket path after start")
+      else
+        @socket_path = new_socket_path
+      end
+
+      @socket_server ||= Control::SocketServer.new(self, @socket_path)
+
+      leftover_applications = @applications.children.dup
+      (configuration['applications'] || {}).each do |name, app_config|
+        if app_config
+          name = name.to_s
+          application = @applications.children.select { |a| a.name == name }.first
+          if application
+            leftover_applications.delete(application)
+          else
+            @logger.info("Adding application #{name}")
+            application = Application.new(self, name)
+            @applications.register(application)
+          end
+          application.configure!(app_config['procfile'], app_config)
+        end
+      end
+      leftover_applications.each do |application|
+        @logger.info("Removing application #{name}")
+        @applications.unregister(application)
+      end
+
+      cgroups_config = configuration['cgroups'] || {}
+
+      new_cgroup_hiearchy_root = cgroups_config['hieararchy_root']
+      new_cgroup_hiearchy_root ||= 'pazuzu'
+      new_cgroup_hiearchy_root.gsub!(/^\/+/, '')
+      new_cgroup_hiearchy_root.gsub!(/\/$/, '')
+      if @cgroup_hieararchy_root and new_cgroup_hiearchy_root != @cgroup_hieararchy_root
+        @logger.warn("Cannot change cgroups hiearchy root after start")
+      else
+        @cgroup_hieararchy_root = new_cgroup_hiearchy_root
+      end
+
+      new_cgroup_subsystems = [cgroups_config['subsystems']].flatten.compact
+      new_cgroup_subsystems ||= %w(memory cpu cpuacct blkio)
+      if @cgroup_subsystems and new_cgroup_subsystems != @cgroup_subsystems
+        @logger.warn("Cannot change cgroups subsystems after start")
+      else
+        @cgroup_subsystems = new_cgroup_subsystems
+      end
+
+      new_cgroups_fs_root_path = cgroups_config['fs_root']
+      new_cgroups_fs_root_path ||= '/sys/fs/cgroup'
+      new_cgroups_fs_root_path.gsub!(/\/$/, '')
+      if @cgroups_fs_root_path and @cgroups_fs_root_path != new_cgroups_fs_root_path
+        @logger.warn("Cannot change cgroups root after start")
+      else
+        @cgroups_fs_root_path = new_cgroups_fs_root_path
+      end
+    end
+
+    def applications
+      @applications.children
+    end
+
+    # Returns a +Cgroup+ object given an instance.
+    def cgroup_for_instance(instance)
+      path = [
+        @cgroup_hieararchy_root,
+        instance.worker.application.name,
+        instance.worker.name,
+        instance.index
+      ].join('/')
+      Cgroup.new(@cgroups_fs_root_path, path, @cgroup_subsystems)
+    end
+
+    attr_reader :logger
+
+    protected
+
+      def on_starting
+        @applications.start!
+        @socket_server.start!
+        runnable_state.started!
+      end
+
+      def on_started
+        @logger.info "Started"
+      end
+
+      def on_stopping
+        @applications.stop!
+        @logger.info "Waiting for applications to stop"
+        begin
+          timeout(10) do
+            loop do
+              break if @applications.run_state == :stopped
+              sleep(0.5)
+            end
+          end
+        rescue Timeout::Error
+          @logger.error "Timed out waiting for applications to stop normally, giving up"
+        rescue SignalException
+          @logger.warn "Interrupted while waiting for applications to stop"
+        end
+        @socket_server.stop!
+        runnable_state.stopped!
+      end
+
+      def on_stopped
+        @logger.info "Stopped"
+      end
+
+      def load_config_from_yaml(file_name, target = {})
+        File.open(file_name, 'r:utf-8') do |file|
+          loaded = YAML.load(file)
+
+          target.merge!(loaded.except('include'))
+
+          if (includes = loaded['include'])
+            includes = [includes] unless includes.is_a?(Array)
+            includes.flat_map { |s| Dir.glob(s) }.uniq.each do |file_name|
+              load_config_from_yaml(file_name, target)
+            end
+          end
+        end
+        target
+      rescue ConfigurationError
+        raise
+      rescue Errno::ENOENT
+        raise ConfigurationError, "Configuration file #{file_name} not found"
+      rescue => e
+        raise ConfigurationError, "Could not read configuration file: #{e}"
+      end
+
+      def expand_includes(configuration)
+        if configuration['include']
+          new_config = {}
+          new_config.merge!(configuration)
+          new_config.delete('include')
+        end
+        configuration
+      end
+
+  end
+
+end