pazuzu 0.1.2

data/.gitignore

@@ -0,0 +1,8 @@
+*~
+*.bak
+/_trash
+*.sublime-workspace
+*.sublime-project
+/.rspec
+/Gemfile.lock
+/.rvmrc

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Pazuzu.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+require "pazuzu/version"
+
+Gem::Specification.new do |s|
+  s.name        = "pazuzu"
+  s.version     = Pazuzu::VERSION
+  s.authors     = ["Alexander Staubo"]
+  s.email       = ["alex@origo.no"]
+  s.homepage    = "http://github.com/origo/pazuzu"
+  s.summary     = s.description = %{Pazuzu is a supervisor daemon that manages pools of application processes as daemons.}
+
+  s.rubyforge_project = "pazuzu"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'activesupport', '~> 3.1.0'
+  s.add_runtime_dependency 'statemachine', '~> 2.0.1'
+  s.add_runtime_dependency 'yajl-ruby', '~> 1.1'
+  s.add_runtime_dependency 'i18n'
+  s.add_runtime_dependency 'scashin133-syslog_logger', '~> 1.7.3'
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+end

data/README.md

@@ -0,0 +1,171 @@
+Pazuzu
+======
+
+Pazuzu is a supervisor daemon that manages pools of application processes as daemons. Pazuzu uses Heroku's `Procfile` format and relies on Linux cgroups to encapsulate groups of process.
+
+                               ....
+                              ......
+                             ..=?$..               .
+             ...            ...N~,....            .....
+        .. ..... .          ...M8O....           ......
+      ......NZ$,..        . ...DD8....          . $O$+....
+         .$8~Z7$. .     ....:N88ZO:Z....        ..8ZOD=..
+     . ...8OO7Z$+....  .  ..88Z:I=8$?..      ....ZO7I$M.....
+     ....DZ+$ZZZ8... . .....?DN878=,~..... . ...N7ODOIDO...
+    ....D=ZZ7Z$$?D.....  ..M$D8D$D8=I~....  ...N$88IDO8O:....
+     ...OOZ$ZO$$?OO.... ...8D8ND8?MM,,... ....8+DZZ7$ZDOO..  .
+    ....OZN7$~8Z$7$$.......88O88$~?DO.......=OZZOO8MDOZODN... ..
+      .,D$7I7ZZ$?+$Z8,......ZDDDND8M,......M$OZNDO$NODM8+I..
+      ..MDO$Z$DMD+~:Z8:......DDNMM?I,....8Z8Z8Z8N?Z$DDO8D...
+      . ~+D7IN?Z=IOD$ONN.....N8N8~$D..IIOZ78Z$8Z$DO?ZMOOM...
+       ...?7DNO=?IDOO8OONDNNNNNNN8OII$I=$Z8Z78D8NZZZZDDD.  ..
+     .....MZ$DNDDN?ZZ8DNN8DDDNNNDO7$I$+++ZOOOOO8$NMDI8...
+      ....:DOO8NOOZOO8DNNO88DO78O87~~77D=7OO$8OO?NZ?M...
+         ...O8ODN$87O8DD8NNND8Z++II:77?7~$$OZOZ$7D+8. .
+           .,8ONNDO~=Z8ZONNNNO=~==:8IN87I=OZZ7ID$7.,..
+          ...,8OND88+DZ8DNNNNNDO87+D7D8$$:7Z7Z8Z?.....
+       ...  ...DNN88NNDONNNNNNNNMZO7$DO8ZZ$7$O7, ...
+           . ...MNDDDDO88DNNNNNDOOMD$DNN=77777,......
+               ..,D8ZDO8NDNNNNDD88D78NND8O,$O$.....
+               ...OONN8ODDNNNNDD8ZOI$DNDZ8Z7OO?..
+            ... .O78O8OZ8DNNNNNNZOI=ODNDOZI~78+...
+               ..8O$OZ88NNNNNNNDD8O+$ONN8DD$IZD...
+               .ZDD8OZZDODNDDNNDD$8I+DDNND87O$$$...
+            ....N7IOZ8O$O$DNNNNND7O$~,DNDOOZI+78...
+             ..=8D8DDNDODZNNNDDNDZOZ:?8DNNZ$~O$8..  .
+             ..8DN88DZDDDDNNNNDND8OOO$D8NN88$?=7?...
+          . ...8OM8N8D8DDNNNNNNNDZO7Z+DDNN?I7IM8O...
+          . ..MDDMODOND88NNN8NDMNO$$Z7ZINNMN$NNIZ,
+            ..8O88888DDODDNNDNNNON78Z+Z8DMN=N:O8Z?..
+           . .8888OD88M8DNNDONNNM+DOD,+Z$88OZIMD=I..
+           . M8OOO8NDO+8NNNOOZZMDNN8DO?$ZZOO8IDDOM... .
+         ....8878+8ZONNNN.DDOZ$..DDNDOIMI7888=8OOO+....
+          ..MO88O8O=8D88..8DOOO..DNN8$.Z88Z=O8Z=DIZ.. .
+          ..DO8ZO8Z$D=D8..ZD8Z. .NDNDM.?O8OZ$?8$8Z8....
+       .  ..DIDDD8OO78D. .Z8DD...,DND$..8OOD87D$$IM=...
+        ...8DDO7OI$D$MO. .OND8,...NDD.. .M88I~N7DIDI..
+        ...DZDMOZ$O88.....DDODM...DDZ.....NZ77O?Z?88...
+       ...8D~D7D8ZO: ....DDNDDM..DDD7......=O8D$DDOD...
+        ..DOD8888....  ..8NNDD,.ZDD87...   ...DOZN+87...    .
+       ...... ......... ,8DND8Z.D$MOO......  ...,O.$$...
+         . ......... ....DNDDN+NDN8+8,...   . ........
+               . . .  ...DDDDD.MDN,7Z$. .         ...
+                     ....DDND:.MDMM.~Z.......     ..
+                  ......D8DDN..8DOZNIZ.........
+            ... ... .7Z8DDDDDI,.NOD=O7.... ..
+     ..    ..........:$$$NDNN87?MNI?IM..........
+      .   ...........~D=?D$ODNNZD8=Z7=~++++++... .
+           ..?????????8ZO7$Z8NNDN$O$?=++++++++....
+          ...???????II$ZOZ7O+7OD$=OOI$=+++++++,...
+          ...I+++++??I+D7O$8$?IM7:88=OO?++++++?..
+          ...7++++++???$ZZZZZZ$Z$Z77~~+I+++++++:
+        .....?+++++??I++8DO8DON87+,+7O$?++++++++...
+        .....?+++++++??++?++?$ODZ$8$O$$Z=+++++++...
+
+Introduction
+------------
+
+Pazuzu controls *applications*. An application consists of *workers*. Each worker has a name and a command line. Each worker may have multiple *instances* of itself running.
+
+For example, a typical application may consist of:
+
+* A Rails app, running under some web server like Thin or Unicorn.
+* A WebSockets app listening on a socket.
+* Some queue-processing daemons.
+* Some maintenance jobs that run now and then.
+
+Applications are configured through a simple text format:
+
+    <key>:<command line>
+
+For example:
+
+    main: bundle exec unicorn -c config/unicorn.rb config.ru
+    queue_processor: bundle exec ruby app/queue_processor.rb
+    web_socket_server: node lib/server.js
+
+Pazuzu will control the lifecycle of each worker. Main points:
+
+* Workers can be started and stopped by issuing appropriate commands.
+* Workers will be monitored and respawned if they crash.
+* If the desired number of workers is changed, new worker instances will be spawned or terminated as needed.
+* Recent output from workers is kept so that it can be viewed.
+* If Pazuzu crashes, workers will continue to run, and will be reattached when Pazuzu is restarted.
+
+Pazuzu leaves the following to external tools:
+
+* Resource limiting. Use the Linux cgroups tools to limit CPU and memory usage.
+* Scaling. Use external tools to monitor load, and invoke the `pazuzu` command to change the number of workers.
+
+Installation
+------------
+
+    gem install pazuzu
+
+Requirements
+------------
+
+* Ruby 1.9.1 or later.
+* Linux 2.6.24 or later with [cgroups](http://www.kernel.org/doc/Documentation/cgroups/) enabled in the kernel configuration.
+* The [libcg](http://libcg.git.sourceforge.net/) binaries, which include `cgcreate` and `cgexec`.
+
+Getting started
+---------------
+
+Create a minimal configuration `/etc/pazuzu/pazuzu.conf`:
+
+    applications:
+      myapp:
+        procfile: /srv/myapp
+
+Create a minimal /srv/myapp/Procfile:
+
+    myworker: sleep 1h    
+
+Start Pazuzu (non-daemonized):
+
+    sudo pazuzud
+
+Configuration file
+------------------
+
+`log_path`: Set the log file. Specify `syslog` to use syslog. Defaults to stderr.
+
+`socket_path`: Where to create a socket for accepting commands. Defaults to `/var/run/pazuzud.socket`.
+
+`include`: Include another configuration file or a number of files. May be either a single file name/glob pattern or an array of file names/glob patterns (eg., `/etc/pazuzu/conf.d/*.conf`). The included files's keys are merged into the current configuration in the order of inclusion.
+
+`cgroups`: Has the following sub-keys:
+
+* `hiearchy_root`: Root of hierarchy for all cgroups created. Defaults to `pazuzu`.
+
+* `subsystems`: An array of cgroups subsystem to attach each cgroup to. Defaults to `memory`, `cpu`, `cpuacct` and `blkio` (essentially all subsystems at the time of writing).
+
+* `fs_root`: Where the cgroups file system is mounted. Defaults to `/sys/fs/cgroup`, which may or may not be the default for your Linux distribution.
+
+`applications`: This is where applications are listed. Each application has its own key. For example:
+
+    applications:
+      foo:
+        procfile: /srv/foo
+
+Each application has the following sub-keys:
+
+* `procfile` (*required*): The path to the procfile. If a directory, it's assumed that it contains a file named `Procfile`.
+* `user`: User name or UID to run the process as. Defaults to the user that Pazuzu runs as.
+* `group`: Group name or GID to run the process as. Defaults to the group that Pazuzu runs as.
+* `workers`: This key allows per-worker overrides. Each worker has its own key, which is the name specified in the `Procfile`. 
+
+For example:
+
+    applications:
+      foo:
+        procfile: /srv/foo
+        workers:
+          bar:
+            num_instances: 2
+
+Each worker has the following sub-keys:
+
+* `command_line`: Override the worker's command line.
+* `num_instances`: The number of instances to run. Defaults to 1.

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/gem_tasks"
+
+desc 'Bump version'
+task :bump do
+  if `git status -uno -s --porcelain | wc -l`.to_i > 0
+    abort "You have uncommitted changed."
+  end
+  text = File.read('lib/pazuzu/version.rb')
+  if text =~ /VERSION = '(.*)'/
+    old_version = $1
+    version_parts = old_version.split('.')
+    version_parts[-1] = version_parts[-1].to_i + 1
+    new_version = version_parts.join('.')
+    text.gsub!(/VERSION = '(.*)'/, "VERSION = '#{new_version}'")
+    File.open('lib/pazuzu/version.rb', 'w') { |f| f << text }
+    (system("git add lib/pazuzu/version.rb") and
+      system("git commit -m 'Bump to #{new_version}.'")) or abort "Failed to commit."
+  else
+    abort "Could not find version number"
+  end
+end

data/bin/pazuzu

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'pazuzu'
+
+controller = Pazuzu::CommandLine::Controller.new
+controller.run!(ARGV)

data/bin/pazuzud

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'pazuzu'
+
+runner = Pazuzu::SupervisorRunner.new
+runner.run!(ARGV)

data/lib/pazuzu.rb

@@ -0,0 +1,37 @@
+require 'set'
+require 'optparse'
+require 'timeout'
+require 'fileutils'
+require 'yaml'
+require 'singleton'
+require 'logger'
+require 'delegate'
+require 'etc'
+require 'thread'
+require 'socket'
+
+require 'statemachine'
+require 'active_support/core_ext/hash'
+require 'yajl/json_gem'
+
+require 'pazuzu/utility/annotated_logger'
+require 'pazuzu/utility/rate_limiter'
+require 'pazuzu/utility/process_spawning'
+require 'pazuzu/utility/runnable'
+require 'pazuzu/utility/runnable_pool'
+require 'pazuzu/utility/output_tailer'
+
+require 'pazuzu/control/socket_server'
+require 'pazuzu/control/socket_client'
+require 'pazuzu/control/protocol'
+
+require 'pazuzu/command_line/controller'
+
+require 'pazuzu/supervisor'
+require 'pazuzu/procfiles'
+require 'pazuzu/application'
+require 'pazuzu/instance'
+require 'pazuzu/worker'
+require 'pazuzu/supervisor_runner'
+require 'pazuzu/version'
+require 'pazuzu/cgroup'

data/lib/pazuzu/application.rb

@@ -0,0 +1,114 @@
+module Pazuzu
+
+  class Application
+
+    include Utility::Runnable
+
+    def initialize(supervisor, name)
+      super()
+      @supervisor = supervisor
+      @name = name
+      @logger = Utility::AnnotatedLogger.new(supervisor.logger, qname)
+      @workers = Utility::RunnablePool.new
+      @mutex = Mutex.new
+    end
+
+    def configure!(procfile_path, configuration)      
+      procfile_path = Procfiles.normalize_procfile_path(procfile_path)
+
+      root_path = File.dirname(procfile_path)
+
+      @user = configuration['user']
+      @group = configuration['group']
+
+      workers_config = configuration['workers'] || {}
+
+      leftover_workers = @workers.children.dup
+
+      procfile = YAML.load(File.open(procfile_path))
+      procfile = {} unless procfile.is_a?(Hash)
+      procfile.each do |name, command_line|
+        name = name.to_s
+
+        worker_config = workers_config[name] || {}
+
+        worker = @workers.children.select { |w| w.name == name }.first
+        if worker
+          leftover_workers.delete(worker)
+        else
+          @logger.info "Adding worker #{name}"
+          worker = Worker.new(self, name)
+        end
+        worker.configure!(root_path, command_line, worker_config)
+        @workers.register(worker)
+      end
+
+      leftover_workers.each do |worker|
+        @logger.info "Removing worker #{worker.name}"
+        @workers.unregister(worker)
+      end
+    end
+
+    def qname
+      @name
+    end
+
+    # Call to set up a forked process according to the application's
+    # configuration.
+    def setup_spawned_process!
+      Utility::ProcessSpawning.set_user_and_group!(@user, @group)
+      Utility::ProcessSpawning.prepare_child_process!
+    end
+
+    def workers
+      @workers.children
+    end
+
+    def log_entries
+      entries = @workers.children.map(&:log_entries)
+      entries.flatten!(1)
+      entries.sort_by! { |(source, time, message)| time }
+      entries
+    end
+
+    attr_reader :name
+    attr_reader :supervisor
+    attr_reader :user
+    attr_reader :group
+
+    private
+
+      def on_starting
+        @logger.info "Starting"
+        @monitor_thread ||= Thread.start { monitor_workers }
+        @workers.start!
+        runnable_state.started!
+      end
+
+      def on_started
+        @logger.info "Started"
+      end
+
+      def on_stopping
+        @logger.info "Stopping"
+        @workers.stop!
+      end
+
+      def monitor_workers
+        while [:starting, :running, :stopping].include?(run_state)
+          if runnable_state.state == :stopping
+            unless @workers.children.any? { |w| w.run_state == :running }
+              runnable_state.stopped!
+            end
+          end
+          sleep(1)
+        end
+      rescue Exception => e
+        @logger.error(e.message)
+      ensure
+        @monitor_thread = nil
+      end
+
+  end
+
+end

data/lib/pazuzu/cgroup.rb

@@ -0,0 +1,73 @@
+module Pazuzu
+
+  class Cgroup
+
+    class CgroupCreationFailed < Exception; end
+
+    class << self
+      # Are cgroups available?
+      def available?
+        File.exist?('/proc/cgroups')
+      end
+    end
+
+    def initialize(fs_root_path, path, subsystems)
+      @fs_root_path = fs_root_path
+      @path = path
+      @subsystems = subsystems
+    end
+
+    def mounted?
+      return @subsystems.any? { |subsystem|
+        Dir.glob(File.join(@fs_root_path, subsystem, @path, "cgroup.procs")).any? { |file_name|
+          File.exists?(file_name)
+        }
+      }
+    end
+
+    def exec(command)
+      unless system("cgcreate -g #{@subsystems.join(',')}:#{@path}")
+        raise CgroupCreationFailed, "Could not create cgroup hiearchy #{@path}"
+      end
+      command = "cgexec -g #{@subsystems.join(',')}:#{@path} #{command}"
+      return Process.exec(command)
+    end
+
+    def pids
+      pids = pids_including_children
+      pids.reject { |pid| pids.include?(parent_pid_for(pid)) }
+    end
+
+    attr_reader :path
+
+    private
+
+      def pids_including_children
+        return @subsystems.flat_map { |subsystem|
+          Dir.glob(File.join(@fs_root_path, subsystem, @path, "cgroup.procs")).flat_map { |file_name|
+            begin
+              File.read(file_name).split("\n").map { |s| Integer(s) }
+            rescue Errno::ENOENT
+              nil
+            end
+          }
+        }.compact.tap(&:uniq!)
+      end
+
+      def parent_pid_for(pid)
+        begin
+          File.readlines("/proc/#{pid}/status").each do |line|
+            return $1.to_i if line =~ /^PPid:\s*(\d+)/i
+          end
+        rescue Errno::ENOENT
+        end
+        nil
+      end
+
+      def subsystem_path(subsystem)
+        File.join(@fs_root_path, subsystem, @path)
+      end
+
+  end
+
+end