trident 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NDRhNzEyMDc3ZGY3NGM3MDAzMzNkNWJmYTM5ODUxOTA4OTljN2YyMg==
+  data.tar.gz: !binary |-
+    MjQ2YWVmMzAwZjEzNTUwZTkwNzJlOGU3YzNhMjc4ZTNlNTU2ZTdhYw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZTc0NGI3MjljOTg4YzI1MDhkMDg0YWVjNTUxY2ZiYjkxN2YyMDkyMGRhYzZm
+    MDRjMmQ4MDA0MmRjMGVjNmM2NWI1OTMwNDUyYTU2ZWE1YjI2MjA3YjZkN2Zm
+    NzY3NWZkZWJiYTBiODM5Njk4MDU1MWZmMGVjOTc3OTk0YmRhZDI=
+  data.tar.gz: !binary |-
+    NTljZGRhZDc2NDBkYmVjYjkzZGM2OTM0M2Q0NmYzYWNlZjc3MmYxNzFkM2Uy
+    YWNkMWIxMzM4MzhkOTBmODVjNjkyMzliOWMyYTUyOTFmMTk3OWJlZTQ0Mjgy
+    NjllMmJlZjkyN2U3NWM4NzlhNjNhZDhjOGVjODZiZjY5ZDMyNDk=

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode
+  
+script: bundle exec rake

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in trident.gemspec
+gemspec
+
+# for code coverage during travis-ci test runs
+gem 'coveralls', :require => false
+
+gem "mocha", :require => false

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Matt Conway
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# Trident
+
+A ruby gem for managing pools of forked workers
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'trident'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install trident
+
+## Usage
+
+After installing the gem, use the 'trident' binary to generate an example configuration file:
+
+    trident --generate-config > config/trident.yml
+
+Edit the file with your desired setup, then run trident to launch all your worker pools
+
+See other command line options with
+    trident --help
+
+
+## TODO
+
+* Add support for reloading the trident config with a HUP signal

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+require 'rake'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.libs.push 'test'
+end
+
+task :default => :test
+
+task :console do
+  Bundler.require
+  require "trident"
+  ARGV.clear
+  require "irb"
+  IRB.start
+end

data/bin/trident

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'trident'
+Trident::CLI.run
+

data/lib/trident.rb

@@ -0,0 +1,8 @@
+require 'gem_logger'
+require "trident/version"
+require "trident/utils"
+require "trident/cli"
+require "trident/pool"
+require "trident/pool_handler"
+require "trident/pool_manager"
+require "trident/signal_handler"

data/lib/trident/cli.rb

@@ -0,0 +1,130 @@
+require 'clamp'
+require 'erb'
+require 'yaml'
+
+module Trident
+  class CLI < Clamp::Command
+
+    include GemLogger::LoggerSupport
+    include Trident::Utils
+
+    def self.description
+      "Starts the trident pool manager"
+    end
+
+    option "--verbose",
+           :flag, "verbose output\n",
+           :default => false
+    option "--generate-config",
+           :flag, "generates an example config file to stdout"
+    option "--config",
+           "FILENAME", "use the given config file\n",
+           :default => "config/trident.yml"
+    option "--logfile",
+           "FILENAME", "log to the given file"
+    option "--pidfile",
+           "FILENAME", "store pid in the given pidfile"
+    option "--daemon",
+           :flag, "run as a daemon",
+           :default => false
+    option "--pool",
+           "POOL", "only run the given pool(s)",
+           :multivalued => true
+
+
+    def execute
+      if generate_config?
+        puts File.read(File.expand_path("../../../trident.example.yml", __FILE__))
+        exit(0)
+      end
+
+      procline "cli", "(initializing)"
+
+      self.logfile = expand_path(logfile)
+      self.pidfile = expand_path(pidfile)
+      self.config = expand_path(config)
+
+      GemLogger.default_logger = Logger.new(logfile ? logfile : STDOUT)
+      GemLogger.default_logger.level = verbose? ? Logger::DEBUG : Logger::INFO
+      $stdout.sync = $stderr.sync = true
+
+      if daemon? && (logfile.nil? || pidfile.nil?)
+        signal_usage_error "--logfile and --pidfile are required when running as a daemon"
+      end
+
+      logger.info "Loading config from: #{config}"
+      config_hash = load_config(config)
+
+      if GC.respond_to?(:copy_on_write_friendly=)
+        GC.copy_on_write_friendly = true
+      end
+
+      daemonize(logfile) if daemon?
+      File.write(pidfile, Process.pid.to_s) if pidfile
+
+      handlers = create_handlers(config_hash['handlers'])
+      pools = create_pools(config_hash['pools'], handlers, pool_list)
+
+      manager = Trident::PoolManager.new(config_hash['application'],
+                                           pools.values,
+                                           config_hash['prefork'] == true)
+      sh_thread = Thread.new { Trident::SignalHandler.start(config_hash['signals'], manager) }
+      manager.start
+      sh_thread.join
+    end
+
+    private
+
+    def project_root
+      @root ||= ENV['BUNDLE_GEMFILE'] ? "#{File.dirname(ENV['BUNDLE_GEMFILE'])}" : "."
+    end
+
+    def expand_path(path)
+      if path && path !~ /^\//
+        File.expand_path("#{project_root}/#{path}")
+      else
+        path
+      end
+    end
+
+    # Configure through yaml file
+    def load_config(path_to_yaml_file)
+      config = YAML::load(ERB.new(IO.read(path_to_yaml_file)).result)
+      config = config[Rails.env.to_s] if defined?(::Rails) && config.has_key?(Rails.env.to_s)
+      config
+    end
+
+    def daemonize(logfile)
+      Process.daemon
+      $stdout.reopen(logfile, "a")
+      $stderr.reopen(logfile, "a")
+    end
+
+    def create_handlers(handlers_config_hash)
+      handlers = {}
+      handlers_config_hash.each do |name, handler_config|
+        handler = Trident::PoolHandler.new(name,
+                                             handler_config['class'],
+                                             handler_config['environment'],
+                                             handler_config['signals'],
+                                             handler_config['options'])
+        handlers[name] = handler
+      end
+      handlers
+    end
+
+    def create_pools(pools_config_hash, handlers, pool_filter=[])
+      pools = {}
+      pools_config_hash.each do |name, pool_config|
+        handler = handlers[pool_config['handler']]
+        raise "No handler defined: #{pool_config['handler']}" unless handler
+
+        next if pool_filter.size > 0 && ! pool_filter.include?(name)
+
+        pool = Trident::Pool.new(name, handler, pool_config['size'], pool_config['options'])
+        pools[name] = pool
+      end
+      pools
+    end
+  end
+end

data/lib/trident/pool.rb

@@ -0,0 +1,108 @@
+module Trident
+  class Pool
+    include GemLogger::LoggerSupport
+    include Trident::Utils
+
+    attr_reader :name, :handler, :size, :options, :workers
+
+    def initialize(name, handler, size, options={})
+      @name = name
+      @handler = handler
+      @size = size
+      @options = options || {}
+      @workers = Set.new
+    end
+
+    def start
+      logger.info "<pool-#{name}> Starting pool"
+      maintain_worker_count('stop_gracefully')
+      logger.info "<pool-#{name}> Pool started with #{workers.size} workers"
+    end
+
+    def stop(action='stop_gracefully')
+      logger.info "<pool-#{name}> Stopping pool"
+      @size = 0
+      maintain_worker_count(action)
+      logger.info "<pool-#{name}> Pool stopped"
+    end
+
+    def wait
+      logger.info "<pool-#{name}> Waiting for pool"
+      cleanup_dead_workers(true)
+      logger.info "<pool-#{name}> Wait complete"
+    end
+
+    def update
+      logger.info "<pool-#{name}> Updating pool"
+      maintain_worker_count('stop_gracefully')
+      logger.info "<pool-#{name}> Pool up to date"
+    end
+
+    private
+
+    def maintain_worker_count(kill_action)
+      cleanup_dead_workers(false)
+
+      if size > workers.size
+        spawn_workers(size - workers.size)
+      elsif size < workers.size
+        kill_workers(workers.size - size, kill_action)
+      else
+        logger.debug "<pool-#{name}> Worker count is correct"
+      end
+    end
+
+    def cleanup_dead_workers(blocking=true)
+      wait_flags = blocking ? 0 : Process::WNOHANG
+      workers.clone.each do |pid|
+        begin
+          wpid = Process.wait(pid, wait_flags)
+        rescue Errno::EINTR
+          logger.warn("<pool-#{name}> Interrupted cleaning up workers, retrying")
+          retry
+        rescue Errno::ECHILD
+          logger.warn("<pool-#{name}> Error cleaning up workers, ignoring")
+          # Calling process.wait on a pid that was already waited on throws
+          # a ECHLD, so may as well remove it from our list of workers
+          wpid = pid
+        end
+        workers.delete(wpid) if wpid
+      end
+    end
+
+    def spawn_workers(count)
+      logger.info "<pool-#{name}> Spawning #{count} workers"
+      count.times do
+        spawn_worker
+      end
+    end
+
+    def kill_workers(count, action)
+      logger.info "<pool-#{name}> Killing #{count} workers with #{action}"
+      workers.to_a[-count, count].each do |pid|
+        kill_worker(pid, action)
+      end
+    end
+
+    def spawn_worker
+      pid = fork do
+        procline "pool-#{name}-worker", "starting handler #{handler.name}"
+        Trident::SignalHandler.reset_for_fork
+        handler.load
+        handler.start(options)
+      end
+      workers << pid
+      logger.info "<pool-#{name}> Spawned worker #{pid}, worker count now at #{workers.size}"
+    end
+
+    def kill_worker(pid, action)
+      sig = handler.signal_for(action)
+      raise "<pool-#{name}> No signal for action: #{action}" unless sig
+      logger.info "<pool-#{name}> Sending signal to worker: #{pid}/#{sig}/#{action}"
+      Process.kill(sig, pid)
+      workers.delete(pid)
+      logger.info "<pool-#{name}> Killed worker #{pid}, worker count now at #{workers.size}"
+    end
+
+  end
+end

data/lib/trident/pool_handler.rb

@@ -0,0 +1,31 @@
+module Trident
+  class PoolHandler
+
+    attr_reader :name, :worker_class_name, :environment, :signal_mappings, :options
+
+    def initialize(name, worker_class_name, environment, signal_mappings, options={})
+      @name = name
+      @worker_class_name = worker_class_name
+      @environment = environment
+      @signal_mappings = signal_mappings
+      @options = options || {}
+    end
+
+    def load
+      eval environment if environment
+    end
+
+    def worker_class
+      self.class.const_get(worker_class_name)
+    end
+
+    def start(opts={})
+      worker_class.new(self.options.merge(opts)).start
+    end
+
+    def signal_for(action)
+      signal_mappings[action] || signal_mappings['default'] || "SIGTERM"
+    end
+
+  end
+end

data/lib/trident/pool_manager.rb

@@ -0,0 +1,79 @@
+module Trident
+  class PoolManager
+    include GemLogger::LoggerSupport
+    include Trident::Utils
+
+    attr_reader :name, :pools, :prefork
+
+    def initialize(name, pools, prefork)
+      logger.info "Initializing pool manager"
+      procline "manager-#{name}", "(initializing)"
+      @name = name
+      @pools = pools
+      @prefork = prefork
+    end
+
+    def start
+      logger.info "Starting pools"
+      load_handlers if prefork
+      pools.each do |pool|
+        pool.start
+      end
+      procline "manager", "managing #{procline_display}"
+    end
+
+    def stop_forcefully
+      stop('stop_forcefully')
+    end
+
+    def stop_gracefully
+      stop('stop_gracefully')
+    end
+
+    # waits for children to exit
+    def wait
+      logger.info "Waiting for pools to exit"
+      procline "manager", "waiting #{procline_display}"
+      pools.each do |pool|
+        pool.wait
+      end
+      :break
+    end
+
+    def update
+      pools.each do |pool|
+        pool.update
+      end
+      procline "manager", "managing #{procline_display}"
+    end
+
+    private
+
+    def procline_display
+      display = ""
+      pools.each_with_index do |pool, i|
+        display << "#{pool.name}#{pool.workers.to_a.inspect}"
+        display << " " if i + 1 < pools.size
+      end
+      display
+    end
+
+    def load_handlers
+      procline "manager", "preforking #{procline_display}"
+      pools.each do |pool|
+        pool.handler.load
+      end
+    end
+
+    # tells all children to stop using action
+    def stop(action)
+      logger.info "Stopping pools: #{action}"
+      procline "manager", "stopping #{procline_display}"
+      pools.each do |pool|
+        pool.stop(action)
+      end
+      :break
+    end
+
+  end
+end