spool 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cd69246ea37212c4be4790fffcf8605c16f31e2b
+  data.tar.gz: 556ff1e0891bd8e83407f837a522f1aa6bd770cc
+SHA512:
+  metadata.gz: c7e8b6b9799b61d376564d6f97edd784477acbb0fc66fffdc916771788f7b7f34659d22ed676ed27954b687a873d829df5647483bae3cfb19bd739abafcb8012
+  data.tar.gz: bbdb9307edea81364ca9eb6732f09686ef8a676273dffc25f38532867a3d58266874ac72c035568fe3322a43e499029d00560d89c73c53afd9662dedd6255193

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-ci
+repo_token: UK3KHfqryIfzhZDncQX9pU4yoYLkjlOgE

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.ruby-gemset

@@ -0,0 +1 @@
+spool

data/.ruby-version

@@ -0,0 +1 @@
+ruby 2.1

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0
+  - 2.1

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spool.gemspec
+gemspec
+
+gem 'coveralls', require: false

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Gabriel Naiman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# Spool
+
+[![Gem Version](https://badge.fury.io/rb/spool.png)](https://rubygems.org/gems/spool)
+[![Build Status](https://travis-ci.org/gabynaiman/spool.png?branch=master)](https://travis-ci.org/gabynaiman/spool)
+[![Coverage Status](https://coveralls.io/repos/gabynaiman/spool/badge.png?branch=master)](https://coveralls.io/r/gabynaiman/spool?branch=master)
+[![Code Climate](https://codeclimate.com/github/gabynaiman/spool.png)](https://codeclimate.com/github/gabynaiman/spool)
+[![Dependency Status](https://gemnasium.com/gabynaiman/spool.png)](https://gemnasium.com/gabynaiman/spool)
+
+Manage and keep alive pool of processes
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'spool'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install spool
+
+## Usage
+
+#### Setup
+
+*Worker*
+
+    loop do
+      puts "#{Process.pid} - #{Time.now}"
+      sleep 1
+    end
+
+*Pool config*
+
+    processes 4
+    command 'ruby worker.rb'
+    # optional
+    dir File.dirname(__FILE__)
+    env VAR1: 'foo', VAR2: 'bar'
+    pidfile 'pool.pid'
+    restart_when { |p| p.memory > 512 }
+
+#### Start
+
+    console:~$ spool pool_config.rb
+
+#### Signals
+
+- **INT/TERM:** quick shutdown, kills all processes immediately
+- **QUIT:** graceful shutdown, waits for processes to finish
+- **HUP:** reloads config file and gracefully restart all processes
+- **USR2:** gracefully restart all processes with current configuration
+- **TTIN:** increment the number of processes by one
+- **TTOU:** decrement the number of processes by one
+
+## Contributing
+
+1. Fork it ( https://github.com/gabynaiman/spool/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,18 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:spec) do |t|
+  t.libs << 'spec'
+  t.pattern = 'spec/**/*_spec.rb'
+  t.verbose = false
+end
+
+desc 'Console'
+task :console do
+  require 'pry'
+  require 'spool'
+  ARGV.clear
+  Pry.start
+end
+
+task default: :spec

data/bin/spool

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+if ARGV.empty? || !File.exists?(ARGV.first)
+  puts "Invalid pool specification file: #{ARGV.first}"
+  exit 1
+end
+
+pid = fork do
+  require File.expand_path('../lib/spool', File.dirname(__FILE__))
+  config = Spool::DSL.configure ARGV.first
+  Spool::Pool.new(config).start
+end
+
+puts "Pool started (#{pid})"

data/lib/spool/configuration.rb

@@ -0,0 +1,25 @@
+module Spool
+  class Configuration
+  
+    attr_accessor :processes, 
+                  :env,
+                  :dir, 
+                  :command, 
+                  :pidfile, 
+                  :restart_condition,
+                  :source_file
+
+    def env
+      @env ||= {}
+    end
+
+    def dir
+      @dir ||= source_file ? File.dirname(source_file) : Dir.pwd
+    end
+
+    def pidfile
+      @pidfile ||= File.join(dir, (source_file ? "#{File.basename(source_file, '.*')}.pid" : 'pool.pid'))
+    end
+
+  end
+end

data/lib/spool/dsl.rb

@@ -0,0 +1,48 @@
+module Spool
+  class DSL
+
+    def self.configure(filename=nil, &block)
+      dsl = new filename, &block
+      dsl.configuration
+    end
+    
+    attr_reader :configuration
+    
+    private
+
+    def initialize(filename=nil, &block)
+      @configuration = Configuration.new
+      if filename
+        configuration.source_file = File.expand_path filename
+        instance_eval IO.read(filename), filename
+      else
+        instance_eval &block
+      end
+    end
+
+    def processes(count)
+      configuration.processes = count
+    end
+
+    def env(env)
+      configuration.env = env
+    end
+
+    def dir(dir)
+      configuration.dir = File.expand_path dir
+    end
+
+    def command(command)
+      configuration.command = command
+    end
+
+    def pidfile(pidfile)
+      configuration.pidfile = pidfile
+    end
+
+    def restart_when(&block)
+      configuration.restart_condition = block
+    end
+
+  end
+end

data/lib/spool/pool.rb

@@ -0,0 +1,117 @@
+module Spool
+  class Pool
+
+    SIGNALS = {
+      INT:  :stop!,
+      TERM: :stop!,
+      QUIT: :stop,
+      HUP:  :reload,
+      USR2: :restart,
+      TTIN: :incr,
+      TTOU: :decr
+    }
+
+    attr_reader :configuration, :runner, :processes
+    
+    def initialize(configuration=nil, &block)
+      @configuration = configuration || DSL.configure(&block)
+      @processes = []
+      @started = false
+    end
+
+    def started?
+      @started
+    end
+
+    def stopped?
+      !started?
+    end
+
+    def start
+      @started = true
+
+      handle_signals
+
+      File.write configuration.pidfile, Process.pid if configuration.pidfile
+
+      configuration.processes.times.map do
+        processes << Spawner.spawn(configuration)
+      end
+
+      while @started
+        check_status
+        sleep 0.05
+      end
+    end
+
+    def stop(timeout=0)
+      processes.each(&:stop)
+      Timeout.timeout(timeout) { wait_for_stopped processes }
+    rescue Timeout::Error
+    ensure
+      stop!
+    end
+
+    def stop!
+      processes.each(&:kill)
+      processes.clear
+      File.delete configuration.pidfile if File.exists? configuration.pidfile
+      @started = false
+    end
+
+    def incr(count=1)
+      configuration.processes += count
+    end
+
+    def decr(count=1)
+      configuration.processes -= count
+    end
+
+    def reload
+      @configuration = DSL.configure configuration.source_file if configuration.source_file
+    end
+
+    def restart
+      processes.each(&:stop)
+    end
+
+    private
+
+    def handle_signals
+      SIGNALS.each do |signal, event|
+        Signal.trap(signal) { send event }
+      end
+    end
+
+    def check_status 
+      return if stopped?
+
+      processes.select(&configuration.restart_condition).each(&:stop) if configuration.restart_condition
+      processes.delete_if { |p| !p.alive? }
+
+      if configuration.processes > processes.count
+        (configuration.processes - processes.count).times do
+          processes << Spawner.spawn(configuration)
+        end
+
+      elsif configuration.processes < processes.count
+        list = processes.take(processes.count - configuration.processes)
+        list.each(&:stop)
+        wait_for_stopped list
+        list.each { |p| processes.delete p }
+
+      end
+
+    rescue
+      retry
+    end
+
+    def wait_for_stopped(processes)
+      while processes.any?(&:alive?)
+        sleep 0.01
+      end
+    end
+
+  end
+
+end

data/lib/spool/spawner.rb

@@ -0,0 +1,56 @@
+module Spool
+  class Spawner
+
+    attr_reader :configuration
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def spawn
+      base_file = File.join Dir.tmpdir, SecureRandom.uuid
+      pid_file = "#{base_file}.pid"
+      out_file = "#{base_file}.out"
+      script_file = "#{base_file}.sh"
+
+      File.write script_file, %Q{
+        #!/usr/bin/env bash
+        #{configuration.command.strip} &
+        echo $! > #{pid_file}
+      }
+
+      ::Process.spawn configuration.env, 
+                      "sh #{script_file}", 
+                      chdir: configuration.dir, 
+                      out: out_file, 
+                      err: out_file
+
+      pid = wait_for_pid pid_file
+
+      Datacenter::Process.new(pid).tap do |process|
+        raise "Invalid command: #{configuration.command}\n#{IO.read(out_file)}" unless process.alive?
+      end
+
+    ensure
+      [script_file, out_file, pid_file].each do |filename|
+        File.delete filename if File.exists? filename
+      end
+    end
+
+    def self.spawn(configuration)
+      new(configuration).spawn
+    end
+
+    private
+
+    def wait_for_pid(pid_file)
+      Timeout.timeout(60) do
+        until File.exists?(pid_file); end
+        IO.read(pid_file).to_i
+      end
+    rescue Timeout::Error
+      nil
+    end
+
+  end
+end

data/lib/spool/version.rb

@@ -0,0 +1,3 @@
+module Spool
+  VERSION = '0.0.1'
+end

data/lib/spool.rb

@@ -0,0 +1,10 @@
+require 'datacenter'
+require 'securerandom'
+require 'timeout'
+require 'tmpdir'
+
+require_relative 'spool/version'
+require_relative 'spool/dsl'
+require_relative 'spool/configuration'
+require_relative 'spool/pool'
+require_relative 'spool/spawner'

data/spec/coverage_helper.rb

@@ -0,0 +1,8 @@
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start

data/spec/dsl_spec.rb

@@ -0,0 +1,37 @@
+require 'minitest_helper'
+
+describe Spool::DSL do
+
+  MockProcess = Struct.new :memory
+  
+  it 'Configure from block' do
+    config = Spool::DSL.configure do
+      processes 10
+      env VAR_1: 1, VAR_2: 2
+      dir '/tmp'
+      command 'tailf file.log'
+      pidfile '/tailf.pid'
+      restart_when { |p| p.memory > 512 }
+    end
+
+    assert_configuration config
+    config.source_file.must_equal nil
+  end
+
+  it 'Configure from file' do
+    config_file = File.expand_path('../sample_config.rb', __FILE__)
+    config = Spool::DSL.configure config_file
+    assert_configuration config
+    config.source_file.must_equal config_file
+  end
+
+  def assert_configuration(config)
+    config.processes.must_equal 10
+    config.env.must_equal VAR_1: 1, VAR_2: 2
+    config.dir.must_equal '/tmp'
+    config.command.must_equal 'tailf file.log'
+    config.restart_condition.call(MockProcess.new(600)).must_equal true
+    config.restart_condition.call(MockProcess.new(100)).must_equal false
+  end
+
+end

data/spec/minitest_helper.rb

@@ -0,0 +1,12 @@
+require 'coverage_helper'
+require 'minitest/autorun'
+require 'turn'
+require 'spool'
+require 'pry-nav'
+require 'benchmark'
+
+Turn.config do |c|
+  c.format = :pretty
+  c.natural = true
+  c.ansi = true
+end

data/spec/pool_spec.rb

@@ -0,0 +1,123 @@
+require 'minitest_helper'
+
+describe Spool::Pool do
+
+  after do
+    @pool.stop! if @pool
+    machine = Datacenter::Machine.new
+    machine.processes('ruby -e').each(&:kill)
+  end
+
+  def start_pool(&block)
+    @pool = Spool::Pool.new(&block).tap do |pool|
+      t = Thread.new { pool.start }
+      t.abort_on_exception = true
+      while pool.processes.count < pool.configuration.processes
+      end
+    end
+  end
+
+  def assert_with_timeout(timeout, &block)
+    Timeout.timeout(timeout) do
+      until block.call
+        sleep 0.01
+      end
+    end
+  end
+
+  it 'Start and stop' do
+    pool = start_pool do
+      processes 1
+      command 'ruby -e "loop do; sleep 1; end"'
+    end
+
+    pool.must_be :started?
+    pool.processes.count.must_equal 1
+    pool.processes[0].must_be :alive?
+
+    process = pool.processes[0]
+
+    pool.stop
+
+    pool.must_be :stopped?
+    pool.processes.must_be_empty
+    process.wont_be :alive?
+  end
+
+  it 'Recover killed process' do
+    pool = start_pool do
+      processes 1
+      command 'ruby -e "loop do; sleep 1; end"'
+    end
+
+    original_process = pool.processes[0]
+    original_process.kill
+
+    original_process.wont_be :alive?
+
+    until pool.processes[0] && pool.processes[0].pid != original_process.pid; end
+
+    pool.processes.count.must_equal 1
+    pool.processes[0].must_be :alive?
+  end
+
+  it 'Stop with timeout' do
+    pool = start_pool do
+      processes 1
+      command 'ruby -e "Signal.trap(:TERM) { sleep 5; exit 0 }; loop { sleep 1 }"'
+    end
+
+    process = pool.processes[0]
+
+    Benchmark.realtime { pool.stop 0.1 }.must_be :<, 1
+
+    pool.must_be :stopped?
+    pool.processes.must_be_empty    
+    process.wont_be :alive?
+  end
+
+  it 'Increase processes' do
+    pool = start_pool do
+      processes 1
+      command 'ruby -e "loop do; sleep 1; end"'
+    end
+
+    pool.processes.count.must_equal 1
+
+    pool.incr 2
+
+    assert_with_timeout(1) { pool.processes.count == 3 }
+  end
+
+  it 'Decrease processes' do
+    pool = start_pool do
+      processes 3
+      command 'ruby -e "loop do; sleep 1; end"'
+    end
+
+    pool.processes.count.must_equal 3
+
+    pool.decr 2
+
+    assert_with_timeout(1) { pool.processes.count == 1 }
+  end
+
+  it 'Change process when satisfied stop condition' do
+    pool = start_pool do
+      processes 1
+      command 'ruby -e "a = 50_000_000.times.to_a; loop do; end"'
+      restart_when { |p| p.memory > 300 }
+    end
+
+    memory = 0
+    process = pool.processes[0]
+    while pool.processes.empty? || pool.processes[0].pid == process.pid
+      memory = process.memory
+    end
+
+    memory.must_be :>, 100
+    pool.processes.count.must_equal 1
+    pool.processes[0].pid.wont_equal process.pid
+  end
+
+end

data/spec/sample_config.rb

@@ -0,0 +1,6 @@
+processes 10
+env VAR_1: 1, VAR_2: 2
+dir '/tmp'
+command 'tailf file.log'
+pidfile '/tailf.pid'
+restart_when { |p| p.memory > 512 }

data/spec/sample_worker.rb

@@ -0,0 +1,4 @@
+loop do
+  puts "#{Process.pid} - #{Time.now}"
+  sleep 1
+end

data/spool.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spool/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'spool'
+  spec.version       = Spool::VERSION
+  spec.authors       = ['Gabriel Naiman']
+  spec.email         = ['gnaiman@keepcon.com']
+  spec.summary       = 'Manage and keep alive pool of processes'
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/gabynaiman/spool'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'datacenter', '~> 0.1.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'minitest', '~> 4.7'
+  spec.add_development_dependency 'turn', '~> 0.9'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'pry-nav'
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: spool
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gabriel Naiman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: datacenter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- gnaiman@keepcon.com
+executables:
+- spool
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/spool
+- lib/spool.rb
+- lib/spool/configuration.rb
+- lib/spool/dsl.rb
+- lib/spool/pool.rb
+- lib/spool/spawner.rb
+- lib/spool/version.rb
+- spec/coverage_helper.rb
+- spec/dsl_spec.rb
+- spec/minitest_helper.rb
+- spec/pool_spec.rb
+- spec/sample_config.rb
+- spec/sample_worker.rb
+- spool.gemspec
+homepage: https://github.com/gabynaiman/spool
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Manage and keep alive pool of processes
+test_files:
+- spec/coverage_helper.rb
+- spec/dsl_spec.rb
+- spec/minitest_helper.rb
+- spec/pool_spec.rb
+- spec/sample_config.rb
+- spec/sample_worker.rb