zkexec 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a532a346ae00de20b532490448e745ae1132a57f
+  data.tar.gz: 08435fec85406659ec1f6266b598d9af96612c9c
+SHA512:
+  metadata.gz: 84d232792462ce12690fb431f57240402e6570e58a5e4cf9c082f329f4211e79c25c9d2c6624d72d0e431db6361c89337fc00f4708bae92a2f3801a73fca6d49
+  data.tar.gz: 1067682fb483530f67a73fd071fd0fa9a57e0eb1ab55dca74253c14b2a57b45dff270f91fc9f3c9fa42555c7f60d77a0b0e2bd9bedb9b947bce6b64a799ddcb3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zkexec.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Kyle Maxwell
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,160 @@
+# zkexec
+
+zkexec is a wrapper around an executable that imports configuration from a zookeeper cluster.  When zookeeper configuration changes, zkexec syncs the configuration, then restarts the executable.  zkexec supports rolling restarts, health checks, and alerts.
+
+Warning: this code is not yet being used in production. 
+
+## Tutorial
+
+### Ante
+
+1. Get a local zookeeper running on :2181 (the default port).
+2. Checkout this git repo and cd into it.
+3. Either `gem install zkexec`, or `bundle; export PATH=$PATH:bin`
+
+### Calling zkexec
+
+We'll start with the contrived example of running a script that acts just like cat, but it prefixes each line with the contents of a file:
+
+    :$ echo -n hello > /tmp/prefix
+    :$ echo world | ./test/libexec/prefixed-cat /tmp/prefix
+    hello world
+
+We want to run this in the console, such that we can type input in stdin, and get our prefixed output.  Here's the invokation (feel free to remove the --silent):
+
+    :$ zkexec --silent --exec "./test/libexec/prefixed-cat /tmp/prefix"
+
+This doesn't do anything special--it just wraps the execution.  Go ahead and run that, then type a few lines.  Ctrl-c away, check the return code.  It should behave just like `prefixed-cat`.
+
+### Updating the config files
+
+Let's put the prefix in zookeeper.  We'll throw it in the root, and use the --mirror flag to let zkexec know about it.
+
+    :$ zookeeper/bin/zkCli.sh -cmd create /prefix hello
+    :$ zkexec --silent \
+         --exec "./test/libexec/prefixed-cat /tmp/prefix" \
+         --mirror /tmp/prefix=/prefix
+    world                                    # typed into stdin
+    hello world                              # stdout    
+
+So, now that we're tracking the config file, lets change it in another console.  Leave the existing terminal running.
+
+    :$ zookeeper/bin/zkCli.sh -cmd set /prefix goodbye
+
+Now, flip back to the original terminal and start typing.  You'll note that now the prefix is `goodbye`!
+
+### Edge cases
+
+#### What if you run a command that terminates?
+
+zkexec will exit with the same error code.  Try it with ``/usr/bin/false`! (oh and I'll take this opportunity to show the default logs that show up on stderr when you don't pass the --silent flag)
+
+    :$ zkexec --exec false # $ zkexec --exec false
+    [2014-04-28 16:41:38 -0700] connecting to localhost:2181
+    [2014-04-28 16:41:38 -0700] connected
+    [2014-04-28 16:41:38 -0700] forking false
+    [2014-04-28 16:41:38 -0700] forked 41804
+    [2014-04-28 16:41:38 -0700] command failed
+    :$ echo $?
+    1
+
+This means that you should have something (monit, etc) watching zkexec.  zkexec is not trying to be an all purpose monitoring solution.
+
+### Adding health checks
+
+A health check is simply an executable that returns 0 when the system is healthy, and non-zero otherwise.  For this tutorial, we'll use `./test/libexec/slowcheck`.  slowcheck takes two arguments, (1) the tcp port to check, and (2) the number of seconds to sleep before checking it.  If and only if the port is open, slowcheck will succeed.  The sleep will be useful later.
+
+Let's append a health check to the previous command
+
+    :$ zkexec \
+         --exec "./test/libexec/prefixed-cat /tmp/prefix" \
+         --mirror /tmp/prefix=/prefix \
+         --health "./test/libexec/slowcheck 2181 1" \
+         --health-delay 1 \
+         --health-interval 1
+
+The health-delay is the interval before the first health check (useful if your script is slow to start), and after the service starts, the health check will run every health-interval seconds.  It's up to you to make the health check script reasonable.  zkexec will happily run absurd scripts without timeouts, etc.
+
+In this case, we're actually checking zookeeper's port. There's nothing special or required about that, it just happened to be convenient filler.
+
+If you aren't running in silent mode, you'll see output like:
+
+    [2014-04-28 16:51:07 -0700] connecting to localhost:2181
+    [2014-04-28 16:51:07 -0700] connected
+    [2014-04-28 16:51:07 -0700] registering callback on /prefix
+    [2014-04-28 16:51:07 -0700] forking ./test/libexec/prefixed-cat /tmp/prefix
+    [2014-04-28 16:51:07 -0700] forked 42333
+    [2014-04-28 16:51:08 -0700] health checking via: ./test/libexec/slowcheck 2181 1
+    [2014-04-28 16:51:09 -0700] successful health check
+    [2014-04-28 16:51:10 -0700] health checking via: ./test/libexec/slowcheck 2181 1
+    [2014-04-28 16:51:12 -0700] successful health check
+
+
+### Adding alerts
+
+An alert is simply an executable that gets called when a health check fails unexpectedly.  Alerts won't trigger within `health-delay` seconds of a restart.
+
+Alerts are provided for your convenience.  In most cases, it's better to just add a dead man's switch to your health check.
+
+#### A better alert scheme
+
+Install a dead man's switch via monit via something similar to this sample code:
+
+    :$ mkdir -p /var/run/foo
+    :$ echo "check file succeeded\_at path /var/run/foo/succeeded\_at 
+         if timestamp < 5 minutes then alert" > /etc/monit/conf.d/foo.conf
+    :$ sudo monit reload 
+    :$
+    :$ zkexec \
+         ...
+         --health "./test/libexec/slowcheck 2181 1 && touch /tmp/succeeded_at" 
+
+### Rolling restarts
+
+Use `--lock NAME` to establish a rolling restart group.  When a config file changes, all wrapped processes sharing the lock name and the zookeeper cluster must acquire the lock before killing the child process.  A zkexec only releases the lock when health checks succeed or the zkexec parent process is killed.
+
+Give it a try locally.  In this case, we'll use the slowness of the health check script to our advantage, because we can watch the restart execute serially via the timestamps of the logs.
+
+Run two or more of the following command in separate terminals:
+
+    :$ zkexec \
+         --exec "./test/libexec/prefixed-cat /tmp/prefix" \
+         --lock foo \
+         --health "test/libexec/slowcheck 2181 10" \
+         --mirror /tmp/prefix=/prefix \
+         --health-delay 20 \
+         --health-interval 1
+
+Then, run the following:
+
+    :$ zookeeper/bin/zkCli.sh -cmd set /prefix woot
+
+You can watch the timestamps of the zkexec processes as they log on stdout, noting that they take turns restarting.
+
+#### Config failures in rolling restart
+
+A config failure during a restart is defined as either (1) the restarted process exiting non-zero, or (2) the restarted process failing health checks for `health-delay` seconds.
+
+A config failure pauses the restart.  If this happens, you should push new config, and if it doesn't fail, the restart will pick up where it left off.
+
+### All options
+
+zkexec doesn't use config files.  All options are documented on the command line `--help`:
+
+    :$ zkexec -h
+    Usage: zkexec [options]
+
+    Run a command, and restart if the config files change on the remote zookeeper.
+
+        -e, --exec COMMAND               Run this command
+        -c, --cluster HOST:PORT,...      Comma-delimited list of zookeeper hosts
+        -H, --health COMMAND             Run this command to health-check
+        -i, --health-interval INTERVAL   Health-check every INTERVAL seconds
+        -d, --health-delay INTERVAL      Wait before starting health checks
+        -m, --mirror LOCAL_PATH=ZK_PATH  Mirror a config file from zookeeper to localhost
+        -l, --lock NAME                  Name of a zk lockfile, used to enforce rolling restarts
+        -a, --alert COMMAND              Run this command if the primary command returns
+                                         falsey or health checks fail for too long
+        -v, --verbose
+        -s, --silent
+        -h, --help                       Show this message

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/zkexec

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
+Thread.abort_on_exception = true
+require "zkexec"
+
+runner = ZkExec::Runner.new
+runner.run(ARGV)

data/lib/zkexec.rb

@@ -0,0 +1,15 @@
+require "zkexec/version"
+require "zkexec/executor"
+require "zkexec/runner"
+
+require "zk"
+
+module ZkExec
+  def log(s) 
+    STDERR.puts("[#{Time.now}] #{s}") unless $silent
+  end
+  
+  def silence!
+    $silent = true
+  end
+end

data/lib/zkexec/executor.rb

@@ -0,0 +1,182 @@
+require "zk"
+require "thread"
+
+module ZkExec
+  class Executor
+    include ZkExec
+    include Process
+    
+    def initialize(options)
+      @cmd             = options[:exec]
+      @cluster         = options[:cluster]
+      @health          = options[:health]
+      @health_interval = options[:health_interval]
+      @health_delay    = options[:health_delay]
+      @mirrors         = options[:mirrors]
+      @alert           = options[:alert]
+      @lock_name       = options[:lock]
+      
+      log "connecting to #{@cluster}"
+      @zk = ZK.new(@cluster, :thread => :per_callback)
+      raise "timeout connecting to #{@cluster}" unless @zk.connected?
+      log "connected"
+      
+      # re-establish watches
+      @on_connected ||= @zk.on_connected do
+        @mirrors.each do |(local, remote)|
+          watch(remote)
+        end
+      end
+      
+      @restart_lock = @lock_name && @zk.exclusive_locker(@lock_name)
+      @local_lock = Mutex.new
+      
+      @mirrors.each do |(local, remote)|
+        log "registering callback on #{remote}"
+        @zk.register(remote) do |event|
+          if event.changed?
+            log "#{remote} changed"
+            copy(local, remote)  
+            kill_to_refork
+          else
+            watch(remote)  
+          end
+        end
+        watch(remote)
+      end
+    end
+
+    private 
+    def copy(local, remote)
+      data = watch(remote)
+      File.open(local, "w") {|f| f.print(data) }
+    rescue ZK::Exceptions::NoNode => e
+      raise "node not found in #{e.message}"
+    end
+    
+    private 
+    def watch(remote)
+      data, stat = *@zk.get(remote, :watch => true)
+      return data
+    rescue ZK::Exceptions::NoNode => e
+      raise "node not found in #{e.message}"
+    end
+    
+    private
+    def with_restart_lock
+      if @restart_lock
+        begin
+          log "waiting on lock: #{@lock_name}"
+          @restart_lock.lock(:wait => true)
+          log "acquired lock: #{@lock_name}"
+          yield
+        ensure
+          @restart_lock.unlock
+          log "released lock: #{@lock_name}"
+        end
+      else
+        yield
+      end
+    end
+    
+    private 
+    def pid_exists?(pid)
+      begin
+        Process.getpgid(pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+    
+    private 
+    def kill_to_refork
+      if @child
+        with_restart_lock do
+          if @health_checks
+            Thread.kill(@health_checks) 
+            @health_checks = nil
+          end
+          @should_refork = true
+          child = @child
+          @child = nil
+          
+          log "killing #{child}"
+          Process.kill("TERM", child)
+          checks_started = Time.now
+          while pid_exists?(child) && Time.now - checks_started < 30
+            sleep 1
+          end
+          if pid_exists?(child)
+            log "force killing #{child}"
+            Process.kill("KILL", child) 
+          end
+
+          log "#{child} terminated"
+          
+          # This intentionally infinite loops on failure, so that we don't propagate bad config
+          health_checks_started = Time.now
+          loop do
+            log "waiting for health check success"
+            if system(@health)
+              log "health checks succeeding"
+              return
+            end
+            if Time.now - health_checks_started > @health_delay
+              log "health checks failing"
+              alert
+            end
+            sleep @health_interval
+          end
+        end
+      end
+    end
+    
+    def alert 
+      if @alert
+        fork { exec(@alert) }
+      end
+    end
+    
+    def start_health_thread
+      @health_checks ||= @health && Thread.new {
+        sleep @health_delay
+        loop do
+          log "health checking via: #{@health}"
+          pid = fork { exec(@health) }
+          wait pid
+          if $?.exitstatus == 0
+            log "successful health check"
+          else
+            log "failed health check, alerting"
+            alert
+          end
+          sleep @health_interval
+        end
+      }
+    end
+
+    public
+    def run
+      Thread.new { execute }
+    end
+
+    def execute
+      @should_refork = true
+      
+      while @should_refork
+        start_health_thread
+        log "forking #{@cmd}"
+        @child = fork { exec @cmd }
+        log "forked #{@child}"
+        @should_refork = false
+        wait @child
+      end
+
+      if $?.exitstatus != 0
+        alert 
+        raise "command failed"
+      end
+    end
+  end
+end

data/lib/zkexec/runner.rb

@@ -0,0 +1,82 @@
+require "optparse"
+
+module ZkExec
+  class Runner
+    include ZkExec
+    
+    def initialize
+    end
+
+    def run(args)
+      options = { :mirrors => [], :cluster => "localhost:2181", :health_interval => 30, :health_delay => 30 }
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: zkexec [options]\n\nRun a command, and restart if the config files change on the remote zookeeper.\n\n"
+
+        opts.on("-e", "--exec COMMAND", "Run this command") do |s|
+          options[:exec] = s
+        end
+        
+        opts.on("-c", "--cluster HOST:PORT,...", "Comma-delimited list of zookeeper hosts") do |s|
+          options[:cluster] = s
+        end
+        
+        opts.on("-H", "--health COMMAND", "Run this command to health-check") do |s|
+          options[:health] = s
+        end
+        
+        opts.on("-i", "--health-interval INTERVAL", "Health-check every INTERVAL seconds") do |s|
+          options[:health_interval] = s.to_f
+        end
+        
+        opts.on("-d", "--health-delay INTERVAL", "Wait before starting health checks") do |s|
+          options[:health_delay] = s.to_f
+        end
+        
+        opts.on("-m", "--mirror LOCAL_PATH=ZK_PATH", "Mirror a config file from zookeeper to localhost") do |s|
+          options[:mirrors] << s.split("=")
+        end
+        
+        opts.on("-l", "--lock NAME", "Name of a zk lockfile, used to enforce rolling restarts") do |s|
+          options[:lock] = s
+        end
+                
+        opts.on("-a", "--alert COMMAND", "Run this command if the primary command returns","falsey or health checks fail for too long") do |s|
+          options[:alert] = s
+        end
+        
+        opts.on("-v", "--verbose") do
+          options[:verbose] = true
+        end
+
+        opts.on("-s", "--silent") do 
+          options[:silent] = true
+        end
+        
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end
+      
+      opts.parse!
+      
+      unless options[:exec]
+        puts "Missing required option --exec.  See --help for usage."
+        exit 1
+      end
+      
+      silence! if options[:silent]
+      
+      begin
+        Executor.new(options).execute
+      rescue => e
+        if options[:verbose]
+          raise
+        else
+          log(e.message)
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/zkexec/version.rb

@@ -0,0 +1,3 @@
+module ZkExec
+  VERSION = "0.0.1"
+end

data/test/libexec/prefixed-cat

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+prefixes = ARGV.map{ |arg| File.read(arg) }
+
+while line = STDIN.gets
+  puts [prefixes + [line]].join(" ")
+end

data/test/libexec/slowcheck

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+if ARGV.length != 2
+  abort "usage: #{$0} PORT SLEEP"
+end
+
+require "socket"
+
+sleep ARGV[1].to_f
+TCPSocket.new("localhost", ARGV[0].to_i).close
+

data/test/zk_exec_test.rb

@@ -0,0 +1,27 @@
+gem "minitest"
+require "minitest/autorun"
+require "minitest/spec"
+require "socket"
+
+TCPSocket.new("localhost", 2181).close \
+  rescue abort "No zookeeper running, please start a local one!"
+
+ZKEXEC = File.expand_path(File.join(File.dirname(__FILE__), "..", "bin", "zkexec"))
+
+describe "zkexec" do
+  it "should fail when run without args" do
+    system("#{ZKEXEC}").must_equal(false)
+  end
+  
+  it "should run with exec" do # note this is /usr/bin/true or equivalent
+    system("#{ZKEXEC} --exec true").must_equal(true)
+  end
+  
+  it "should keep the inner response code" do # note this is /usr/bin/false or equivalent
+    system("#{ZKEXEC} --exec false").must_equal(false)
+  end
+  
+  it "should alert on failure" do
+    `#{ZKEXEC} --exec false --alert 'echo FOO'`.must_match(/FOO/)
+  end
+end

data/zkexec.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'zkexec/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "zkexec"
+  spec.version       = ZkExec::VERSION
+  spec.authors       = ["Kyle Maxwell"]
+  spec.email         = ["kyle@kylemaxwell.com"]
+  spec.summary       = %q{Run a process in a wrapper that manages config files from zookeeper}
+  spec.description   = %q{Run a process in a wrapper that manages config files from zookeeper}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "zk", "1.9.4"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: zkexec
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.9.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.9.4
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Run a process in a wrapper that manages config files from zookeeper
+email:
+- kyle@kylemaxwell.com
+executables:
+- zkexec
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/zkexec
+- lib/zkexec.rb
+- lib/zkexec/executor.rb
+- lib/zkexec/runner.rb
+- lib/zkexec/version.rb
+- test/libexec/prefixed-cat
+- test/libexec/slowcheck
+- test/zk_exec_test.rb
+- zkexec.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Run a process in a wrapper that manages config files from zookeeper
+test_files:
+- test/libexec/prefixed-cat
+- test/libexec/slowcheck
+- test/zk_exec_test.rb