amiral 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p194@amiral"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.16.6 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in amiral.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2012 Pierre-Yves Ritschard <pyr@spootnik.org>
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.			  

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/amiral.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'amiral/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "amiral"
+  gem.version       = Amiral::VERSION
+  gem.authors       = ["Pierre-Yves Ritschard"]
+  gem.email         = ["pyr@spootnik.org"]
+  gem.description   = %q{simple command and control based on redis}
+  gem.summary       = %q{issue commands on a fleet of hosts}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency 'redis'
+  gem.add_dependency 'json'
+  gem.add_dependency 'facter'
+  gem.add_dependency 'awesome_print'
+  gem.add_dependency 'uuidtools'
+  gem.add_dependency 'daemons'
+end

data/bin/amiral.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require 'rubygems'
+require 'redis'
+require 'amiral'
+require 'optparse'
+require 'ostruct'
+require 'logger'
+
+options = OpenStruct.new
+options.logfile = "amiral.log"
+options.host = "localhost"
+options.port = 6379
+options.match_spec = {:all => true}
+options.foreground = false
+options.rundir = "."
+options.loglevel = Logger::INFO
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: amiral-agent [options]"
+  
+  opts.separator ""
+
+  opts.on("-k", "--private-key [keypath]", String, "SSH private key path") do |keypath|
+    tab = keypath.split(':')
+    if tab.length > 1
+      options.keytype = tab[0].to_sym
+      options.keypath = tab[1]
+    else
+      options.keytype = :dss
+      options.keypath = keypath
+    end
+  end
+
+  opts.on("-l", "--log-level [level]", String, "log level") do |level|
+    case level
+    when "debug"
+      options.loglevel = Logger::DEBUG
+    when "info"
+      options.loglevel = Logger::INFO
+    when "warn"
+      options.loglevel = Logger::WARN
+    when "error"
+      options.loglevel = Logger::ERROR
+    end
+  end
+
+  opts.on("-c", "--redis-host [host]", String, "redis host") do |host|
+    options.host = host
+  end
+
+  opts.on("-p", "--redis-port [port]", OptionParser::DecimalInteger,
+          "redis port") do |port|
+    options.port = port
+  end
+
+  opts.on("-x", "--exchange [exchange]", String,
+          "redis pubsub exchange name") do |exchange|
+    options.exchange = exchange
+  end
+
+  opts.on("-q", "--response-queue [queue]", String,
+          "redis response queue name") do |rqueue|
+    options.queue_response = rqueue
+  end
+
+  opts.on("-a", "--ack-queue [queue]", String,
+          "redis acknowledgement queue name") do |aqueue|
+    options.queue_ack = aqueue
+  end
+
+  opts.on("-r", "--rundir [directory]", String, "agent run directory") do |rundir|
+    options.rundir = rundir
+  end
+
+  opts.on("-f", "--[no]-foreground") do |foreground|
+    options.foreground = foreground
+  end
+
+  opts.on("-m", "--match [filter]", String, "node match filter") do |matcher|
+    options.match_spec = Amiral::Matcher.new(matcher).spec
+  end
+
+  opts.on("-T", "--timeouts [timeouts]", String,
+          "acknowledgement and response timeouts") do |timeouts|
+    tab = timeouts.split(',')
+    raise "timeouts need two values" unless tab.length == 2
+    options.ack_timeout = tab[0].to_i
+    options.reponse_timeout = tab[1].to_i
+  end
+end.parse!
+
+options.redis_in = Redis.new :host => options.host, :port => options.port
+options.redis_out = Redis.new :host => options.host, :port => options.port
+
+classes = {
+  'agent' => Amiral::Agent,
+  'controller' => Amiral::Controller
+}
+
+role = ARGV.shift
+
+raise "no such role: #{role}" unless klass = classes[role]
+
+klass.new(options.marshal_dump).tap{|o| o.run(ARGV)}.show

data/lib/amiral/version.rb

@@ -0,0 +1,3 @@
+module Amiral
+  VERSION = '0.1.0'
+end

data/lib/amiral.rb

@@ -0,0 +1,379 @@
+require 'rubygems'
+require 'redis'
+require 'openssl'
+require 'base64'
+require 'json'
+require 'facter'
+require 'uuidtools'
+require 'awesome_print'
+require 'daemons'
+require 'logger'
+
+module Amiral
+  KEYTYPES = {
+    :dss => OpenSSL::PKey::DSA,
+    :rsa => OpenSSL::PKey::RSA
+  }
+
+  module Providers
+    class Uptime
+
+      PATTERN = /^([0-9:]+) up (.*),[ \t]+([0-9]+) users,[ \t]+load average: ([0-9.]+), ([0-9.]+), ([0-9.]+)/
+      def execute message
+        uptime = `uptime`.strip
+        
+        if uptime =~ PATTERN
+          {
+            :exit => 0,
+            :time => $1,
+            :since => $2,
+            :users => $3,
+            :averages => [$4, $5, $6],
+            :short => uptime,
+          }
+        else
+          {
+            :exit => 1,
+            :short => "could not parse uptime",
+            :data => {:uptime => uptime}
+          }
+        end
+      end
+    end
+
+    class Facts
+      def execute message
+        facts = Facter.to_hash
+        {
+          :exit => 0,
+          :facts => facts,
+          :short => "returned #{facts.length} facts"
+        }
+      end
+    end
+
+    class Ping
+      def execute message
+        {:exit => 0, :short => "alive"}
+      end
+    end
+
+    class ProviderList
+      def execute message
+        {:exit => 0, :short => PROVIDERS.keys}
+      end
+    end
+
+    class Invalid
+      def execute message
+        {:exit => 1, :short => "invalid provider requested"}
+      end
+    end
+  end
+
+  PROVIDERS = {
+    "uptime"        => Amiral::Providers::Uptime,
+    "facts"         => Amiral::Providers::Facts,
+    "provider_list" => Amiral::Providers::ProviderList,
+    "ping"          => Amiral::Providers::Ping,
+    "invalid"       => Amiral::Providers::Invalid
+  }
+
+  module Message
+    
+    def serialize data
+      json = data.to_json
+      digest = OpenSSL::Digest::SHA1.digest(json)
+      sig = Base64.encode64(@privkey.syssign(digest)).chomp
+      "#{sig}:#{json}"
+    end
+
+    def deserialize data
+      (sig, json) = data.split(':', 2)
+      raise "wrong format" unless (sig && json)
+      sig = Base64.decode64(sig)
+      unless @privkey.sysverify(OpenSSL::Digest::SHA1.digest(json), sig)
+        raise "invalid signature"
+      end
+      JSON.parse json
+    end
+
+    def hostname
+      `hostname`.strip
+    end
+
+    def uuidgen
+      UUIDTools::UUID.random_create.to_s
+    end
+
+    def timestamp
+      Time.new.to_i
+    end
+  end
+
+  class Matcher
+
+    attr_accessor :spec
+
+    def match
+      return true if (@spec.has_key? :all && @spec[:all])
+      
+      if @spec.has_key? :hostname
+        return false unless `hostname`.chomp =~ Regexp.new(@spec[:hostname])
+      end
+
+      ## XXX: platform
+
+      if @spec.has_key? :provider
+        return false unless PROVIDERS.has_key? @spec[:provider]
+      end
+
+      if @spec.has_key? :facts
+        @spec[:facts].map do |k,v|
+          return false unless Facter.value(k).to_s =~ Regexp.new(v)
+        end
+      end
+      true
+    end
+
+    def initialize match_spec
+      case match_spec.class.to_s
+      when "String"
+        @spec = match_spec.split(',').map do |expr|
+          (lval, rval) = expr.split('=', 2)
+          case lval
+          when 'all'
+            { :all => true }
+          when /^fact:/
+            { :facts => { lval.split(':', 2)[1] => rval } }
+          when /^hostname/
+            { :hostname => rval }
+          when /^provider/
+            { :provider => rval }
+          when /^platform/
+            { :platform => rval }
+          else
+            raise "unsupported match filter type: #{lval}"
+          end
+        end.reduce do |e1,e2|
+          e1.merge(e2) do |e,c1,c2|
+            c1.merge c2
+          end
+        end
+      when "Hash"
+        @spec = match_spec.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+      else
+        raise "don't know how to create match from #{match_spec} (#{match_spec.class})"
+      end
+    end
+    
+    def serialize
+      @spec.to_json
+    end
+    
+  end
+  
+  class Agent
+    include Message
+
+    def initialize settings
+      @settings = {
+        :redis_in => Redis.new,
+        :redis_out => Redis.new,
+        :exchange => "req",
+        :queue_ack => "ack",
+        :queue_response => "res",
+        :keytype => :dss,
+        :hostname => hostname,
+        :loglevel => Logger::INFO,
+        :rundir => "."
+      }.merge settings
+
+      @logger = Logger.new(STDOUT)
+      @logger.level = @settings[:loglevel]
+
+      raise "need key path" unless @settings[:keypath]
+
+      @privkey = File.open @settings[:keypath] do |file| 
+        KEYTYPES[@settings[:keytype]].new file
+      end
+
+      @redis_in = @settings[:redis_in]
+      @redis_out = @settings[:redis_out]
+    end
+
+    def run args
+      begin
+        Daemons.daemonize({:backtrace => true,
+                            :ontop => @settings[:foreground],
+                            :dir_mode => :normal,
+                            :dir => @settings[:rundir],
+                            :app_name => "amiral-agent",
+                            :log_output => true})
+        listen
+      rescue SystemExit => err
+        @logger.debug("agent starting")
+      rescue Exception => err
+        @logger.fatal("caught exception, exiting")
+        @logger.fatal(err)
+      end
+    end
+
+    def show
+    end
+
+    def listen
+      exchange = @settings[:exchange]
+      @redis_in.subscribe exchange do |on|
+        
+        on.message do |x, payload|
+
+          @logger.debug "incoming message"
+          message = deserialize payload
+        
+          if agent_matches? message['match']
+            @logger.info "valid request type: #{message['command']['provider']}"
+            uuid = uuidgen
+            send_ack(:in_reply_to => message['reply_to'],
+                     :uuid => uuid,
+                     :status => "start")
+            begin
+              out = execute message
+            rescue Exception => e
+              @logger.error "provider crashed: #{e}"
+              @logger.error e
+            end
+            send_response(:in_reply_to => message['reply_to'],
+                          :uuid => uuid,
+                          :status => "complete",
+                          :output => out)
+          else
+            send_ack(:in_reply_to => message['reply_to'],
+                     :status => "noop")
+          end
+        end
+      end
+    end
+
+    def agent_matches? matcher
+      m = Matcher.new matcher
+      m.match
+    end
+
+    def execute message
+      (PROVIDERS[message['command']['provider']] ||
+       Amiral::Providers::Invalid).new.execute message
+    end
+    
+    def send_ack response
+      send_payload @settings[:queue_ack], response
+    end
+
+    def send_response response
+      send_payload @settings[:queue_response], response
+    end
+
+    def send_payload q, response
+      response[:hostname] = @settings[:hostname]
+      @redis_out.lpush q, serialize(response)
+    end
+  end
+
+  class Controller
+    include Message
+
+    def initialize settings
+      @settings = {
+        :redis => Redis.new,
+        :exchange => "req",
+        :queue_ack => "ack",
+        :queue_response => "res",
+        :ack_timeout => 2,
+        :response_timeout => 5,
+        :match_spec => {:all => true},
+        :keytype => :dss,
+      }.merge settings
+
+      raise "need key path" unless @settings[:keypath]
+
+      @privkey = File.open @settings[:keypath] do |file| 
+        KEYTYPES[@settings[:keytype]].new file
+      end
+
+      @redis = @settings[:redis]
+    end
+
+    def run args
+      provider = args.shift
+      request provider, {}
+    end
+
+    def request provider, args
+
+      uuid = uuidgen
+      request = {
+        :command => {
+          :provider => provider,
+          :args => args
+        },
+        :reply_to => uuid,
+        :match => @settings[:match_spec]
+      }
+      
+      @redis.publish @settings[:exchange], serialize(request)
+
+      # pop items for 2 seconds
+      acks = []
+      puts "accepting acknowledgements for #{@settings[:ack_timeout]} seconds"
+      limit = timestamp + @settings[:ack_timeout]
+
+      while timestamp < limit do
+        t = timestamp
+        t = (limit - t == 0) ? 1 : (limit - t)
+        (q, data) = @redis.brpop(@settings[:queue_ack], t) 
+        if data
+          ack = deserialize data
+          acks << ack if ack['in_reply_to'] == uuid
+        end
+      end
+
+      pos_acks = acks.keep_if do |a|
+        a['status'] == 'start'
+      end
+
+      remaining = pos_acks.length
+      puts "got #{remaining}/#{acks.length} positive acknowledgements"
+      
+      # wait for responses now
+      @responses = []
+      limit = timestamp + @settings[:response_timeout]
+      while timestamp < limit && remaining > 0 do
+        t = timestamp
+        t = (limit - t == 0) ? 1 : (limit - t)
+        (q, data) = @redis.brpop(@settings[:queue_response], t)
+
+        if data
+          res = deserialize data
+          if res['in_reply_to'] == uuid
+            @responses << res
+            remaining -= 1
+          end
+        end
+      end
+
+      puts "got #{@responses.length}/#{pos_acks.length} responses"
+      @responses
+    end
+
+    def show
+      @responses.map do |node|
+        if node['output']['exit'] == 0
+          printf "%20s: %s\n", node['hostname'], node['output']['short']
+        else
+          printf "%20s: failed!\n", node['hostname']
+          ap node
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: amiral
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Pierre-Yves Ritschard
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: simple command and control based on redis
+email:
+- pyr@spootnik.org
+executables:
+- amiral.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- LICENSE.txt
+- Rakefile
+- amiral.gemspec
+- bin/amiral.rb
+- lib/amiral.rb
+- lib/amiral/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: issue commands on a fleet of hosts
+test_files: []