auto-consul 0.0.1

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'aws-sdk'
+
+group :development do
+  gem 'rake',              '~> 10.1.0'
+  gem 'pry',               '~> 0.9.0'
+end
+
+group :test do
+  gem 'rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,36 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (1.40.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    json (1.8.1)
+    method_source (0.8.2)
+    mini_portile (0.5.3)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.1.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    slop (3.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk
+  pry (~> 0.9.0)
+  rake (~> 10.1.0)
+  rspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Ethan Rowe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,132 @@
+auto-consul
+===========
+
+Ruby gem for bootstrapping consul cluster members
+
+# Example usage
+
+Given two vagrant boxes, each with consul and auto-consul installed.
+
+Export your AWS keys into the environment in each:
+
+    ```
+    export AWS_ACCESS_KEY_ID=...
+    export AWS_SECRET_ACCESS_KEY=...
+    ```
+
+This will allow the AWS SDK to pick them up.
+
+The server, screen A:
+
+    auto-consul -r s3://my-bucket/consul/test-cluster \
+                -a 192.168.50.100 \
+                -n server1 \
+                run
+
+Then, server screen B:
+
+    while true; do
+        auto-consul -r s3://my-bucket/consul/test-cluster \
+                    -a 192.168.50.100 \
+                    -n server1 \
+                    heartbeat
+        sleep 60
+    done
+
+The first launches the agent, the latter checks its run status and
+issues a heartbeat to the specified S3 bucket.
+
+Because this is the first server, there will be no heartbeats in the
+bucket (assuming a fresh bucket/key combination).  Therefore, the agent
+will be launched in server mode, along with the bootstrap option to
+initialize the raft cluster for state management.
+
+Look in the S3 bucket above, under "servers", and you should see
+a timestamped entry like "20140516092731-server1".  This is produced
+by the "heartbeat" command and allows new agents to discover active
+members of the cluster for joining.
+
+Having seen the server heartbeat, go to the agent vagrant box, and
+do something similar.  Screen A:
+
+    auto-consul -r s3://my-bucket/consul/test-cluster \
+                -a 192.168.50.101 \
+                -n agent1 \
+                run
+
+In this case, the agent will discover the server via its heartbeat.  It
+will know that we have enough servers (it defaults to only wanting one;
+that's fine for dev/testing but not good for availability) and thus
+simply join as a normal agent.
+
+Screen B:
+
+    while true; do
+        auto-consul -r s3://my-bucket/consul/test-cluster \
+                    -a 192.168.50.101 \
+                    -n agent1 \
+                    heartbeat
+        sleep 60
+    done
+
+This generates heartbeats like the server did, but while the server
+sends heartbeats both to "servers" and "agents" in the bucket, the
+normal agent sends heartbeats only to "agents".
+
+# Mode determination
+
+Given a desired number of servers (defaulting to 1) and a registry
+(for now, an S3-style URL), the basic algorithm is:
+
+- Are there enough servers?
+  - Yes: Be an agent.  Done.
+  - No: are there no servers?
+    - Yes: Be a server with bootstrap mode.  Done.
+    - No: Be a server without bootstrap mode, joining with others.  Done.
+
+There is very obviously a race condition in the determination of node
+mode.  In practice, it should be easy enough to coordinate things such
+that the race doesn't cause problems.  Longer term, we'll need to revise
+the mode determination logic to use a backend supporting optimistic
+locking or some such.  (A compare-and-swap pattern would work fine; consul
+itself would allow for this given one existing server).
+
+## Heartbeats and membership
+
+The heartbeats give us a rough indication of cluster membership.  The
+tool uses an expiry time (in seconds) to determine which heartbeats are
+still active, and will purge any expired heartbeats from the registry
+whenever it encounters them.
+
+Each heartbeat tells us:
+- The node's name within the consul cluster
+- The timestamp of the heartbeat (the freshness)
+- The IP at which the node can be reached for cluster join operations.
+
+For now, it is necessary to run the heartbeat utility in parallel to the
+run utility.  In subsequent work we may want to have these things coordinated
+by one daemon, but given the experimental nature of this project it's not
+worth caring about just yet.
+
+The heartbeat asks consul for its status and from that determines if it
+is running as a server or regular agent (or if it is running at all).  If
+consul is not running at all, no heartbeat will be emitted.
+
+The default expiry is 120 seconds.  It is recommended that heartbeats fire
+at half that duration (60 seconds).
+
+# Cluster join
+
+After the node mode is determined, it's necessary (except in the case of
+a bootstrap-mode server) to join a cluster by contacting an extant member.
+
+This is the primary purpose of the heartbeat registry; a server-mode node
+will find the IP (from the active heartbeats) of a *server*, and use that
+IP to join the cluster.  An agent-mode node will find the IP of an *agent*
+for the join operation.
+
+In a production-ready tool, we would have a monitor on the registry and
+keep trying new hosts until a join succeeds.  But in this experimental
+phase, it just picks the first member in the relevant list and uses that.
+If that member is actually down, then the join simply won't work.
+

data/bin/auto-consul

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+
+# vim: set filetype=ruby;
+
+require 'auto-consul'
+require 'optparse'
+require 'socket'
+require 'ostruct'
+
+class UnknownCommandException < Exception
+end
+
+class Command < OpenStruct
+  def local
+    @local ||= AutoConsul::Local.bind_to_path(data)
+  end
+
+  def cluster
+    @cluster ||= AutoConsul::Cluster.new registry
+  end
+
+  def state
+    @state ||= AutoConsul::RunState::CLIProvider.new
+  end
+
+  def do_set_mode
+    cluster.set_mode! local, expiry, servers
+  end
+
+  def do_run
+    if local.mode.nil?
+      do_set_mode
+    end
+    do_direct_run
+  end
+
+  def do_direct_run
+    runner = :no_op
+    runner = :run_agent! if local.agent?
+    runner = :run_server! if local.server?
+    runner = AutoConsul::Runner.method(runner)
+    runner.call node, addr, expiry, local, cluster
+  end
+
+  def do_heartbeat
+    if state.running?
+      cluster.servers.heartbeat! node, addr, expiry if state.server?
+      cluster.agents.heartbeat! node, addr, expiry if state.agent?
+    end
+  end
+
+  def execute cmd
+    command = "do_#{cmd}".to_sym
+    if respond_to? command
+      send command
+    else
+      raise UnknownCommandException.new("Unknown command: #{cmd}")
+    end
+  end
+end
+
+runner = Command.new(:data    => '/tmp/consul/state',
+                     :dc      => 'dc1',
+                     :expiry  => 120,
+                     :servers => 1,
+                     :node    => Socket.gethostname.split('.', 2)[0])
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: auto-consul [options] COMMAND"
+
+  opts.on("-r", "--registry URL", String, "The cluster registry URL") do |u|
+    runner.registry = u
+  end
+
+  opts.on("-d", "--data-dir PATH", String, "The path where local state will be preserved.") do |d|
+    runner.data_dir = d
+  end
+
+  opts.on("-a", "--address IPADDR", String, "The IP address to bind to and announce for cluster communication.") do |a|
+    runner.addr = a
+  end
+
+  opts.on("-n", "--node NAME", String, "The unique name by which the node identifies itself within the cluster.") do |n|
+    runner.node = n
+  end
+
+  opts.on("-e", "--expiry SECONDS", Integer, "The expiration time (in seconds) for registry heartbeats") do |e|
+    runner.expiry = e.to_i
+  end
+
+  opts.on("-s", "--servers NUMBER", Integer, "The desired number of consul servers.") do |s|
+    runner.servers = s.to_i
+  end
+
+  opts.on_tail('-h', '--help', "Show this help message.") do
+    puts opts
+    exit
+  end
+end
+
+parser.parse!
+
+begin
+  runner.execute(ARGV.shift)
+rescue UnknownCommandException => e
+  puts e.message
+  puts parser
+  exit 2
+end
+

data/lib/auto-consul/cluster.rb

@@ -0,0 +1,53 @@
+require 'uri'
+
+module AutoConsul
+  class Cluster
+    def self.get_provider_for_uri uri_string
+      uri = URI(uri_string)
+      Registry.supported_schemes[uri.scheme.downcase].new uri
+    end
+
+    attr_reader :uri_string
+
+    def initialize uri
+      @uri_string = uri
+    end
+
+    def servers
+      @servers ||= self.class.get_provider_for_uri File.join(uri_string, 'servers')
+    end
+
+    def agents
+      @agents ||= self.class.get_provider_for_uri File.join(uri_string, 'agents')
+    end
+
+    def set_mode! local_state, expiry, desired_servers=1
+      if servers.members(expiry).size < desired_servers
+        local_state.set_server!
+      else
+        local_state.set_agent!
+      end
+    end
+
+    module Registry
+      def self.supported_schemes
+        constants.inject({}) do |a, const|
+          if const.to_s =~ /^(.+?)Provider$/
+            a[$1.downcase] = const_get(const)
+          end
+          a
+        end
+      end
+
+      class Provider
+        attr_reader :uri
+
+        def initialize uri
+          @uri = uri
+        end
+      end
+    end
+  end
+end
+
+require_relative 'providers/s3'

data/lib/auto-consul/local.rb

@@ -0,0 +1,78 @@
+require 'fileutils'
+
+module AutoConsul
+  module Local
+    class FileSystemState
+      def initialize path
+        unless File.directory? path
+          FileUtils.mkdir_p path
+        end
+
+        @path = path
+      end
+
+      def path
+        @path
+      end
+
+      def mode_path
+        File.join(path, 'mode')
+      end
+
+      def set_server!
+        set_mode 'server'
+      end
+
+      def set_agent!
+        set_mode 'agent'
+      end
+
+      def set_mode mode
+        File.open(mode_path, 'w') do |f|
+          f.write mode
+        end
+      end
+
+      VALID_MODES = {
+        'agent' => 'agent',
+        'server' => 'server',
+      }
+
+      def self.determine_mode mode_file
+        if File.file? mode_file
+          value = File.open(mode_file, 'r') {|f| f.read }
+          VALID_MODES[value]
+        else
+          nil
+        end
+      end
+
+      def mode
+        if @mode.nil?
+          @mode = self.class.determine_mode mode_path
+        end
+        @mode
+      end
+
+      def server?
+        mode == 'server'
+      end
+
+      def agent?
+        mode == 'agent'
+      end
+
+      def data_path
+        if not (m = mode).nil?
+          File.join(path, mode)
+        else
+          nil
+        end
+      end
+    end
+
+    def self.bind_to_path path
+      FileSystemState.new path
+    end
+  end
+end

data/lib/auto-consul/providers/s3.rb

@@ -0,0 +1,108 @@
+require 'aws-sdk'
+
+module AutoConsul::Cluster::Registry
+  class S3Provider < Provider
+    class S3Member
+      attr_reader :s3_object, :identifier, :time
+
+      def initialize s3obj
+        @s3_object = s3obj
+        @time, @identifier = S3Provider.from_key_base(File.basename(s3obj.key))
+        @data_read = false
+      end
+
+      def data
+        if not @data_read
+          @data = s3_object.read
+          @data_read = true
+        end
+        @data
+      end
+    end
+
+    def s3
+      @s3 ||= self.class.get_s3
+    end
+
+    def self.get_s3
+      AWS::S3.new
+    end
+
+    def bucket_name
+      uri.host
+    end
+
+    def bucket
+      @bucket ||= s3.buckets[bucket_name]
+    end
+
+    def key_prefix
+      uri.path[1..-1]
+    end
+
+    def now
+      Time.now
+    end
+
+    KEY_TIMESTAMP_FORMAT = '%Y%m%d%H%M%S'
+
+    def self.write_stamp time
+      time.dup.utc.strftime KEY_TIMESTAMP_FORMAT
+    end
+
+    def self.read_stamp stamp
+      t = Time.strptime(stamp, KEY_TIMESTAMP_FORMAT)
+      Time.utc(t.year, t.month, t.day, t.hour, t.min, t.sec, 0)
+    end
+
+    def self.to_key_base time, identifier
+      "#{write_stamp time}-#{identifier.to_s}"
+    end
+
+    def self.from_key_base key_base
+      stamp, identifier = key_base.split('-', 2)
+      [read_stamp(stamp), identifier]
+    end
+
+    def write_key time, identity
+      File.join(key_prefix, self.class.to_key_base(time, identity))
+    end
+
+    def heartbeat! identity, data, expiry=nil
+      result = bucket.objects[write_key now, identity].write data
+      purge!(expiry) unless expiry.nil?
+      result
+    end
+
+    def purge! expiry
+      min_key = File.join(key_prefix, "#{self.class.write_stamp(Time.now - expiry + 1)}-")
+      bucket.objects.with_prefix(key_prefix).delete_if do |s3obj|
+        s3obj.key < min_key
+      end
+    end
+
+    def members expiry
+      deletes, actives = [], {}
+      # The expiry gives an exclusive boundary, not an inclusive,
+      # so the minimal allowable key must begin one second after the
+      # specified expiry (given a resolution of seconds).
+      min_time = Time.now.utc - expiry + 1
+      min_key = File.join(key_prefix, min_time.strftime('%Y%m%d%H%M%S-'))
+      bucket.objects.with_prefix(key_prefix).each do |obj|
+        if obj.key < min_key
+          deletes << obj
+        else
+          o = S3Member.new(obj)
+          actives[o.identifier] = o
+        end
+      end
+      deletes! deletes
+      actives.values.sort_by {|m| [m.time, m.identifier]}
+    end
+
+    def deletes! deletes
+      bucket.objects.delete deletes if deletes.size > 0
+    end
+  end
+end
+

data/lib/auto-consul/run_state/cli.rb

@@ -0,0 +1,61 @@
+module AutoConsul::RunState
+  class CLIProvider
+    AGENT_MASK = 0b1
+    SERVER_MASK = 0b10
+
+    def check_run_state
+      result = 0
+      r, w = IO.pipe
+      if system("consul info", :out => w)
+        w.close
+        result = flags_from_output r
+        r.close
+      end
+      result
+    end
+
+    def flags_from_output stream
+      consul_block = false
+      result = 0
+      stream.each do |line|
+        if line =~ /^consul:\s/
+          consul_block = true
+          result = AGENT_MASK
+          break
+        end
+      end
+
+      if consul_block
+        stream.each do |line|
+          # Exit condition from consul block
+          break if line !~ /^\s+/
+
+          if line =~ /^\s+server\s+=\s+true/
+            result |= SERVER_MASK
+            break
+          end
+        end
+      end
+      result
+    end
+
+    def run_state
+      if @run_state.nil?
+        @run_state = check_run_state
+      end
+      @run_state
+    end
+
+    def agent?
+      (run_state & AGENT_MASK) > 0
+    end
+
+    def server?
+      (run_state & SERVER_MASK) > 0
+    end
+
+    def running?
+      run_state > 0
+    end
+  end
+end

data/lib/auto-consul/run_state.rb

@@ -0,0 +1,5 @@
+module AutoConsul::RunState
+end
+
+require_relative 'run_state/cli'
+

data/lib/auto-consul/runner.rb

@@ -0,0 +1,56 @@
+module AutoConsul
+  module Runner
+    SLEEP_INTERVAL = 2
+    RETRIES = 5
+
+    def self.launch_and_join(agent_args, remote_ip=nil)
+      pid = spawn(*(['consul', 'agent'] + agent_args))
+
+      # We really need to check that is running, but later.
+      return nil unless verify_running(pid)
+
+      if not remote_ip.nil?
+        join remote_ip
+      end
+
+      pid
+    end
+
+    def self.verify_running pid
+      RETRIES.times do |i|
+        sleep SLEEP_INTERVAL + (SLEEP_INTERVAL * i)
+        return true if system('consul', 'info')
+      end
+      return false
+    end
+
+    def self.join remote_ip
+      system('consul', 'join', remote_ip)
+    end
+
+    def self.pick_joining_host hosts
+      # Lets randomize this later.
+      hosts[0].data
+    end
+
+    def self.run_agent! identity, bind_ip, expiry, local_state, registry
+      remote_ip = pick_joining_host(registry.agents.members(expiry))
+      pid = launch_and_join(['-bind', bind_ip,
+                             '-data-dir', local_state.data_path,
+                             '-node', identity], remote_ip)
+      Process.wait pid
+    end
+
+    def self.run_server! identity, bind_ip, expiry, local_state, registry
+      members = registry.servers.members(expiry)
+      remote_ip = members.size > 0 ? pick_joining_host(members) : nil
+
+      args = ['-bind', bind_ip, '-data-dir', local_state.data_path, '-node', identity, '-server']
+      args << '-bootstrap' if members.size < 1
+
+      pid = launch_and_join(args, remote_ip)
+
+      Process.wait pid unless pid.nil?
+    end
+  end
+end

data/lib/auto-consul.rb

@@ -0,0 +1,8 @@
+module AutoConsul
+end
+
+require 'auto-consul/local'
+require 'auto-consul/cluster'
+require 'auto-consul/run_state'
+require 'auto-consul/runner'
+