centurion 1.0.6

data/centurion.gemspec

@@ -0,0 +1,40 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'centurion/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'centurion'
+  spec.version       = Centurion::VERSION
+  spec.authors       = [
+    'Nic Benders', 'Karl Matthias', 'Andrew Bloomgarden', 'Aaron Bento',
+    'Paul Showalter', 'David Kerr', 'Jonathan Owens', 'Jon Guymon',
+    'Merlyn Albery-Speyer', 'Amjith Ramanujam', 'David Celis', 'Emily Hyland',
+    'Bryan Stearns']
+  spec.email         = [
+    'nic@newrelic.com', 'kmatthias@newrelic.com', 'andrew@newrelic.com',
+    'aaron@newrelic.com', 'poeslacker@gmail.com', 'dkerr@newrelic.com',
+    'jonathan@newrelic.com', 'jon@newrelic.com', 'merlyn@newrelic.com',
+    'amjith@newrelic.com', 'dcelis@newrelic.com', 'ehyland@newrelic.com',
+    'bryan@newrelic.com']
+  spec.summary       = %q{Deploy images to a fleet of Docker servers}
+  spec.homepage      = 'https://github.com/newrelic/centurion'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'trollop'
+  spec.add_dependency 'excon', '~> 0.33'
+  spec.add_dependency 'logger-colors'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'simplecov'
+
+  spec.required_ruby_version = '>= 1.9.3'
+end

data/lib/capistrano_dsl.rb

@@ -0,0 +1,91 @@
+# This file borrows heavily from the Capistrano project, so although much of
+# the code has been re-written at this point, we include their license here
+# as a reminder. NOTE that THIS LICENSE ONLY APPLIES TO THIS FILE itself, not
+# to the rest of the project.
+#
+# ORIGINAL CAPISTRANO LICENSE FOLLOWS:
+#
+# MIT License (MIT)
+# 
+# Copyright (c) 2012-2013 Tom Clements, Lee Hambley
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'singleton'
+
+module Capistrano
+  module DSL
+    module Env
+      class CurrentEnvironmentNotSetError < RuntimeError; end
+
+      class Store < Hash
+        include Singleton
+      end
+
+      def env
+        Store.instance
+      end
+
+      def fetch(key, default=nil, &block)
+        env[current_environment][key] || default
+      end
+
+      def any?(key)
+        value = fetch(key)
+        if value && value.respond_to?(:any?)
+          value.any?
+        else
+          !fetch(key).nil?
+        end
+      end
+
+      def set(key, value)
+        env[current_environment][key] = value
+      end
+
+      def delete(key)
+        env[current_environment].delete(key)
+      end
+
+      def set_current_environment(environment)
+        env[:current_environment] = environment
+        env[environment] ||= {}
+      end
+
+      def current_environment
+        raise CurrentEnvironmentNotSetError.new('Must set current environment') unless env[:current_environment]
+        env[:current_environment]
+      end
+
+      def clear_env
+        env.clear
+      end
+    end
+  end
+end
+
+module Capistrano
+  module DSL
+    include Env
+
+    def invoke(task, *args)
+      Rake::Task[task].invoke(*args)
+    end
+  end
+end

data/lib/centurion.rb

@@ -0,0 +1,5 @@
+Dir[File.join(File.dirname(__FILE__), 'centurion', '*')].each do |file|
+  require file
+end
+
+module Centurion; end

data/lib/centurion/deploy.rb

@@ -0,0 +1,145 @@
+require 'excon'
+
+module Centurion; end
+
+module Centurion::Deploy
+  FAILED_CONTAINER_VALIDATION = 100
+
+  def stop_containers(target_server, port_bindings)
+    public_port    = public_port_for(port_bindings)
+    old_containers = target_server.find_containers_by_public_port(public_port)
+    info "Stopping container(s): #{old_containers.inspect}"
+
+    old_containers.each do |old_container|
+      info "Stopping old container #{old_container['Id'][0..7]} (#{old_container['Names'].join(',')})"
+      target_server.stop_container(old_container['Id'])
+    end
+  end
+
+  def wait_for_http_status_ok(target_server, port, endpoint, image_id, tag, sleep_time=5, retries=12)
+    info 'Waiting for the port to come up'
+    1.upto(retries) do
+      if container_up?(target_server, port) && http_status_ok?(target_server, port, endpoint)
+        info 'Container is up!'
+        break
+      end
+
+      info "Waiting #{sleep_time} seconds to test the #{endpoint} endpoint..."
+      sleep(sleep_time)
+    end
+
+    unless http_status_ok?(target_server, port, endpoint)
+      error "Failed to validate started container on #{target_server}:#{port}"
+      exit(FAILED_CONTAINER_VALIDATION)
+    end
+  end
+
+  def container_up?(target_server, port)
+    # The API returns a record set like this:
+    #[{"Command"=>"script/run ", "Created"=>1394470428, "Id"=>"41a68bda6eb0a5bb78bbde19363e543f9c4f0e845a3eb130a6253972051bffb0", "Image"=>"quay.io/newrelic/rubicon:5f23ac3fad7979cd1efdc9295e0d8c5707d1c806", "Names"=>["/happy_pike"], "Ports"=>[{"IP"=>"0.0.0.0", "PrivatePort"=>80, "PublicPort"=>8484, "Type"=>"tcp"}], "Status"=>"Up 13 seconds"}]
+
+    running_containers = target_server.find_containers_by_public_port(port)
+    container = running_containers.pop
+
+    unless running_containers.empty?
+      # This _should_ never happen, but...
+      error "More than one container is bound to port #{port} on #{target_server}!"
+      return false
+    end
+
+    if container && container['Ports'].any? { |bind| bind['PublicPort'].to_i == port.to_i }
+      info "Found container up for #{Time.now.to_i - container['Created'].to_i} seconds"
+      return true
+    end
+
+    false
+  end
+
+  def http_status_ok?(target_server, port, endpoint)
+    url      = "http://#{target_server.hostname}:#{port}#{endpoint}"
+    response = begin
+      Excon.get(url)
+    rescue Excon::Errors::SocketError
+      warn "Failed to connect to #{url}, no socket open."
+      nil
+    end
+
+    return false unless response
+    return true if response.status >= 200 && response.status < 300
+
+    warn "Got HTTP status: #{response.status}" 
+    false
+  end
+
+  def wait_for_load_balancer_check_interval
+    sleep(fetch(:rolling_deploy_check_interval, 5))
+  end
+
+  def cleanup_containers(target_server, port_bindings)
+    public_port    = public_port_for(port_bindings)
+    old_containers = target_server.old_containers_for_port(public_port)
+    old_containers.shift(2)
+
+    info "Public port #{public_port}"
+    old_containers.each do |old_container|
+      info "Removing old container #{old_container['Id'][0..7]} (#{old_container['Names'].join(',')})"
+      target_server.remove_container(old_container['Id'])
+    end
+  end
+
+  def container_config_for(target_server, image_id, port_bindings=nil, env_vars=nil)
+    container_config = {
+      'Image'        => image_id,
+      'Hostname'     => target_server.hostname,
+    }
+
+    if port_bindings
+      container_config['ExposedPorts'] = { port_bindings.keys.first => {} }
+    end
+
+    if env_vars
+      container_config['Env'] = env_vars.map { |k,v| "#{k}=#{v}" }
+    end
+
+    container_config
+  end
+
+  def start_new_container(target_server, image_id, port_bindings, volumes, env_vars=nil)
+    container_config = container_config_for(target_server, image_id, port_bindings, env_vars)
+    start_container_with_config(target_server, volumes, port_bindings, container_config)
+  end
+
+  def launch_console(target_server, image_id, port_bindings, volumes, env_vars=nil)
+    container_config = container_config_for(target_server, image_id, port_bindings, env_vars).merge(
+      'Cmd'         => [ '/bin/bash' ],
+      'AttachStdin' => true,
+      'Tty'         => true,
+      'OpenStdin'   => true,
+    )
+
+    container = start_container_with_config(target_server, volumes, port_bindings, container_config)
+
+    target_server.attach(container['Id'])
+  end
+
+  private
+  
+  def start_container_with_config(target_server, volumes, port_bindings, container_config)
+    info "Creating new container for #{container_config['Image'][0..7]}"
+    new_container = target_server.create_container(container_config)
+
+    host_config = { 
+      'PortBindings' => port_bindings 
+    }
+    # Map some host volumes if needed
+    host_config['Binds'] = volumes if volumes && !volumes.empty?
+
+    info "Starting new container #{new_container['Id'][0..7]}"
+    target_server.start_container(new_container['Id'], host_config)
+
+    info "Inspecting new container #{new_container['Id'][0..7]}:"
+    info target_server.inspect_container(new_container['Id'])
+
+    new_container
+  end
+end

data/lib/centurion/deploy_dsl.rb

@@ -0,0 +1,94 @@
+require_relative 'docker_server_group'
+require 'uri'
+
+module Centurion::DeployDSL
+  def on_each_docker_host(&block)
+    Centurion::DockerServerGroup.new(fetch(:hosts, []), fetch(:docker_path)).tap do |hosts|
+      hosts.each { |host| block.call(host) }
+    end
+  end
+
+  def env_vars(new_vars)
+    current = fetch(:env_vars, {})
+    new_vars.each_pair do |new_key, new_value|
+      current[new_key.to_s] = new_value
+    end
+    set(:env_vars, current)
+  end
+
+  def host(hostname)
+    current = fetch(:hosts, [])
+    current << hostname
+    set(:hosts, current)
+  end
+
+  def localhost
+    # DOCKER_HOST is like 'tcp://127.0.0.1:4243'
+    docker_host_uri = URI.parse(ENV['DOCKER_HOST'] || "tcp://127.0.0.1")
+    host_and_port = [docker_host_uri.host, docker_host_uri.port].compact.join(':')
+    host(host_and_port)
+  end
+
+  def host_port(port, options)
+    validate_options_keys(options, [ :host_ip, :container_port, :type ])
+    require_options_keys(options,  [ :container_port ])
+
+    add_to_bindings(
+      options[:host_ip] || '0.0.0.0', 
+      options[:container_port],
+      port,
+      options[:type] || 'tcp'
+    )
+  end
+
+  def public_port_for(port_bindings)
+    # {'80/tcp'=>[{'HostIp'=>'0.0.0.0', 'HostPort'=>'80'}]}
+    first_port_binding = port_bindings.values.first
+    first_port_binding.first['HostPort']
+  end
+
+  def host_volume(volume, options)
+    validate_options_keys(options, [ :container_volume ])
+    require_options_keys(options,  [ :container_volume ])
+
+    binds            = fetch(:binds, [])
+    container_volume = options[:container_volume]
+
+    binds << "#{volume}:#{container_volume}"
+    set(:binds, binds)
+  end
+
+  def get_current_tags_for(image)
+    hosts = Centurion::DockerServerGroup.new(fetch(:hosts), fetch(:docker_path))
+    hosts.inject([]) do |memo, target_server|
+      tags = target_server.current_tags_for(image)
+      memo += [{ server: target_server.hostname, tags: tags }] if tags
+      memo
+    end
+  end
+
+  private
+
+  def add_to_bindings(host_ip, container_port, port, type='tcp')
+    set(:port_bindings, fetch(:port_bindings, {}).tap do |bindings|
+      bindings["#{container_port.to_s}/#{type}"] = [
+        {'HostIp' => host_ip, 'HostPort' => port.to_s}
+      ]
+      bindings
+    end)
+  end
+
+  def validate_options_keys(options, valid_keys)
+    unless options.keys.all? { |k| valid_keys.include?(k) }
+      raise ArgumentError.new('Options passed with invalid key!')
+    end
+  end
+
+  def require_options_keys(options, required_keys)
+    missing = required_keys.reject { |k| options.keys.include?(k) }
+
+    unless missing.empty?
+      raise ArgumentError.new("Options must contain #{missing.inspect}")
+    end
+  end
+end

data/lib/centurion/docker_registry.rb

@@ -0,0 +1,35 @@
+require 'excon'
+require 'json'
+require 'uri'
+
+module Centurion; end
+
+class Centurion::DockerRegistry
+  def initialize()
+    # @base_uri = "https://staging-docker-registry.nr-ops.net"
+    @base_uri = 'http://chi-docker-registry.nr-ops.net'
+  end
+  
+  def digest_for_tag( repository, tag)
+    path = "/v1/repositories/#{repository}/tags/#{tag}"
+    $stderr.puts "GET: #{path.inspect}"
+    response = Excon.get(
+      @base_uri + path,
+      :headers => { "Content-Type" => "application/json" }
+    )
+    raise response.inspect unless response.status == 200
+
+    # This hack is stupid, and I hate it. But it works around the fact that
+    # the Docker Registry will return a base JSON String, which the Ruby parser
+    # refuses (possibly correctly) to handle
+    JSON.load('[' + response.body + ']').first
+  end
+  
+  def respository_tags( respository )
+    path = "/v1/repositories/#{respository}/tags"
+    $stderr.puts "GET: #{path.inspect}"
+    response = Excon.get(@base_uri + path)
+    raise response.inspect unless response.status == 200
+    JSON.load(response.body)
+  end
+end

data/lib/centurion/docker_server.rb

@@ -0,0 +1,58 @@
+require 'pty'
+require 'forwardable'
+
+require_relative 'logging'
+require_relative 'docker_via_api'
+require_relative 'docker_via_cli'
+
+module Centurion; end
+
+class Centurion::DockerServer
+  include Centurion::Logging
+  extend Forwardable
+
+  attr_reader :hostname, :port
+
+  def_delegators :docker_via_api, :create_container, :inspect_container,
+                 :inspect_image, :ps, :start_container, :stop_container,
+                 :old_containers_for_port, :remove_container
+  def_delegators :docker_via_cli, :pull, :tail, :attach
+
+  def initialize(host, docker_path)
+    @docker_path = docker_path
+    @hostname, @port = host.split(':')
+    @port ||= '4243'
+  end
+
+  def current_tags_for(image)
+    running_containers = ps.select { |c| c['Image'] =~ /#{image}/ }
+    return [] if running_containers.empty?
+
+    parse_image_tags_for(running_containers)
+  end
+
+  def find_containers_by_public_port(public_port, type='tcp')
+    ps.select do |container|
+      if container['Ports']
+        container['Ports'].find do |port|
+          port['PublicPort'] == public_port.to_i && port['Type'] == type
+        end
+      end
+    end
+  end
+
+  private
+
+  def docker_via_api
+    @docker_via_api ||= Centurion::DockerViaApi.new(@hostname, @port)
+  end
+
+  def docker_via_cli
+    @docker_via_cli ||= Centurion::DockerViaCli.new(@hostname, @port, @docker_path)
+  end
+
+  def parse_image_tags_for(running_containers)
+    running_container_names = running_containers.map { |c| c['Image'] }
+    running_container_names.map { |name| name.split(/:/).last } # (image, tag)
+  end
+end