rory-deploy 1.8.4.1

data/Rakefile

@@ -0,0 +1,15 @@
+$: << File.expand_path("lib")
+require 'bundler/gem_tasks'
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.rspec_opts = %w[--color --format=documentation]
+    t.pattern = "spec/**/*_spec.rb"
+  end
+
+  task :default => [:spec]
+rescue LoadError
+  # don't generate Rspec tasks if we don't have it installed
+end

data/bin/rory

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require_relative '../lib/centurion'
+require 'capistrano_dsl'
+
+self.extend Capistrano::DSL
+self.extend Centurion::DeployDSL
+self.extend Centurion::Logging
+
+#
+# Initialize Rake engine
+#
+require 'rake'
+Rake.application.options.trace = true
+
+task_dir = File.expand_path(File.join(File.dirname(__FILE__), *%w{.. lib tasks}))
+Dir.glob(File.join(task_dir, '*.rake')).each { |file| load file }
+
+#
+# Trollop option setup
+#
+require 'trollop'
+
+opts = Trollop::options do
+  opt :project,          'project (blog, forums...)',            type: String, required: true,  short: '-p'
+  opt :environment,      "environment (production, staging...)", type: String, required: true,  short: '-e'
+  opt :action,           'action (deploy, list...)',             type: String, default: 'list', short: '-a'
+  opt :image,            'image (yourco/project...)',            type: String, short: '-i'
+  opt :tag,              'tag (latest...)',                      type: String, short: '-t'
+  opt :hosts,            'hosts, comma separated',               type: String, short: '-h'
+  opt :docker_path,      'path to docker executable',            type: String, default: 'docker', short: '-d'
+  opt :no_pull,          'Skip the pull_image step',             type: :flag,  default: false, short: '-n'
+  opt :registry_user,    'user for registry auth',               type: String, short: :none
+  opt :registry_password,'password for registry auth',           type: String, short: :none
+  opt :override_env,     'override environment variables, comma separated', type: String
+end
+
+set_current_environment(opts[:environment].to_sym)
+set :project, opts[:project]
+set :environment, opts[:environment]
+
+# Load the per-project config and execute the task for the current environment
+projects_dir = File.join(Dir.getwd(), 'config', 'centurion')
+config_file  = "#{opts[:project]}.rake"
+if File.exists?(File.join(projects_dir, config_file))
+  load File.join(File.join(projects_dir, config_file))
+elsif File.exists?(config_file)
+  load config_file
+else
+  raise "Can't find '#{config_file}'!"
+end
+invoke("environment:#{opts[:environment]}")
+
+# Override the config with command line values if given
+set :image, opts[:image] if opts[:image]
+set :tag,   opts[:tag] if opts[:tag]
+set :hosts, opts[:hosts].split(",") if opts[:hosts]
+set :name,  opts[:project].gsub(/_/, '-')
+
+# Override environment variables when specified
+if opts[:override_env]
+  opts[:override_env].split(',').each do |envvar|
+    key, value = envvar.split('=')
+    env_vars(key => value)
+  end
+end
+
+# Default tag should be "latest"
+set :tag, 'latest' unless any?(:tag)
+set :docker_registry, Centurion::DockerRegistry::OFFICIAL_URL unless any?(:docker_registry)
+
+# Specify a path to docker executable
+set :docker_path, opts[:docker_path]
+
+set :no_pull, opts[:no_pull_given]
+set :registry_user, opts[:registry_user] if opts[:registry_user]
+set :registry_password, opts[:registry_password] if opts[:registry_password]
+
+invoke('centurion:setup')
+invoke(opts[:action])

data/bin/rory-gen-config

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'trollop'
+
+opts = Trollop::options do
+  opt :project,       'The short name (no spaces) of the project to scaffold', required: true, type: String
+  opt :registry_base, 'The base url for your registry (ex: example.com/yourcompany/) slash terminated', default: ''
+  opt :centurion_dir, 'The base dir for centurion configs', default: File.join(Dir.getwd, 'config', 'centurion')
+end
+
+project_file  = File.join(opts[:centurion_dir], "#{opts[:project]}.rake")
+
+unless Dir.exists?(opts[:centurion_dir])
+  puts "Creating #{opts[:centurion_dir]}"
+  FileUtils.mkdir_p(opts[:centurion_dir])
+end
+
+unless File.exists?(project_file)
+  puts "Writing example config to #{project_file}"
+  File.write(project_file, <<-EOS.gsub(/^\s{4}/, ''))
+    namespace :environment do
+      task :common do
+        set :image, '#{opts[:registry_base]}#{opts[:project]}'
+        # Point this to an appropriate health check endpoint for rolling deploys (defaults to '/')
+        # set :status_endpoint, '/status/check'
+
+        # Example on how to change docker registry to Dogestry.
+        # This requires:
+        # - aws_access_key_id
+        # - aws_secret_key
+        # - s3_bucket
+        #
+        # And optionally:
+        # - s3_region (default to us-east-1)
+        #
+        # registry :dogestry
+        # set :aws_access_key_id, 'abc123'
+        # set :aws_secret_key, 'xyz'
+        # set :s3_bucket, 'bucket-for-docker-images'
+      end
+
+      desc 'Staging environment'
+      task :staging => :common do
+        set_current_environment(:staging)
+        # env_vars YOUR_ENV: 'staging'
+        # host_port 10234, container_port: 9292
+        # host 'docker-server-staging-1.example.com'
+        # host 'docker-server-staging-2.example.com'
+
+        # You can assign different docker daemon port. Example:
+        # host 'docker-server-staging-3.example.com:4243'
+      end
+
+      desc 'Production environment'
+      task :production => :common do
+        set_current_environment(:production)
+        # env_vars YOUR_ENV: 'production'
+        # host_port 23235, container_port: 9293
+        # host 'docker-server-prod-1.example.com'
+        # host 'docker-server-prod-2.example.com'
+        # host_volume '/mnt/volume1', container_volume: '/mnt/volume1'
+      end
+    end
+  EOS
+end
+
+gemfile = File.join(opts[:centurion_dir], *%w{.. .. Gemfile})
+unless File.exists?(gemfile)
+  raise "Error creating Gemfile: $!" unless system("bash -c 'cd #{File.dirname(gemfile)} && bundle init'")
+end
+
+unless File.read(gemfile) =~ /centurion/s
+  puts 'Adding Centurion to the Gemfile'
+  File.open(gemfile, 'a') { |f| f << "gem 'centurion'" }
+  puts "\n\nRemember to run `bundle install` before running Centurion\n\n"
+end
+
+puts 'Done!'

data/lib/capistrano_dsl.rb

@@ -0,0 +1,91 @@
+# This file borrows heavily from the Capistrano project, so although much of
+# the code has been re-written at this point, we include their license here
+# as a reminder. NOTE that THIS LICENSE ONLY APPLIES TO THIS FILE itself, not
+# to the rest of the project.
+#
+# ORIGINAL CAPISTRANO LICENSE FOLLOWS:
+#
+# MIT License (MIT)
+# 
+# Copyright (c) 2012-2013 Tom Clements, Lee Hambley
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'singleton'
+
+module Capistrano
+  module DSL
+    module Env
+      class CurrentEnvironmentNotSetError < RuntimeError; end
+
+      class Store < Hash
+        include Singleton
+      end
+
+      def env
+        Store.instance
+      end
+
+      def fetch(key, default=nil, &block)
+        env[current_environment][key] || default
+      end
+
+      def any?(key)
+        value = fetch(key)
+        if value && value.respond_to?(:any?)
+          value.any?
+        else
+          !fetch(key).nil?
+        end
+      end
+
+      def set(key, value)
+        env[current_environment][key] = value
+      end
+
+      def delete(key)
+        env[current_environment].delete(key)
+      end
+
+      def set_current_environment(environment)
+        env[:current_environment] = environment
+        env[environment] ||= {}
+      end
+
+      def current_environment
+        raise CurrentEnvironmentNotSetError.new('Must set current environment') unless env[:current_environment]
+        env[:current_environment]
+      end
+
+      def clear_env
+        env.clear
+      end
+    end
+  end
+end
+
+module Capistrano
+  module DSL
+    include Env
+
+    def invoke(task, *args)
+      Rake::Task[task].invoke(*args)
+    end
+  end
+end

data/lib/centurion.rb

@@ -0,0 +1,9 @@
+Dir[File.join(File.dirname(__FILE__), 'core_ext', '*')].each do |file|
+  require file
+end
+
+Dir[File.join(File.dirname(__FILE__), 'centurion', '*')].each do |file|
+  require file
+end
+
+module Centurion; end

data/lib/centurion/deploy.rb

@@ -0,0 +1,139 @@
+require 'excon'
+require 'socket'
+
+module Centurion; end
+
+module Centurion::Deploy
+  FAILED_CONTAINER_VALIDATION = 100
+
+  def stop_containers(target_server, service, timeout = 30)
+    old_containers = if service.public_ports.nil? || service.public_ports.empty?
+      target_server.find_containers_by_name(service.name)
+    else
+      target_server.find_containers_by_public_port(service.public_ports.first)
+    end
+
+    info "Stopping container(s): #{old_containers.inspect}"
+
+    old_containers.each do |old_container|
+      info "Stopping old container #{old_container['Id'][0..7]} (#{old_container['Names'].join(',')})"
+      target_server.stop_container(old_container['Id'], timeout)
+    end
+  end
+
+  def wait_for_health_check_ok(health_check_method, target_server, container_id, port, endpoint, image_id, tag, sleep_time=5, retries=12)
+    info 'Waiting for the port to come up'
+    1.upto(retries) do
+      if container_up?(target_server, container_id) && health_check_method.call(target_server, port, endpoint)
+        info 'Container is up!'
+        break
+      end
+
+      info "Waiting #{sleep_time} seconds to test the #{endpoint} endpoint..."
+      sleep(sleep_time)
+    end
+
+    unless health_check_method.call(target_server, port, endpoint)
+      error "Failed to validate started container on #{target_server.hostname}:#{port}"
+      exit(FAILED_CONTAINER_VALIDATION)
+    end
+  end
+
+  def container_up?(target_server, container_id)
+    # The API returns a record set like this:
+    #[{"Command"=>"script/run ", "Created"=>1394470428, "Id"=>"41a68bda6eb0a5bb78bbde19363e543f9c4f0e845a3eb130a6253972051bffb0", "Image"=>"quay.io/newrelic/rubicon:5f23ac3fad7979cd1efdc9295e0d8c5707d1c806", "Names"=>["/happy_pike"], "Ports"=>[{"IP"=>"0.0.0.0", "PrivatePort"=>80, "PublicPort"=>8484, "Type"=>"tcp"}], "Status"=>"Up 13 seconds"}]
+
+    container = target_server.find_container_by_id(container_id)
+
+    if container
+      info "Found container up for #{Time.now.to_i - container['Created'].to_i} seconds"
+      return true
+    end
+
+    false
+  end
+
+  def http_status_ok?(target_server, port, endpoint)
+    url      = "http://#{target_server.hostname}:#{port}#{endpoint}"
+    response = begin
+      Excon.get(url, headers: {'Accept' => '*/*'})
+    rescue Excon::Errors::SocketError
+      warn "Failed to connect to #{url}, no socket open."
+      nil
+    end
+
+    return false unless response
+    return true if response.status >= 200 && response.status < 300
+
+    warn "Got HTTP status: #{response.status}"
+    false
+  end
+
+  def wait_for_load_balancer_check_interval
+    sleep(fetch(:rolling_deploy_check_interval, 5))
+  end
+
+  def cleanup_containers(target_server, service)
+    old_containers = target_server.old_containers_for_name(service.name)
+    old_containers.shift(2)
+
+    info "Service name #{service.name}"
+    old_containers.each do |old_container|
+      info "Removing old container #{old_container['Id'][0..7]} (#{old_container['Names'].join(',')})"
+      target_server.remove_container(old_container['Id'])
+    end
+  end
+
+  def hostname_proc
+    hostname = fetch(:container_hostname)
+    return nil if hostname.nil?
+
+    if hostname.respond_to?(:call)
+      hostname
+    else
+      ->(h) { hostname }
+    end
+  end
+
+  def start_new_container(server, service, restart_policy)
+    container_config = service.build_config(server.hostname, &hostname_proc)
+    info "Creating new container for #{container_config['Image']}"
+    container = server.create_container(container_config, service.name)
+
+    host_config = service.build_host_config(restart_policy)
+
+    info "Starting new container #{container['Id'][0..7]}"
+    server.start_container(container['Id'], host_config)
+
+    info "Inspecting new container #{container['Id'][0..7]}:"
+    (server.inspect_container(container['Id']) || {}).each_pair do |key,value|
+      info "\t#{key} => #{value.inspect}"
+    end
+
+    container
+  end
+
+  def launch_console(server, service)
+    container_config = service.build_console_config(server.hostname)
+    info "Creating new container for #{container_config['Image'][0..7]}"
+
+    container = server.create_container(container_config, service.name)
+
+    host_config = service.build_host_config
+
+    info "Starting new container #{container['Id'][0..7]}"
+    server.start_container(container['Id'], host_config)
+
+    server.attach(container['Id'])
+  end
+
+  def enter_container(server, service)
+    container = if service.public_ports.nil? || service.public_ports.empty?
+      server.find_containers_by_name(service.name).first
+    else
+      server.find_containers_by_public_port(service.public_ports.first).first
+    end
+
+    server.exec_it(container["Id"], "/bin/bash")
+  end
+end

data/lib/centurion/deploy_dsl.rb

@@ -0,0 +1,180 @@
+require_relative 'docker_server_group'
+require_relative 'docker_server'
+require_relative 'service'
+require 'uri'
+
+module Centurion::DeployDSL
+  def on_each_docker_host(&block)
+    build_server_group.tap { |hosts| hosts.each { |host| block.call(host) } }
+  end
+
+  def on_first_docker_host(&block)
+    build_server_group.tap { |hosts| block.call(hosts.first) }
+  end
+
+  def env_vars(new_vars)
+    current = fetch(:env_vars, {})
+    new_vars.each_pair do |new_key, new_value|
+      current[new_key.to_s] = new_value.to_s
+    end
+    set(:env_vars, current)
+  end
+
+  def add_capability(new_cap_adds)
+    if !valid_capability?(new_cap_adds)
+      abort("Invalid capability addition #{new_cap_adds} specified.")
+    end
+    current = fetch(:cap_adds, [])
+    set(:cap_adds, current << new_cap_adds)
+  end
+
+  def drop_capability(new_cap_drops)
+    if !valid_capability?(new_cap_drops)
+      abort("Invalid capability drop #{new_cap_drops} specified.")
+    end
+    current = fetch(:cap_drops, [])
+    set(:cap_drops, current << new_cap_drops)
+  end
+
+  def host(hostname)
+    current = fetch(:hosts, [])
+    current << hostname
+    set(:hosts, current)
+  end
+
+  def memory(memory)
+    set(:memory, memory)
+  end
+
+  def cpu_shares(cpu_shares)
+    set(:cpu_shares, cpu_shares)
+  end
+
+  def command(command)
+    set(:command, command)
+  end
+
+  def localhost
+    # DOCKER_HOST is like 'tcp://127.0.0.1:2375'
+    docker_host_uri = URI.parse(ENV['DOCKER_HOST'] || "tcp://127.0.0.1")
+    host_and_port = [docker_host_uri.host, docker_host_uri.port].compact.join(':')
+    host(host_and_port)
+  end
+
+  def host_port(port, options)
+    validate_options_keys(options, [ :host_ip, :container_port, :type ])
+    require_options_keys(options,  [ :container_port ])
+
+    set(:port_bindings, fetch(:port_bindings, []).tap do |bindings|
+      bindings << Centurion::Service::PortBinding.new(port, options[:container_port], options[:type] || 'tcp', options[:host_ip])
+    end)
+  end
+
+  def network_mode(mode)
+    if %w(bridge host).include?(mode) or mode =~ /container.*/
+      set(:network_mode, mode)
+    else
+      abort("invalid value for network_mode: #{mode}, value must be one of 'bridge', 'host', or 'container:<name|id>'")
+    end
+  end
+
+  def public_port_for(port_bindings)
+    # port_bindings = [#<struct Centurion::Service::PortBinding
+    #   host_port=17090,
+    #   container_port=80,
+    #   type="tcp",
+    #   host_ip=nil>]
+    port_bindings.first.host_port
+  end
+
+  def host_volume(volume, options)
+    validate_options_keys(options, [ :container_volume ])
+    require_options_keys(options,  [ :container_volume ])
+
+    set(:binds, fetch(:binds, []).tap do |volumes|
+      volumes << Centurion::Service::Volume.new(volume, options[:container_volume])
+    end)
+  end
+
+  def get_current_tags_for(image)
+    build_server_group.inject([]) do |memo, target_server|
+      tags = target_server.current_tags_for(image)
+      memo += [{ server: target_server.hostname, tags: tags }] if tags
+      memo
+    end
+  end
+
+  def registry(type)
+    set(:registry, type.to_s)
+  end
+
+  def health_check(method)
+   abort("Health check expects a callable (lambda, proc, method), but #{method.class} was specified")  unless method.respond_to?(:call)
+   set(:health_check, method)
+  end
+
+  def extra_host(ip, name)
+    current = fetch(:extra_hosts, [])
+    current.push("#{name}:#{ip}")
+    set(:extra_hosts, current)
+  end
+
+  def defined_service
+    Centurion::Service.from_env
+  end
+
+  def defined_health_check
+    Centurion::HealthCheck.new(fetch(:health_check, method(:http_status_ok?)),
+                               fetch(:status_endpoint, '/'),
+                               fetch(:rolling_deploy_wait_time, 5),
+                               fetch(:rolling_deploy_retries, 24))
+  end
+
+  def defined_restart_policy
+    Centurion::Service::RestartPolicy.new(fetch(:restart_policy_name, 'on-failure'), fetch(:restart_policy_max_retry_count, 10))
+  end
+
+  private
+
+  def build_server_group
+    hosts, docker_path = fetch(:hosts, []), fetch(:docker_path)
+    Centurion::DockerServerGroup.new(hosts, docker_path, build_tls_params)
+  end
+
+  def validate_options_keys(options, valid_keys)
+    unless options.keys.all? { |k| valid_keys.include?(k) }
+      raise ArgumentError.new('Options passed with invalid key!')
+    end
+  end
+
+  def require_options_keys(options, required_keys)
+    missing = required_keys.reject { |k| options.keys.include?(k) }
+
+    unless missing.empty?
+      raise ArgumentError.new("Options must contain #{missing.inspect}")
+    end
+  end
+
+  def valid_capability?(capability)
+    %w(ALL SETPCAP SYS_MODULE SYS_RAWIO SYS_PACCT SYS_ADMIN SYS_NICE
+       SYS_RESOURCE SYS_TIME SYS_TTY_CONFIG MKNOD AUDIT_WRITE AUDIT_CONTROL
+       MAC_OVERRIDE MAC_ADMIN NET_ADMIN SYSLOG CHOWN NET_RAW DAC_OVERRIDE FOWNER
+       DAC_READ_SEARCH FSETID KILL SETGID SETUID LINUX_IMMUTABLE
+       NET_BIND_SERVICE NET_BROADCAST IPC_LOCK IPC_OWNER SYS_CHROOT SYS_PTRACE
+       SYS_BOOT LEASE SETFCAP WAKE_ALARM BLOCK_SUSPEND).include?(capability)
+  end
+
+  def tls_paths_available?
+    Centurion::DockerViaCli.tls_keys.all? { |key| fetch(key).present? }
+  end
+
+  def build_tls_params
+    return {} unless fetch(:tlsverify)
+    {
+      tls: fetch(:tlsverify || tls_paths_available?),
+      tlscacert: fetch(:tlscacert),
+      tlscert: fetch(:tlscert),
+      tlskey: fetch(:tlskey)
+    }
+  end
+end