master-cap 0.1

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/Readme.markdown

@@ -0,0 +1,101 @@
+# What is it ?
+
+Here are [capistrano](https://github.com/capistrano/capistrano/wiki) extensions to be used with [master-chef](https://github.com/octo-technology/master-chef)
+
+# How to use it
+
+* Add ``master-cap`` to your Gemfile
+* Load ``master-cap`` in your Capfile: ``require 'master-cap/topology-directory.rb'``
+* Create a subdirectory named ``topology``, and add your toplogy YAML files into
+* Enjoy :)
+
+# Topology file
+
+Example of a topology file: ``integ.yml``
+
+```yml
+:topology:
+  :app:
+    :hostname: my_app_server.mydomain.net
+    :type: linux_chef
+    :roles:
+      - app_server
+  :db:
+    :hostname: db_server.mydomain.net
+    :type: linux_chef
+    :roles:
+      - db_server
+  :redis:
+    :hostname: redise.mydomain.net
+    :type: linux_chef
+    :roles:
+      - redis_server
+:cap_override:
+  :my_specific_cap_param: 'toto'
+:default_role_list:
+  - base
+```
+
+# Capistrano commands
+
+## Node selection
+
+* ``cap integ show``
+* ``cap app-integ show``
+* ``cap integ_db_server show``
+
+## SSH command
+
+* ``cap integ check``: try to connect on each nodes with ssh
+* ``cap integ ssh_cmd -s cmd=uname``: exec ``uname``command on each nodes
+
+# Master-chef binding
+
+## Configuration
+
+If you use only master-chef (no custom repo with Chef recipes), you have nothing to do.
+
+If you use a custom repo, please add following lines before requiring master-cap
+
+```ruby
+set :git_repos_manager_class, 'SimpleGitReposManager'
+
+set :git_repos, [
+  {
+    :url => "http://github.com/octo-technology/master-chef.git",
+    :ref => "aa597911b6d394dff27338c825aa966105cb6607",
+  },
+  {
+    :url => "git@github.com:xxxx/yyyy.git",
+    :local_path => "../yyyy",
+  }
+]
+```
+
+Note:
+* ``:ref`` is used to lock master-chef version
+* ``:local_path`` is used to run ``chef:local``
+
+# Cap commands
+
+* ``cap integ chef``: run chef on all servers
+* ``cap app-integ chef:local``: run chef on app server, using local recipes. Usefull for debugging.
+* ``cap app-integ chef:generate_local_json``: generate the ``local.json`` file using current topology.
+* ``cap app-integ chef:stack``: output last chef stack trace
+* ``cap integ chef:stack``: purge all master-chef git repos caches
+
+# License
+
+Copyright 2012 Bertrand Paquet
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/lib/master-cap/chef.rb

@@ -0,0 +1,105 @@
+require 'tempfile'
+require 'json'
+
+require File.join(File.dirname(__FILE__), 'git_repos_manager.rb')
+
+Capistrano::Configuration.instance.load do
+
+  namespace :chef do
+
+    set :master_chef_path, fetch(:master_chef_path, '../master-chef')
+    set :git_repos_manager, Object.const_get(fetch(:git_repos_manager_class, 'EmptyGitReposManager')).new(self)
+
+    task :generate_local_json do
+      set :user, chef_user
+      env = check_only_one_env
+      find_servers(:roles => chef_role).each do |s|
+        env, node = find_node s.host
+        r = []
+        r += (TOPOLOGY[env][:default_role_list] || []) unless node[:no_default_role]
+        r += node[:roles]
+        json = JSON.pretty_generate({
+          :repos => {
+            :git => git_repos_manager.list,
+          },
+          :run_list => r.map{|x| "role[#{x}]"},
+          :node_config => {
+            :topology_node_name => node[:topology_name]
+          }
+        })
+        puts json
+        f = Tempfile.new File.basename("local_json_#{name}")
+        f.write json
+        f.close
+        upload f.path, "/etc/chef/local.json", {:hosts => [s]}
+      end
+      git_repos_manager.list.each do |git_repo|
+        if git_repo =~ /^.+@.+:.+\.git$/
+          run "sudo ssh -o StrictHostKeyChecking=no #{git_repo.split(':')[0]} echo toto || true > /dev/null 2>&1", :roles => chef_role
+        end
+      end
+    end
+
+    def get_prefix
+      prefix = ""
+      prefix += "http_proxy=#{http_proxy} https_proxy=#{http_proxy}" if exists? :http_proxy
+      prefix
+    end
+
+    task :upload_git_tag_override, :roles => :linux_chef do
+      set :user, chef_user
+      env = check_only_one_env
+
+      git_tag_override = git_repos_manager.compute_override(env)
+
+      if git_tag_override
+        f = Tempfile.new File.basename("git_tag_override")
+        f.write JSON.dump(git_tag_override)
+        f.close
+
+        upload_to_root f.path, "/etc/chef/local.json.git_tag_override"
+      end
+
+    end
+
+    task :upload_topology, :roles => :linux_chef  do
+      set :user, chef_user
+      env = check_only_one_env
+
+      f = Tempfile.new File.basename("topology_env")
+      f.write YAML.dump(TOPOLOGY[env])
+      f.close
+      upload_to_root f.path, "/etc/chef/topology.yml"
+    end
+
+    task :default, :roles => chef_role  do
+      set :user, chef_user
+      upload_topology
+      upload_git_tag_override
+      run "#{get_prefix} /etc/chef/update.sh"
+    end
+
+    task :stack, :roles => chef_role  do
+      set :user, chef_user
+      run "sudo cat /var/chef/cache/chef-stacktrace.out"
+    end
+
+    task :purge_cache, :roles => chef_role do
+      set :user, chef_user
+      run "sudo rm -rf /var/chef/cache/git_repos"
+    end
+
+    task :local, :roles => chef_role  do
+      set :user, chef_user
+      upload_topology
+      find_servers(:roles => chef_role).each do |x|
+        prefix = ""
+        prefix = "PROXY=#{http_proxy}" if exists? :http_proxy
+        command = "sh -c \"#{prefix} #{master_chef_path}/runtime/chef_local.rb #{x} #{git_repos_manager.compute_local_path}\""
+        abort unless system command
+      end
+    end
+
+  end
+
+end

data/lib/master-cap/default_translation_strategy.rb

@@ -0,0 +1,14 @@
+
+class DefaultTranslationStrategy
+
+  def node_name env, name, node, topology
+    return name.to_s if topology[:no_node_suffix]
+    "#{name}-#{env}"
+  end
+
+  def node_hostname env, name, node, topology
+    return node[:hostname] if node[:hostname]
+    raise "No hostname in #{node}"
+  end
+
+end

data/lib/master-cap/git_repos_manager.rb

@@ -0,0 +1,48 @@
+
+class EmptyGitReposManager
+
+  def initialize cap
+    @cap = cap
+  end
+
+  def compute_override env
+    if @cap.exists? :master_chef_version
+      return {"http://github.com/octo-technology/master-chef.git" => @cap.master_chef_version}
+    end
+    nil
+  end
+
+  def list
+    ["http://github.com/octo-technology/master-chef.git"]
+  end
+
+  def compute_local_path
+    ""
+  end
+
+end
+
+class SimpleGitReposManager
+
+  def initialize cap
+    @cap = cap
+    @repos = @cap.fetch(:git_repos, [])
+  end
+
+  def compute_override env
+    result = {}
+    @repos.each do |x|
+      result[x[:url]] = x[:ref] if x[:ref]
+    end
+    result.size == 0 ? nil : result
+  end
+
+  def list
+    @repos.map{|x| x[:url]}
+  end
+
+  def compute_local_path
+    @repos.map{|x| x[:local_path] ? File.expand_path(x[:local_path]) : ""}.join(' ')
+  end
+
+end

data/lib/master-cap/misc.rb

@@ -0,0 +1,90 @@
+
+require 'timeout'
+
+unless Capistrano::Configuration.respond_to?(:instance)
+  abort "master-cap requires Capistrano 2"
+end
+
+Capistrano::Configuration.instance.load do
+
+  def exec_local_with_timeout cmd, timeout
+    pid = Process.fork { exec cmd }
+    begin
+      Timeout.timeout(timeout) do
+        Process.wait(pid)
+        raise "Wrong return code for #{cmd} : #{$?.exitstatus}" unless $?.exitstatus == 0
+      end
+    rescue Timeout::Error
+      Process.kill('TERM', pid)
+      raise "Timeout when executing #{cmd}"
+    end
+  end
+
+  def exec_local cmd
+    begin
+      abort "#{cmd} failed. Aborting..." unless system cmd
+    rescue
+      abort "#{cmd} failed. Aborting..."
+    end
+  end
+
+  def capture_local cmd
+    begin
+      result = %x{#{cmd}}
+      abort "#{cmd} failed. Aborting..." unless $? == 0
+      result
+    rescue
+      abort "#{cmd} failed. Aborting..."
+    end
+  end
+
+  def upload_to_root local, remote, options = {}
+    tmp_file = "/tmp/#{File.basename(remote)}"
+    upload local, tmp_file, options
+    run_root "mv #{tmp_file} #{remote}"
+  end
+
+  def run_root cmd
+    run "sudo sh -c '#{cmd}'"
+  end
+
+  def error msg
+    abort "Error : #{msg}"
+  end
+
+  def fill s, k
+    s.length >= k ? s : fill(s.to_s + " ", k)
+  end
+
+  def multiple_capture command, options = nil
+    result = {}
+    launch = true
+    launch = false if options && options[:hosts] && options[:hosts] == []
+    if launch
+      begin
+        parallel(options) do |session|
+          session.else command do |channel, stream, data|
+            env, name, node = find_node(channel.properties[:server].host)
+            if block_given?
+              result[name] = yield name, node, data
+            else
+              result[name] = data
+            end
+          end
+        end
+      rescue Capistrano::ConnectionError => e
+        puts "\e[31m#{e}\e[0m"
+      end
+    end
+    if options
+      default_value = options[:default_value]
+      if default_value
+        find_servers.map{|x| find_node x.host}.each do |env, name, node|
+          result[name] = default_value unless result[name]
+        end
+      end
+    end
+    result
+  end
+
+end

data/lib/master-cap/topology-directory.rb

@@ -0,0 +1,64 @@
+
+require File.join(File.dirname(__FILE__), 'topology.rb')
+
+Capistrano::Configuration.instance.load do
+
+  topology_directory = fetch(:topology_directory, 'topology')
+
+  task :load_topology do
+    Dir["#{topology_directory}/*.yml"].each do |f|
+      env = File.basename(f).split('.')[0]
+      TOPOLOGY[env] = YAML.load(File.read(f))
+      nodes = []
+      roles_map = {}
+      default_role_nodes = []
+      TOPOLOGY[env][:topology].each do |k, v|
+        v[:topology_name] = translation_strategy.node_name(env, k, v, TOPOLOGY[env])
+        v[:topology_hostname] = begin translation_strategy.node_hostname(env, k, v, TOPOLOGY[env]) rescue nil end
+        next unless v[:topology_hostname]
+        node_roles = v[:roles].map{|x| x.to_sym}
+        node_roles << v[:type].to_sym if v[:type]
+        n = {:name => k.to_s, :host => v[:topology_hostname], :roles => node_roles}
+        nodes << n
+        default_role_nodes << n unless v[:no_default_role]
+        task v[:topology_name] do
+          server n[:host], *node_roles if n[:host]
+          load_cap_override env
+        end
+        node_roles.each do |r|
+          roles_map[r] = [] unless roles_map[r]
+          roles_map[r] << n
+        end
+      end
+      roles_map.each do |r, v|
+        task "#{env}_#{r}" do
+          v.each do |k|
+            server k[:host], *(k[:roles]) if k[:host]
+            load_cap_override env
+          end
+        end
+      end
+      task "#{env}_default_role" do
+        default_role_nodes.each do |k|
+          server k[:host], *(k[:roles])
+          load_cap_override env
+        end
+      end
+      task env do
+        nodes.each do |k|
+          server k[:host], *(k[:roles]) if k[:host]
+          load_cap_override env
+        end
+      end
+    end
+  end
+
+  on :load, "load_topology"
+
+  def load_cap_override env
+    TOPOLOGY[env][:cap_override].each do |k, v|
+      set k, v
+    end if TOPOLOGY[env][:cap_override]
+  end
+
+end

data/lib/master-cap/topology.rb

@@ -0,0 +1,71 @@
+
+require File.join(File.dirname(__FILE__), 'misc.rb')
+require File.join(File.dirname(__FILE__), 'default_translation_strategy.rb')
+
+unless Capistrano::Configuration.respond_to?(:instance)
+  abort "master-cap requires Capistrano 2"
+end
+
+TOPOLOGY = {}
+
+Capistrano::Configuration.instance.load do
+
+  set :chef_role, fetch(:chef_role, :linux_chef)
+  set :chef_user, fetch(:chef_user, "chef")
+  set :translation_strategy, Object.const_get(fetch(:translation_strategy_class, 'DefaultTranslationStrategy')).new
+
+  task :check do
+    find_servers(:roles => chef_role).sort_by{|s| s.host}.each do |s|
+      env, node = find_node s.host
+      begin
+        exec_local_with_timeout "ssh -o StrictHostKeyChecking=no #{chef_user}@#{node[:topology_hostname]} uname > /dev/null 2>&1", fetch(:check_timeout, 10)
+        puts "OK    : #{node[:topology_hostname]}"
+      rescue
+        puts "ERROR : Unable to join #{node[:topology_hostname]}"
+      end
+    end
+  end
+
+  task :ssh_cmd, :roles => chef_role do
+    error "Please specify command with -s cmd=" unless exists? :cmd
+    set :user, chef_user
+    run cmd
+  end
+
+  task :show do
+    ss = find_servers
+    puts "Number of selected servers: #{ss.length}"
+    ss.each do |s|
+      env, node = find_node s.host
+      puts "#{fill(node[:topology_name], 30)} #{fill(node[:topology_hostname], 50)} [#{node[:roles].sort.join(',')}]"
+    end
+  end
+
+  def find_node node_name
+    TOPOLOGY.each do |k, v|
+      v[:topology].each do |name, node|
+        return [k, node] if translation_strategy.node_hostname(k, name, node, v) == node_name
+      end
+    end
+    error "Node not found #{node_name}"
+  end
+
+def check_only_one_env servers = nil
+  servers = find_servers unless servers
+  env_list = {}
+  servers.each do |s|
+    env, name, node = find_node(s.is_a?(String) ? s : s.host)
+    env_list[env] = :toto
+  end
+  error "Please, do not launch this command without env" if env_list.keys.size == 0
+  error "Please, do not launch this command on two env : #{env_list.keys.join(' ')}" if env_list.keys.size != 1
+  env = env_list.keys.first
+
+  check_only_one_env_callback(env, servers) if exists? :check_only_one_env_callback
+
+  env
+end
+
+end
+
+require File.join(File.dirname(__FILE__), 'chef.rb')

data/master-cap.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+
+  s.name        = "master-cap"
+  s.version     = "0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Bertrand Paquet"]
+  s.email       = ["bertrand.paquet@gmail.com"]
+  s.homepage    = "http://github.com/bpaquet/master-cap"
+  s.summary     = "Capistrano tasks designed to work with master-chef"
+  s.description = "Capistrano tasks designed to work with master-chef"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.extra_rdoc_files = [
+    "Readme.markdown"
+  ]
+
+  s.specification_version = 3
+  s.add_runtime_dependency(%q<capistrano>, [">= 2"])
+  s.add_runtime_dependency(%q<json>)
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: master-cap
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Bertrand Paquet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capistrano
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Capistrano tasks designed to work with master-chef
+email:
+- bertrand.paquet@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- Readme.markdown
+files:
+- Rakefile
+- Readme.markdown
+- lib/master-cap/chef.rb
+- lib/master-cap/default_translation_strategy.rb
+- lib/master-cap/git_repos_manager.rb
+- lib/master-cap/misc.rb
+- lib/master-cap/topology-directory.rb
+- lib/master-cap/topology.rb
+- master-cap.gemspec
+homepage: http://github.com/bpaquet/master-cap
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Capistrano tasks designed to work with master-chef
+test_files: []