amoeba_deploy_tools 0.0.1 → 0.0.2

data/lib/amoeba_deploy_tools/commands/node.rb

@@ -0,0 +1,119 @@
+require 'json'
+
+module AmoebaDeployTools
+  class Node < Command
+    include AmoebaDeployTools::Concerns::SSH
+
+    desc 'bootstrap', 'Setup the node initially (run the first time you want to deploy a node)'
+    long_desc <<-LONGDESC
+      `bootstrap` will load Chef and the `basenode` recipe on the node. This is effectively the way
+      to setup a node initially. Subsequent runs can use the `node push` command.
+    LONGDESC
+    option :version, desc: 'Chef version to bootstrap', default: '11.8.2'
+    option :interactive, desc: 'Run interactively (useful for entering passwords)', type: :boolean, default: false
+    def bootstrap
+      logger.info 'Starting `bootstrap`!'
+
+      refresh
+      knife_solo :prepare, 'bootstrap-version' => options[:version], ssh: true, interactive: options[:interactive]
+
+      knife_solo :cook, ssh: true, include_private_key: true, interactive: options[:interactive] do |j|
+        j.run_list = ['role[base]']
+      end
+
+      force_deployer
+
+      pull
+
+      logger.warn 'Node bootstrapped successfully, you can now push to the node:'
+      logger.warn "\tamoeba node push --node #{options[:node]}\n"
+    end
+
+    desc 'force_deployer', 'Forces the deploy user to be `deploy` or that specified in node.json'
+    def force_deployer
+      logger.info 'Starting force_deployer'
+      data_bag(:nodes)[node.name] = { deployment: { user: node.depoyment_.user || 'deploy' } }
+      puts "Remote node: #{remote_node}"
+    end
+
+    desc 'push', 'Push any changes to the node'
+    def push
+      logger.info 'Starting push...'
+      refresh
+      knife_solo :cook, ssh: true, include_private_key: true
+      pull
+    end
+
+    desc 'pull', 'Pull down node state and store in local node databag (run automatically after push)'
+    def pull
+      logger.info 'Starting `pull`!'
+      force_deployer unless remote_node.deployment_.user
+
+      raw_json = ssh_run('sudo cat ~deploy/node.json', silent: true)
+      data_bag(:nodes)[node.name]  = JSON.load raw_json
+    end
+
+    desc 'list', 'Show available nodes in kitchen'
+    def list
+      inside_kitchen do
+        puts Dir.glob('nodes/*.json').sort.map {|n| File.basename(n).sub(/\.json$/, '')}
+      end
+    end
+
+    desc 'refresh', 'Refresh data bags based on node config. Note this is normally run automatically.'
+    long_desc <<-DESC
+      Normally, you should not need to run `refresh`. It is run automatically before every `push` or
+      `bootstrap`. This command prepares data bags for the node. Presently, this is only used for
+      SSH keys. Thus, `refresh` will go through all the `authorized_keys` folders and generate
+      data_bags for each user. These data_bags are then used by the Chef Cookbooks during pushes.
+    DESC
+    def refresh
+      logger.info "Starting `refresh`!"
+      inside_kitchen do
+        # Handle authorized_keys
+        logger.debug '# Refreshing authorized_keys'
+        Dir.glob('authorized_keys/*') do |user_dir|
+          if File.directory? user_dir
+            user_name = File.basename(user_dir)
+            logger.info "Processing SSH keys for user #{user_name}."
+            user_bag = data_bag(:authorized_keys)[user_name]
+            user_bag[:keys] = []
+
+            Dir.glob(File.join(user_dir, '*')) do |key_file|
+              logger.debug "Reading key file: #{key_file}"
+              user_bag[:keys] << File.read(key_file).strip
+            end
+
+            logger.info "Writing #{user_bag.options[:filename]}"
+            user_bag.save
+          else
+            logger.info "Ignoring file in authorized_keys (must be inside a directory): #{f}"
+          end
+        end
+
+        logger.debug '# Ensuring bundle is up to date'
+        # Handle bundler, ensure it's up to date
+        unless system('bundle check > /dev/null 2>&1')
+          logger.info "Bundle out of date! Running bundle update..."
+          Cocaine::CommandLine.new('bundle', 'install').run
+        end
+      end
+    end
+
+    desc 'exec "CMD"', 'Execute given command (via SSH) on node, as deploy user (can sudo)'
+    def exec(cmd)
+      ssh_run(cmd, interactive: true)
+    end
+
+    desc 'ssh', 'SSHs to the node as the deploy user (can sudo)'
+    def ssh
+      exec nil
+    end
+
+    desc 'sudo "CMD"', 'Executes the given command as root on the node'
+    def sudo(cmd)
+      # pull args off of cmd
+      exec("sudo #{cmd}")
+    end
+  end
+end

data/lib/amoeba_deploy_tools/commands/ssl.rb

@@ -0,0 +1,45 @@
+module AmoebaDeployTools
+  class Ssl < Command
+    # Needed for knife solo stuff
+    include AmoebaDeployTools::Concerns::SSH
+
+    desc 'import', 'Import an SSL certificate and add to encrypted data bag (encryption key `default` used unless specified)'
+    option :privateKey, desc: 'Name of the private key to use. Will create if missing', default: 'default'
+    option :key, desc: 'SSL private key file name / path', required: true
+    option :cert, desc: 'SSL public certificate file name / path', required: true
+    option :ca, desc: 'SSL intermediary CA certificate name / path'
+    def import(cert_name=nil)
+      logger.debug "Starting SSL import!"
+      validate_chef_id!(cert_name)
+
+      private_key = options[:privateKey]
+
+      json_data = { id: cert_name }
+      [:key, :cert, :ca].each do |c|
+        # read certificates before we get in the kitchen
+        if File.exist? options[c]
+          json_data[c] = File.read(options[c])
+        else
+          logger.error "Cannot find certificate file to import (ignoring): #{options[c]}"
+          options[c] = nil
+        end
+      end
+
+      # Ensure key exists
+      unless config.private_keys_[private_key]
+        logger.warn "Private key missing: #{options[:privateKey]}, running `amoeba key create #{options[:privateKey]}`"
+        AmoebaDeployTools::Key.new.create(options[:privateKey])
+      end
+
+      inside_kitchen do
+        # Import to certs databag
+        with_tmpfile( json_data.to_json, name: [cert_name, '.json'] ) do |file_name|
+          knife_solo "data bag create certs #{cert_name}",
+                     'json-file' => file_name,
+                     'secret' => "'#{config.reload!.private_keys[private_key]}'"
+        end
+      end
+    end
+
+  end
+end

data/lib/amoeba_deploy_tools/config.rb

@@ -0,0 +1,133 @@
+require 'yaml'
+require 'json'
+require 'fileutils'
+require 'hashie/mash'
+
+module AmoebaDeployTools
+  class Config < Hashie::Mash
+    def self.load(filename, opts={})
+      opts[:filename] = File.expand_path filename
+      Config.new.tap do |c|
+        c.restore(opts)
+      end
+    end
+
+    def self.create(filename, opts={})
+      opts.merge!({
+        filename: File.expand_path(filename),
+        create: true
+      })
+
+      Config.new.tap do |c|
+        c.options(opts)
+      end
+    end
+
+    def options(opts=nil)
+      @opts ||= { format: :yaml }
+      @opts.merge! opts if opts
+      @opts
+    end
+
+    def restore(opts=nil)
+      options(opts)
+
+      return unless filename = options[:filename]
+
+      self.clear.deep_merge! deserialize(File.read(filename))
+
+      self
+    rescue Errno::ENOENT
+      @new_file = true
+      FileUtils.touch(filename) and retry if options[:create]
+    end
+
+    def reload!
+      restore(filename: options[:filename])
+    end
+
+    def new_file?
+      !!@new_file
+    end
+
+    def save(opts=nil)
+      options(opts)
+
+      return unless filename = options[:filename]
+
+      File.open(filename, 'w') do |fh|
+        fh.write(serialize(self.to_hash))
+      end
+
+      self
+    rescue Errno::ENOENT
+      FileUtils.touch(filename) and retry if options[:create]
+    end
+
+    def [](k)
+      chain = k.to_s.split('.')
+      cur = self
+
+      return super if chain.count <= 1
+
+      for c in chain[0..-2]
+        if cur and cur.key? c
+          cur = cur.regular_reader(c)
+        else
+          return
+        end
+      end
+
+      cur[chain[-1]]
+    end
+
+    def []=(k, v)
+      chain = k.to_s.split('.')
+      cur = self
+
+      return super if chain.count <= 1
+
+      for c in chain[0..-2]
+        cur = cur.initializing_reader(c)
+      end
+
+      cur[chain[-1]] = v
+    end
+
+    def flatten
+      flat = {}
+
+      each do |k1, v1|
+        if v1.class == self.class
+          v1.flatten.each do |k2, v2|
+            flat["#{k1}.#{k2}"] = v2
+          end
+        else
+          flat[k1] = v1
+        end
+      end
+
+      flat
+    end
+
+
+    def to_s
+      to_hash.to_s
+    end
+
+    protected
+
+    @@formats = {
+      json: JSON,
+      yaml: YAML
+    }
+
+    def serialize(d)
+      @@formats[options[:format]].dump(d)
+    end
+
+    def deserialize(d)
+      @@formats[options[:format]].load(d) || {}
+    end
+  end
+end

data/lib/amoeba_deploy_tools/data_bag.rb

@@ -0,0 +1,30 @@
+require 'json'
+
+module AmoebaDeployTools
+  class DataBag
+    def initialize(bag, kitchen)
+      @bag_dir = File.join(kitchen, 'data_bags', bag.to_s)
+      Dir.mkdir @bag_dir unless Dir.exists? @bag_dir
+    end
+
+    def []=(id, item)
+      bag_item = DataBagItem.create(item_filename(id), format: :json)
+      bag_item.clear.deep_merge!(item.to_hash)
+      bag_item.id = id
+      bag_item.save
+    end
+
+    def [](id)
+      DataBagItem.load(item_filename(id), format: :json, create: true).tap do |i|
+        i.id = id
+      end
+    end
+
+    def item_filename(id)
+      File.join(@bag_dir, "#{id}.json")
+    end
+  end
+
+  class DataBagItem < Config
+  end
+end

data/lib/amoeba_deploy_tools/helpers.rb

@@ -0,0 +1,51 @@
+require 'tempfile'
+require 'pathname'
+require 'tmpdir'
+
+def with_tmpfile(content=nil, options={} )
+  name = options[:name] || 'amoebatmp'
+
+  tmpf = Tempfile.new name
+  tmpf.write content
+  tmpf.close
+
+  results = yield tmpf.path, tmpf
+
+  tmpf.unlink
+  results
+end
+
+def in_tmpdir
+  Dir.mktmpdir do |tmpd|
+    Dir.chdir tmpd do
+      yield
+    end
+  end
+end
+
+def dedent(s)
+  indent = s.split("\n").reject {|l| l =~ /^\s*$/}.map {|l| l.index /\S/ }.min
+  s.sub(/^\n/, '').gsub(/ +$/, '').gsub(/^ {#{indent}}/, '')
+end
+
+def indent(s, indent=4)
+  s.gsub(/^/, ' ' * indent)
+end
+
+def say_bold(text)
+  say text, :bold
+end
+
+def say_fatal(text)
+  say text, :red
+  exit 1
+end
+
+class Exception
+  def bt
+    backtrace.map do |l|
+      cwd = Dir.pwd
+      l.start_with?(cwd) ? l.sub(cwd, '.') : l
+    end
+  end
+end

data/lib/amoeba_deploy_tools/interactive_cocaine_runner.rb

@@ -0,0 +1,27 @@
+# coding: UTF-8
+require 'pty'
+
+module AmoebaDeployTools
+  class InteractiveCocaineRunner
+    def self.supported?
+      true
+    end
+
+    def supported?
+      self.class.supported?
+    end
+
+    def call(command, env = {})
+      with_modified_environment(env) do
+        system(command)
+      end
+      ''
+    end
+
+    private
+
+    def with_modified_environment(env, &block)
+      ClimateControl.modify(env, &block)
+    end
+  end
+end

data/lib/amoeba_deploy_tools/logger.rb

@@ -0,0 +1,67 @@
+require 'logger'
+
+module AmoebaDeployTools
+  class Logger < ::Logger
+    module Colors
+      VERSION = '1.0.0'
+
+      NOTHING      = '0;0'
+      BLACK        = '0;30'
+      RED          = '0;31'
+      GREEN        = '0;32'
+      BROWN        = '0;33'
+      BLUE         = '0;34'
+      PURPLE       = '0;35'
+      CYAN         = '0;36'
+      LIGHT_GRAY   = '0;37'
+      DARK_GRAY    = '1;30'
+      LIGHT_RED    = '1;31'
+      LIGHT_GREEN  = '1;32'
+      YELLOW       = '1;33'
+      LIGHT_BLUE   = '1;34'
+      LIGHT_PURPLE = '1;35'
+      LIGHT_CYAN   = '1;36'
+      WHITE        = '1;37'
+
+      SCHEMA = {
+        STDOUT => %w[nothing green brown red purple cyan],
+        STDERR => %w[nothing green yellow light_red light_purple light_cyan],
+      }
+    end
+
+    class SimpleFormatter < ::Logger::Formatter
+      def call(severity, time, progname, msg)
+        "%s\n" % msg2str(msg)
+      end
+    end
+
+    def format_message(level, *args)
+      if Logger::Colors::SCHEMA[@logdev.dev]
+        color = begin
+          Logger::Colors.const_get \
+            Logger::Colors::SCHEMA[@logdev.dev][Logger.const_get(level.sub "ANY","UNKNOWN")].to_s.upcase
+        rescue NameError
+          "0;0"
+        end
+        "\e[#{ color }m#{ super(level, *args) }\e[0;0m"
+      else
+        super(level, *args)
+      end
+    end
+
+    def initialize(logdev)
+      super
+
+      @formatter = SimpleFormatter.new
+    end
+
+    @@instance = self.new(STDOUT)
+
+    def self.instance
+      return @@instance
+    end
+
+    # Prevent folks from instantiating one
+    private_class_method :new
+  end
+end

data/lib/amoeba_deploy_tools/noisey_cocaine_runner.rb

@@ -0,0 +1,42 @@
+# coding: UTF-8
+require 'pty'
+
+module AmoebaDeployTools
+  class NoiseyCocaineRunner
+    def self.supported?
+      true
+    end
+
+    def supported?
+      self.class.supported?
+    end
+
+    def call(command, env = {})
+      buffer = ''
+      with_modified_environment(env) do
+        begin
+          # Use PTY.spawn so we don't buffer anything. r will contain the output (stdout & stderr)
+          PTY.spawn(command) do |r,w,pid|
+            begin
+              r.each { |line| Cocaine::CommandLine.logger.debug line.strip; buffer << line }
+            rescue Errno::EIO
+              # Output is done
+            end
+            # Note: This requires ruby 1.9.2!
+            Process.wait(pid) # Wait for the process to die (so $? is set)
+          end
+        rescue PTY::ChildExited
+          # The command is done
+          # $!.status.exitstatus would likely contain the exit code
+        end
+      end
+      buffer
+    end
+
+    private
+
+    def with_modified_environment(env, &block)
+      ClimateControl.modify(env, &block)
+    end
+  end
+end

data/lib/amoeba_deploy_tools/version.rb

@@ -0,0 +1,3 @@
+module AmoebaDeployTools
+  VERSION = "0.0.2"
+end

data/spec/amoeba_deploy_tools/command_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe AmoebaDeployTools::Command do
+end

data/spec/amoeba_deploy_tools/commands_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+require 'stringio'
+require 'fileutils'
+
+class AmoebaDeployTools
+  describe 'Amoeba command' do
+    def run_cmd(argv)
+      $stdout = StringIO.new
+      $stderr = StringIO.new
+
+      object = Amoeba.new(*argv).tap {|c| c.run(false) }
+
+      @stdout = $stdout.tap {|f| f.seek(0) }.read
+      @stderr = $stderr.tap {|f| f.seek(0) }.read
+    ensure
+      $stdout = STDOUT
+      $stderr = STDERR
+
+      return object
+    end
+
+    def self.context(*a, **kw)
+      if a.first.is_a? Array
+        kw[:argv] = a[0]
+        a[0] = "[#{a[0].join(' ')}]"
+      end
+
+      super
+    end
+
+    @@subject = nil
+    def rerun_cmd
+      @@subject = run_cmd(example.metadata[:argv])
+    end
+
+    subject { rerun_cmd }
+    before { rerun_cmd }
+
+    shared_context require_kitchen: true do
+      around do |example|
+        in_tmpdir do
+          %w(
+            .amoeba
+            .amoeba/nodes
+            .amoeba/roles
+          ).map {|p| Dir.mkdir p}
+
+          example.metadata[:nodes].each do |d|
+            FileUtils.touch ".amoeba/nodes/#{d}.json"
+          end if example.metadata[:nodes].is_a? Enumerable
+
+          example.run
+        end
+      end
+
+      context 'when missing kitchen' do
+        before { FileUtils.rm_rf '.amoeba' }
+
+        its (:status) { should eq(1) }
+      end
+    end
+
+    context %w(help) do
+      its (:cmd) { should eq(:amoeba) }
+      its (:subcmd) { should eq(:help) }
+
+      its (:status) { should eq(1) }
+    end
+
+    context %w(node list), :require_kitchen do
+      its (:cmd) { should eq(:node) }
+      its (:subcmd) { should eq(:list) }
+
+      context 'without nodes' do
+        it 'returns empty' do
+          expect(@stdout).to eq('')
+        end
+
+        its (:status) { should eq(0) }
+      end
+
+      context 'with nodes', nodes: %w( a.example.com b.example.com ) do
+        it 'lists nodes' do
+          expect(@stdout).to eq(example.metadata[:nodes].map {|n| n + "\n"}.join)
+        end
+
+        its (:status) { should eq(0) }
+      end
+    end
+  end
+end

data/spec/amoeba_deploy_tools/config_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'yaml'
+
+describe AmoebaDeployTools::Config do
+  let (:config) { subject }
+
+  let :given do
+    {'foo' => 'bar'}
+  end
+
+  before merge: true do
+    config.merge! given
+  end
+
+  it 'saves config files', :merge do
+    with_tmpfile do |f|
+      config.save filename: f
+    end
+
+    expect(config).to eq(given)
+  end
+
+  it 'loads config files' do
+    with_tmpfile YAML.dump(given) do |f|
+      config.restore filename: f
+    end
+
+    expect(config).to eq(given)
+  end
+
+  context 'with a nested structure' do
+    let :given do
+      {'foo' => {'bar' => {'baz' => 'garply'}}}
+    end
+
+    it 'expands dotted keys' do
+      config['foo.bar.baz'] = 'garply'
+
+      expect(config).to eq(given)
+    end
+
+    it 'flattens into dotted keys', :merge do
+      expect(config.flatten).to eq({
+        'foo.bar.baz' => 'garply'
+      })
+    end
+  end
+end

data/spec/amoeba_deploy_tools/helpers_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe :dedent do
+  it 'correctly removes indentation' do
+    expect(dedent %{
+    foo
+      bar
+    baz
+        garply
+    }).to eq("foo\n  bar\nbaz\n    garply\n")
+  end
+end
+
+describe :indent do
+  it 'indents each line by the specified amount' do
+    expect(indent("foo\n  bar\nbaz\n    garply\n", 3)).to eq("   foo\n     bar\n   baz\n       garply\n")
+  end
+end