minionizer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82bef054c00cc362c8c6bde7b0557b1ec7ea8966
-  data.tar.gz: 496df270bcb0c2c4fe65d3bb291ef5d9d9673e69
+  metadata.gz: f0f6f2ff87e40cfacb8163853951e4b6d2851dd3
+  data.tar.gz: aaa855643097d24799b62ed3b6f1fc187670a224
 SHA512:
-  metadata.gz: b437e19e86a6bc97c546ba32193a77812c0710c50b375927ddd0c49b129fab0e96b4b91ab9e6456a409b10061e7ca04398220bb125c4a85a7c56382d06d67621
-  data.tar.gz: ef1cc521975e07f8fbee54cd3e3f5415d0610f2dfb6bc6cb5a0f5b79725d05942f4c9f1baf8e915729ecb6a9d32a9bdf111c936f233f7c76dd9abf1797668588
+  metadata.gz: caf5ec31a1e30efac84dc898af40c4a7d9e2e247fcfdae6110d5bb93c9ab657f48348f664f5dc0dec5a40721c2181f85f4bc337e055e989ccf38abbcde596f70
+  data.tar.gz: 6a366d4b175747590bc289741068d20523b32716580ecae1244f4dd84aa2d91d4bbaf013f1632ca123fd528369932aef0d7f71cdd749ea1221c40ad1e4a44476

data/.travis.yml

@@ -1,6 +1,6 @@
 language: ruby
 rvm:
   - "2.0.0"
-  - "2.1.1"
+  - "2.1.3"
 # uncomment this line if your project needs to run something other than `rake`:
 # # script: bundle exec rspec spec

data/Gemfile.lock

@@ -1,50 +1,59 @@
 PATH
   remote: .
   specs:
-    minionizer (0.0.1)
+    minionizer (0.2.0)
       activesupport (~> 4.1)
+      binding_of_caller (~> 0.7)
+      net-scp (~> 1.2)
       net-ssh (~> 2.9)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    activesupport (4.1.1)
+    activesupport (4.1.6)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.1)
       tzinfo (~> 1.1)
-    coveralls (0.7.0)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    coveralls (0.7.1)
       multi_json (~> 1.3)
       rest-client
       simplecov (>= 0.7)
       term-ansicolor
       thor
-    docile (1.1.3)
-    fakefs (0.5.2)
-    i18n (0.6.9)
+    debug_inspector (0.0.2)
+    docile (1.1.5)
+    fakefs (0.6.0)
+    i18n (0.6.11)
     json (1.8.1)
     metaclass (0.0.4)
-    mime-types (2.2)
-    minitest (5.3.3)
-    mocha (1.0.0)
+    mime-types (2.4.3)
+    minitest (5.4.2)
+    mocha (1.1.0)
       metaclass (~> 0.0.1)
-    multi_json (1.10.0)
-    net-ssh (2.9.0)
-    rake (10.3.1)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
-    simplecov (0.8.2)
+    multi_json (1.10.1)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.1)
+    netrc (0.8.0)
+    rake (10.3.2)
+    rest-client (1.7.2)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    simplecov (0.9.1)
       docile (~> 1.1.0)
-      multi_json
+      multi_json (~> 1.0)
       simplecov-html (~> 0.8.0)
     simplecov-html (0.8.0)
     term-ansicolor (1.3.0)
       tins (~> 1.0)
     thor (0.19.1)
-    thread_safe (0.3.3)
-    tins (1.1.0)
-    tzinfo (1.1.0)
+    thread_safe (0.3.4)
+    tins (1.3.3)
+    tzinfo (1.2.2)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -54,5 +63,6 @@ DEPENDENCIES
   coveralls
   fakefs (~> 0.5)
   minionizer!
+  minitest (~> 5.4)
   mocha (~> 1.0)
   rake (~> 10.3)

data/README.md

@@ -9,9 +9,9 @@
 Minionizer aims to be a light weight, yet powerful, server provisioning tool with minimum learning
 curve.
 
-Minionizer is still in alpha development and is not yet ready for anything resembling production use.
+Minionizer is still in beta development and is not yet ready for anything resembling production use.
 
-# Overview
+## Overview
 
 Minionizer allows you keep all of your provisioning "recipies" for a set of servers, along with any
 data those recipies may need (such as config files), in a single git repository.
@@ -31,27 +31,30 @@ packages, etc. You can use these core commands to build more complex recipies, o
 minionizer plugins that WILL BE available, such as posgresql installationi/upgrade, ruby
 installation/upgrade, etc. 
 
-# Installation
+## Installation
 
     gem install minionizer
 
-# Usage
+## Demonstration
 
-## Setup a new provisioning project in the current folder.
-Note: This step doesn't actually work yet.
+A demonstration repo is available at [https://github.com/jsgarvin/minionizer-demo](https://github.com/jsgarvin/minionizer-demo).
+
+## Usage
+
+### Setup a new provisioning project in the current folder.
 
     minionize --init subfolder_name
 
 Creates `subfolder_name` and initializes it with some initial folders and files to get you started.
 
-## Modify config/minions.yml
+### Modify config/minions.yml
 
 The minions.yml file is where you define what servers this project will manage and what roles
 each server will play.
 
 You will probably want assign each server multiple roles, such as `['production', 'db']`.
 
-## Create role instructions
+### Create role instructions
 
 A sample role file WILL BE provided in the ./roles folder to get you started. Each role file defines
 what servers assigned that role should do on each (re)provisioning.
@@ -61,7 +64,7 @@ file. You will likely have some roles, such as "production", that are mearly a m
 several servers together and won't have a corresponding role file.  You will need at least one role,
 though, such as "db" or "webserver", that will have a corresponding role file.
 
-## Provision Servers
+### Provision Servers
 
 To provision all of the servers that are assigned a particular role, run...
 
@@ -79,7 +82,7 @@ or
 This will loop through each role that is assigned to just that server, and any corresponding role
 files will be run.
 
-# Contribute
+## Contribute
 
 To contribute to Minionizer development you will need to install [vagrant](http://www.vagrantup.com/)
 and [VirtualBox](https://www.virtualbox.org/) in order to be able to run acceptance tests.

data/Rakefile

@@ -14,6 +14,7 @@ end
 
 namespace :test do
   namespace :vm do
+    desc 'Start Test VM'
     task :start do
       relay_output(vagrant_command(:up))
       unless snapshot_plugin_installed?
@@ -24,6 +25,7 @@ namespace :test do
         relay_output(vagrant_command('snapshot take blank-test-slate'))
       end
     end
+    desc 'Stop Test VM'
     task :stop do
       relay_output(vagrant_command(:halt))
     end

data/bin/minionize

@@ -1,3 +1,13 @@
 #!/usr/bin/env ruby
 require_relative '../lib/minionizer'
-Minionizer::Minionization.new(ARGV, Minionizer::Configuration.instance).call
+if ARGV[0] == '--init'
+  Minionizer::Initialization.create(ARGV[1])
+elsif ['-gk','--generate-key'].include?(ARGV[0])
+  Minionizer::Cryptographer.new.generate_key
+elsif ['-es','--encrypt-secrets'].include?(ARGV[0])
+  Minionizer::Cryptographer.new.encrypt_secrets
+elsif ['-ds','--decrypt-secrets'].include?(ARGV[0])
+  Minionizer::Cryptographer.new.decrypt_secrets
+else
+  Minionizer::Minionization.new(ARGV, Minionizer::Configuration.instance).call
+end

data/lib/core/file_injection.rb

@@ -3,7 +3,7 @@ module Minionizer
 
     def call
       session.exec("mkdir --parents #{target_directory}")
-      session.exec("echo '#{contents_from(source_path)}' > #{target_path}")
+      session.scp(string_io_creator.new(contents), target_path)
       session.exec("chmod #{mode} #{target_path}") if respond_to?(:mode)
       session.exec("chown #{owner} #{target_path}") if respond_to?(:owner)
       session.exec("chgrp #{group} #{target_path}") if respond_to?(:group)
@@ -17,8 +17,36 @@ module Minionizer
       File.dirname(target_path)
     end
 
-    def contents_from(source)
-      File.open(source).read.strip
+    def contents
+      options[:contents] ||= processed_contents_from_source_path
+    end
+
+    def processed_contents_from_source_path
+      if source_file_requires_erb_processing?
+        ERB.new(contents_from_source_path).result(erb_binding)
+      else
+        contents_from_source_path
+      end
+    end
+
+    def source_file_requires_erb_processing?
+      File.extname(source_path) == '.erb'
+    end
+
+    def contents_from_source_path
+      File.open(source_path).read.strip
+    end
+
+    def erb_binding
+      level = 1
+      until binding.of_caller(level).eval('self.class') != self.class
+        level += 1
+      end
+      binding.of_caller(level)
+    end
+
+    def string_io_creator
+      options[:string_io_creator] ||= StringIO
     end
   end
 end

data/lib/core/public_ssh_key_injection.rb

@@ -2,36 +2,57 @@ module Minionizer
   class PublicSshKeyInjection < TaskTemplate
 
     def call
+      folder_creation.call
       file_injection.call
-    ensure
-      temp_file.unlink
     end
 
     #######
     private
     #######
 
+    def folder_creation
+      @folder_creation ||= folder_creation_creator.new(session, folder_creation_options)
+    end
+
     def file_injection
       @file_injection ||= file_injection_creator.new(session, file_injection_options)
     end
 
+    def folder_creation_creator
+      options[:folder_creation_creator] ||= FolderCreation
+    end
+
     def file_injection_creator
       options[:file_injection_creator] ||= FileInjection
     end
 
+    def folder_creation_options
+      {
+        path: target_folder,
+        owner: target_username,
+        group: target_username,
+        mode: 700
+      }
+    end
+
     def file_injection_options
       {
-        source_path: temp_file.path,
-        target_path: "~#{target_username}/.ssh/authorized_keys",
+        contents: combined_keys,
+        target_path: "#{target_folder}/authorized_keys",
         owner: target_username,
-        group: target_username
+        group: target_username,
+        mode: 600
       }
     end
 
-    def temp_file
-      @temp_file ||= Tempfile.new('MinionizerPublicKeys').tap do |temp_file|
+    def target_folder
+      "/home/#{target_username}/.ssh"
+    end
+
+    def combined_keys
+      String.new.tap do |string|
         Dir.glob("data/public_keys/*.pubkey") do |key_file|
-          temp_file.puts File.open(key_file).read
+          string << File.open(key_file).read
         end
       end
     end

data/lib/core/task_template.rb

@@ -19,6 +19,9 @@ module Minionizer
       options.key?(method_name) || super
     end
 
+    def download_file(source, destination)
+      session.exec(%Q{bash -c "mkdir -p #{destination}; cd #{destination}; wget -nc #{source}"})
+    end
   end
 end
 

data/lib/minionizer/command_execution.rb

@@ -4,11 +4,12 @@ module Minionizer
     class CommandError < StandardError; end
     class InvocationError < StandardError; end
 
-    attr_reader :connection, :command
+    attr_reader :connection, :command, :options
 
-    def initialize(connection, command)
+    def initialize(connection, command, options={})
       @connection = connection
       @command = command
+      @options = options
     end
 
     def call
@@ -27,6 +28,7 @@ module Minionizer
     end
 
     def execute_command_inside_channel(channel)
+      inform("Running: #{command}")
       channel.exec(command) do |_, success|
         if success
           compile_results(channel)
@@ -51,10 +53,21 @@ module Minionizer
     end
 
     def compile_results(channel)
-      channel.on_data { |_, data| results[:stdout] += data.strip }
-      channel.on_extended_data { |_, data| results[:stderr] += data.to_s }
+      read_stdout(channel)
+      channel.on_extended_data { |_, _, data| results[:stderr] += data.to_s }
       channel.on_request('exit-status') { |_,data| results[:exit_code] = data.read_long }
       channel.on_request('exit-signal') { |_,data| results[:exit_signal] = data.read_string }
     end
+
+    def read_stdout(channel)
+      channel.on_data do |_, data|
+        inform("STDOUT: #{data}")
+        results[:stdout] += data.strip
+      end
+    end
+
+    def inform(message)
+      puts message if options[:verbose]
+    end
   end
 end

data/lib/minionizer/cryptographer.rb

@@ -0,0 +1,84 @@
+module Minionizer
+  class Cryptographer
+    KEYRING_PATH = './data/public_keyring.gpg'
+    SECRET_FOLDER  = './data/secrets'
+    SAFE_FOLDER  = './data/encrypted_secrets'
+
+    def generate_key
+      system("#{gpg_command} --gen-key")
+    end
+
+    def encrypt_secrets
+      Dir.foreach(SECRET_FOLDER) do |filename|
+        next if ['.','..'].include?(filename)
+        encrypt_and_rehash(filename)
+      end
+    end
+
+    def decrypt_secrets
+      Dir.foreach(SAFE_FOLDER) do |filename|
+        next unless filename.match(/\.enc$/)
+        decrypt(filename)
+      end
+    end
+
+    private
+
+    def encrypt_and_rehash(filename)
+      if shas_match?(filename)
+        puts "Skipping: #{filename} (shas match)"
+      else
+        puts "Encrypting: #{filename}"
+        system("#{gpg_command} --output #{SAFE_FOLDER}/#{filename}.enc #{recipient_args.join(' ')} --encrypt #{SECRET_FOLDER}/#{filename}")
+        hash(filename)
+      end
+    end
+
+    def hash(filename)
+      puts "Hashing: #{filename}"
+      system("sha512sum #{SECRET_FOLDER}/#{filename} > #{SAFE_FOLDER}/#{filename}.sha")
+    end
+
+    def decrypt(filename)
+      if shas_match?(decrypted_filename(filename))
+        puts "Skipping: #{filename} (shas match)"
+      else
+        puts "Decrypting: #{filename}"
+        system("#{gpg_command} --output #{SECRET_FOLDER}/#{decrypted_filename(filename)} --decrypt #{SAFE_FOLDER}/#{filename}")
+      end
+    end
+
+    def recipient_args
+      recipient_fingerprints.split("\n").map {|fingerprint| "--recipient #{fingerprint}" }
+    end
+
+    def recipient_fingerprints
+      "fingerprint\n"
+      `#{gpg_command} --fingerprint --with-colons | grep fpr | cut -d ":" -f10`
+    end
+
+    def decrypted_filename(filename)
+      filename.match(/(.+)\.enc$/)[1]
+    end
+
+    def gpg_command
+      "gpg --keyring #{KEYRING_PATH} --no-default-keyring"
+    end
+
+    def shas_match?(filename)
+      File.exists?(stored_sha_path(filename)) && local_sha(filename) == stored_sha(filename)
+    end
+
+    def local_sha(filename)
+      `sha512sum #{SECRET_FOLDER}/#{filename}`
+    end
+
+    def stored_sha(filename)
+      `cat #{stored_sha_path(filename)}`
+    end
+
+    def stored_sha_path(filename)
+      "#{SAFE_FOLDER}/#{filename}.sha"
+    end
+  end
+end

data/lib/minionizer/initialization.rb

@@ -0,0 +1,55 @@
+module Minionizer
+  class Initialization
+    SUBFOLDERS = [
+      '', #root folder
+      '/config',
+      '/data/encrypted_secrets',
+      '/data/public_keys',
+      '/data/secrets',
+      '/lib',
+      '/roles'
+    ]
+
+    attr_reader :path
+
+    def self.create(path)
+      new(path).save
+    end
+
+    def initialize(path)
+      @path = path
+    end
+
+    def save
+      create_directory_structure
+      touch_config
+      ignore_secrets
+    end
+
+    private
+
+    def create_directory_structure
+      SUBFOLDERS.each do |folder|
+        system("mkdir -p #{path}#{folder}")
+      end
+    end
+
+    def touch_config
+      system("touch #{path}/config/minions.yml")
+    end
+
+    def ignore_secrets
+      system("touch #{gitignore_path}")
+      system("echo 'data/secrets' > #{gitignore_path}") unless secrets_ignored?
+    end
+
+    def secrets_ignored?
+      File.readlines(gitignore_path).grep(/^data\/secrets$/).any?
+    end
+
+    def gitignore_path
+      "#{path}/.gitignore"
+    end
+
+  end
+end

data/lib/minionizer/session.rb

@@ -1,12 +1,14 @@
+require 'securerandom'
 module Minionizer
   class Session
-    attr_reader :fqdn, :username, :password, :connector, :command_executor
+    attr_reader :fqdn, :username, :password, :ssh_connector, :scp_connector, :command_executor
 
-    def initialize(fqdn, credentials, connector = Net::SSH, command_executor = CommandExecution)
+    def initialize(fqdn, credentials, ssh_connector=Net::SSH, scp_connector=Net::SCP, command_executor=CommandExecution)
       @fqdn = fqdn
       @username = credentials['username']
       @password = credentials['password']
-      @connector = connector
+      @ssh_connector = ssh_connector
+      @scp_connector = scp_connector
       @command_executor = command_executor
     end
 
@@ -22,26 +24,33 @@ module Minionizer
     end
 
     def exec(*commands)
-      results = commands.map { |command| execution(command).call }
+      options = commands.last.is_a?(Hash) ? commands.pop : {}
+      results = commands.map { |command| execution(command, options).call }
       results.length == 1 ? results.first : results
     end
 
+    def scp(local_path, remote_path)
+      if with_sudo?
+        tmp_filename = "#{SecureRandom.hex}.minionizer_tempfile"
+        scp_connection.upload!(local_path, "#{tmp_filename}")
+        exec("mv #{tmp_filename} #{remote_path}")
+      else
+        scp_connection.upload!(local_path, remote_path)
+      end
+    end
+
     #######
     private
     #######
 
-    def execution(command)
+    def execution(command, options={})
       if with_sudo?
-        command_executor.new(connection, prefix_sudo(command))
+        command_executor.new(ssh_connection, prefix_sudo(command), options)
       else
-        command_executor.new(connection, command)
+        command_executor.new(ssh_connection, command, options)
       end
     end
 
-    def connection
-      @connection ||= connector.start(fqdn, username, password: password)
-    end
-
     def prefix_sudo(command)
       %Q{sudo bash -c "#{command}"}
     end
@@ -50,5 +59,13 @@ module Minionizer
       @with_sudo
     end
 
+    def ssh_connection
+      @ssh_connection ||= ssh_connector.start(fqdn, username, password: password)
+    end
+
+    def scp_connection
+      @scp_connection ||= scp_connector.start(fqdn, username, password: password)
+    end
+
   end
 end

data/lib/minionizer/version.rb

@@ -1,3 +1,3 @@
 module Minionizer
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/minionizer.rb

@@ -1,6 +1,8 @@
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/inflector'
+require 'binding_of_caller'
 require 'erb'
+require 'net/scp'
 require 'net/ssh'
 require 'singleton'
 require 'yaml'

data/minionizer.gemspec

@@ -11,10 +11,13 @@ Gem::Specification.new do |s|
   s.description = %q{Minionizer aims to be a light weight server provisioning tool without bloat or steep learning curves.}
 
   s.add_dependency('activesupport', '~> 4.1')
+  s.add_dependency('binding_of_caller', '~> 0.7')
   s.add_dependency('net-ssh', '~> 2.9')
+  s.add_dependency('net-scp', '~> 1.2')
 
   s.add_development_dependency('fakefs', '~> 0.5')
   s.add_development_dependency('mocha', '~> 1.0')
+  s.add_development_dependency('minitest', '~> 5.4')
   s.add_development_dependency('rake', '~> 10.3')
 
   s.files = `git ls-files`.split("\n")

data/test/integration/core_library_test.rb

@@ -88,8 +88,8 @@ module Minionizer
           group: ownername
         }}
         let(:code) { %Q{
-          session.sudo do
-            Minionizer::FileInjection.new( session, #{options}).call
+          session.sudo do |sudo_session|
+            Minionizer::FileInjection.new(sudo_session, #{options}).call
           end
         } }
 
@@ -107,6 +107,7 @@ module Minionizer
         it 'injects a file' do
           2.times { assert_throws(:high_five) { minionization.call } }
           assert_file_exists(target_path)
+          assert_equal('FooBar', session.sudo("cat #{target_path}")[:stdout])
           mode = session.exec("stat --format=%a #{target_path}")[:stdout]
           assert_equal('700',mode)
           owner = session.exec("stat --format=%U #{target_path}")[:stdout]
@@ -137,11 +138,15 @@ module Minionizer
         after do
           File.delete("#{source_path}/foobar.pubkey")
           File.delete("#{source_path}/foobaz.pubkey")
+          skip unless minion_available?
+          session.sudo("userdel #{target_username}")
         end
 
         it 'injects public keys' do
           2.times { assert_throws(:high_five) { minionization.call } }
-          assert_file_exists("~#{target_username}/.ssh/authorized_keys")
+          assert_file_exists("/home/#{target_username}/.ssh/authorized_keys")
+          assert_match(/FooBar/, session.sudo("cat ~#{target_username}/.ssh/authorized_keys")[:stdout])
+          assert_match(/FooBaz/, session.sudo("cat ~#{target_username}/.ssh/authorized_keys")[:stdout])
         end
       end
 
@@ -173,7 +178,7 @@ module Minionizer
       end
 
       def link_exists?(path, parameter = :e)
-        session.exec("[ -#{parameter} #{path} ] && echo 'yes' || echo 'no'")[:stdout] == 'yes'
+        session.sudo("[ -#{parameter} #{path} ] && echo 'yes' || echo 'no'")[:stdout] == 'yes'
       end
 
       def assert_user_exists(username)

data/test/test_helper.rb

@@ -4,7 +4,6 @@ require 'coveralls'
 require 'minitest/autorun'
 require 'fakefs/safe'
 require 'socket'
-require 'tempfile'
 require 'timeout'
 
 if Coveralls.will_run?