ec2_bootstrap 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 50318b3c980d39aa1255261e3721ebcd5b167731
+  data.tar.gz: 4a43edf57231b9a9b5368f933f6925e3e08ce995
+SHA512:
+  metadata.gz: 1395ad633953e8d5c09a39e55c0a81e0089df51dae759fb1672373556d80223150d86a21eae4dcb16fb3e06affeddf97eb149cb07d600c60a101bcd9222a9443
+  data.tar.gz: ad1a5bacefbeb3a9de43c9885fc6c0e50ab994bde1e37abf6b71a1c85c2aad1cdd7546ed68c9b0f62d16e93a48abe3999f23821c6919c295bc21f974b80b8626

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Cozy Services Ltd.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# Ec2Bootstrap
+
+A simple wrapper for the EC2 Knife plugin to automate the creation of new EC2 instances.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ec2_bootstrap'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install ec2_bootstrap
+
+## Configuration
+
+Requires AWS credentials in a format compatible with the [EC2 Knife plugin](https://github.com/chef/knife-ec2/blob/master/README.md).
+
+Also requires a YAML config file that looks like the example `config.example.yml`. The config file must include a top-level `nodes` key whose value is an array of hashes. Each node must include the keys `node_name` and `knife_ec2_flags`.
+
+You may also want to include some form of cloud-init config. To do this, you can do one of two things:
+
+1. Include a top-level `cloud_config` key with the contents you'd like in the cloud config files used for each node, and ec2_bootstrap will generate separate config files for each node that all include the node's hostname and fqdn.
+2. If you'd prefer to write your own cloud config files, you can include the cloud config's path in a `user-data` key in the node's `knife_ec2_flags` hash.
+
+For any `knife_EC2_flags` values that are lists, they need to be formatted as one long string with values separated by commas. Ex: `security-group-ids: sg-12345678,sg-abcdef12`.
+
+## Usage
+
+	$ bundle exec ec2_bootstrap -c CONFIG_FILE [options]
+
+Passing in a config file with `-c` is required.
+
+By default, `ec2_bootstrap` is set to dryrun mode, where `ec2_bootstrap` will
+print out what config would be used to create a new EC2 instance without
+actually creating the instance.
+
+	$ bundle exec ec2_bootstrap --help
+	# help menu
+	$ bundle exec ec2_bootstrap -c CONFIG_FILE
+	# runs ec2_bootstrap in dryrun mode
+	$ bundle exec ec2_bootstrap -c CONFIG_FILE --no-dryrun
+	# runs ec2_bootstrap and actually creates a new EC2 instance
+	$ bundle exec ec2_bootstrap -c CONFIG_CILE -v
+	# runs ec2_bootstrap in verbose mode
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/ec2_bootstrap/fork )
+2. Install dependencies (`bundle install`)
+3. Create your feature branch (`git checkout -b my-new-feature`)
+4. Make your changes.
+5. Run the tests and make sure they pass (`bundle exec rspec`)
+6. Commit your changes (`git commit -am 'Add some feature'`)
+7. Push to the branch (`git push origin my-new-feature`)
+8. Create a new pull request.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/ec2_bootstrap

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'optparse'
+require 'ec2_bootstrap'
+
+options = {
+	dryrun: true,
+	verbose: false
+}
+
+OptionParser.new do |opts|
+	opts.banner = 'Usage: ec2_bootstrap -c CONFIG_FILE [options]'
+
+	opts.on('-c', '--config CONFIG_FILE',
+		'Path to custom instance config') do |file|
+			options[:config] = file
+	end
+
+	opts.on('-d', '--[no-]dryrun',
+		'Print out changes without actually creating a new EC2 instance		(default: true)') do |dryrun|
+			options[:dryrun] = dryrun
+	end
+
+	opts.on('-v', '--[no-]verbose',
+		'Run in verbose mode		(default: false)') do |verbose|
+			options[:verbose] = verbose
+	end
+end.parse!
+
+config_file = options.delete(:config)
+
+unless config_file
+	puts "You must specify a config file!"
+	exit 1
+end
+
+EC2Bootstrap.from_config_file(config_file, options[:dryrun], options[:verbose]).create_instances

data/lib/ec2_bootstrap/instance.rb

@@ -0,0 +1,64 @@
+require 'json'
+
+class EC2Bootstrap
+	class Instance
+
+		attr_accessor :name
+		attr_accessor :knife_ec2_flags
+
+		def initialize(instance_name:, knife_ec2_flags:, logger:, domain: nil, json_attributes_file:nil)
+			@name = instance_name
+			@json_attributes_file = json_attributes_file
+			@knife_ec2_flags = build_knife_ec2_flags_hash(knife_ec2_flags)
+			@logger = logger
+			@domain = domain
+		end
+
+		def build_knife_ec2_flags_hash(knife_ec2_flags)
+			knife_ec2_flags['json-attributes'] = "'#{self.load_json_attributes(@json_attributes_file)}'" if @json_attributes_file
+
+			additional_knife_flags = {
+				'node-name' => @name,
+				'tags' => "Name=#{@name}"
+			}
+
+			return knife_ec2_flags.merge(additional_knife_flags)
+		end
+
+		# Load the JSON and then dump it back out to ensure it's valid JSON.
+		# Also makes the JSON easier to read when printing out the command in
+		# verbose mode by removing all newlines.
+		def load_json_attributes(file_path)
+			return JSON.dump(JSON.load(File.read(file_path)))
+		end
+
+		def format_knife_shell_command
+			prefix = 'knife ec2 server create '
+			knife_flag_array = @knife_ec2_flags.map {|key, value| ['--' + key, value]}.flatten.compact
+			return prefix + knife_flag_array.join(' ')
+		end
+
+		def generate_cloud_config(cloud_config, dryrun)
+			cloud_config['hostname'] = @name
+			cloud_config['fqdn'] = "#{@name}.#{@domain}" if @domain
+
+			formatted_cloud_config = cloud_config.to_yaml.gsub('---', '#cloud-config')
+			cloud_config_path = "cloud_config_#{@name}.txt"
+
+			if dryrun
+				msg = "If this weren't a dry run, I would write the following contents to #{cloud_config_path}:\n#{formatted_cloud_config}"
+				@logger.debug(msg)
+			else
+				self.write_cloud_config_to_file(cloud_config_path, formatted_cloud_config)
+				@logger.debug("Wrote cloud config to #{cloud_config_path}.")
+			end
+
+			@knife_ec2_flags['user-data'] = cloud_config_path
+		end
+
+		def write_cloud_config_to_file(path, contents)
+			File.open(path, 'w') {|f| f.write(contents)}
+		end
+
+	end
+end

data/lib/ec2_bootstrap/version.rb

@@ -0,0 +1,3 @@
+module Ec2Bootstrap
+  VERSION = "1.0.0"
+end

data/lib/ec2_bootstrap.rb

@@ -0,0 +1,75 @@
+require 'yaml'
+require 'open3'
+require 'logger'
+
+require 'ec2_bootstrap/version'
+require 'ec2_bootstrap/instance'
+
+class EC2Bootstrap
+
+	attr_accessor :cloud_config
+	attr_accessor :instances
+	attr_accessor :dryrun
+
+	def initialize(config, dryrun=true, verbose=false)
+		@logger = Logger.new(STDOUT)
+		verbose ? @logger.level = Logger::DEBUG : @logger.level = Logger::INFO
+
+		@cloud_config = config['cloud_config']
+		@instances = self.make_instances(config['instances'])
+		@dryrun = dryrun
+	end
+
+	def self.from_config(config, *args)
+		instances = config['instances']
+		raise KeyError, "Config file is missing 'instances' key." unless instances
+		raise TypeError, "'instances' config must be an array of hashes." unless instances.is_a?(Array) && instances.first.is_a?(Hash)
+		config['instances'] = instances.map {|i| i.map {|key, value| [key.to_sym, value]}.to_h}
+
+		return self.new(config, *args)
+	end
+
+	def self.from_config_file(config_path, *args)
+		config = YAML.load(File.read(config_path))
+
+		self.from_config(config, *args)
+	end
+
+	def make_instances(instances_config)
+		return instances_config.map {|i| self.instance_class.new(i.merge(logger: @logger))}
+	end
+
+	def instance_class
+		return Instance
+	end
+
+	def create_instances
+		@logger.debug("This was a dry run. No EC2 instances were created.") if @dryrun
+
+		@instances.each do |instance|
+			@logger.debug("Instance name: #{instance.name}")
+
+			instance.generate_cloud_config(@cloud_config, @dryrun) if @cloud_config
+
+			knife_shell_command = instance.format_knife_shell_command
+			@logger.debug("Knife shell command:\n#{knife_shell_command}")
+			
+			unless @dryrun
+				status = self.shell_out_command(knife_shell_command)
+				return status
+			end
+		end
+	end
+
+	def shell_out_command(command)
+		STDOUT.sync = true
+		Open3::popen2e(command) do |stdin, stdout_and_stderr, wait_thr|
+			while (line = stdout_and_stderr.gets) do
+				@logger.info(line.strip)
+			end
+			status = wait_thr.value
+			@logger.info("status: #{status}")
+			return status
+		end
+	end
+end

data/spec/ec2_bootstrap/instance_spec.rb

@@ -0,0 +1,43 @@
+require_relative '../spec_helper'
+
+describe 'EC2Bootstrap::Instance' do
+
+	let(:knife_flags_hash) do
+		{
+			'availability-zone' => 'us-east-1a',
+			'environment' => 'production',
+			'flavor' => 'm4.large'
+		}
+	end
+
+	let(:logger) do
+		logger = Logger.new(STDOUT)
+		logger.level = Logger::WARN
+		logger
+	end
+
+	let(:instance) do
+		EC2Bootstrap::InstanceMock.new(
+			instance_name: 'pumpkin',
+			domain: 'chocolate.muffins.com',
+			knife_ec2_flags: knife_flags_hash,
+			logger: logger
+		)
+	end
+
+	it 'properly formats the knife shell command' do
+		knife_command = 'knife ec2 server create --availability-zone us-east-1a --environment production --flavor m4.large'
+
+		expect(instance.format_knife_shell_command).to include(knife_command)
+	end
+
+	it 'can generate its own cloud config' do
+		cloud_config = {
+			'manage_etc_hosts': 'true',
+			'bootcmd': ['do stuff', 'do some more stuff']
+		}
+
+		expect(instance.generate_cloud_config(cloud_config, false)).to eq("cloud_config_#{instance.name}.txt")
+	end
+
+end

data/spec/ec2_bootstrap_spec.rb

@@ -0,0 +1,95 @@
+require_relative 'spec_helper'
+
+describe 'EC2Bootstrap' do
+
+	let(:yaml_content) do
+		{
+			'cloud_config' => {
+				'manage_etc_hosts' => 'true',
+				'bootcmd' => ['do stuff', 'do some more stuff']
+			},
+			'instances' => [
+				{
+					'instance_name' => 'cat',
+					'knife_ec2_flags' => {},
+					'domain' => 'cats.com'
+				},
+				{
+					'instance_name' => 'mouse',
+					'json_attributes_file' => {},
+					'knife_ec2_flags' => {}
+				},
+				{
+					'instance_name' => 'whale',
+					'knife_ec2_flags' => {}
+				}
+			]
+		}
+	end
+
+	context 'loading from the yaml config' do
+
+		it "raises a KeyError if the yaml content lacks an 'instances' key" do
+			yaml = yaml_content.reject {|k,v| k == 'instances'}
+
+			expect {EC2BootstrapMock.from_config(yaml, false)}.to raise_error(KeyError)
+		end
+
+		it "raises a TypeError if the 'instances' value is not an array of hashes" do
+			yaml = yaml_content.merge({'instances' => ['a thing', 'another thing']})
+
+			expect {EC2BootstrapMock.from_config(yaml, false)}.to raise_error(TypeError)
+		end
+
+		it 'loads successfully if the yaml content is properly formatted' do
+			bootstrap = EC2BootstrapMock.from_config(yaml_content, false)
+
+			expect(bootstrap.cloud_config).to eq(yaml_content['cloud_config'])
+			expect(bootstrap.instances).to be_an(Array)
+			expect(bootstrap.instances.first).to be_a(EC2Bootstrap::InstanceMock)
+			expect(bootstrap.dryrun).to be_falsey
+		end
+
+	end
+
+	context 'creating instances' do
+		context 'generating cloud config' do
+
+			it "doesn't generate cloud config if it wasn't included at the top level of the yaml config" do
+				yaml = yaml_content.reject {|k,v| k == 'cloud_config'}
+				bootstrap = EC2BootstrapMock.from_config(yaml, false)
+				instance = bootstrap.instances.first
+
+				expect(instance).to_not receive(:generate_cloud_config)
+				bootstrap.create_instances
+			end
+
+			it 'generates cloud config if it was included at the top level in the yaml config' do
+				bootstrap = EC2BootstrapMock.from_config(yaml_content, false)
+				instance = bootstrap.instances.first
+
+				expect(instance).to receive(:generate_cloud_config)
+				bootstrap.create_instances
+			end
+
+		end
+
+		context 'shelling out knife EC2 command' do
+
+			it "doesn't shell out if it's a dryrun" do
+				bootstrap = EC2BootstrapMock.from_config(yaml_content, true)
+
+				expect(bootstrap).to_not receive(:shell_out_command)
+				bootstrap.create_instances
+			end
+
+			it "shells out if it's not a dryrun" do
+				bootstrap = EC2BootstrapMock.from_config(yaml_content, false)
+
+				expect(bootstrap).to receive(:shell_out_command)
+				bootstrap.create_instances
+			end
+		end
+	end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,33 @@
+require 'ec2_bootstrap'
+require 'rspec'
+
+class EC2BootstrapMock < EC2Bootstrap
+
+	def instance_class
+		return InstanceMock
+	end
+
+	def shell_out_command(command)
+		return 0
+	end
+
+end
+
+class EC2Bootstrap
+	class InstanceMock < Instance
+
+		def write_cloud_config_to_file(path, content)
+			return content.bytesize
+		end
+
+		def load_json_attributes(json)
+			return json
+		end
+
+	end
+end
+
+RSpec.configure do |config|
+	config.run_all_when_everything_filtered = true
+	config.filter_run :focus
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: ec2_bootstrap
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Cozy Services Ltd.
+- Rachel King
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.5'
+- !ruby/object:Gem::Dependency
+  name: knife-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+description: Bootstrap EC2 instances with custom config.
+email:
+- opensource@cozy.co
+executables:
+- ec2_bootstrap
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ec2_bootstrap
+- lib/ec2_bootstrap.rb
+- lib/ec2_bootstrap/instance.rb
+- lib/ec2_bootstrap/version.rb
+- spec/ec2_bootstrap/instance_spec.rb
+- spec/ec2_bootstrap_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/CozyCo/ec2_bootstrap
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Bootstrap EC2 instances with custom config.
+test_files:
+- spec/ec2_bootstrap/instance_spec.rb
+- spec/ec2_bootstrap_spec.rb
+- spec/spec_helper.rb