image_builder 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 96fdbf490de1409c12b7a35690ec161016076e46
+  data.tar.gz: 4cf8e63dd9fc03672579739c72ee8e6a95571718
+SHA512:
+  metadata.gz: 4a69a23cb0a72a8cb509511c4848f79b8534222e67252384d5d43ec5a4eac99cab99001d99ddf5d80655b5ffa06a4501aefd47a58c44a0b8ee3d76713e6b0766
+  data.tar.gz: 8bc8a87b2541b9e78c51bed13ddad2f0227e267ebefd53ba41aca9cd4157cdb492ee1acacc1143c8022cd0be5537c24da79c1038ebe684bbbea18ae359f199f5

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rubocop.yml

@@ -0,0 +1,16 @@
+AllCops:
+  Exclude:
+    - 'image_builder.gemspec'
+    - spec/fixtures/**
+
+Metrics/LineLength:
+  Max: 132
+
+Metrics/MethodLength:
+  Max: 25
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/PerceivedComplexity:
+  Max: 10

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in image_builder.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Mike Morris
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# ImageBuilder
+
+A gem to build operating system images for various platforms.  At initial release, this gem supports
+building images using packer to build images for the AWS platform
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'image_builder'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install image_builder
+
+## Usage
+
+This is how you use the gem, should probably write something useful here.
+But since it's just a library gem that basically just wraps the packer utility,
+read this code, and the packer documentation to figure out what to do
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/image_builder/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,29 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+gem_dir = ::File.dirname(__FILE__)
+
+desc 'Rubocop'
+task rc: [:rubocop]
+task :rubocop do
+  sh "bundle exec rubocop -D #{gem_dir}"
+end
+
+task default: [:rc] do
+  Rake::Task[:spec].invoke('progress')
+end
+
+desc 'Run rspec'
+RSpec::Core::RakeTask.new(:spec, [:format, :tags]) do |t, args|
+  format = args[:format] || 'documentation'
+  tags = args[:tags] || []
+  t.verbose = false
+  t.fail_on_error = true
+  t.rspec_opts = "--format=#{format}"
+
+  tags.flatten.compact.each do |tag|
+    t.rspec_opts += " --tag #{tag}"
+  end
+
+  t.ruby_opts = '-W0'
+end

data/bin/image_builder

@@ -0,0 +1,234 @@
+#!/usr/bin/env ruby
+
+require 'commander'
+require 'cloudutil/aws/ec2'
+require 'chef/node'
+require 'json'
+
+# All sorts of funkiness happens if I try to put these in the
+# top-level image_builder module, and try to require just that
+require 'image_builder/version'
+require 'image_builder/backends/packer'
+require 'image_builder/builders/aws_ebs'
+require 'image_builder/provisioners/chef_client'
+require 'image_builder/provisioners/chef_solo'
+require 'image_builder/provisioners/shell'
+require 'image_builder/post_processors/vagrant'
+
+def prog_name
+  ::File.basename(__FILE__)
+end
+
+def attr_missing?(attr_value)
+  attr_value.nil? || attr_value.strip.empty?
+end
+
+def read_attrs(attr_file)
+  # I didn't want to have to dip into the chef gem to do this,
+  # but it appears to be the best way to get attributes
+  node = Chef::Node.build('image-builder')
+  node.from_file(::File.expand_path(attr_file))
+  node.attributes.merged_attributes
+end
+
+def create_chef_client_provisioner(knife_file, chef_attrs, options)
+  chef_client_prov = ImageBuilder::Provisioners::Chef::Client.from_file(knife_file)
+  chef_client_prov.chef_environment = chef_attrs['amibakery']['build']['chef_env']
+  chef_client_prov.node_name = "#{prog_name}_{{uuid}}"
+  chef_client_prov.run_list  = options.r_run_list
+  chef_client_prov.json = JSON.parse(options.j_json) # Placeholder for custom json used to pass in build-specific data
+  chef_client_prov
+end
+
+def resolve_sec_grps(grp_ary, util_obj)
+  grps = []
+
+  grp_ary.each do |g|
+    grps << util_obj.resolve_security_group_id(g)
+  end
+
+  grps.flatten.compact.uniq
+end
+
+def format_string(str, subs = {})
+  str % subs unless str.nil?
+end
+
+def format_tags(hash, subs = {})
+  new_hash = hash
+
+  unless hash.nil?
+    hash.each_pair do |k, v|
+      new_hash[k] = format_string(v, subs)
+    end
+  end
+
+  new_hash
+end
+
+# rubocop:disable Metrics/MethodLength
+def create_builders(attrs, options)
+  # Create builder objects, resolve AWS names/tags to ids along the way
+  builders = []
+  cu = Cloudutil::AWS::EC2.new
+
+  build_attrs = attrs['amibakery']['build']
+  %w(centos ubuntu).each do |p|
+    build_attrs[p].keys.each do |v|
+      plat_attrs = build_attrs[p][v]
+      root_type  = plat_attrs['root_dev']
+      virt_type  = plat_attrs['virt_type']
+      sub_hash   = { p: p, v: v, root_type: root_type, virt_type: virt_type }
+
+      b = ImageBuilder::Builders::AWS::EBS.new
+      b.name = "#{p}#{v}-#{root_type}"
+      b.ami_name = format_string(options.n_ami_name, sub_hash)
+      b.instance_type = build_attrs['inst_type']
+      b.source_ami = cu.resolve_ami_id(plat_attrs['ami_id'])
+      b.ssh_username = plat_attrs['build_user']
+      b.ami_description = format_string(options.ami_description, sub_hash)
+      b.ami_regions = options.c_copy_regions
+      b.ami_users = options.u_users
+      b.ami_virtualization_type = virt_type
+      b.security_group_ids = resolve_sec_grps([build_attrs['sec_grp_id']].flatten, cu)
+      b.ssh_private_ip = true
+      b.subnet_id = cu.resolve_subnet_id(build_attrs['subnet_id'])
+      b.tags = { Name: b.ami_name }.merge(format_tags(options.tags.clone, sub_hash))
+      b.launch_block_device_mappings = b.class.default_launch_block_device_mappings
+      b.ami_block_device_mappings = b.class.default_block_device_mappings
+
+      builders << b
+    end
+  end
+
+  builders
+end
+# rubocop:enable Metrics/MethodLength
+
+def run_packer(builders, provisioners, postprocessors, path = nil)
+  ImageBuilder::Backends::Packer.packer_path(path) unless path.nil? || path.strip.empty?
+  packer = ImageBuilder::Backends::Packer.new
+
+  builders.each do |b|
+    packer.add_builder(b)
+  end
+
+  provisioners.each do |p|
+    packer.add_provisioner(p)
+  end
+
+  postprocessors.each do |p|
+    packer.add_post_processor(p)
+  end
+
+  packer.build(['-machine-readable'])
+end
+
+def convert_tags(tag_opt)
+  # Should only ever be an empty Hash (option default),
+  # or an Array from the command-line
+  hash = {}
+
+  if tag_opt.is_a? Array
+    tag_opt.each do |t|
+      (tag, value) = t.split('=', 2)
+      hash[tag] = value
+    end
+  end
+
+  hash
+end
+
+# NOTE: Array options are in format '--opt arg1,arg2' (comma-seperated, no space!)
+Commander.configure do
+  program :name, prog_name
+  program :version, ImageBuilder::VERSION
+  program :description, 'A program to generate AWS AMIs with Chef and Packer.'
+    .concat(' It is very opinionated about how it works, so use it as a reference, and write your own')
+
+  default_command :build
+  global_option('-d', '--debug', 'Provide debugging output')
+
+  command :build do |c|
+    c.syntax = "#{prog_name} [build] [OPTIONS]"
+    c.description = 'A program to build AWS AMIs using Chef and Packer. This is the default action to run, if not supplied'
+
+    c.option '-r, --run-list RUN_LIST', Array, 'The run-list for Chef to execute to provision the node'
+    c.option '-c, --copy-regions REGIONS', Array, 'A list of regions to copy the generated AMI to'
+    c.option '-u, --users USERS', Array, 'A list of users to grant access to the generated AMI'
+    c.option '-s, --serverspec-path SS_PATH', String, 'The path to the serverspec directory to use to validate the build'
+    c.option '-k, --knife-file KNIFE_FILE', String, 'The path to a knife config file to use to configure a Chef provisioner'
+    c.option '-a, --attrib-file ATTRIB_FILE', String, 'The path to a Chef recipes attribute file to use to configure an AWS builder'
+    c.option '-p, --packer-path PACKER_PATH', String, 'The path to the packer executable, if it\'s in a non-standard location'
+    c.option '-j, --json JSON', String, 'A string of JSON text to pass along to the chef-client provisioner'
+    c.option '-n, --ami-name NAME', String, 'The name of the ami to create'
+    c.option '--tags TAGS', Array, 'Tags to apply to the AMI (Name tag is auto-generated as --ami-name value)'
+    c.option '--ami-description DESC', String, 'A description of the ami'
+    c.option '--cleanup-run-list RUN_LIST', Array, 'A run-list used to clean up the provisioned node before AMI creation'
+    c.option '--[no-]vagrant-box', 'Create a Vagrant box defintion after the build completes'
+
+    c.action do |_args, options|
+      options.default vagrant_box: false
+      options.default tags: {}
+
+      knife_file  = options.k_knife_file
+      attrib_file = options.a_attrib_file
+
+      # Do a fixup for --tags arg to convert from Array to Hash
+      options.tags = convert_tags(options.tags)
+
+      # Check to make sure -k, -a, & -n args are given, since it's missing from doing in Commander
+      err_msgs = []
+
+      err_msgs << 'Missing required -k option' if attr_missing? knife_file
+      err_msgs << 'Missing required -a option' if attr_missing? attrib_file
+      err_msgs << 'Missing required -n option' if attr_missing? options.n_ami_name
+
+      fail ArgumentError, err_msgs.join("\n") unless err_msgs.empty?
+
+      # create AWS builder info via data from -a option
+      chef_attrs = read_attrs(attrib_file)
+      builders = create_builders(chef_attrs, options)
+
+      # create chef-client provisioner from knife.rb and chef attributes
+      chef_client_prov = create_chef_client_provisioner(knife_file, chef_attrs, options)
+
+      dir_prov = ImageBuilder::Provisioners::Shell.new
+      dir_prov.inline = ['sudo mkdir -p -m 777 /etc/chef /tmp/packer-chef-client']
+
+      # Do 'require' here, to avoid collisions with potential unqualified references to the File class
+      require 'image_builder/provisioners/file'
+
+      unless attr_missing? chef_client_prov.encrypted_data_bag_secret_path
+        sec_prov = ImageBuilder::Provisioners::File.new
+        sec_prov.source = chef_client_prov.encrypted_data_bag_secret_path
+        sec_prov.destination = '/etc/chef/encrypted_data_bag_secret'
+      end
+
+      unless attr_missing? options.s_serverspec_path
+        ss_file_prov = ImageBuilder::Provisioners::File.new
+        ss_file_prov.source = options.s_serverspec_path
+        ss_file_prov.destination = '/var/tmp'
+
+        ss_shell_prov = ImageBuilder::Provisioners::Shell.new
+        ss_shell_prov.inline = ['/var/tmp/serverspec/serverspec.sh']
+      end
+
+      unless options.cleanup_run_list.nil?
+        cleanup_prov = ImageBuilder::Provisioners::Chef::Solo.new
+        cleanup_prov.run_list = [options.cleanup_run_list].flatten
+        cleanup_prov.remote_cookbook_paths = ['/var/chef/cache/cookbooks']
+        cleanup_prov.skip_install = true
+      end
+
+      # add vagrant post-processor, if requested
+      vagrant = ImageBuilder::PostProcessors::Vagrant.new if options.vagrant_box
+
+      # order matters!
+      provisioners = [dir_prov, sec_prov, chef_client_prov, ss_file_prov, ss_shell_prov, cleanup_prov].compact
+
+      # run packer
+      run_packer(builders, provisioners, [vagrant].compact, options.p_packer_path)
+    end
+  end
+end

data/image_builder.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'image_builder/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'image_builder'
+  spec.version       = ImageBuilder::VERSION
+  spec.authors       = ['Mike Morris']
+  spec.email         = ['michael.m.morris@pearson.com']
+  spec.summary       = 'A gem to create operating system images using various methods'
+  spec.description   = IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+
+  spec.add_dependency 'mixlib-shellout'
+  spec.add_dependency 'commander'
+  spec.add_dependency 'cloudutil'
+  spec.add_dependency 'chef', '~> 11.12'
+end

data/lib/image_builder.rb

@@ -0,0 +1,38 @@
+require 'image_builder/version'
+
+# Generic top-level module comment
+module ImageBuilder
+  protected
+
+  def attr_to_hash(src_hash, attr_sym, required = false)
+    val = send(attr_sym)
+
+    if required
+      src_hash[attr_sym] = val
+    else
+      unless val.nil?
+        if val.respond_to? :empty?
+          v = check_empty(val)
+          src_hash[attr_sym] = v unless v.nil? # rubocop:disable Metrics/BlockNesting
+        else
+          # Not nil, and doesn't support empty?, so assign
+          src_hash[attr_sym] = val
+        end
+      end
+    end
+  end
+
+  private
+
+  def check_empty(val)
+    if val.is_a? String
+      # Strip whitespace surrounding strings before testing for empty
+      return val unless val.strip.empty?
+    else
+      # Array, Hash, etc...
+      return val unless val.empty?
+    end
+
+    nil
+  end
+end