pupistry 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e3280a3e4dd9385278ae57d3ce8ac72ada37c0f
-  data.tar.gz: 1625107171f0d871a16b35008fac3c5e4ba3db63
+  metadata.gz: 19803e351862b0c1f835d5a9091f94cd980adb1e
+  data.tar.gz: bd052ce0fe5a40568c75fa53ab157653f70afaa8
 SHA512:
-  metadata.gz: 388aeb48985189832a1acd9f3e469cf07ec3a3eb784dee51fea8be3fae325dac439802727d8e7efcf124304e035ed9b53450051c0a2df3cdb4e35b7db5378b60
-  data.tar.gz: c8c90250e40b22b82c9192a96dc3f1d572d84b6b4f01c7303be889ffaadee86ea266fa76c800f521338696ad7537875c83c6ab49a8a75a196e2c6db9d6de455d
+  metadata.gz: 4ca14cd1082df16d9b2ade7d5e6d839abbd681195d8fe118ef5fe601540900ef067fabd7582dd436fc951031a4e1674debc10299c7e751c42caddb593a112fb4
+  data.tar.gz: b25c20084c6243a78eefcdd5163db87aa47ecc8a5116405177a39d68d5d14001005e8a496b810c05e068af621360329f944f88d3d2df26e08746308c4693ebe2

data/README.md

@@ -376,13 +376,44 @@ The bootstrap script goal is to get you from stock OS to running Pupistry and
 doing your first Puppet run. After that - it's up to you and your Puppet
 skills to make your node actually do something useful. :-)
 
+
+## 5. (optional) Baking an image with Packer
+
 Note that the node initialisation process is still susceptable to weaknesses
 such as a bug in a new version of Puppet or Pupistry, or changes to the OS
 packages. If this is a concern/issue for you and you want complete reliability,
 then use the user data to build a host pre-loaded with Puppet and Pupistry and
-then create an image of it using a tool like Packer. Doing this, you can make
-it possible to build all the way to Puppet execution with no dependencies on any
-third parties other than your VM provider and AWS S3.
+then create an image of it using a tool like [Packer](www.packer.io). Doing
+this, you can make it possible to build all the way to Puppet execution with no
+dependencies on any third parties other than your VM provider and AWS S3.
+
+Pupistry includes support for generating some Packer examples that you can
+either use as-is or built upon to meet your own needs. You can list all the
+available Packer templates with:
+
+    pupistry packer
+
+You can select a template and generate a Packer file by specifying the template
+and the output file on the command line:
+
+    pupistry packer --template aws_amazon-any --file packer.json
+
+Once the file has been generated, you can build your packer environment with
+the `packer build` command. Note that some templates will require additional
+variables to be passed to them at run time, for example the AWS template
+requires a VPC ID and subnet ID specific to your account.
+
+    packer build \
+      -var 'aws_vpc_id=vpc-example' \
+      -var 'aws_subnet_id=subnet-example' \
+      output.json
+
+By default any Packer machines are built with the hostname of "packer" which
+allows you to specifically target them with your manifests. If you don't do any
+targetting, the default manifests will be applied.
+
+Templates tend to have other customisable variables, check the available
+options and their defaults with `packer inspect output.json`.
 
 
 # Tutorials

data/exe/pupistry

@@ -215,6 +215,44 @@ class CLI < Thor
     end
   end
 
+  desc 'packer', 'Generate a packer template for a particular provider with OS bootstrap script included'
+  method_option :template, type: :string, desc: 'The template you want to generate'
+  method_option :file, type: :string, desc: 'File to write the generated packer template into'
+  def packer
+    # Thor seems to force class options to be defined repeatedly? :-/
+    if options[:verbose]
+      $logger.level = Logger::DEBUG
+    else
+      $logger.level = Logger::INFO
+    end
+
+    if options[:config]
+      Pupistry::Config.load(options[:config])
+    else
+      Pupistry::Config.find_and_load
+    end
+
+    if options[:template]
+      $logger.info "Generating packer template #{options[:template]}"
+
+      templates = Pupistry::Packer.new
+      templates.build options[:template]
+
+      if options[:file]
+        templates.output_file options[:file]
+      else
+        templates.output_plain
+      end
+    else
+      templates = Pupistry::Packer.new
+      templates.list
+
+      puts '--'
+      puts 'Tip: Run `pupistry packer --template example` to generate a specific template'
+    end
+  end
+
+
   ## Other Commands
 
   desc 'setup', 'Write a template configuration file'

data/lib/pupistry/bootstrap.rb

@@ -82,6 +82,12 @@ module Pupistry
       end
     end
 
+    def output_array
+      # Return the output as an array of lines. Useful by other internal
+      # methods such as Packer templates.
+      @contents.split(/\n/)
+    end
+
     def output_plain
       # Do nothing clever, just output the template data.
       puts '-- Bootstrap Start --'

data/lib/pupistry/packer.rb

@@ -0,0 +1,116 @@
+# rubocop:disable Style/Documentation, Style/GlobalVars
+require 'rubygems'
+require 'erubis'
+
+module Pupistry
+  # Pupistry::Packer
+
+  class Packer
+    attr_accessor :template_dir
+    attr_accessor :contents
+
+    def initialize
+      # We need to find where the templates are located - either it should be
+      # in the current working directory, or if we are an installed gem, we
+      # can try the gem's installed path.
+
+      if Dir.exist?('resources/packer/')
+        # Use local PWD version first if possible
+        @template_dir = Dir.pwd
+      else
+        # Check for GEM installed location
+        begin
+          @template_dir = Gem::Specification.find_by_name('pupistry').gem_dir
+        rescue Gem::LoadError
+          $logger.error "Unable to find packer templates directory, doesn't appear we are running from project dir nor as a Gem"
+          return false
+        end
+      end
+
+      @template_dir = @template_dir.chomp('/') + '/resources/packer/'
+
+      if Dir.exist?(@template_dir)
+        $logger.debug "Using directory #{@template_dir} for packer templates"
+      else
+        $logger.error "Unable to find packer templates dir at #{@template_dir}, unable to proceed."
+        return false
+      end
+    end
+
+    def list
+      # Simply glob the templates directory and list their names.
+      $logger.debug 'Finding all available templates'
+
+      Dir.glob("#{@template_dir}/*.erb").each do |file|
+        puts "- #{File.basename(file, '.json.erb')}"
+      end
+    end
+
+    def build(template)
+      # Build a template with the configured parameters already to go and save
+      # into the object, so it can be outputted in the desired format.
+
+      $logger.debug "Generating a packer template using #{template}"
+
+      unless File.exist?("#{@template_dir}/#{template}.json.erb")
+        $logger.error 'The requested template does not exist, unable to build'
+        return 0
+      end
+
+      # Extract the OS bootstrap name from the template filename, we can then
+      # generate the bootstrap commands to be inserted inline into the packer
+      # configuration.
+
+      matches = template.match(/^\S*_(\S*)$/)
+
+      if matches[1]
+        $logger.debug "Fetching bootstrap data for #{matches[1]}..."
+      else
+        $logger.error 'Unable to parse the packer filename properly'
+        return 0
+      end
+
+      bootstrap = Pupistry::Bootstrap.new
+      unless bootstrap.build matches[1]
+        $logger.error 'An unexpected error occured when building the bootstrap data to go inside Packer'
+      end
+
+      # Pass the values we care about to the template
+      template_values = {
+        bootstrap_commands: bootstrap.output_array
+      }
+
+      # Generate template using ERB
+      begin
+        @contents = Erubis::Eruby.new(File.read("#{@template_dir}/#{template}.json.erb")).result(template_values)
+      rescue StandardError => e
+        $logger.error 'An unexpected error occured when trying to generate the packer template'
+        raise e
+      end
+    end
+
+    def output_plain
+      # Do nothing clever, just output the template data.
+      puts '-- Packer Start --'
+      puts @contents
+      puts '-- Packer End --'
+      puts 'Tip: add --file output.json to write out the packer file directly and then run with `packer build output.json`'
+    end
+
+    def output_file(filename)
+      # Write the template to the specified file
+      begin
+        File.open(filename, 'w') do |f|
+          f.puts @contents
+        end
+      rescue StandardError => e
+        $logger.error "An unexpected erorr occured when attempting to write the template to #{filename}"
+        raise e
+      else
+        $logger.info "Wrote template into file #{filename} successfully."
+      end
+    end
+  end
+end
+
+# vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent

data/lib/pupistry/version.rb

@@ -1,3 +1,3 @@
 module Pupistry
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
 end

data/lib/pupistry.rb

@@ -4,6 +4,7 @@ require 'pupistry/artifact'
 require 'pupistry/bootstrap'
 require 'pupistry/config'
 require 'pupistry/gpg'
+require 'pupistry/packer'
 require 'pupistry/storage_aws'
 
 # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent

data/resources/bootstrap/freebsd-10.erb

@@ -12,6 +12,8 @@
 # * We can't rely on Bash, since it's not available in FreeBSD by default.
 #
 
+setenv PATH /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
 env ASSUME_ALWAYS_YES=YES pkg bootstrap
 env ASSUME_ALWAYS_YES=YES pkg upgrade --yes
 env ASSUME_ALWAYS_YES=YES pkg install --yes ruby devel/ruby-gems puppet gnupg
@@ -37,5 +39,5 @@ agent:
 EOF
 chmod 700 /usr/local/etc/pupistry
 chmod 700 /usr/local/etc/puppetlabs/code/environments
-pupistry apply --verbose
+/usr/local/bin/pupistry apply --verbose
 

data/resources/packer/PACKER_NOTES.md

@@ -0,0 +1,49 @@
+# Packer Templates
+
+This directory contains templates for use with Packer (https://www.packer.io/).
+
+It can be very useful to use Packer with Pupistry, since it allows you to
+create your own image with Puppet, Pupistry and updates already loaded which
+is very useful when doing autoscaling and you need fast, consistent startup
+times.
+
+The packer templates provided will build an image which has Pupistry installed
+and will apply any manifests that match hostname of "packer". This should give
+you a good general purpose image, but if you want to autoscale a particular app
+you may wish to build packer images using specific hostnames to match your
+Puppet manifests
+
+Additional packer templates for major platforms are always welcome. Please
+submit a pull request for review and if acceptable, will be merged.
+
+
+# Usage
+
+Refer to the main application README.md file for usage information.
+
+
+# Development Notes
+
+The filenames of the templates must be in the format of
+PLATFORM_OPERATINGSYSTEM.json.erb, this is intentional since OPERATINGSYSTEM
+then matches one of the OSes in the bootstrap directory and we can
+automatically populate the inline shell commands.
+
+When debugging broken packer template runs, add -debug to the build command
+to have control over stepping through the build process. This will give you
+the ability to log into the instance before it gets terminated to do any
+debugging on the system if needed.
+
+
+# Examples
+
+See the "aws_amazon-any.json.erb" template for an example on how the templates
+should be written for AWS.
+
+
+# Life Span
+
+Any distribution that is EOL and no longer supported by either the distribution
+or by Puppetlabs will be subject to removal to keep the bootstrap selection
+modern and clean. Pull requests to clean up cruft are accepted.
+

data/resources/packer/aws_amazon-any.json.erb

@@ -0,0 +1,38 @@
+{
+  "variables": {
+    "aws_access_key": "",
+    "aws_secret_key": "",
+    "aws_ami": "ami-fd9cecc7",
+    "aws_region": "ap-southeast-2",
+    "aws_ami_name": "pupistry aws_amazon-any {{isotime \"2006-01-02\"}}",
+    "aws_vpc_id": null,
+    "aws_subnet_id": null,
+    "hostname": "packer"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "region": "{{user `aws_region`}}",
+    "source_ami": "{{user `aws_ami`}}",
+    "instance_type": "t2.micro",
+    "ssh_username": "ec2-user",
+    "vpc_id": "{{user `aws_vpc_id`}}",
+    "subnet_id": "{{user `aws_subnet_id`}}",
+    "ami_name": "{{user `aws_ami_name`}}"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["sudo hostname {{user `hostname`}}"]
+  },
+  {
+    "type": "shell",
+    "inline_shebang": "/bin/bash -x",
+    "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo '{{ .Path }}'",
+    "inline": <%= bootstrap_commands %>
+  },
+  {
+    "type": "shell",
+    "inline": ["sudo rm -rf /tmp/*"]
+  }]
+}

data/resources/packer/aws_freebsd-10.json.erb

@@ -0,0 +1,39 @@
+{
+  "variables": {
+    "aws_access_key": "",
+    "aws_secret_key": "",
+    "aws_ami": "ami-193f5123",
+    "aws_region": "ap-southeast-2",
+    "aws_ami_name": "pupistry aws_freebsd-10 {{isotime \"2006-01-02\"}}",
+    "aws_vpc_id": null,
+    "aws_subnet_id": null,
+    "hostname": "packer"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "region": "{{user `aws_region`}}",
+    "source_ami": "{{user `aws_ami`}}",
+    "instance_type": "t2.micro",
+    "ssh_username": "ec2-user",
+    "ssh_timeout": "15m",
+    "vpc_id": "{{user `aws_vpc_id`}}",
+    "subnet_id": "{{user `aws_subnet_id`}}",
+    "ami_name": "{{user `aws_ami_name`}}"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["su -m root -c 'hostname {{user `hostname`}}'"]
+  },
+  {
+    "type": "shell",
+    "inline_shebang": "/bin/tcsh",
+    "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} su -m root -c '/bin/tcsh {{ .Path }}'",
+    "inline": <%= bootstrap_commands %>
+  },
+  {
+    "type": "shell",
+    "inline": ["su -m root -c 'rm -rf /tmp/*'"]
+  }]
+}

data/resources/packer/aws_ubuntu-14.04.json.erb

@@ -0,0 +1,38 @@
+{
+  "variables": {
+    "aws_access_key": "",
+    "aws_secret_key": "",
+    "aws_ami": "ami-19c8b123",
+    "aws_region": "ap-southeast-2",
+    "aws_ami_name": "pupistry aws_ubuntu-14.04 {{isotime \"2006-01-02\"}}",
+    "aws_vpc_id": null,
+    "aws_subnet_id": null,
+    "hostname": "packer"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "region": "{{user `aws_region`}}",
+    "source_ami": "{{user `aws_ami`}}",
+    "instance_type": "t2.micro",
+    "ssh_username": "ubuntu",
+    "vpc_id": "{{user `aws_vpc_id`}}",
+    "subnet_id": "{{user `aws_subnet_id`}}",
+    "ami_name": "{{user `aws_ami_name`}}"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["sudo hostname {{user `hostname`}}"]
+  },
+  {
+    "type": "shell",
+    "inline_shebang": "/bin/bash -x",
+    "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo '{{ .Path }}'",
+    "inline": <%= bootstrap_commands %>
+  },
+  {
+    "type": "shell",
+    "inline": ["sudo rm -rf /tmp/*"]
+  }]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupistry
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Jethro Carr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-05-31 00:00:00.000000000 Z
+date: 2015-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -177,6 +177,7 @@ files:
 - lib/pupistry/bootstrap.rb
 - lib/pupistry/config.rb
 - lib/pupistry/gpg.rb
+- lib/pupistry/packer.rb
 - lib/pupistry/storage_aws.rb
 - lib/pupistry/version.rb
 - lib/pupistry.rb
@@ -190,8 +191,10 @@ files:
 - resources/bootstrap/fedora-any.erb
 - resources/bootstrap/freebsd-10.erb
 - resources/bootstrap/ubuntu-14.04.erb
-- resources/packer/amazon_linux.json
-- resources/packer/test_user_data.txt
+- resources/packer/aws_amazon-any.json.erb
+- resources/packer/aws_freebsd-10.json.erb
+- resources/packer/aws_ubuntu-14.04.json.erb
+- resources/packer/PACKER_NOTES.md
 - README.md
 - settings.example.yaml
 homepage: https://github.com/jethrocarr/pupistry

data/resources/packer/amazon_linux.json

@@ -1,17 +0,0 @@
-{
-  "variables": {
-    "aws_access_key": "",
-    "aws_secret_key": ""
-  },
-  "builders": [{
-    "type": "amazon-ebs",
-    "access_key": "{{user `aws_access_key`}}",
-    "secret_key": "{{user `aws_secret_key`}}",
-    "region": "ap-southeast-2",
-    "source_ami": "ami-fd9cecc7",
-    "instance_type": "m3.medium",
-    "ssh_username": "ec2-user",
-    "ami_name": "pupistry default {{timestamp}}",
-    "user_data": "test_user_data.txt"
-  }]
-}

data/resources/packer/test_user_data.txt

@@ -1,33 +0,0 @@
--- Bootstrap Start --
-IyEvYmluL2Jhc2ggLXgKIyBUaGlzIGJvb3RzdHJhcCBpcyBzcGVjaWZjaWFs
-bHkgZm9yIEFtYXpvbidzIExpbnV4IEFNSXMsIGlmIHlvdSBhcmUgdXNpbmcK
-IyBvdGhlciBkaXN0cmlidXRpb25zIGxpa2UgVWJ1bnR1IG9yIENlbnRPUyBv
-biBBV1MsIHVzZSB0aG9zZSBib290c3RyYXAKIyB0ZW1wbGF0ZXMuCiMKIyBB
-bWF6b24gTGludXggaXMgYmFzZWQgb24gUkhFTCwgYnV0IGhhcyBhIGxvdCBt
-b3JlIHZhcmlhdGlvbnMgdGhhdCBvdGhlcgojIGNsb25lcyBsaWtlIENlbnRP
-Uywgc3VjaCBhcyBzaGlwcGluZyB3aXRoIG11bHRpcGxlIHZlcnNpb25zIG9m
-IFB1cHBldAojIGFuZCBSdWJ5IC0gd2hpY2ggaXMgdXNlZnVsLCBidXQgY2Fu
-IGFsc28gbWFrZSBsaWZlLi4uLiBpbnRlcmVzdGluZy4KKApleGVjIDE+ID4o
-bG9nZ2VyIC1zIC10IHVzZXItZGF0YSkgMj4mMQoKZXhwb3J0IFBBVEg9JFBB
-VEg6L3Vzci9sb2NhbC9iaW4KCnl1bSB1cGRhdGUgLS1hc3N1bWV5ZXMKeXVt
-IGluc3RhbGwgLS1hc3N1bWV5ZXMgcHVwcGV0MyBydWJ5LWRldmVsIHJ1Ynln
-ZW1zIGdjYyB6bGliLWRldmVsIGxpYnhtbDItZGV2ZWwgcGF0Y2ggZ251cGcy
-CgojIE5vdCBzdXJlIHdoeSB0aGlzIGRvZXNuJ3QgZ2V0IHB1bGxlZCBkb3du
-IHByb3Blcmx5LCBtYXliZSBpdCdzIGNvcmUgYW5kCiMgQW1hem9uIGRpZG4n
-dCBwYWNrYWdlIGl0IHByb3Blcmx5PyBOZWVkIGl0IGZvciBUaG9yIHdoaWNo
-IGlzIHVzZWQgYnkgUHVwaXN0cnkKZ2VtIGluc3RhbGwgaW8tY29uc29sZQoK
-Z2VtIGluc3RhbGwgcHVwaXN0cnkKbWtkaXIgLXAgL2V0Yy9wdXBpc3RyeQpt
-a2RpciAtcCAvZXRjL3B1cHBldGxhYnMvY29kZS9lbnZpcm9ubWVudHMKY2F0
-ID4gL2V0Yy9wdXBpc3RyeS9zZXR0aW5ncy55YW1sIDw8ICJFT0YiCmdlbmVy
-YWw6CiAgYXBwX2NhY2hlOiB+Ly5wdXBpc3RyeS9jYWNoZQogIHMzX2J1Y2tl
-dDogcHVwaXN0cnktcmVzb3VyY2VzCiAgczNfcHJlZml4OiAKICBncGdfZGlz
-YWJsZTogZmFsc2UKICBncGdfc2lnbmluZ19rZXk6IEE4MEFDQTZGCmFnZW50
-OgogIHB1cHBldGNvZGU6IC9ldGMvcHVwcGV0bGFicy9jb2RlL2Vudmlyb25t
-ZW50cwogIGFjY2Vzc19rZXlfaWQ6IEFLSUFKMlhKQ01TNFJBUDNCVFpBCiAg
-c2VjcmV0X2FjY2Vzc19rZXk6IHdoQWlFaCtZV1hzaUw4Y0NONitLSFovT1Zs
-M1VCK3ZZb0FHZEd4eDkKICByZWdpb246IGFwLXNvdXRoZWFzdC0yCiAgcHJv
-eHlfdXJpOiAKICBkYWVtb25fZnJlcXVlbmN5OiA2MAogIGRhZW1vbl9taW5p
-bWFsOiB0cnVlCkVPRgpjaG1vZCA3MDAgL2V0Yy9wdXBpc3RyeS9zZXR0aW5n
-cy55YW1sCmNobW9kIDcwMCAvZXRjL3B1cHBldGxhYnMvY29kZS9lbnZpcm9u
-bWVudHMKcHVwaXN0cnkgYXBwbHkgLS12ZXJib3NlCgopCg==
--- Bootstrap End --