pupistry 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2438d4ff627cb0c9719ae22d33a026833365769
-  data.tar.gz: ac00b03d3b81ad8bcac5f45d6d3bbc01fcfae733
+  metadata.gz: 2d77b057ae7f6a793146456653336f4d8518ad9c
+  data.tar.gz: 883a456f63a65d166c6c9fb27d94b249ff64d10d
 SHA512:
-  metadata.gz: 91bcf3aecc457d6794b9b3f1c52cec5551ac185d8c69e84d295c38a4b0565a91b810601e121159e9476a16009b16bbf3f7969a84ee79b990a9fef878fcb47594
-  data.tar.gz: c42a271c5b29e972b1295f9e4891332472340065a14b9327d70670e31be3fba08463c78b6cfebcf73d8818fa23b3ccae3f153c995659dd109c12698acdc4093b
+  metadata.gz: f502b9aea495831ccfa58e14ade4d146cf272c23aa039f94421ed1ec135ad42e8d4459302afbc66e975d0cc15dc5b4241724c339c7f9f6aecedcbb0100a017bb
+  data.tar.gz: aa02112d087ff67fe57b075c02571beaa607a194bfef37cd042cbaeb2ad54a7c6a8f8076ef68b8cede0fdf22e2b1ebbb22745cc4454ac1c5ee827744d4828877

data/README.md

@@ -20,14 +20,15 @@ servers.
 Pupistry builds on the functionality offered by the r10k workflow but rather
 than requiring the implementing of site-specific custom bootstrap and custom
 workflow mechanisms, Pupistry executes r10k, assembles the combined modules
-and then generates a compress artifact file. It then signs the artifact with
-GPG and uploads it into an Amazon S3 bucket along with a manifest file.
+and then generates a compress artifact file. It then optionally signs the
+artifact with GPG and finally uploads it into an Amazon S3 bucket along with a
+manifest file.
 
 The masterless Puppet machines then just run a Pupistry job which checks for a
 new version of the manifest file. If there is, it downloads the new artifact
-and does a GPG validation before applying it and running Puppet. To make life
-even easier, Pupistry will even spit out bootstrap files for your platform
-which sets up each server from scratch to pull and run the artifacts.
+and does an optional GPG validation before applying it and running Puppet. To
+make life even easier, Pupistry will even spit out bootstrap files for your
+platform which sets up each server from scratch to pull and run the artifacts.
 
 Essentially Pupistry is intended to be a robust solution for masterless Puppet
 deployments and makes it trivial for beginners to get started with Puppet.
@@ -210,7 +211,6 @@ Alternatively if you like living on the edge, download this repository and run:
     gem install pupistry-VERSION.gem
     pupistry setup
 
-TODO: Currently setup not implemented, copy the sample file provided.
 
 ## 2. S3 Bucket
 
@@ -343,6 +343,31 @@ and running masterless Puppet environment using Pupistry. It covers the very
 basics of setting up your r10k environment.
 
 
+# GPG Notes
+
+GPG can be a bit of a beast to setup and get used to. Pupistry tries to make
+the signing and key sharing process as simple as possible, but setting up GPG
+on your platform and creating your key is beyond the scope of this document.
+
+If you are being asked for your GPG password for every `pupistry push` even in
+rapid succession, then you may need to setup gpg-agent so it can keep you
+logged in for short durations to get a better balance of security vs usability.
+
+Currently Pupistry supports a 1:1 approach, where the key used to sign the
+artifact is the key used to verify it. Pull requests to add support for signing
+and verifying against a keyring list would be welcome to make it easier for
+teams to use GPG without having everyone with a single master key.
+
+Note that GPG isn't vital for security - you still have end-to-end transport
+security between your build machine and your servers via HTTPS/TLS to and from
+the S3 bucket, all that GPG does is prevent anyone who managed to break into
+your S3 bucket from pushing their own Puppet manifests out.
+
+Generally S3 is secure (assuming no bugs in AWS itself), any likely exploit
+would be from you accidentally sharing your IAM credentials in the wrong place,
+or an exploited build server.
+
+
 # Caveats & Future Plans
 
 ## Use r10k
@@ -444,6 +469,10 @@ issue tracker is fine, but pull requests speak louder than words. :-)
 If you find a bug or need support, please use the issue tracker rather than
 personal emails to the author.
 
+Feel free to grep the source for "TODO" comments on various tasks that
+need doing.
+
+
 
 # Author
 

data/bin/pupistry

@@ -376,7 +376,11 @@ class CLI < Thor
       raise e
     end
 
-    # Tell the user to edit it
+    # TODO: This is where I'd like to do things more cleverly. Currently we
+    # just tell the user to edit the configuration file, but would be really
+    # cool to write a setup wizard that helps them complete the configuration
+    # file and validates the input (eg AWS keys).
+
     if ENV['EDITOR']
       $logger.info "Now open the config file with `#{ENV['EDITOR']} #{config_dest}` and set your configuration values before running Pupistry."
     else

data/lib/pupistry.rb

@@ -1,8 +1,8 @@
-require 'pupistry/config'
-require 'pupistry/bootstrap'
 require 'pupistry/artifact'
+require 'pupistry/bootstrap'
+require 'pupistry/config'
+require 'pupistry/gpg'
 require 'pupistry/storage_aws'
 
 
-
 # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent

data/lib/pupistry/artifact.rb

@@ -3,6 +3,7 @@ require 'yaml'
 require 'time'
 require 'digest'
 require 'fileutils'
+require 'base64'
 
 module Pupistry
   # Pupistry::Artifact
@@ -215,9 +216,28 @@ module Pupistry
         $logger.warn "You have GPG signing *disabled*, whilst not critical it does weaken your security."
         $logger.warn "Skipping signing step..."
       else
-        $logger.info "GPG signing the artifact with configured key"
 
-        # TODO: should probably write this bit!
+        gpgsig = Pupistry::GPG.new @checksum
+
+        # Sign the artifact
+        unless gpgsig.artifact_sign
+          $logger.fatal "Unable to proceed with an unsigned artifact"
+          exit 0
+        end
+
+        # Verify the signature - we want to make sure what we've just signed
+        # can actually be validated properly :-)
+        unless gpgsig.artifact_verify
+          $logger.fatal "Whilst a signature was generated, it was unable to be validated. This would suggest a bug of some kind."
+          exit 0
+        end
+
+        # Save the signature to the manifest
+        unless gpgsig.signature_save
+          $logger.fatal "Unable to write the signature into the manifest file for the artifact."
+          exit 0
+        end
+
       end
 
 
@@ -313,7 +333,7 @@ module Pupistry
         "version"   => @checksum, 
         "date"      => Time.new.inspect,
         "builduser" => ENV['USER'] || 'unlabled',
-        "gpgsig"    => 'unsighed',
+        "gpgsig"    => 'unsigned',
       }
 
       begin
@@ -387,6 +407,22 @@ module Pupistry
         raise "Application bug, trying to install no artifact"
       end
 
+      # Validate the artifact if GPG is enabled.
+      if $config["general"]["gpg_disable"] == true
+        $logger.warn "You have GPG validation *disabled*, whilst not critical it does weaken your security."
+        $logger.warn "Skipping validation step..."
+      else
+
+        gpgsig = Pupistry::GPG.new @checksum
+
+        unless gpgsig.artifact_verify
+          $logger.fatal "The GPG signature could not be validated for the artifact. This could be a bug, a file corruption or a POSSIBLE SECURITY ISSUE such as maliciously modified content."
+          raise "Fatal unexpected error"
+        end
+
+      end
+
+
       # Make sure the artifact has been unpacked
       unless Dir.exists?($config["general"]["app_cache"] + "/artifacts/unpacked.#{@checksum}")
         $logger.error "The unpacked directory expected for #{@checksum} does not appear to exist or is not readable"

data/lib/pupistry/gpg.rb

@@ -0,0 +1,283 @@
+require 'rubygems'
+require 'yaml'
+require 'fileutils'
+require 'base64'
+
+module Pupistry
+  # Pupistry::GPG
+
+  class GPG
+    # All the functions needed for manipulating the GPG signatures
+    attr_accessor :checksum
+    attr_accessor :signature
+
+    def initialize checksum
+
+      # Need a checksum to do signing for
+      if checksum
+        @checksum = checksum
+      else
+        $logger.fatal "Probable bug, need a checksum provided with GPG validation"
+        exit 0
+      end
+
+      # Make sure that we have GPG available
+      unless system("gpg --version >> /dev/null 2>&1")
+        $logger.fatal "'gpg' command is not available, unable to do any signature creation or verification."
+        exit 0
+      end
+
+    end
+
+
+    # Sign the artifact and return the signature. Does not validation of the signature.
+    #
+    # false   Failure
+    # base64  Encoded signature
+    #
+    def artifact_sign
+      @signature = 'unsigned'
+
+      # Clean up the existing signature file
+      signature_cleanup
+
+
+      Dir.chdir("#{$config["general"]["app_cache"]}/artifacts/") do
+
+        # Generate the signature file and pick up the signature data
+        unless system "gpg --use-agent --detach-sign artifact.#{@checksum}.tar.gz"
+          $logger.error "Unable to sign the artifact, an unexpected failure occured. No file uploaded."
+          return false
+        end
+
+        if File.exists?("artifact.#{@checksum}.tar.gz.sig")
+          $logger.info "A signature file was successfully generated."
+        else
+          $logger.error "A signature file was NOT generated."
+          return false
+        end
+
+        # Convert the signature into base64. It's easier to bundle all the
+        # metadata into a single file and extracting it out when needed, than
+        # having to keep track of yet-another-file. Because we encode into
+        # ASCII here, no need to call GPG with --armor either.
+        
+        @signature = Base64.encode64(File.read("artifact.#{@checksum}.tar.gz.sig"))
+
+        unless @signature
+          $logger.error "An unexpected issue occured and no signature was generated"
+          return false
+        end
+      end
+
+
+      # Make sure the public key has been uploaded if it hasn't already
+      pubkey_upload
+
+      return @signature
+    end
+
+
+    # Verify the signature for a particular artifact.
+    #
+    # true  Signature is legit
+    # false Signature is invalid (security issue!)
+    #
+    def artifact_verify
+
+      Dir.chdir("#{$config["general"]["app_cache"]}/artifacts/") do
+
+        if File.exists?("artifact.#{@checksum}.tar.gz.sig")
+          $logger.debug "Signature already extracted on disk, running verify...."
+        else
+          $logger.debug "Extracting signature from manifest data..."
+          signature_extract
+        end
+
+        # Verify the signature
+        unless pubkey_exists?
+          pubkey_install
+        end
+
+        output_verify = `gpg --quiet --status-fd 1 --verify artifact.#{@checksum}.tar.gz.sig 2>&1`
+
+        # Cleanup on disk file
+        signature_cleanup
+
+        # Was it valid?
+        output_verify.each_line do |line|
+
+          if /\[GNUPG:\]\sGOODSIG\s[A-Z0-9]*#{$config["general"]["gpg_signing_key"]}\s/.match(line)
+            $logger.info "Artifact #{@checksum} has a valid signature belonging to #{$config["general"]["gpg_signing_key"]}"
+            return true
+          end
+
+          if /\[GNUPG:\]\sBADSIG\s/.match(line)
+            $logger.fatal "Artifact #{@checksum} has AN INVALID GPG SECURITY SIGNATURE and could be CORRUPT or TAMPERED with."
+            exit 0
+          end
+
+        end
+
+        # Unexpected error
+        $logger.error "An unexpected validation issue occured, see below debug information:"
+
+        output_verify.each_line do |line|
+          $logger.error "GPG: #{line}"
+        end
+
+      end
+
+      # Something went wrong
+      $logger.fatal "Artifact #{@checksum} COULD NOT BE GPG VALIDATED and could be CORRUPT or TAMPERED with."
+      exit 0
+
+    end
+
+
+    # Generally we should clean up old signature files before and after using them
+    #
+    def signature_cleanup
+      FileUtils.rm("#{$config["general"]["app_cache"]}/artifacts/artifact.#{@checksum}.tar.gz.sig", :force => true)
+    end
+
+
+    # Extract the signature from the manifest file and write it to file in native binary format.
+    #
+    # false     Unable to extract
+    # unsigned  Manifest shows that the artifact is not signed
+    # base64    Encoded signature
+    #
+    def signature_extract
+      begin
+        manifest            = YAML::load(File.open($config["general"]["app_cache"] + "/artifacts/manifest.#{@checksum}.yaml"))
+        
+        if manifest['gpgsig']
+          # We have the base64 version
+          @signature = manifest['gpgsig']
+
+          # Decode the base64 and write the signature file
+          File.write("#{$config["general"]["app_cache"]}/artifacts/artifact.#{@checksum}.tar.gz.sig", Base64.decode64(@signature))
+
+          return @signature
+        else
+          return false
+        end
+
+      rescue Exception => e
+        $logger.error "Something unexpected occured when reading the manifest file"
+        raise e
+      end
+
+    end
+
+
+    # Save the signature into the manifest file
+    #
+    def signature_save
+      begin
+        manifest            = YAML::load(File.open($config["general"]["app_cache"] + "/artifacts/manifest.#{@checksum}.yaml"))
+        manifest['gpgsig']  = @signature
+
+        File.open("#{$config["general"]["app_cache"]}/artifacts/manifest.#{@checksum}.yaml",'w') do |fh|
+          fh.write YAML::dump(manifest)
+        end
+
+        return true
+
+      rescue Exception => e
+        $logger.error "Something unexpected occured when updating the manifest file with GPG signature"
+        return false
+      end
+
+    end
+
+
+    # Check if the public key is installed on this machine?
+    #
+    def pubkey_exists?
+
+      # We prefix with 0x to avoid matching on strings in key names
+      if system "gpg --status-fd a --list-keys 0x#{$config["general"]["gpg_signing_key"]} 2>&1 >> /dev/null"
+        $logger.debug "Public key exists on this system"
+        return true
+      else
+        $logger.debug "Public key does not exist on this system"
+        return false
+      end
+    end
+    
+
+    # Extract & upload the public key to the s3 bucket for other users
+    #
+    def pubkey_upload
+      unless File.exists?("#{$config["general"]["app_cache"]}/artifacts/#{$config["general"]["gpg_signing_key"]}.publickey")
+
+        # GPG key does not exist locally, we therefore assume it's not in the S3
+        # bucket either, so we should export out and upload. Technically this may
+        # result in a few extra uploads (once for any new machine using Pupistry)
+        # but it doesn't cause any issue and saves me writing more code ;-)
+        
+        $logger.info "Exporting GPG key #{$config["general"]["gpg_signing_key"]} and uploading to S3 bucket..."
+
+        # If it doesn't exist on this machine, then we're a bit stuck!
+        unless pubkey_exists?
+          $logger.error "The public key #{$config["general"]["gpg_signing_key"]} does not exist on this system, so unable to export it out"
+          return false
+        end
+
+        # Export out key
+        unless system "gpg --export --armour 0x#{$config["general"]["gpg_signing_key"]} > #{$config["general"]["app_cache"]}/artifacts/#{$config["general"]["gpg_signing_key"]}.publickey"
+         $logger.error "A fault occured when trying to export the GPG key"
+         return false
+       end
+
+
+        # Upload
+        s3 = Pupistry::Storage_AWS.new 'build'
+
+        unless s3.upload "#{$config["general"]["app_cache"]}/artifacts/#{$config["general"]["gpg_signing_key"]}.publickey", "#{$config["general"]["gpg_signing_key"]}.publickey"
+          $logger.error "Unable to upload GPG key to S3 bucket"
+          return false
+        end
+
+      end
+
+    end
+
+
+    # Install the public key. This is a potential avenue for exploit, if a
+    # machine is being built for the first time, it has no existing trust of
+    # the GPG key, other than transit encryption to the S3 bucket. To protect
+    # against attacks at the bootstrap time, you should pre-load your machine
+    # images with the public GPG key.
+    #
+    # For those users who trade off some security for convienence, we install
+    # the GPG public key for them direct from the S3 repo.
+    #
+    def pubkey_install
+      begin
+        $logger.warn "Installing GPG key #{$config["general"]["gpg_signing_key"]}..."
+
+        s3 = Pupistry::Storage_AWS.new 'agent'
+
+        unless s3.download "#{$config["general"]["gpg_signing_key"]}.publickey", "#{$config["general"]["app_cache"]}/artifacts/#{$config["general"]["gpg_signing_key"]}.publickey"
+          $logger.error "Unable to download GPG key from S3 bucket, this will prevent validation of signature"
+          return false
+        end
+
+        unless system "gpg --import < #{$config["general"]["app_cache"]}/artifacts/#{$config["general"]["gpg_signing_key"]}.publickey"
+         $logger.error "A fault occured when trying to import the GPG key"
+         return false
+       end
+
+      rescue Exception => e
+        $logger.error "Something unexpected occured when installing the GPG public key"
+        return false
+      end
+    end
+
+  end 
+end
+
+# vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent

data/lib/pupistry/storage_aws.rb

@@ -74,6 +74,13 @@ module Pupistry
       rescue AWS::S3::Errors::SignatureDoesNotMatch => e
         $logger.error "IAM signature error when accessing #{$config["general"]["s3_bucket"]}, probably invalid IAM credentials"
         raise e
+
+      rescue AWS::S3::Errors::MissingCredentialsError => e
+        $logger.error "AWS credentials not supplied. You must either:"
+        $logger.error "a) Specify them in the config file for Pupistry"
+        $logger.error "b) Use IAM roles with an EC2 instance."
+        $logger.error "c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY"
+        return false
       
       rescue Exception => e
         raise e
@@ -124,6 +131,13 @@ module Pupistry
         $logger.error "IAM signature error when accessing #{$config["general"]["s3_bucket"]}, probably invalid IAM credentials"
         raise e
 
+      rescue AWS::S3::Errors::MissingCredentialsError => e
+        $logger.error "AWS credentials not supplied. You must either:"
+        $logger.error "a) Specify them in the config file for Pupistry"
+        $logger.error "b) Use IAM roles with an EC2 instance."
+        $logger.error "c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY"
+        return false
+     
       rescue Exception => e
         raise e
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupistry
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Jethro Carr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-22 00:00:00.000000000 Z
+date: 2015-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1
@@ -78,6 +78,7 @@ files:
 - lib/pupistry/artifact.rb
 - lib/pupistry/bootstrap.rb
 - lib/pupistry/config.rb
+- lib/pupistry/gpg.rb
 - lib/pupistry/storage_aws.rb
 - README.md
 - settings.example.yaml