jackal-stacks 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b71b344dfac97c00282df6e96bde5f3d413a74e
+  data.tar.gz: 9b787df9e775186d5333de90f3eacfd9076a1135
+SHA512:
+  metadata.gz: 55393fe0bb6bfd1b26e6841eac37a4674b384560a90688bfa8094c7fc3ec01a13841cbd0ed8e265136ba90c811bd4c4c4a1cbba1ebcc4a141e88c875ede9609d
+  data.tar.gz: dadc256d8c3b29ec838e608141f4a088577ff593790c3aad5d9b22e3526136d893dc990b78f5c2e9ab897f842cd44b9ab4edfd9e41c5dbbb495f0d1101fd1df8

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing
+
+## Branches
+
+### `master` branch
+
+The master branch is the current stable released version.
+
+### `develop` branch
+
+The develop branch is the current edge of development.
+
+## Pull requests
+
+* https://github.com/carnivore-rb/jackal-stacks/pulls
+
+Please base all pull requests of the `develop` branch. Merges to
+`master` only occur through the `develop` branch. Pull requests
+based on `master` will likely be cherry picked.
+
+## Issues
+
+Need to report an issue? Use the github issues:
+
+* https://github.com/carnivore-rb/jackal-stacks/issues

data/LICENSE

@@ -0,0 +1,13 @@
+   Copyright 2015 Chris Roberts
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,15 @@
+# Jackal Stacks
+
+Build stacks \o/
+
+## Requirements
+
+This service assumes the payload information provided via the code
+fetcher service.
+
+## Configuration
+
+# Info
+
+* Repository: https://github.com/carnivore-rb/jackal-stacks
+* IRC: Freenode @ #carnivore

data/jackal-stacks.gemspec

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'jackal-stacks/version'
+Gem::Specification.new do |s|
+  s.name = 'jackal-stacks'
+  s.version = Jackal::Stacks::VERSION.version
+  s.summary = 'Message processing helper'
+  s.author = 'Chris Roberts'
+  s.email = 'code@chrisroberts.org'
+  s.homepage = 'https://github.com/carnivore-rb/jackal-stacks'
+  s.description = 'Build stuff!'
+  s.require_path = 'lib'
+  s.license = 'Apache 2.0'
+  s.add_dependency 'jackal'
+  s.add_dependency 'librarian-chef'
+  s.add_dependency 'sfn'
+  s.files = Dir['lib/**/*'] + %w(jackal-stacks.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
+end

data/lib/jackal-stacks.rb

@@ -0,0 +1,13 @@
+require 'sfn'
+require 'jackal'
+
+module Jackal
+  module Stacks
+    autoload :Builder, 'jackal-stacks/builder'
+    autoload :StackCommon, 'jackal-stacks/stack_common'
+    autoload :Wrecker, 'jackal-stacks/wrecker'
+  end
+end
+
+require 'jackal-stacks/formatter'
+require 'jackal-stacks/version'

data/lib/jackal-stacks/builder.rb

@@ -0,0 +1,242 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    # Stack builder
+    class Builder < Callback
+
+      include StackCommon
+
+      # Setup callback
+      def setup(*_)
+        require 'sfn'
+        require 'bogo-ui'
+        require 'stringio'
+        require 'openssl'
+        # Ensure we can build the API at startup
+        stacks_api
+      end
+
+      # Determine validity of message
+      #
+      # @param message [Carnivore::Message]
+      # @return [Truthy, Falsey]
+      def valid?(message)
+        super do |payload|
+          payload.get(:data, :stacks, :builder) &&
+            payload.get(:data, :stacks, :template) &&
+            payload.get(:data, :stacks, :asset) &&
+            allowed?(payload)
+        end
+      end
+
+      # @return [String] working directory
+      def working_directory
+        memoize(:working_directory, :direct) do
+          FileUtils.mkdir_p(dir = config.fetch(:working_directory, '/tmp/jackal-stack-builder'))
+          dir
+        end
+      end
+
+      # @return [String] fresh working directory
+      def workspace(payload)
+        File.join(working_directory, payload[:id])
+      end
+
+      # Build or update stacks
+      #
+      # @param message [Carnivore::Message]
+      def execute(message)
+        failure_wrap(message) do |payload|
+          directory = asset_store.unpack(asset_store.get(payload.get(:data, :stacks, :asset)), workspace(payload))
+          begin
+            unless(payload.get(:data, :stacks, :name))
+              payload.set(:data, :stacks, :name, stack_name(payload))
+            end
+            store_stable_asset(payload, directory)
+            begin
+              stack = stacks_api.stacks.get(payload.get(:data, :stacks, :name))
+            rescue
+              stack = nil
+            end
+            if(stack)
+              info "Stack currently exists. Applying update [#{stack}]"
+              run_stack(payload, directory, :update)
+              payload.set(:data, :stacks, :updated, true)
+            else
+              info "Stack does not currently exist. Building new stack [#{payload.get(:data, :stacks, :name)}]"
+              init_provider(payload)
+              run_stack(payload, directory, :create)
+              payload.set(:data, :stacks, :created, true)
+            end
+          ensure
+            FileUtils.rm_rf(directory)
+          end
+          job_completed(:stacks, payload, message)
+        end
+      end
+
+      # Build configuration arguments for Sfn::Command execution
+      #
+      # @param payload [Smash]
+      # @param directory [String] directory to unpacked asset
+      # @return [Smash] stack command options hash
+      def build_stack_args(payload, directory)
+        Smash.new(
+          :base_directory => File.join(directory, 'cloudformation'),
+          :parameters => load_stack_parameters(payload, directory),
+          :ui => Bogo::Ui.new(
+            :app_name => 'JackalStacks',
+            :defaults => true,
+            :output_to => StringIO.new('')
+          ),
+          :interactive_parameters => false,
+          :nesting_bucket => config.get(:orchestration, :bucket_name),
+          :apply_nesting => true,
+          :processing => true,
+          :options => {
+            :disable_rollback => true,
+            :capabilities => ['CAPABILITY_IAM']
+          },
+          :credentials => config.get(:orchestration, :api, :credentials),
+          :file => payload.get(:data, :stacks, :template),
+          :file_path_prompt => false,
+          :poll => false
+        )
+      end
+
+      # Extract any custom parameters from asset store if available,
+      # and merge any parameters provided via payload, and finally
+      # merge any parameters provided via configuration
+      #
+      # @param payload [Smash]
+      # @param directory [String]
+      # @note parameter precedence:
+      #   * Hook URL parameters
+      #   * Payload parameters
+      #   * Stacks file parameters
+      #   * Service configuration parameters
+      def load_stack_parameters(payload, directory)
+        params = Smash.new
+        stacks_file = load_stacks_file(payload, directory)
+        s_namespace = determine_namespace(payload)
+        template = payload.get(:data, :stacks, :template)
+        params.deep_merge!(payload.fetch(:data, :webhook, :query, :stacks, :parameters, Smash.new))
+        params.deep_merge!(payload.fetch(:data, :stacks, :parameters, Smash.new))
+        params.deep_merge!(
+          stacks_file.fetch(s_namespace, template, :parameters,
+            stacks_file.fetch(:default, template, :parameters, Smash.new)
+          )
+        )
+        params.deep_merge!(
+          config.fetch(:parameter_overrides, s_namespace, template,
+            config.fetch(:parameter_overrides, :default, template, Smash.new)
+          )
+        )
+        params
+      end
+
+      # Parse the `.stacks` file if available
+      #
+      # @param payload [Smash]
+      # @param directory [String] path to unpacked asset directory
+      # @return [Smash]
+      def load_stacks_file(payload, directory)
+        stacks_path = File.join(directory, '.stacks')
+        if(File.exists?(stacks_path))
+          Bogo::Config.new(file_path).data
+        else
+          Smash.new
+        end
+      end
+
+      # Perform stack action
+      #
+      # @param payload [Smash]
+      # @param directory [String] directory to unpacked asset
+      # @param action [Symbol, String] :create or :update
+      # @return [TrueClass]
+      def run_stack(payload, directory, action)
+        unless([:create, :update].include?(action.to_sym))
+          abort ArgumentError.new("Invalid action argument `#{action}`. Expecting `create` or `update`!")
+        end
+        args = build_stack_args(payload, directory)
+        stack_name = payload.get(:data, :stacks, :name)
+        Sfn::Command.const_get(action.to_s.capitalize).new(args, [stack_name]).execute!
+        true
+      end
+
+      # Check if this payload is allowed to be processed based on
+      # defined restrictions within the configuration
+      #
+      # @param payload [Smash]
+      # @return [TrueClass, FalseClass]
+      def allowed?(payload)
+        !!determine_namespace(payload)
+      end
+
+      # Initialize provider if instructed via config
+      #
+      # @param payload [Smash]
+      # @note this currently init's chef related items
+      def init_provider(payload)
+        if(config.get(:init, :chef, :validator) || config.get(:init, :chef, :encrypted_secret))
+          bucket = stacks_api.api_for(:storage).buckets.get(config.get(:orchestration, :bucket_name))
+          validator_name = name_for(payload, 'validator.pem')
+          if(config.get(:init, :chef, :validator) && bucket.files.get(validator_name).nil?)
+            file = bucket.files.build(:name => validator_name)
+            file.body = OpenSSL::PKey::RSA.new(2048).export
+            file.save
+          end
+          secret_name = name_for(payload, 'encrypted_data_bag_secret')
+          if(config.get(:init, :chef, :encrypted_secret) && bucket.files.get(secret_name).nil?)
+            file = bucket.files.build(:name => secret_name)
+            file.body = SecureRandom.base64(2048)
+            file.save
+          end
+        end
+      end
+
+      # Store stable asset in object store
+      #
+      # @param payload [Smash]
+      def store_stable_asset(payload, directory)
+        if(config.get(:init, :stable))
+          ['rm -rf .librarian Gemfile Gemfile.lock', 'librarian-chef install', 'rm -rf tmp'].each do |cmd|
+            info "Starting stable store pre-pack command: #{cmd}"
+            process_manager.process(payload[:id], cmd) do |process|
+              process.cwd = directory
+              process.io.inherit!
+              process.start
+              process.wait
+              unless(process.exit_code == 0)
+                raise 'Cookbook install failed!'
+              end
+            end
+            info "Completed stable store pre-pack command: #{cmd}"
+          end
+          debug "Starting stable asset upload for #{payload[:id]}"
+          bucket = stacks_api.api_for(:storage).buckets.get(config.get(:orchestration, :bucket_name))
+          stable_name = name_for(payload, 'stable.zip')
+          file = bucket.files.get(stable_name) || bucket.files.build(:name => stable_name)
+          file.body = asset_store.pack(directory)
+          file.save
+        end
+      end
+
+      # Provide prefixed key name for asset
+      #
+      # @param payload [Smash]
+      # @param asset_name [String
+      # @return [String]
+      # @note this is currently a no-op and thus are shared across
+      #   stacks. currently is stubbed for completion of template and
+      #   interaction logic
+      def name_for(payload, asset_name)
+        File.join(determine_namespace(payload), asset_name)
+        asset_name
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/formatter.rb

@@ -0,0 +1,4 @@
+require 'jackal-stacks/formatter/webhook'
+require 'jackal-stacks/formatter/code_fetcher'
+require 'jackal-stacks/formatter/github'
+require 'jackal-stacks/formatter/slack'

data/lib/jackal-stacks/formatter/code_fetcher.rb

@@ -0,0 +1,30 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    module Formatter
+      # Format code fetcher data for stacks
+      class CodeFetcher < Jackal::Formatter
+
+        SOURCE = 'code_fetcher'
+        DESTINATION = 'stacks'
+
+        # Format payload
+        #
+        # @param payload [Smash]
+        def format(payload)
+          if(payload.get(:data, :code_fetcher))
+            payload.set(:data, :stacks, :asset,
+              payload.get(:data, :code_fetcher, :asset)
+            )
+            payload.set(:data, :stacks, :reference,
+              payload.get(:data, :code_fetcher, :info, :reference)
+            )
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/formatter/github.rb

@@ -0,0 +1,39 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    module Formatter
+      # Format github data for stacks
+      class Github < Jackal::Formatter
+
+        # Source service
+        SOURCE = :github
+        # Destination service
+        DESTINATION = :stacks
+
+        # Valid github events
+        VALID_EVENTS = %w(create delete push)
+
+        # Format payload
+        #
+        # @param payload [Smash]
+        def format(payload)
+          if(VALID_EVENTS.include?(payload.get(:data, :github, :event)))
+            if(payload.get(:data, :github, :query, :template))
+              payload.set(:data, :stacks, :template,
+                payload.get(:data, :github, :query, :template)
+              )
+            end
+            if(payload.get(:data, :github, :event) == 'delete')
+              payload.set(:data, :stacks, :wrecker, true)
+            else
+              payload.set(:data, :stacks, :builder, true)
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/formatter/slack.rb

@@ -0,0 +1,43 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    module Formatter
+      # Format result for slack notification
+      class Slack < Jackal::Formatter
+
+        # Source service
+        SOURCE = 'stacks'
+        # Destination service
+        DESTINATION = 'slack'
+
+        NOTIFY_ON = %w(created updated destroyed)
+
+        # Format payload
+        #
+        # @param payload [Smash]
+        def format(payload)
+          if(payload.get(:data, :stacks))
+            unless((notify = payload.fetch(:data, :stacks, {}).keys & NOTIFY_ON).empty?)
+              msgs = payload.fetch(:data, :slack, :messages, [])
+              msgs << Smash.new(
+                :description => "Stacks result: #{notify.first}",
+                :message => [
+                  "Stack has been #{notify.first} [name: #{payload.get(:data, :stacks, :name)}]",
+                  "* Template: #{payload.get(:data, :stacks, :template)}",
+                  "* Repository: #{payload.get(:data, :code_fetcher, :info, :owner)}/#{payload.get(:data, :code_fetcher, :info, :name)}",
+                  "* Reference: #{payload.get(:data, :code_fetcher, :info, :reference)}",
+                  "* SHA: #{payload.get(:data, :code_fetcher, :info, :commit_sha)}"
+                ].join("\n"),
+                :color => :good
+              )
+              payload.set(:data, :slack, :messages, msgs)
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/formatter/webhook.rb

@@ -0,0 +1,27 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    module Formatter
+      # Format webhook data for stacks
+      class Webhook < Jackal::Formatter
+
+        SOURCE = 'webhook'
+        DESTINATION = 'stacks'
+
+        # Format payload
+        #
+        # @param payload [Smash]
+        def format(payload)
+          if(payload.get(:data, :webhook, :query, :template))
+            payload.set(:data, :stacks, :template,
+              payload.get(:data, :webhook, :query, :template)
+            )
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/stack_common.rb

@@ -0,0 +1,42 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    module StackCommon
+
+      # @return [Miasma::Models::Orchestration]
+      def stacks_api
+        memoize(:stacks_api, :direct) do
+          Miasma.api(
+            config.fetch(
+              :orchestration, :api, Smash.new
+            ).merge(
+              :type => :orchestration
+            )
+          )
+        end
+      end
+
+      # Determine namespace key to use for accessing parameters
+      #
+      # @param payload [Smash]
+      # @return [String]
+      # @note if not match found, `default` will return
+      def determine_namespace(payload)
+        config.fetch(:mappings, Smash.new).map do |ns, glob|
+          ns if File.fnmatch?(glob, payload.get(:data, :stacks, :reference).to_s)
+        end.compact.first
+      end
+
+      # Generate stack name based on payload
+      #
+      # @param payload [Smash]
+      # @return [String] stack name
+      def stack_name(payload)
+        s_namespace = determine_namespace(payload)
+        "#{s_namespace}-#{payload.get(:data, :stacks, :template).sub(/\.[a-z]+$/, '').gsub(/[^A-Za-z0-9\-]/, '-')}"
+      end
+
+    end
+  end
+end

data/lib/jackal-stacks/version.rb

@@ -0,0 +1,6 @@
+module Jackal
+  module Stacks
+    # Current library version
+    VERSION = Gem::Version.new('0.1.0')
+  end
+end

data/lib/jackal-stacks/wrecker.rb

@@ -0,0 +1,46 @@
+require 'jackal-stacks'
+
+module Jackal
+  module Stacks
+    # Stack destroyer
+    class Wrecker < Callback
+
+      include StackCommon
+
+      # Setup callback
+      def setup(*_)
+        stacks_api
+      end
+
+      # Determine validity of message
+      #
+      # @param message [Carnivore::Message]
+      # @return [Truthy, Falsey]
+      def valid?(message)
+        super do |payload|
+          payload.get(:data, :stacks, :wrecker)
+        end
+      end
+
+      # Build or update stacks
+      #
+      # @param message [Carnivore::Message]
+      def execute(message)
+        failure_wrap(message) do |payload|
+          s_name = stack_name(payload)
+          stack = stacks_api.stacks.get(s_name)
+          if(stack)
+            info "Stack currently exists. Destroying. [#{stack.name}]"
+            stack.destroy
+            payload.set(:data, :stacks, :destroyed, true)
+            job_completed(:stacks, payload, message)
+          else
+            error "Failed to locate requested stack for destruction [#{s_name}]"
+            failed(payload, message, 'Requested stack does not exist')
+          end
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: jackal-stacks
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jackal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: librarian-chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sfn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Build stuff!
+email: code@chrisroberts.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTING.md
+- LICENSE
+- README.md
+- jackal-stacks.gemspec
+- lib/jackal-stacks.rb
+- lib/jackal-stacks/builder.rb
+- lib/jackal-stacks/formatter.rb
+- lib/jackal-stacks/formatter/code_fetcher.rb
+- lib/jackal-stacks/formatter/github.rb
+- lib/jackal-stacks/formatter/slack.rb
+- lib/jackal-stacks/formatter/webhook.rb
+- lib/jackal-stacks/stack_common.rb
+- lib/jackal-stacks/version.rb
+- lib/jackal-stacks/wrecker.rb
+homepage: https://github.com/carnivore-rb/jackal-stacks
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Message processing helper
+test_files: []
+has_rdoc: