kumo_config 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kumo_config.gemspec
+gemspec

data/README.md

@@ -0,0 +1,121 @@
+# KumoConfig [![Build status](https://badge.buildkite.com/1d42699430881063b95a0082c8f23f0acc6d5bad5909716720.svg)](https://buildkite.com/redbubble/kumo-config-gem) [![Code Climate](https://codeclimate.com/github/redbubble/kumo_keisei_config/badges/gpa.svg)](https://codeclimate.com/github/redbubble/kumo_config_gem)
+
+A utility for resolving environment configuration.
+
+## Installation
+
+This gem is automatically installed in the kumo container, so any `apply-env` or `deploy` scripts have access to it.
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kumo_config'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kumo_config
+
+## Usage
+
+### Basic Usage
+
+The basic usage will give you an EnvironmentConfig based on the paths you specify.
+
+`config_path`: Where environments' configuration files live, e.g. `production.yml`, `staging.yml`, `production_secrets.yml`, `staging_secrets.yml`
+`template_path`: Where the CloudFormation template lives - **TODO: THIS BELONGS IN KUMO_KEISEI, NOT HERE**
+`params_template_file_path`: The location of the template for submitting paramters to CloudFormation. **AGAIN, THIS DOES NOT BELONG HERE**
+`injected_config`: A Hash containing any configuration you want to inject at runtime rather than loading from a file. For example, you might want to pull some settings from a VPC.
+
+```ruby
+EnvironmentConfig.new(
+  config_path: File.join('/app', 'env', 'config'),
+  template_path: File.join('/app', 'env', 'cloudformation', 'redbubble.json'),
+  params_template_file_path: File.join('/app', 'env', 'cloudformation', 'redbubble.yml.erb')
+  injected_config: { key: 'value' }
+)
+```
+
+### Configuration Hierarchy
+
+Configuration will be loaded from the following sources:
+
+1. `common.yml` and `common_secrets.yml` if they exist.
+2. `{environment}.yml` and `{environment}_secrets.yml` or `development.yml` and `development_secrets.yml` if environment specific config does not exist.
+
+### Injecting Configuration
+
+You can also inject configuration at run time by adding it to the object provided to the `apply!` call:
+
+```ruby
+stack_config = {
+  config_path: File.join('/app', 'env', 'config'),
+  template_path: File.join('/app', 'env', 'cloudformation', 'myapp.json'),
+  injected_config: {
+    'Seed' => random_seed,
+  }
+}
+stack.apply!(stack_config)
+```
+
+### Getting the configuration and secrets without an `apply!`
+
+If you need to inspect the configuration without applying a stack, call `config`:
+```ruby
+stack_config = {
+  config_path: File.join('/app', 'env', 'config'),
+  template_path: File.join('/app', 'env', 'cloudformation', 'myapp.json'),
+  injected_config: {
+    'Seed' => random_seed,
+  }
+}
+marshalled_config = stack.config(stack_config)
+marshalled_secrets = stack.plain_text_secrets(stack_config)
+
+if marshalled_config['DB_HOST'].start_with? '192.' then
+  passwd = marshalled_secrets['DB_PASS']
+  ...
+end
+```
+
+## Dependencies
+
+#### Ruby Versions
+
+This gem is tested with Ruby (MRI) versions 1.9.3 and 2.2.3.
+
+## Release
+
+1. Upgrade version in VERSION
+2. Run ./script/release-gem
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/kumo_config/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+### Automated AWS Integration Tests - TODO: REVISE
+
+You can test the Cloudformation responsibilities of this gem by extending the integration tests at `spec/integration`.
+
+To run these tests you need a properly configured AWS environment (with `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` set) and then run `./script/integration_test.sh`.
+
+If you run this within a Buildkite job then you will have a stack named "kumokeisei-test-$buildnumber" created and torn down for each integration test context. If you run this outside of a Buildkite job then the stack will be named "kumokeisei-test-$username".
+
+### Manual testing with Kumo Tools container
+
+Changes to the gem can be manually tested end to end in a project that uses the gem (i.e. http-wala).
+
+1. First start the dev-tools container: `kumo tools debug non-production`
+1. gem install specific_install
+1. Re-install the gem: `gem specific_install https://github.com/redbubble/kumo_keisei_gem.git -b <your_branch>`
+1. Fire up a console: `irb`
+1. Require the gem: `require "kumo_keisei"`
+1. Interact with the gem's classes. `KumoKeisei::Stack.new(...).apply!`

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/kumo_config.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+version = File.read(File.expand_path('../VERSION', __FILE__)).strip
+
+Gem::Specification.new do |spec|
+  spec.name          = "kumo_config"
+  spec.version       = version
+  spec.authors       = ["Redbubble"]
+  spec.email         = ["delivery-engineering@redbubble.com"]
+  spec.summary       = "A utility for reading Kumo configuration"
+  spec.homepage      = "http://redbubble.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'kumo_ki'
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.4"
+  spec.add_development_dependency "pry", "~> 0.10"
+end

data/lib/kumo_config/environment_config.rb

@@ -0,0 +1,125 @@
+require 'kumo_ki'
+require 'logger'
+require 'yaml'
+
+require_relative 'file_loader'
+
+module KumoConfig
+  # Environment Configuration for a cloud formation stack
+  class EnvironmentConfig
+    class ConfigurationError < StandardError; end
+
+    LOGGER = Logger.new(STDOUT)
+
+    attr_reader :app_name, :env_name
+
+    def initialize(options, logger = LOGGER)
+      @env_name = options[:env_name]
+      @params_template_file_path = options[:params_template_file_path]
+      @injected_config = options[:injected_config] || {}
+      @log = logger
+
+      if options[:config_path]
+          @config_file_loader = KumoConfig::FileLoader.new(config_dir_path: options[:config_path])
+      elsif options[:config_dir_path]
+          @log.warn "[DEPRECATION] `:config_dir_path` is deprecated, please pass in `:config_path` instead"
+          @config_file_loader = KumoConfig::FileLoader.new(config_dir_path: options[:config_dir_path])
+      end
+
+    end
+
+    def production?
+      env_name == 'production'
+    end
+
+    def development?
+      !%w(production staging).include? env_name
+    end
+
+    def plain_text_secrets
+      @plain_text_secrets ||= decrypt_secrets(encrypted_secrets)
+    end
+
+    def config
+      # a hash of all settings that apply to this environment
+      @config ||= common_config.merge(env_config).merge(@injected_config)
+    end
+
+    def get_binding
+      binding
+    end
+
+    private
+
+    def kms
+      @kms ||= KumoKi::KMS.new
+    end
+
+    def params_template_erb
+      return nil unless @params_template_file_path && File.exist?(@params_template_file_path)
+      template_file_loader = KumoConfig::FileLoader.new(config_dir_path: File.dirname(@params_template_file_path))
+      template_file_loader.load_erb(File.basename(@params_template_file_path))
+    end
+
+    def decrypt_secrets(secrets)
+      Hash[
+        secrets.map do |name, cipher_text|
+          @log.debug "Decrypting '#{name}'"
+          decrypt_cipher name, cipher_text
+        end
+      ]
+    end
+
+    def decrypt_cipher(name, cipher_text)
+      if cipher_text.start_with? '[ENC,'
+        begin
+          [name, kms.decrypt(cipher_text[5, cipher_text.size]).to_s]
+        rescue
+          @log.error "Error decrypting secret '#{name}'"
+          raise
+        end
+      else
+        [name, cipher_text]
+      end
+    end
+
+    def env_config_file_name
+      "#{env_name}.yml"
+    end
+
+    def env_secrets_file_name
+      "#{env_name}_secrets.yml"
+    end
+
+    def encrypted_secrets
+      encrypted_common_secrets.merge(encrypted_env_secrets)
+    end
+
+    def encrypted_common_secrets
+      @config_file_loader.load_hash('common_secrets.yml')
+    end
+
+    def encrypted_env_secrets
+      secrets = @config_file_loader.load_hash(env_secrets_file_name)
+
+      if !secrets.empty?
+        secrets
+      else
+        @config_file_loader.load_hash('development_secrets.yml')
+      end
+    end
+
+    def common_config
+      @config_file_loader.load_hash('common.yml')
+    end
+
+    def env_config
+      config = @config_file_loader.load_hash(env_config_file_name)
+      if !config.empty?
+        config
+      else
+        @config_file_loader.load_hash('development.yml')
+      end
+    end
+  end
+end

data/lib/kumo_config/file_loader.rb

@@ -0,0 +1,38 @@
+require 'erb'
+
+module KumoConfig
+  class FileLoader
+    def initialize(options)
+      @config_dir_path = options[:config_dir_path]
+    end
+
+    def load_hash(file_name, optional = true)
+      # reads a file presuming it's a yml in form of key: value, returning it as a hash
+      path = file_path(file_name)
+
+      begin
+        YAML::load(File.read(path))
+      rescue Errno::ENOENT => ex
+        # file not found, return empty dictionary if that is ok
+        return {} if optional
+        raise ex
+      rescue StandardError => ex
+        # this is an error we weren't expecting
+        raise ex
+      end
+    end
+
+    def load_erb(file_name)
+      # loads a file, constructs an ERB object from it and returns the ERB object
+      # DOES NOT RENDER A RESULT!!
+      path = file_path(file_name)
+      ERB.new(File.read(path))
+    end
+
+    private
+
+    def file_path(file_name)
+      File.join(@config_dir_path, file_name)
+    end
+  end
+end

data/lib/kumo_config.rb

@@ -0,0 +1 @@
+require_relative "kumo_config/environment_config"

data/script/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+bundle install
+if [[ -z "$KUMO_CONFIG_VERSION" && -n "$BUILDKITE_BUILD_NUMBER" ]]; then
+  export KUMO_CONFIG_VERSION="$BUILDKITE_BUILD_NUMBER"
+fi
+
+echo "--- :wind_chime: Building gem :wind_chime:"
+
+gem build kumo_config.gemspec

data/script/integration_test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+echo "--- :clock1: :clock2: running specs :clock3: :clock4:"
+bundle install && bundle exec rspec --pattern "spec/integration/*_spec.rb"
+
+function inline_image {
+  printf '\033]1338;url='"$1"';alt='"$2"'\a\n'
+}
+
+echo "+++ Done! :thumbsup: :shipit:"
+inline_image "https://giftoppr.desktopprassets.com/uploads/f828c372186b5fa80a1c553adbcd4bc4d331396b/tumblr_m2cg550aq21ql201ao1_500.gif" "Yuss"

data/script/unit_test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+echo "--- :clock1: :clock2: running specs :clock3: :clock4:"
+bundle install && bundle exec rspec --exclude-pattern "spec/integration/*_spec.rb"
+
+function inline_image {
+  printf '\033]1338;url='"$1"';alt='"$2"'\a\n'
+}
+
+echo "+++ Done! :thumbsup: :shipit:"
+inline_image "https://giftoppr.desktopprassets.com/uploads/f828c372186b5fa80a1c553adbcd4bc4d331396b/tumblr_m2cg550aq21ql201ao1_500.gif" "Yuss"

data/spec/integration/config_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe 'KumoConfig' do
+  let(:config_path) { File.join(File.dirname(__FILE__), 'fixtures') }
+  let(:environment_name) { 'production' }
+
+  subject { KumoConfig::EnvironmentConfig.new(
+    env_name: environment_name,
+    config_path: config_path
+  ) }
+
+  it 'resolves the configuration for the specified environment' do
+    expect(subject.config['production_key']).to eq('production_value')
+  end
+
+  it 'uses commmon configuration if nothing is specified in an environment or injected config' do
+    expect(subject.config['common_key']).to eq('common_value')
+  end
+
+  it 'gives the environment config precedence over common config' do
+    expect(subject.config['overridden_key']).to eq('production override value')
+  end
+
+  it 'resolves secret configuration into a Hash' do
+    expect(subject.plain_text_secrets['production_secret_key']).to eq('production_secret_value')
+  end
+
+  it 'uses commmon secret configuration if nothing is specified in the environment' do
+    expect(subject.plain_text_secrets['common_secret_key']).to eq('common_secret_value')
+  end
+
+  it 'gives the environment secret precedence over common secrets' do
+    expect(subject.plain_text_secrets['overridden_secret']).to eq('production_override_secret')
+  end
+
+  context 'when there is injected configuration' do
+    subject { KumoConfig::EnvironmentConfig.new(
+      env_name: environment_name,
+      config_path: config_path,
+      injected_config: {
+        'injected_key' => 'injected value',
+        'overridden_key' => 'injected override value'
+      }
+    ) }
+
+    it 'resolves injected configuration items' do
+      expect(subject.config['injected_key']).to eq('injected value')
+    end
+
+    it 'gives injected config precedence over config from files' do
+      expect(subject.config['overridden_key']).to eq('injected override value')
+    end
+  end
+
+
+  context 'when an environment is specified for which we do not have configuration files' do
+    let(:environment_name) { 'nonsense' }
+
+    it 'uses development environment as a default' do
+      expect(subject.config['development_key']).to eq('development value')
+    end
+
+    it 'uses development secrets' do
+      expect(subject.plain_text_secrets['development_secret']).to eq('development_secret_value')
+    end
+  end
+end

data/spec/integration/fixtures/common.yml

@@ -0,0 +1,2 @@
+common_key: common_value
+overridden_key: original value

data/spec/integration/fixtures/common_secrets.yml

@@ -0,0 +1,11 @@
+common_secret_key: >
+    [ENC,AQECAHiRAIBw6aG5PR5XWhLpFs0CtWZCoVenu5eErfT6U2j+PgAAAHEwbwYJ
+    KoZIhvcNAQcGoGIwYAIBADBbBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEE
+    DFIEC1gYWtlJQlCFnwIBEIAuKctvUp9pni1Ij1M5tbyBkAms4mcKedYIRzUO
+    FASJonP8HpTe11oL0gbQQFP5vA==
+
+overridden_secret: >
+    [ENC,AQECAHiRAIBw6aG5PR5XWhLpFs0CtWZCoVenu5eErfT6U2j+PgAAAG4wbAYJ
+    KoZIhvcNAQcGoF8wXQIBADBYBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEE
+    DPBwsLbzuz9H4WL7twIBEIAre1We74YNLNu3xKLLvF3J5UJD8eH1+7IHn0II
+    Bne6CbUPITvPc2N1Hq9KYA==

data/spec/integration/fixtures/development.yml

@@ -0,0 +1 @@
+development_key: development value

data/spec/integration/fixtures/development_secrets.yml

@@ -0,0 +1,5 @@
+development_secret: >
+    [ENC,AQECAHiRAIBw6aG5PR5XWhLpFs0CtWZCoVenu5eErfT6U2j+PgAAAHYwdAYJ
+    KoZIhvcNAQcGoGcwZQIBADBgBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEE
+    DMcUwkdxmGkeBjJf8AIBEIAztkU80u2FKQ1o+72j3vQE2Haxv0L5tdrG0nBr
+    reDdZ/miq66KN92ijTTPWwrrrJH6GGkz

data/spec/integration/fixtures/production.yml

@@ -0,0 +1,2 @@
+production_key: production_value
+overridden_key: production override value

data/spec/integration/fixtures/production_secrets.yml

@@ -0,0 +1,11 @@
+production_secret_key: >
+    [ENC,AQECAHiRAIBw6aG5PR5XWhLpFs0CtWZCoVenu5eErfT6U2j+PgAAAHUwcwYJ
+    KoZIhvcNAQcGoGYwZAIBADBfBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEE
+    DOYHUsqhXlV2Chl9AQIBEIAyPvjZe1pWhGOZ6RGP8srNB1oMC/4JwX+nYdjC
+    +i1U5PusiCrRERJdk3B0Cm4yIz3GexY=
+
+overridden_secret: >
+    [ENC,AQECAHiRAIBw6aG5PR5XWhLpFs0CtWZCoVenu5eErfT6U2j+PgAAAHgwdgYJ
+    KoZIhvcNAQcGoGkwZwIBADBiBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEE
+    DEgzP81EmQgMXTuNFwIBEIA1lkR9cGUtdAOQjT3S19RKKL3w2KgIHWnrZh05
+    tM25nWeufrLe4wldF73/4zSXfEZqlxycfZ0=