makemespiffy 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9ca618f5b75c3126fe81406bc1730fad8669d9d5
+  data.tar.gz: c4c7144a3327bed3ea0482bd1f310946a9d6e219
+SHA512:
+  metadata.gz: 410bf36d64d0abe4c23febe497eb4e3a4a2b377ec9b69684c7e7b7266d6e444a1779bf7f970d8c2d32e6931ea3e685eabbad829e06eaf3ad3cc05146c8ce9695
+  data.tar.gz: 2c765d06fc74cdbf31718611e818c79718946dcbd3d7fc45eaae41f30fc408a6f7743fc8b5c41e9aa8e1a75a98ee4a29715f01372d78725cab84b3d7964c0fed

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,3 @@
+--format progress
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+script: bundle exec rspec
+notifications:
+  email:
+    recipients:
+      - drnicwilliams@gmail.com
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in makemespiffy.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Dr Nic Williams
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+Make Me Spiffy
+==============
+
+Convert a flat BOSH manifest for something into a set of Spiff templates.
+
+This is being developed to help migrate flat BOSH manifests into a standardised Concourse pipeline that could deploy anything.
+
+The plan
+--------
+
+Given a BOSH manifest `manifest.yml`, run the command multiple times to extract data into spiff templates:
+
+```
+makemespiffy manifest.yml name environment/name.yml meta.name
+```
+
+This will take the root level key `name` from `manifest.yml` and replace it with `(( meta.name ))`.
+
+```
+name: manifest-name
+```
+
+Becomes:
+
+```yaml
+meta:
+  name: (( merge ))
+
+name: (( meta.name ))
+```
+
+It will also create `environment/name.yml` (if not yet created) and add the extracted value:
+
+```yaml
+meta:
+  name: manifest-name
+```
+
+Multiple fields can be extracted into the same target file.
+
+Reference items from lists by their `name:` field (like `spiff` itself does):
+
+```
+makemespiffy manifest.yml "jobs.runner_z1.instances" environment/scaling.yml meta.instances.runner_z1
+```
+
+Complex objects can be extracted too:
+
+```
+makemespiffy manifest.yml "networks.cf1.subnets" environment/networking.yml meta.subnets.cf1
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/makemespiffy

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'makemespiffy'
+
+input_manifest_path=ARGV[0]
+input_scope=ARGV[1]
+output_spiff_template_path=ARGV[2]
+output_meta_scope=ARGV[3]
+
+if ARGV[3].nil? || ARGV[3].length == 0
+  $stderr.puts "USAGE: makemespiffy manifest.yml properties.something templates/properties.yml meta.something"
+  exit 1
+end
+
+input_manifest = MakeMeSpiffy::InputManifest.from_file(input_manifest_path)
+value = input_manifest.spiffy(input_scope, output_meta_scope)
+input_manifest.insert_scope_value(output_meta_scope, "(( merge ))")
+
+output_manifest = MakeMeSpiffy::OutputTemplateManifest.from_file(output_spiff_template_path)
+output_manifest.insert_scope_value(output_meta_scope, value)
+
+#
+# (( x.y.z )) is rendered as "(( x.y.z ))" in to_yaml; so explicitly
+# construct the YAMLTree
+#
+# Thanks @tenderlove for https://gist.github.com/tenderlove/8c3988b8f797dfee0a3a
+#
+def tree o, options = {}
+  visitor = Psych::Visitors::YAMLTree.create options
+  visitor << o
+  visitor.tree
+end
+
+t = tree(input_manifest.manifest)
+# find scalar nodes and modify them
+t.grep(Psych::Nodes::Scalar).each do |node|
+  node.plain  = true
+  node.quoted = false
+  node.style  = Psych::Nodes::Scalar::PLAIN
+end
+
+File.open(input_manifest_path, "w") do |file|
+  file << t.yaml
+end
+File.open(output_spiff_template_path, "w") do |file|
+  file << output_manifest.manifest.to_yaml
+end

data/lib/makemespiffy.rb

@@ -0,0 +1,7 @@
+module MakeMeSpiffy
+  # Your code goes here...
+end
+
+require "makemespiffy/version"
+require "makemespiffy/input_manifest"
+require "makemespiffy/output_template_manifest"

data/lib/makemespiffy/input_manifest.rb

@@ -0,0 +1,58 @@
+require "yaml"
+
+module MakeMeSpiffy
+  class InputManifest
+    attr_reader :manifest
+
+    class UnknownScopeError < Exception; end
+
+    def self.from_file(manifest_path)
+      file = YAML.load_file(manifest_path)
+      self.new(file)
+    end
+
+    def initialize(bosh_manifest_yaml)
+      @manifest = bosh_manifest_yaml
+    end
+
+    # Primary method to replace a chunk of manifest with a (( meta.scope ))
+    def spiffy(extraction_scope, meta_scope)
+      part, *other_parts = extraction_scope.split('.')
+
+      if manifest.is_a?(Array)
+        part_value = manifest.find { |item| item["name"] == part }
+      else
+        part_value = manifest[part]
+      end
+
+      if other_parts.size == 0
+        if part_value
+          manifest[extraction_scope] = "(( #{meta_scope} ))"
+        end
+        return part_value
+      else
+        unless part_value
+          raise UnknownScopeError, extraction_scope
+        end
+        inner_manifest = InputManifest.new(part_value)
+        return inner_manifest.spiffy(other_parts.join("."), meta_scope)
+      end
+    end
+
+    # Usage: insert_scope_value("meta.foo.bar", 1234)
+    # Will add the following into manifest YAML
+    #     meta:
+    #       foo:
+    #         bar: 1234
+    def insert_scope_value(meta_scope, value)
+      parts = meta_scope.split('.')
+      submanifest = manifest
+      scoping_parts, key = parts[0..-2], parts[-1]
+      for part in scoping_parts
+        submanifest[part] ||= {}
+        submanifest = submanifest[part]
+      end
+      submanifest[key] = value
+    end
+  end
+end

data/lib/makemespiffy/output_template_manifest.rb

@@ -0,0 +1,36 @@
+require "yaml"
+
+module MakeMeSpiffy
+  class OutputTemplateManifest
+    attr_reader :manifest
+
+    def self.from_file(manifest_path)
+      if File.exist?(manifest_path)
+        file = YAML.load_file(manifest_path)
+      else
+        file = {}
+      end
+      self.new(file)
+    end
+
+    def initialize(bosh_manifest_yaml)
+      @manifest = bosh_manifest_yaml
+    end
+
+    # Usage: insert_scope_value("meta.foo.bar", 1234)
+    # Will add the following into manifest YAML
+    #     meta:
+    #       foo:
+    #         bar: 1234
+    def insert_scope_value(meta_scope, value)
+      parts = meta_scope.split('.')
+      submanifest = manifest
+      scoping_parts, key = parts[0..-2], parts[-1]
+      for part in scoping_parts
+        submanifest[part] ||= {}
+        submanifest = submanifest[part]
+      end
+      submanifest[key] = value
+    end
+  end
+end

data/lib/makemespiffy/version.rb

@@ -0,0 +1,3 @@
+module MakeMeSpiffy
+  VERSION = "0.1.0"
+end

data/makemespiffy.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'makemespiffy/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "makemespiffy"
+  spec.version       = MakeMeSpiffy::VERSION
+  spec.authors       = ["Dr Nic Williams"]
+  spec.email         = ["drnicwilliams@gmail.com"]
+  spec.summary       = %q{Convert a flat BOSH manifest for something into a set of Spiff templates.}
+  spec.description   = %q{Convert a flat BOSH manifest for something into a set of Spiff templates.}
+  spec.homepage      = "https://github.com/cloudfoundry-community/makemespiffy"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

data/spec/fixtures/concourse.yml

@@ -0,0 +1,121 @@
+---
+name: concourse
+
+director_uuid: 3b9806e6-f540-11e4-a153-6c4008a663f0
+
+releases:
+  - name: concourse
+    version: latest
+  - name: garden-linux
+    version: latest
+
+jobs:
+  - name: web
+    instances: 1
+    resource_pool: concourse
+    networks:
+      - name: concourse
+        static_ips: &web-ips [10.244.8.2]
+    persistent_disk: 1024 # for consul
+    templates:
+      - {release: concourse, name: consul-agent}
+      - {release: concourse, name: atc}
+      - {release: concourse, name: tsa}
+    properties:
+      atc:
+        development_mode: true
+        publicly_viewable: true
+        postgresql:
+          database: &atc-db atc
+          role: &atc-role
+            name: atc
+            password: dummy-postgres-password
+
+      consul:
+        agent:
+          mode: server
+
+  - name: db
+    instances: 1
+    resource_pool: concourse
+    networks: [{name: concourse}]
+    persistent_disk: 10240
+    templates:
+      - {release: concourse, name: consul-agent}
+      - {release: concourse, name: postgresql}
+    properties:
+      postgresql:
+        databases: [{name: *atc-db}]
+        roles: [*atc-role]
+
+      consul:
+        agent:
+          servers: {lan: *web-ips}
+
+  - name: worker
+    instances: 1
+    resource_pool: concourse
+    networks: [{name: concourse}]
+    templates:
+      - {release: concourse, name: consul-agent}
+      - {release: garden-linux, name: garden}
+      - {release: concourse, name: groundcrew}
+    properties:
+      garden:
+        # cannot enforce quotas in bosh-lite
+        disk_quota_enabled: false
+
+        listen_network: tcp
+        listen_address: 0.0.0.0:7777
+
+        allow_host_access: true
+
+      consul:
+        agent:
+          servers: {lan: *web-ips}
+
+networks:
+  - name: concourse
+    subnets:
+      # network with static ip used for web
+      - range: 10.244.8.0/30
+        reserved: [10.244.8.1]
+        static: [10.244.8.2]
+        cloud_properties: {}
+
+      # networks for dynamic ips (db, workers, compilation vms)
+      - range: 10.244.8.4/30
+        reserved: [10.244.8.5]
+        cloud_properties: {}
+      - range: 10.244.8.8/30
+        reserved: [10.244.8.9]
+        cloud_properties: {}
+      - range: 10.244.8.12/30
+        reserved: [10.244.8.13]
+        cloud_properties: {}
+      - range: 10.244.8.16/30
+        reserved: [10.244.8.17]
+        cloud_properties: {}
+      - range: 10.244.8.20/30
+        reserved: [10.244.8.21]
+        cloud_properties: {}
+
+resource_pools:
+  - name: concourse
+    network: concourse
+    cloud_properties: {}
+    stemcell:
+      name: bosh-warden-boshlite-ubuntu-trusty-go_agent
+      version: latest
+
+compilation:
+  workers: 3
+  network: concourse
+  cloud_properties: {}
+
+update:
+  canaries: 1
+  max_in_flight: 3
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000

data/spec/fixtures/just_name.yml

@@ -0,0 +1,2 @@
+---
+name: concourse

data/spec/input_manifest_spec.rb

@@ -0,0 +1,62 @@
+describe MakeMeSpiffy::InputManifest do
+  describe 'concourse.yml' do
+    let(:fixture) { fixture_path("concourse.yml") }
+    subject { MakeMeSpiffy::InputManifest.from_file(fixture) }
+
+    describe "simple scope" do
+      it { expect(subject.manifest["name"]).to eq "concourse" }
+      describe "spiffy" do
+        before do
+          @value = subject.spiffy("name", "meta.name")
+        end
+        it {expect(subject.manifest["name"]).to eq "(( meta.name ))"}
+        it {expect(@value).to eq "concourse"}
+      end
+
+      describe "insert_scope_value" do
+        before do
+          subject.insert_scope_value("meta.name", "(( merge ))")
+        end
+        it {expect(subject.manifest["meta"]).to_not be_nil }
+        it {expect(subject.manifest["meta"]["name"]).to eq("(( merge ))") }
+      end
+    end
+
+    describe "nested scope" do
+      it { expect(subject.manifest["compilation"]["workers"]).to eq 3 }
+      describe "spiffy" do
+        before do
+          @value = subject.spiffy("compilation.workers", "meta.instances.workers")
+        end
+        it {expect(subject.manifest["compilation"]["workers"]).to eq "(( meta.instances.workers ))"}
+        it {expect(@value).to eq 3}
+      end
+
+      describe "insert_scope_value" do
+        before do
+          subject.insert_scope_value("meta.instances.workers", "(( merge ))")
+        end
+        it {expect(subject.manifest["meta"]).to_not be_nil }
+        it {expect(subject.manifest["meta"]["instances"]).to_not be_nil }
+        it {expect(subject.manifest["meta"]["instances"]["workers"]).to eq("(( merge ))") }
+      end
+
+      describe "bad input scope" do
+        xit { expect(subject.manifest["compilation"]["foobar"]).to be_nil }
+        xit { expect(subject.manifest["foobar"]["workers"]).to be_nil }
+      end
+    end
+
+    describe "complex scope through named array" do
+      it { expect(subject.manifest["jobs"][0]["name"]).to eq "web" }
+      describe "spiffy" do
+        before do
+          @value = subject.spiffy("jobs.web.instances", "meta.instances.web")
+        end
+        it {expect(subject.manifest["jobs"][0]["name"]).to eq "web"}
+        it {expect(subject.manifest["jobs"][0]["instances"]).to eq "(( meta.instances.web ))"}
+        it {expect(@value).to eq 1}
+      end
+    end
+  end
+end

data/spec/integration_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+cd $DIR/..
+
+set -e
+
+source_manifest=${source_manifest:-spec/fixtures/concourse.yml}
+manifest=tmp/manifest.yml
+makemespiffy() {
+  bundle exec bin/makemespiffy $@
+}
+
+rm -rf tmp
+mkdir -p tmp
+cp ${source_manifest} tmp/manifest.yml
+
+makemespiffy ${manifest} name tmp/name.yml meta.name
+makemespiffy ${manifest} director_uuid tmp/director.yml meta.director_uuid
+makemespiffy ${manifest} releases tmp/stub.yml meta.releases
+makemespiffy ${manifest} resource_pools.concourse.stemcell tmp/stub.yml meta.stemcell
+makemespiffy ${manifest} networks.concourse.subnets tmp/networks.yml meta.subnets
+makemespiffy ${manifest} jobs.web.instances tmp/scaling.yml meta.instances.web
+makemespiffy ${manifest} jobs.worker.instances tmp/scaling.yml meta.instances.worker
+
+spiff merge tmp/manifest.yml tmp/networks.yml tmp/scaling.yml tmp/name.yml tmp/director.yml tmp/stub.yml > tmp/spiffy.yml
+spiff diff ${source_manifest} tmp/spiffy.yml

data/spec/output_template_manifest_spec.rb

@@ -0,0 +1,13 @@
+describe MakeMeSpiffy::OutputTemplateManifest do
+  describe 'concourse.yml' do
+    subject { MakeMeSpiffy::OutputTemplateManifest.from_file("tmp/new.yml") }
+
+    it { expect(subject.manifest).to eq({}) }
+    it {
+      subject.insert_scope_value("meta.top.bottom", 123)
+      expect(subject.manifest["meta"]).not_to be_nil
+      expect(subject.manifest["meta"]["top"]).not_to be_nil
+      expect(subject.manifest["meta"]["top"]["bottom"]).to eq(123)
+    }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,26 @@
+# Copyright (c) 2015 Stark & Wayne, LLC
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test)
+
+$:.unshift(File.expand_path("../../lib", __FILE__))
+
+require "rspec/core"
+
+require "makemespiffy"
+
+# load all files in spec/support/* (but not lower down)
+Dir[File.dirname(__FILE__) + '/support/*'].each do |path|
+  require path unless File.directory?(path)
+end
+
+def fixture_path(path)
+  File.join(File.expand_path("../fixtures", __FILE__), path)
+end
+
+RSpec.configure do |c|
+  c.color = true
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: makemespiffy
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dr Nic Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Convert a flat BOSH manifest for something into a set of Spiff templates.
+email:
+- drnicwilliams@gmail.com
+executables:
+- makemespiffy
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/makemespiffy
+- lib/makemespiffy.rb
+- lib/makemespiffy/input_manifest.rb
+- lib/makemespiffy/output_template_manifest.rb
+- lib/makemespiffy/version.rb
+- makemespiffy.gemspec
+- spec/fixtures/concourse.yml
+- spec/fixtures/just_name.yml
+- spec/input_manifest_spec.rb
+- spec/integration_test.sh
+- spec/output_template_manifest_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/cloudfoundry-community/makemespiffy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Convert a flat BOSH manifest for something into a set of Spiff templates.
+test_files:
+- spec/fixtures/concourse.yml
+- spec/fixtures/just_name.yml
+- spec/input_manifest_spec.rb
+- spec/integration_test.sh
+- spec/output_template_manifest_spec.rb
+- spec/spec_helper.rb