bosh-workspace 0.7.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NjQwYmNmNWQzMDgxYjhmNTFkOGYxNGJmMTNjOWE1ODZiM2U5ZmJlYg==
+  data.tar.gz: !binary |-
+    Y2Y1M2Y2MGZjNmY5MDFhY2NiN2RhYTk3MzliZDA3ZDg5NzFlZWIxMg==
+SHA512:
+  metadata.gz: !binary |-
+    NGE4ZWQzNjVmY2ZkZDgwOGE3Y2QyNzVlMGUxMGZkMjBkYmE2ZGJlNzFhY2M0
+    YjE2MmY5OThlOWI1MDcxNmJiNTRkNWM0OWM1MWUwMTNlNzYxODcxNjI5OGJh
+    N2M0OGI5OTEwOTM4M2U5ZTkyNTdmMjE3MTAwOTQwNThkYWZmMzk=
+  data.tar.gz: !binary |-
+    NGFhZmE1YmMxNzdiZDZhZGFiMjYzNGZiN2UwNmFkZmYyZTZjMzdjMTNlODAz
+    ZjI5NjIyMzQ3NDQzZTRmZTQzNGNmYmM0YWI4MjI2ODBhMzdiNGE1NTc5Yzgy
+    OTk1YWFlODU5ZGYwNTBiNDNjMTJlMDkzODg5Y2I4YTYxMzQyNDY=

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.generated_manifests
+*.DS_Store

data/.rspec

@@ -0,0 +1,3 @@
+--format progress
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+bosh-manifests

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3-p484

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bosh-manifests.gemspec
+gemspec
+
+group :development do
+  gem "guard-rspec"
+end

data/Guardfile

@@ -0,0 +1,7 @@
+guard 'rspec', spec_paths: ["spec"] do
+  notification :off
+  watch(%r{^spec/(.+_spec)\.rb$})
+  watch(%r{^lib/bosh/cli/commands/(.+)\.rb$})    { |m| "spec/plugin_spec.rb" }
+  watch(%r{^lib/bosh/cloudfoundry/(.+)\.rb$})    { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Ruben Koster
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,70 @@
+# Bosh workspace
+
+## Design Goals
+Enabling BOSH operators to share common configuration between different deployments.
+For example having a `QA` and `production` deployment for which only networking differs.
+
+## Installation
+Before you start make sure ruby, bundler and spiff are available on your system.
+Instructions for installing spiff can found [here](https://github.com/cloudfoundry-incubator/spiff#installation).
+
+Add this line to your application's Gemfile:
+
+    gem 'bosh-workspace'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bosh-workspace
+
+
+## Usage
+This BOSH plugin improves the deployments work-flow,
+by extending some of the build in commands bosh commands.
+
+**Set deployment**
+Sets the current deployment. Will search in `./deployments`.
+```
+bosh deployment cf-warden
+```
+
+**Prepare deployment**
+Resolves upstream templates (via releases).
+Resolves and uploads releases/stemcells.
+```
+bosh prepare deployment
+```
+
+**Deploy**
+Merges the specified templates into one deployment manifests using spiff.
+And uses this file to initiate a `deploy`.
+```
+bosh deploy
+```
+
+### Deployment file structure
+A deployment file has the following structure:
+
+**name:**
+The name of your deployment
+
+**director_uuid:**
+The director_uuid of the targeted BOSH director
+
+**releases:**
+Array of releases used for resolving upstream templates
+
+**meta:**
+The meta hash is the last file merged into the final deployment file.
+It can be used to define properties deployment specific properties.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bosh-workspace.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bosh/workspace/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bosh-workspace"
+  spec.version       = Bosh::Manifests::VERSION
+  spec.authors       = ["Ruben Koster"]
+  spec.email         = ["rkoster@starkandwayne.com"]
+  spec.description   = %q{Manage your bosh workspace}
+  spec.summary       = %q{Manage your bosh workspace}
+  spec.homepage      = "http://starkandwayne.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.0.0'
+
+  spec.add_runtime_dependency "bosh_cli",  ">= 1.1722.0"
+  spec.add_runtime_dependency "git", "~> 1.2.6"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec", "~> 2.13.0"
+  spec.add_development_dependency "rspec-fire"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "archive-zip", "~> 0.6.0"
+end

data/lib/bosh/cli/commands/01_make_manifest.rb

@@ -0,0 +1,74 @@
+require "bosh/workspace"
+
+module Bosh::Cli::Command
+  class Manifests < Base
+    include Bosh::Cli::Validation
+    include Bosh::Manifests
+    include ProjectDeploymentHelper
+
+    # Hack to unregister original deployment command
+    Bosh::Cli::Config.instance_eval("@commands.delete('deployment')")
+
+    usage "deployment"
+    desc "Get/set current deployment"
+    def set_current(filename = nil)
+      unless filename.nil?
+        deployment = find_deployment(filename)
+
+        if project_deployment_file?(deployment)
+          self.project_deployment = deployment
+          validate_project_deployment
+          filename = project_deployment.merged_file
+          create_placeholder_deployment unless File.exists?(filename)
+        end
+      end
+
+      deployment_cmd(options).set_current(filename)
+    end
+
+    usage "prepare deployment"
+    desc "Resolve deployment requirements"
+    def prepare
+      require_project_deployment
+      auth_required
+
+      release_manager.update_release_repos
+
+      # TODO: Implement
+      # current_deployment.stemcells.each do |stemcell|
+      #   stemcell.upload unless stemcell.exists?
+      # end
+    end
+
+    # Hack to unregister original deploy command
+    Bosh::Cli::Config.instance_eval("@commands.delete('deploy')")
+
+    usage "deploy"
+    desc "Deploy according to the currently selected deployment manifest"
+    option "--recreate", "recreate all VMs in deployment"
+    def deploy
+      if project_deployment?
+        require_project_deployment
+        build_project_deployment
+      end
+
+      deployment_cmd(options).perform
+    end
+
+    private
+
+    def deployment_cmd(options = {})
+      cmd ||= Bosh::Cli::Command::Deployment.new
+      options.each do |key, value|
+        cmd.add_option key.to_sym, value
+      end
+      cmd
+    end
+
+    def release_manager
+      @release_manager ||= begin
+        ReleaseManager.new(project_deployment.releases, work_dir)
+      end
+    end
+  end
+end

data/lib/bosh/workspace/deployment_manifest.rb

@@ -0,0 +1,67 @@
+module Bosh::Manifests
+  class DeploymentManifest
+    include Bosh::Cli::Validation
+    attr_writer :director_uuid
+    attr_reader :file
+
+    def initialize(file)
+      @file = file
+      err("Deployment file does not exist: #{file}") unless File.exist? @file
+      @manifest = Psych.load(File.read(@file))
+    end
+
+    def perform_validation(options = {})
+      if @manifest.is_a?(Hash)
+        unless @manifest.has_key?("name") && @manifest["name"].is_a?(String)
+          errors << "Manifest should contain a name"
+        end
+
+        unless @manifest.has_key?("director_uuid") && @manifest["director_uuid"].is_a?(String)
+          errors << "Manifest should contain a director_uuid"
+        end
+
+        unless @manifest.has_key?("templates") && @manifest["templates"].is_a?(Array)
+          errors << "Manifest should contain templates"
+        end
+
+        unless @manifest.has_key?("releases") && @manifest["releases"].is_a?(Array)
+          errors << "Manifest should contain releases"
+        end
+
+        unless @manifest.has_key?("meta") && @manifest["meta"].is_a?(Hash)
+          errors << "Manifest should contain meta hash"
+        end
+      else
+        errors << "Manifest should be a hash"
+      end
+    end
+
+    def director_uuid
+      @director_uuid || @manifest["director_uuid"]
+    end
+
+    def merged_file
+      @merged_file ||= begin
+        path = File.join(file_dirname, "../.deployments", file_basename)
+        FileUtils.mkpath File.dirname(path)
+        File.expand_path path
+      end
+    end
+
+    %w[name templates releases meta].each do |var|
+      define_method var do
+        @manifest[var]
+      end
+    end
+
+    private
+
+    def file_basename
+      File.basename(@file)
+    end
+
+    def file_dirname
+      File.dirname(@file)
+    end
+  end
+end

data/lib/bosh/workspace/helpers/project_deployment_helper.rb

@@ -0,0 +1,103 @@
+module Bosh::Manifests
+  module ProjectDeploymentHelper
+
+    def project_deployment?
+      File.exists?(project_deployment_file) &&
+        project_deployment_file?(project_deployment_file)
+    end
+
+    def project_deployment
+      @project_deployment ||= DeploymentManifest.new(project_deployment_file)
+    end
+
+    def project_deployment=(deployment)
+      @project_deployment = DeploymentManifest.new(deployment)
+    end
+
+    def project_deployment_file?(deployment)
+      Psych.load(File.read(deployment)).has_key?("templates")
+    end
+
+    def require_project_deployment
+      unless project_deployment?
+        err("Deployment is not a project deployment: #{deployment}")
+      end
+      validate_project_deployment
+    end
+
+    def create_placeholder_deployment
+      resolve_director_uuid
+
+      File.open(project_deployment.merged_file, "w") do |file|
+        file.write(placeholder_deployment_content)
+      end
+    end
+
+    def validate_project_deployment
+      unless project_deployment.valid?
+        say("Validation errors:".make_red)
+        project_deployment.errors.each { |error| say("- #{error}") }
+        err("'#{project_deployment.file}' is not valid".make_red)
+      end
+    end
+
+    def build_project_deployment
+      resolve_director_uuid
+
+      say("Generating deployment manifest")
+      ManifestBuilder.build(project_deployment, work_dir)
+    end
+
+    def resolve_director_uuid
+      use_targeted_director_uuid if director_uuid_current?
+    end
+
+    private
+
+    def use_targeted_director_uuid
+      no_warden_error unless warden_cpi?
+      project_deployment.director_uuid = bosh_uuid
+    end
+
+    def no_warden_error
+      say("Please put 'director_uuid: #{bosh_uuid}' in '#{deployment}'")
+      err("'director_uuid: current' may not be used in production")
+    end
+
+    def director_uuid_current?
+      project_deployment.director_uuid == "current"
+    end
+
+    def warden_cpi?
+      bosh_status["cpi"] == "warden"
+    end
+
+    def project_deployment_file
+      @project_deployment_file ||= begin
+        path = File.join(deployment_dir, "../deployments", deployment_basename)
+        File.expand_path path
+      end
+    end
+
+    def deployment_dir
+      File.dirname(deployment)
+    end
+
+    def deployment_basename
+      File.basename(deployment)
+    end
+
+    def placeholder_deployment_content
+      { "director_uuid" => project_deployment.director_uuid }.to_yaml +
+        "# Don't edit; placeholder deployment for: #{project_deployment.file}"
+    end
+
+    def bosh_status
+      @bosh_status ||= director.get_status
+    end
+
+    def bosh_uuid
+      bosh_status["uuid"]
+    end
+  end
+end

data/lib/bosh/workspace/helpers/spiff_helper.rb

@@ -0,0 +1,33 @@
+module Bosh::Manifests
+  module SpiffHelper
+    include Bosh::Exec
+
+    def spiff_merge(templates, target_file)
+      spiff(:merge, templates) do |output|
+        File.open(target_file, 'w') { |file| file.write(output) }
+      end
+    end
+
+    private
+
+    def spiff(verb, params)
+      cmd = ["spiff", verb.to_s] + params + ["2>&1"]
+      sh(cmd.join(" "), :yield => :on_false) do |result|
+        spiff_not_found if result.not_found?
+        spiff_failed(result.command, result.output) if result.failed?
+        yield result.output
+      end
+    end
+
+    def spiff_not_found
+      say("Can't find spiff in $PATH".make_red)
+      say("Go to spiff.cfapps.io for installation instructions")
+      err("Please make sure spiff is installed")
+    end
+
+    def spiff_failed(command, output)
+      say("Command failed: '#{command}'")
+      err(output)
+    end
+  end
+end

data/lib/bosh/workspace/manifest_builder.rb

@@ -0,0 +1,68 @@
+module Bosh::Manifests
+  class ManifestBuilder
+    include Bosh::Manifests::SpiffHelper
+
+    def self.build(manifest, work_dir)
+      manifest_builder = ManifestBuilder.new(manifest, work_dir)
+      manifest_builder.merge_templates
+    end
+
+    def initialize(manifest, work_dir)
+      @manifest = manifest
+      @work_dir = work_dir
+    end
+
+    def merge_templates
+      spiff_merge spiff_template_paths, @manifest.merged_file
+    end
+
+    private
+
+    def spiff_template_paths
+      spiff_templates = template_paths
+      spiff_templates << stub_file_path
+    end
+
+    def template_paths
+      @manifest.templates.map { |t| template_path(t) }
+    end
+
+    def stub_file_path
+      path = hidden_file_path(:stubs)
+      File.open(path, 'w') { |file| file.write(stub_file_content) }
+      path
+    end
+
+    def stub_file_content
+      {
+        "director_uuid" => @manifest.director_uuid,
+        "releases" => filterd_releases,
+        "meta" => @manifest.meta
+      }.to_yaml
+    end
+
+    def filterd_releases
+      allowed_keys = %w[name version]
+      @manifest.releases.map do |release|
+        release.select { |key| allowed_keys.include?(key) }
+      end
+    end
+
+    def hidden_file_path(type)
+      File.join(hidden_dir_path(type), "#{@manifest.name}.yml")
+    end
+
+    def hidden_dir_path(name)
+      dir = File.join(@work_dir, ".#{name.to_s}")
+      Dir.mkdir(dir) unless File.exists? dir
+      dir
+    end
+
+    # TODO move to template validator which should have access to @work_dir
+    def template_path(template)
+      path = File.join(@work_dir, "templates", template)
+      err("Template does not exist: #{template}") unless File.exists? path
+      path
+    end
+  end
+end

data/lib/bosh/workspace/release.rb

@@ -0,0 +1,58 @@
+require "git"
+module Bosh::Manifests
+  class Release
+    def initialize(release, releases_dir)
+      @name = release["name"]
+      @version = release["version"]
+      @git_uri = release["git"]
+      @work_dir = File.join(releases_dir)
+    end
+
+    def checkout_current_version
+      repo.pull
+      repo.checkout(current_version_ref)
+    end
+
+    private
+
+    def repo
+      if File.directory?(repo_dir)
+        @repo ||= Git.open(repo_dir)
+      else
+        FileUtils.mkdir_p(@work_dir)
+        @repo = Git.clone(@git_uri, @name, path: @work_dir)
+      end
+    end
+
+    def current_version_ref
+      repo.log().object("releases/#{available_versions[current_version]}")
+    end
+
+    def current_version
+      if @version == "latest"
+        available_versions.keys.sort.last
+      else
+        version = @version.to_i
+        unless available_versions[version]
+          err("Could not find version: #{@version} for release: #{@name}")
+        end
+        version
+      end
+    end
+
+    # transforms releases/foo-1.yml, releases/bar-2.yml to:
+    # { "1" => foo-1.yml, "2" => bar-2.yml }
+    def available_versions
+      @available_versions ||= begin
+        Hash[Dir[File.join(repo_dir, "releases", "*.yml")].
+        reject { |f| f[/index.yml/] }.
+        map { |dir| File.basename(dir) }.
+        map { |version| [ version[/(\d+)/].to_i, version ] }]
+      end
+    end
+
+    def repo_dir
+      File.join(@work_dir, @name)
+    end
+  end
+end

data/lib/bosh/workspace/release_manager.rb

@@ -0,0 +1,14 @@
+module Bosh::Manifests
+  class ReleaseManager
+    def initialize(releases, work_dir)
+      releases_dir = File.join(work_dir, ".releases")
+      @releases = releases.map { |r| Release.new(r, releases_dir) }
+    end
+
+    def update_release_repos
+      @releases.each do |release|
+        release.checkout_current_version
+      end
+    end
+  end
+end

data/lib/bosh/workspace/version.rb

@@ -0,0 +1,5 @@
+module Bosh
+  module Manifests
+    VERSION = "0.7.0"
+  end
+end

data/lib/bosh/workspace.rb

@@ -0,0 +1,13 @@
+module Bosh; module Manifests; end; end
+
+require "cli/core_ext"
+require "cli/validation"
+
+require "bosh/workspace/helpers/spiff_helper"
+require "bosh/workspace/helpers/project_deployment_helper"
+
+require "bosh/workspace/manifest_builder"
+require "bosh/workspace/release_manager"
+require "bosh/workspace/release"
+require "bosh/workspace/deployment_manifest"
+require "bosh/workspace/version"

data/spec/assets/manifests-repo/.manifests/foo.yml

@@ -0,0 +1 @@
+name: foo

data/spec/assets/manifests-repo/deployments/bar.yml

@@ -0,0 +1 @@
+name: bar

data/spec/assets/manifests-repo/deployments/foo.yml

@@ -0,0 +1,4 @@
+name: foo
+templates:
+  - foo/bar.yml
+meta: {}