bosh-workspace 0.8.5 → 0.9.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3dbff47b8f17bd6be608e2994e74bd9f22c509f3
-  data.tar.gz: 73dd2a4c5c3e90094a9b3967bda00c8460491c65
+  metadata.gz: a0e593085c05c3aa75cd00f1dd9d94782a3b8688
+  data.tar.gz: 55aaae5f63c4035d13650769a99605d90e678959
 SHA512:
-  metadata.gz: 81a9e37ecfe5b7c7983b9ecd3934a94af6984759868c2fba97eff004e9817e9decc1f509db9337491021b6ad06e0d78b65090bf57e7b9f91deb9ec99ec4420af
-  data.tar.gz: 3d1911a2617d98c87506b11a762d786b6564057adc1aa6f1f1149ef2277d682516242e458afb8adc5250dd2db33aadb0eed0cc237206658c6345d82475fb3893
+  metadata.gz: 4880764b4fa20f940a5ed3242f8317e58dfc5faeec1f35205b1b1e2dfb2fa729413831eabf71406bf116365d7861e68ca4da3cf3370d6d010454a33777f3d54c
+  data.tar.gz: 08daea7f845dcc9f5f9c8531b1504d88c6832cc1847dcbf49204830fd9e232bc087cd3efaca31b5afac293978de9d1f94f752f4fdd77c3ccf7e050a1be8d4398

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 *.generated_manifests
 *.DS_Store
+/spec/assets/manifests-repo/.releases/*

data/.rspec

@@ -1,3 +1,2 @@
---format progress
 --color
---require spec_helper
+--require spec_helper

data/.ruby-gemset

@@ -1 +1 @@
-bosh-manifests
+bosh-workspace

data/.travis.yml

@@ -1,4 +1,6 @@
 language: ruby
+cache: bundler
+sudo: false
 rvm:
   - 2.0
 addons:

data/Guardfile

@@ -1,7 +1,7 @@
-guard :rspec, cmd: 'rspec' do
-  notification :off
+guard :rspec, cmd: 'rspec', notification: false do
   watch(%r{^spec/(.+_spec)\.rb$})
   watch(%r{^lib/bosh/cli/commands/(.+)\.rb$})    { |m| "spec/commands/#{m[1]}_spec.rb" }
   watch(%r{^lib/bosh/workspace/(.+)\.rb$})    { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^lib/bosh/workspace/(.+)\.rake$})    { |m| "spec/#{m[1]}_task_spec.rb" }
   watch('spec/spec_helper.rb')  { "spec" }
 end

data/README.md

@@ -1,5 +1,5 @@
 # Bosh workspace
-[![Build Status](https://img.shields.io/travis/swisscom/bosh-workspace/master.svg?style=flat-square)](https://travis-ci.org/swisscom/bosh-workspace) [![Test Coverage](https://img.shields.io/codeclimate/coverage/github/rkoster/bosh-workspace.svg?style=flat-square)](https://codeclimate.com/github/rkoster/bosh-workspace) [![Code Climate](https://img.shields.io/codeclimate/github/rkoster/bosh-workspace.svg?style=flat-square)](https://codeclimate.com/github/rkoster/bosh-workspace) [![Dependency Status](https://img.shields.io/gemnasium/swisscom/bosh-workspace.svg?style=flat-square)](https://gemnasium.com/swisscom/bosh-workspace)
+[![Build Status](https://img.shields.io/travis/cloudfoundry-incubator/bosh-workspace/master.svg?style=flat-square)](https://travis-ci.org/cloudfoundry-incubator/bosh-workspace) [![Test Coverage](https://img.shields.io/codeclimate/coverage/github/rkoster/bosh-workspace.svg?style=flat-square)](https://codeclimate.com/github/rkoster/bosh-workspace) [![Code Climate](https://img.shields.io/codeclimate/github/rkoster/bosh-workspace.svg?style=flat-square)](https://codeclimate.com/github/rkoster/bosh-workspace) [![Dependency Status](https://img.shields.io/gemnasium/cloudfoundry-incubator/bosh-workspace.svg?style=flat-square)](https://gemnasium.com/cloudfoundry-incubator/bosh-workspace)
 
 
 This is a `bosh` cli plugin for creating reproducible and upgradable deployments.
@@ -18,9 +18,9 @@ cd fg-boshworkspace
 Lets create the initial files & directories.
 ```
 mkdir deployments templates
-echo 'source "https://rubygems.org"\n\ngem "bosh-workspace"' > Gemfile
-echo "2.0.0" > .ruby-version
-echo '.stemcells*\n.deployments*\n.releases*\n.stubs*\n' > .gitignore
+echo -e 'source "https://rubygems.org"\n\ngem "bosh-workspace"' > Gemfile
+echo "2.1.0" > .ruby-version
+echo -e '.stemcells*\n.deployments*\n.releases*\n.stubs*\n' > .gitignore
 ```
 
 Now install the gems by running bundler.

data/bosh-workspace.gemspec

@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "bosh_common",  ">= 1.2682.0"
   spec.add_runtime_dependency "semi_semantic", "~> 1.1.0"
   spec.add_runtime_dependency "membrane", "~> 1.1.0"
-  spec.add_runtime_dependency "git", "~> 1.2.6"
+  spec.add_runtime_dependency "hashdiff", "~> 0.2.1"
+  spec.add_runtime_dependency "rugged", "~> 0.22.0b5"
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rspec", "~> 3.1.0"

data/lib/bosh/cli/commands/deployment_patch.rb

@@ -0,0 +1,96 @@
+require "bosh/workspace"
+
+module Bosh::Cli::Command
+  class DeploymentPatch < Base
+    include Bosh::Workspace::ProjectDeploymentHelper
+    include Bosh::Workspace::GitCredenialsHelper
+
+    usage "create deployment patch"
+    desc "Extract patch from the current directory and optionally writes to file"
+    def create(deployment_patch)
+      require_project_deployment
+      current_deployment_patch.to_file(deployment_patch)
+      say "Wrote patch to #{deployment_patch}"
+    end
+
+    usage "apply deployment patch"
+    desc "Apply a build patch to the current working directory"
+    option "--dry-run", "only show the changes without applying them"
+    option "--no-commit", "do not commit applied changes"
+    def apply(deployment_patch)
+      require_project_deployment
+      @patch = Bosh::Workspace::DeploymentPatch.from_file(deployment_patch)
+      validate_deployment_patch(@patch, deployment_patch)
+
+      if current_deployment_patch.changes?(@patch)
+        if !options[:dry_run]
+          fetch_repo(templates_dir) if @patch.templates_ref
+          @patch.apply(current_deployment_file, templates_dir)
+          commit_all unless options[:no_commit]
+          say "Successfully applied deployment patch:"
+        else
+          say "Deployment patch:"
+        end
+
+        say patch_changes_table
+      else
+        say "No changes, nothing to do"
+      end
+    end
+
+    private
+
+    def templates_dir
+      File.join(Dir.getwd, 'templates')
+    end
+
+    def current_deployment_file
+      @current_deployment_file ||= project_deployment.file
+    end
+
+    def current_deployment_patch
+      @current_deployment_patch ||= begin
+        Bosh::Workspace::DeploymentPatch
+          .create(current_deployment_file, templates_dir)
+      end
+    end
+
+    def validate_deployment_patch(patch, file)
+      unless patch.valid?
+        say("Validation errors:".make_red)
+        patch.errors.each { |error| say("- #{error}") }
+        err("'#{file}' is not valid".make_red)
+      end
+    end
+
+    def commit_all
+      index = repo.index
+      index.read_tree(repo.head.target.tree)
+      index.add_all
+      options = {
+        tree: index.write_tree(repo),
+        message: changes_message,
+        update_ref: 'HEAD'
+      }
+      Rugged::Commit.create(repo, options)
+    end
+
+    def repo
+      @repo ||= Rugged::Repository.new(Dir.getwd)
+    end
+
+    def patch_changes
+      @patch_changes ||= current_deployment_patch.changes(@patch)
+    end
+
+    def changes_message
+      "Applied " + patch_changes.map { |k, v| "#{k} #{v}" }.join(', ').to_s
+    end
+
+    def patch_changes_table
+      table do |t|
+        patch_changes.each { |k, v| t << [k.to_s, v] }
+      end
+    end
+  end
+end

data/lib/bosh/cli/commands/prepare.rb

@@ -5,6 +5,7 @@ module Bosh::Cli::Command
     include Bosh::Cli::Validation
     include Bosh::Workspace
     include ProjectDeploymentHelper
+    include GitCredenialsHelper
     include ReleaseHelper
     include StemcellHelper
 
@@ -25,7 +26,8 @@ module Bosh::Cli::Command
 
     def prepare_release_repos
       project_deployment_releases.each do |release|
-        say "Cloning release '#{release.name.make_green}' to satisfy template references"
+        say "Fetching release '#{release.name.make_green}' to satisfy template references"
+        fetch_or_clone_repo(release.repo_dir, release.git_url)
         release.update_repo
         msg = "Version '#{release.version.to_s.make_green}'"
         msg = "Ref '#{release.ref.make_green}'" if release.ref
@@ -64,11 +66,11 @@ module Bosh::Cli::Command
         cached_stemcell_upload(stemcell)
       end
     end
-    
-    def cached_stemcell_upload(stemcell) 
+
+    def cached_stemcell_upload(stemcell)
       unless stemcell.downloaded?
         say "Downloading '#{stemcell.name_version.make_green}'"
-        stemcell_download(stemcell.file_name) 
+        stemcell_download(stemcell.file_name)
       end
       say "Uploading '#{stemcell.name_version.make_green}'"
       stemcell_upload(stemcell.file)

data/lib/bosh/workspace/credentials.rb

@@ -0,0 +1,30 @@
+module Bosh::Workspace
+  class Credentials
+    include Bosh::Cli::Validation
+
+    def initialize(file)
+      @raw_credentials = YAML.load_file file
+    end
+
+    def perform_validation(options = {})
+      Schemas::Credentials.new.validate @raw_credentials
+    rescue Membrane::SchemaValidationError => e
+      errors << e.message
+    end
+
+    def find_by_url(url)
+      credentials[url]
+    end
+
+    private
+
+    def credentials
+      @credentials ||= begin
+        Hash[@raw_credentials.map do |c|
+          [c.delete('url'),
+          c.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }]
+        end]
+      end
+    end
+  end
+end

data/lib/bosh/workspace/deployment_patch.rb

@@ -0,0 +1,90 @@
+module Bosh::Workspace
+  class DeploymentPatch
+    include Bosh::Cli::Validation
+    attr_reader :stemcells, :releases, :templates_ref
+
+    def self.create(deployment_file, templates_dir)
+      if File.exist? File.join(templates_dir, '.git')
+        ref = Rugged::Repository.new(templates_dir).head.target.oid
+      end
+      deployment = YAML.load_file deployment_file
+      new(deployment["stemcells"], deployment["releases"], ref)
+    end
+
+    def self.from_file(patch_file)
+      a = YAML.load_file patch_file
+      new(a["stemcells"], a["releases"], a["templates_ref"])
+    end
+
+    def initialize(stemcells, releases, templates_ref)
+      @stemcells = stemcells
+      @releases = releases
+      @templates_ref = templates_ref
+    end
+
+    def perform_validation(options = {})
+      Schemas::DeploymentPatch.new.validate to_hash
+    rescue Membrane::SchemaValidationError => e
+      errors << e.message
+    end
+
+    def to_hash
+      { "stemcells" => stemcells, "releases" => releases }
+        .tap { |h| h["templates_ref"] = templates_ref if templates_ref }
+    end
+
+    def to_yaml
+      to_hash.to_yaml
+    end
+
+    def to_file(patch_file)
+      IO.write(patch_file, to_yaml)
+    end
+
+    def apply(deployment_file, templates_dir)
+      checkout_submodule(templates_dir, templates_ref) if templates_ref
+      deployment = YAML.load_file deployment_file
+      deployment.merge! 'stemcells' => stemcells, 'releases' => releases
+      IO.write(deployment_file, deployment.to_yaml)
+    end
+
+    def changes(patch)
+      {
+        stemcells: item_changes(stemcells, patch.stemcells),
+        releases: item_changes(releases, patch.releases),
+        templates_ref: item_changes(templates_ref, patch.templates_ref)
+      }.reject { |_, v| v.empty? }
+    end
+
+    def changes?(patch)
+      !changes(patch).empty?
+    end
+
+    private
+
+    def item_changes(old, new)
+      return [] unless old
+      old, new = presentify_item(old), presentify_item(new)
+      changes = HashDiff.diff(old, new).map { |a| a.join(' ').squeeze(' ') }
+      presentify_changes(changes).join(', ')
+    end
+
+    def presentify_changes(changes)
+      translations = { '~' => "changed", '-' => 'removed', '+' => 'added' }
+      changes.map { |l| l.gsub(/^./) { |m| translations.fetch(m, m) } }
+    end
+
+    def presentify_item(item)
+      item.is_a?(Array) ? versions_hash(item) : item[0..6]
+    end
+
+    def versions_hash(array)
+      Hash[array.map { |v| [v["name"], v["version"]]}]
+    end
+
+    def checkout_submodule(dir, ref)
+      repo = Rugged::Repository.new(dir)
+      repo.checkout ref, strategy: :force
+    end
+  end
+end

data/lib/bosh/workspace/helpers/git_credentials_helper.rb

@@ -0,0 +1,95 @@
+module Bosh::Workspace
+  module GitCredenialsHelper
+    REFSPEC = ['HEAD:refs/remotes/origin/HEAD']
+
+    def fetch_or_clone_repo(dir, url)
+      repo = File.exist?(dir) ? open_repo(dir) : init_repo(dir, url)
+      fetch_and_checkout(repo)
+    end
+
+    def fetch_repo(dir)
+      fetch_and_checkout(open_repo(dir))
+    end
+
+    private
+
+    def fetch_and_checkout(repo)
+      url = repo.remotes['origin'].url
+      repo.fetch('origin', REFSPEC, connection_options_for(repo, url))
+      repo.checkout 'refs/remotes/origin/HEAD', strategy: :force
+    end
+
+    def connection_options_for(repo, url)
+      return {} if check_connection(repo, url)
+
+      options = { credentials: require_credetials_for(url) }
+      unless check_connection(repo, url, options)
+        say "Using credentials from: #{credentials_file}"
+        err "Invalid credentials for: #{url}"
+      end
+      options
+    end
+
+    def check_connection(repo, url, options = {})
+      repo.remotes.create_anonymous(url).check_connection(:fetch, options)
+    end
+
+    def init_repo(dir, url)
+      FileUtils.mkdir_p File.dirname(dir)
+      Rugged::Repository.init_at(dir).tap do |repo|
+        repo.remotes.create('origin', url)
+      end
+    end
+
+    def open_repo(dir)
+      Rugged::Repository.new(dir)
+    end
+
+    def credentials
+      @credentials ||= Credentials.new(credentials_file)
+    end
+
+    def require_credetials_for(url)
+      unless File.exist? credentials_file
+        say("Authentication is required for: #{url}".make_red)
+        err("Credentials file does not exist: #{credentials_file}".make_red)
+      end
+      unless credentials.valid?
+        say("Validation errors:".make_red)
+        credentials.errors.each { |error| say("- #{error}") }
+        err("'#{credentials_file}' is not valid".make_red)
+      end
+      if creds = credentials.find_by_url(url)
+        load_git_credentials(creds)
+      else
+        say("Credential look up failed in: #{credentials_file}")
+        err("No credentials found for: #{url}".make_red)
+      end
+    end
+
+    def credentials_file
+      File.join work_dir, '.credentials.yml'
+    end
+
+    def load_git_credentials(credentials)
+      case credentials.keys
+      when %i(private_key)
+        options = {
+          username: 'git',
+          privatekey: temp_key_file(credentials[:private_key])
+        }
+        Rugged::Credentials::SshKey.new(options)
+      when %i(username password)
+        Rugged::Credentials::UserPassword.new(credentials)
+      end
+    end
+
+    def temp_key_file(key)
+      file = Tempfile.new('sshkey')
+      file.write key
+      file.close
+      File.chmod(0600, file.path)
+      file.path
+    end
+  end
+end

data/lib/bosh/workspace/helpers/spiff_helper.rb

@@ -11,6 +11,7 @@ module Bosh::Workspace
     private
 
     def spiff(verb, params)
+      params.map!(&:shellescape)
       cmd = ["spiff", verb.to_s] + params + ["2>&1"]
       sh(cmd.join(" "), :yield => :on_false) do |result|
         spiff_not_found if result.not_found?

data/lib/bosh/workspace/project_deployment.rb

@@ -1,59 +1,17 @@
-require "membrane"
 module Bosh::Workspace
   class ProjectDeployment
     include Bosh::Cli::Validation
     attr_writer :director_uuid
     attr_reader :file
 
-    RELEASE_SCHEMA = Membrane::SchemaParser.parse do
-      {
-        "name"    => String,
-        "version" => enum(Integer, "latest"),
-        optional("ref") => enum(String),
-        "git"     => String,
-      }
-    end
-
-    class StemcellVersionValidator < Membrane::Schemas::Base
-      def validate(object)
-        return if object.is_a? Integer
-        return if object.is_a? Float
-        return if object == "latest"
-        return if object.to_s =~ /^\d+\.\d+$/
-        raise Membrane::SchemaValidationError.new(
-          "Should match: latest, version.patch or version. Given: #{object}")
-      end
-    end
-
-    STEMCELL_SCHEMA = Membrane::SchemaParser.parse do
-      {
-        "name"    => String,
-        "version" => StemcellVersionValidator.new
-      }
-    end
-
-    UUID_REGEX = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
-
-    PROJECT_DEPLOYMENT_SCHEMA = Membrane::SchemaParser.parse do
-      {
-        "name"                  => String,
-        "director_uuid"         => enum(UUID_REGEX, "current"),
-        optional("domain_name") => String,
-        "releases"              => [RELEASE_SCHEMA],
-        "stemcells"             => [STEMCELL_SCHEMA],
-        "templates"             => [String],
-        "meta"                  => Hash
-      }
-    end
-
     def initialize(file)
       @file = file
       err("Deployment file does not exist: #{file}") unless File.exist? @file
-      @manifest = Psych.load(File.read(@file))
+      @manifest = YAML.load_file @file
     end
 
     def perform_validation(options = {})
-      PROJECT_DEPLOYMENT_SCHEMA.validate @manifest
+      Schemas::ProjectDeployment.new.validate @manifest
     rescue Membrane::SchemaValidationError => e
       errors << e.message
     end

data/lib/bosh/workspace/release.rb

@@ -1,21 +1,17 @@
-require "git"
 module Bosh::Workspace
   class Release
-    attr_reader :name, :git_uri, :repo_dir, :ref
+    attr_reader :name, :git_url, :repo_dir
 
     def initialize(release, releases_dir)
       @name = release["name"]
       @ref = release["ref"]
       @spec_version = release["version"]
-      @git_uri = release["git"]
+      @git_url = release["git"]
       @repo_dir = File.join(releases_dir, @name)
-      init_repo
     end
 
     def update_repo
-      @repo.fetch "origin", tags: true
-      @repo.checkout last_commit
-      @repo.checkout ref || version_ref
+      repo.checkout ref || version_ref, strategy: :force
     end
 
     def manifest_file
@@ -38,34 +34,35 @@ module Bosh::Workspace
       @spec_version.to_i
     end
 
+    def ref
+      @ref && repo.lookup(@ref).oid
+    end
+
     private
 
+    def repo
+      @repo ||= Rugged::Repository.new(repo_dir)
+    end
+
     # transforms releases/foo-1.yml, releases/bar-2.yml to:
     # { "1" => foo-1.yml, "2" => bar-2.yml }
     def final_releases
       @final_releases ||= begin
-        Hash[Dir[File.join(repo_dir, "releases", "*.yml")]
+        new_style_repo = File.directory?(File.join(repo_dir, "releases", @name))
+        releases_dir = new_style_repo ? "releases/#{@name}" : "releases"
+
+        Hash[Dir[File.join(repo_dir, releases_dir, "*.yml")]
           .reject { |f| f[/index.yml/] }
-          .map { |dir| File.join("releases", File.basename(dir)) }
+          .map { |dir| File.join(releases_dir, File.basename(dir)) }
           .map { |version| [version[/(\d+)/].to_i, version] }]
       end
     end
 
-    def last_commit
-      @repo.log.object("origin").first
-    end
-
     def version_ref
-      @repo.log.object(manifest).first
-    end
-
-    def init_repo
-      if File.directory?(repo_dir)
-        @repo ||= Git.open(repo_dir)
-      else
-        releases_dir = File.dirname(repo_dir)
-        FileUtils.mkdir_p(releases_dir)
-        @repo = Git.clone(@git_uri, @name, path: releases_dir)
+      # figure out the last commit which changed the release manifest file
+      # in other words the commit sha of the final release
+      repo.walk(repo.head.target) do |commit|
+        return commit.oid if commit.tree.path(manifest)
       end
     end
   end

data/lib/bosh/workspace/schemas/credentials.rb

@@ -0,0 +1,27 @@
+module Bosh::Workspace
+  module Schemas
+    class Credentials < Membrane::Schemas::Base
+      def validate(object)
+        Membrane::SchemaParser.parse do
+          [enum(UsernamePassword.new, SshKey.new)]
+        end.validate object
+      end
+
+      class UsernamePassword < Membrane::Schemas::Base
+        def validate(object)
+          Membrane::SchemaParser.parse do
+            { "url" => String, "username" => String, "password" => String }
+          end.validate object
+        end
+      end
+
+      class SshKey < Membrane::Schemas::Base
+        def validate(object)
+          Membrane::SchemaParser.parse do
+            { "url" => String, "private_key" => String }
+          end.validate object
+        end
+      end
+    end
+  end
+end

data/lib/bosh/workspace/schemas/deployment_patch.rb

@@ -0,0 +1,15 @@
+module Bosh::Workspace
+  module Schemas
+    class DeploymentPatch < Membrane::Schemas::Base
+      def validate(object)
+        Membrane::SchemaParser.parse do
+          {
+            "stemcells"               => Stemcells.new,
+            "releases"                => Releases.new,
+            optional("templates_ref") => String
+          }
+        end.validate object
+      end
+    end
+  end
+end

data/lib/bosh/workspace/schemas/project_deployment.rb

@@ -0,0 +1,21 @@
+module Bosh::Workspace
+  module Schemas
+    class ProjectDeployment < Membrane::Schemas::Base
+      UUID_REGEX = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+
+      def validate(object)
+        Membrane::SchemaParser.parse do
+          {
+            "name"                  => String,
+            "director_uuid"         => enum(UUID_REGEX, "current"),
+            optional("domain_name") => String,
+            "releases"              => Releases.new,
+            "stemcells"             => Stemcells.new,
+            "templates"             => [String],
+            "meta"                  => Hash
+          }
+        end.validate object
+      end
+    end
+  end
+end

data/lib/bosh/workspace/schemas/releases.rb

@@ -0,0 +1,16 @@
+module Bosh::Workspace
+  module Schemas
+    class Releases < Membrane::Schemas::Base
+      def validate(object)
+        Membrane::SchemaParser.parse do
+          [{
+            "name"          => String,
+            "version"       => enum(Integer, "latest"),
+            optional("ref") => enum(String),
+            "git"           => String,
+          }]
+        end.validate object
+      end
+    end
+  end
+end