bosh-workspace 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c532d7cdf4e457cb635fa36d895f286ea586222d
-  data.tar.gz: 4378d7c1ee1a9bb859db2a55ceb3d48b1a6f90a8
+  metadata.gz: 095b4cec01561a415d908bc83c6a86e5a6861bac
+  data.tar.gz: 3038bc5ecd5756cf3672bbdc15405eee574a39c1
 SHA512:
-  metadata.gz: ccef1b2f1067ba4aeb70f036ac8550d9cb0a39a521402b18f55980101800257849f321d121277b4016b7be2eaa91d5c6e2d37e6cffc1bb966c87d64257f25ea3
-  data.tar.gz: 424aad5e9d34336c21119d4383b19b324352a96954606f0033b8738d803080575b6136c0f3e4614c92f47442cf75e07f80890cc28c15110dddc7ca296dc9c5a2
+  metadata.gz: 835a7af3d529d8264b273157ce4c2f80e5cdbe73ed3c8b817f469edebd874e730ff64110942b2f2aeb962930ac4fb0a3c39ef62445d9e0bb4c5ebbc04f32876c
+  data.tar.gz: 75e12938c90cea7260a8c2d16a6098d6c189bf9569431bf889e5cce7b23c70b57fafe297aa396b205e70cc54dc40adba7f47ec755007a9408cb60e3cd78bbd7d

data/bosh-workspace.gemspec

@@ -20,16 +20,17 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_runtime_dependency "bosh_cli",  ">= 1.2905.0"
-  spec.add_runtime_dependency "bosh_common",  ">= 1.2905.0"
+  spec.add_runtime_dependency "bosh_cli", ">= 1.2905.0"
+  spec.add_runtime_dependency "bosh_common", ">= 1.2905.0"
+  spec.add_runtime_dependency "bosh-template", ">= 1.2905.0"
   spec.add_runtime_dependency "semi_semantic", "~> 1.1.0"
   spec.add_runtime_dependency "membrane", "~> 1.1.0"
   spec.add_runtime_dependency "hashdiff", "~> 0.2.1"
   spec.add_runtime_dependency "rugged", "~> 0.23.0b3"
 
   spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "rspec", "~> 3.1.0"
-  spec.add_development_dependency "rspec-its", '~> 1.1.0'
+  spec.add_development_dependency "rspec", "~> 3.3.0"
+  spec.add_development_dependency "rspec-its", '~> 1.2.0'
   spec.add_development_dependency "rake"
   spec.add_development_dependency "archive-zip", "~> 0.7.0"
 end

data/lib/bosh/cli/commands/deployment_patch.rb

@@ -3,7 +3,6 @@ require "bosh/workspace"
 module Bosh::Cli::Command
   class DeploymentPatch < Base
     include Bosh::Workspace::ProjectDeploymentHelper
-    include Bosh::Workspace::GitCredentialsHelper
 
     usage "create deployment patch"
     desc "Extract patch from the current directory and optionally writes to file"

data/lib/bosh/cli/commands/prepare.rb

@@ -5,15 +5,16 @@ module Bosh::Cli::Command
     include Bosh::Cli::Validation
     include Bosh::Workspace
     include ProjectDeploymentHelper
-    include GitCredentialsHelper
     include ReleaseHelper
     include StemcellHelper
 
     usage "prepare deployment"
     desc "Resolve deployment requirements"
+    option "--local", "only perform local git operations (don't fetch remote)"
     def prepare
       require_project_deployment
       auth_required
+      offline! if options[:local]
       nl
       prepare_release_repos
       nl
@@ -28,19 +29,18 @@ module Bosh::Cli::Command
       project_deployment_releases.each do |release|
         require_git_url_error if release.git_url.nil?
         say "Fetching release '#{release.name.make_green}' to satisfy template references"
-        fetch_or_clone_repo(release.repo_dir, release.git_url)
         release.update_repo
-        release.required_submodules.each do |submodule|
-          fetch_or_clone_repo(File.join(release.repo_dir, submodule.path), submodule.url)
-          release.update_submodule(submodule)
-        end
-        msg = "Version '#{release.version.to_s.make_green}'"
-        msg = "Ref '#{release.ref.make_green}'" if release.ref
-        say "#{msg} has been checkout into:"
-        say "- #{release.repo_dir}"
+        print_prepare_release_repo_message(release)
       end
     end
 
+    def print_prepare_release_repo_message(release)
+      msg = "Version '#{release.version.to_s.make_green}'"
+      msg = "Ref '#{release.ref.make_green}'" if release.ref
+      say "#{msg} has been checkout into:"
+      say "- #{release.repo_dir}"
+    end
+
     def require_git_url_error
       say "`bosh prepare deployment' can not be used:"
       err("`git:' is missing from `release:'".make_red)
@@ -58,7 +58,7 @@ module Bosh::Cli::Command
         say "Skipping upload"
       elsif release.url
         say "Uploading '#{release.url}'"
-        release_remote(release.url, release.release_dir)
+        release_upload_from_url(release.url)
       else
         say "Uploading '#{release.name_version.make_green}'"
         release_upload(release.manifest_file, release.release_dir)
@@ -82,6 +82,7 @@ module Bosh::Cli::Command
 
     def cached_stemcell_upload(stemcell)
       unless stemcell.downloaded?
+        err "Stemcell not available offline: #{stemcell.file_name}" if offline?
         say "Downloading '#{stemcell.name_version.make_green}'"
         stemcell_download(stemcell.file_name)
       end

data/lib/bosh/workspace.rb

@@ -6,14 +6,15 @@ require "rugged"
 require "hashdiff"
 require "cli/core_ext"
 require "cli/validation"
+require "bosh/template/renderer"
 
-require "bosh/workspace/helpers/spiff_helper"
 require "bosh/workspace/helpers/project_deployment_helper"
-require "bosh/workspace/helpers/git_credentials_helper"
 require "bosh/workspace/helpers/release_helper"
 require "bosh/workspace/helpers/stemcell_helper"
 require "bosh/workspace/helpers/dns_helper"
+require "bosh/workspace/helpers/git_protocol_helper"
 
+require "bosh/workspace/merge_tool"
 require "bosh/workspace/schemas/project_deployment"
 require "bosh/workspace/schemas/deployment_patch"
 require "bosh/workspace/schemas/releases"
@@ -28,5 +29,5 @@ require "bosh/workspace/project_deployment"
 require "bosh/workspace/stub_file"
 require "bosh/workspace/deployment_patch"
 require "bosh/workspace/credentials"
-require "bosh/workspace/git_remote_url"
+require "bosh/workspace/git_credentials_provider"
 require "bosh/workspace/version"

data/lib/bosh/workspace/credentials.rb

@@ -1,13 +1,15 @@
 module Bosh::Workspace
   class Credentials
     include Bosh::Cli::Validation
+    include GitProtocolHelper
+    attr_reader :raw_credentials
 
     def initialize(file)
       @raw_credentials = YAML.load_file file
     end
 
     def perform_validation(options = {})
-      Schemas::Credentials.new.validate @raw_credentials
+      Schemas::Credentials.new.validate raw_credentials
     rescue Membrane::SchemaValidationError => e
       errors << e.message
     end
@@ -16,15 +18,20 @@ module Bosh::Workspace
       credentials[url]
     end
 
+    def url_protocols
+      Hash[raw_credentials.map { |c| [c['url'], git_protocol_from_url(c['url'])] }]
+    end
+
     private
 
     def credentials
       @credentials ||= begin
-        Hash[@raw_credentials.map do |c|
-          [c.delete('url'),
-          c.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }]
-        end]
+        Hash[raw_credentials.map { |c| [c.delete('url'), symbolize_keys(c)] }]
       end
     end
+
+    def symbolize_keys(hash)
+      hash.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }
+    end
   end
 end

data/lib/bosh/workspace/git_credentials_provider.rb

@@ -0,0 +1,80 @@
+module Bosh::Workspace
+  class GitCredentialsProvider
+    attr_reader :credentials_file
+
+    def initialize(credentials_file)
+      @credentials_file = credentials_file
+    end
+
+    def callback
+      proc do |url, user, allowed_types|
+        require_credentials_file_for!(url)
+        validate_credentials!
+        validate_url_protocol_support!
+        credentials_for(url, user, allowed_types)
+      end
+    end
+
+    private
+
+    def credentials
+      @credentials ||= Credentials.new(@credentials_file)
+    end
+
+    def require_credentials_file_for!(url)
+      return if File.exist? @credentials_file
+      say("Authentication is required for: #{url}".make_red)
+      err("Credentials file does not exist: #{@credentials_file}".make_red)
+    end
+
+    def validate_credentials!
+      return if credentials.valid?
+      say("Validation errors:".make_red)
+      credentials.errors.each { |error| say("- #{error}") }
+      err("'#{credentials_file}' is not valid".make_red)
+    end
+
+    def validate_url_protocol_support!
+      credentials.url_protocols.each do |url, protocol|
+        next if Rugged.features.include? protocol
+        say("Please reinstall Rugged gem with #{protocol} support: http://git.io/veiyJ")
+        err("Rugged requires #{protocol} support for: #{url}")
+      end
+    end
+
+    def credentials_for(url, user, allowed_types)
+      if creds = credentials.find_by_url(url)
+        load_git_credentials(creds, user, allowed_types)
+      else
+        say("Credential look up failed in: #{credentials_file}")
+        err("No credentials found for: #{url}".make_red)
+      end
+    end
+
+    def load_git_credentials(credentials, user, allowed_types)
+      case allowed_types
+      when %i(ssh_key)
+        key_file = temp_key_file(credentials[:private_key])
+        Rugged::Credentials::SshKey.new username: user, privatekey: key_file
+      when %i(plain_text)
+        Rugged::Credentials::UserPassword.new(credentials)
+      end
+    end
+
+    def temp_key_file(key)
+      file = Tempfile.new('sshkey')
+      file.write key
+      file.close
+      File.chmod(0600, file.path)
+      file.path
+    end
+
+    def say(*args)
+      super
+    end
+
+    def err(*args)
+      super
+    end
+  end
+end

data/lib/bosh/workspace/{git_remote_url.rb → helpers/git_protocol_helper.rb}

@@ -1,11 +1,7 @@
 module Bosh::Workspace
-  class GitRemoteUrl
-    def initialize(url)
-      @url = url
-    end
-
-    def protocol()
-      case @url
+  module GitProtocolHelper
+    def git_protocol_from_url(url)
+      case url
       when /^git:/
         return :git
       when /^https:/
@@ -15,7 +11,7 @@ module Bosh::Workspace
       when /(@.+:|^ssh:)/
         return :ssh
       else
-        raise "Unsupported protocol for remote git url: #{@url}"
+        return nil
       end
     end
   end

data/lib/bosh/workspace/helpers/project_deployment_helper.rb

@@ -15,7 +15,7 @@ module Bosh::Workspace
     end
 
     def project_deployment_file?(deployment)
-      Psych.load(ERB.new(File.read(deployment)).result).has_key?("templates")
+      ProjectDeployment.new(deployment).manifest.has_key?("templates")
     end
 
     def require_project_deployment
@@ -46,7 +46,7 @@ module Bosh::Workspace
 
       say("Generating deployment manifest")
       ManifestBuilder.build(project_deployment, work_dir)
-      
+
       if domain_name = project_deployment.domain_name
         say("Transforming to dynamic networking (dns)")
         DnsHelper.transform(project_deployment.merged_file, domain_name)
@@ -57,6 +57,14 @@ module Bosh::Workspace
       use_targeted_director_uuid if director_uuid_current?
     end
 
+    def offline!
+      @offline = true
+    end
+
+    def offline?
+      @offline
+    end
+
     private
 
     def use_targeted_director_uuid

data/lib/bosh/workspace/helpers/release_helper.rb

@@ -7,10 +7,13 @@ module Bosh::Workspace
       remote_release && remote_release["versions"].include?(version.to_s)
     end
 
-    def release_upload(manifest_file_or_release_url, release_dir)
-      Dir.chdir(release_dir) do
-        release_cmd.upload(manifest_file_or_release_url)
-      end
+    def release_upload_from_url(release_url)
+      upload_release_cmd.upload(release_url)
+    end
+
+    def release_upload(manifest_file, release_dir)
+      release_tarball = create_release(manifest_file, release_dir)
+      upload_release_cmd.upload(release_tarball)
     end
 
     def releases_dir
@@ -20,15 +23,38 @@ module Bosh::Workspace
     end
 
     def project_deployment_releases
-      @releases ||= begin
-        project_deployment.releases.map { |r| Release.new(r, releases_dir) }
+      project_deployment.releases.map do |r|
+        Release.new(r, releases_dir, credentials_callback, offline: offline?)
       end
     end
 
     private
 
-    def release_cmd
+    def create_release(release_manifest, release_dir)
+      release_tarball = release_manifest.sub('yml', 'tgz')
+      return release_tarball if File.exist?(release_tarball)
+      err "Final release tarball missing: #{release_tarball}" if offline?
+      create_release_cmd(release_dir).create(release_manifest)
+      release_tarball
+    end
+
+    def credentials_callback
+      @callback ||= GitCredentialsProvider.new(credentials_file).callback
+    end
+
+    def credentials_file
+      File.join(work_dir, '.credentials.yml')
+    end
+
+    def upload_release_cmd
       Bosh::Cli::Command::Release::UploadRelease.new
     end
+
+    def create_release_cmd(release_dir)
+      Bosh::Cli::Command::Release::CreateRelease.new.tap do |r|
+        r.add_option(:with_tarball, true)
+        r.add_option(:dir, release_dir)
+      end
+    end
   end
 end

data/lib/bosh/workspace/manifest_builder.rb

@@ -1,6 +1,6 @@
 module Bosh::Workspace
   class ManifestBuilder
-    include Bosh::Workspace::SpiffHelper
+    attr_reader :merge_tool
 
     def self.build(project_deployment, work_dir)
       manifest_builder = ManifestBuilder.new(project_deployment, work_dir)
@@ -9,22 +9,20 @@ module Bosh::Workspace
 
     def initialize(project_deployment, work_dir)
       @project_deployment = project_deployment
+      @merge_tool = @project_deployment.merge_tool
       @work_dir = work_dir
     end
 
     def merge_templates
-      spiff_merge spiff_template_paths, @project_deployment.merged_file
+      @merge_tool.merge(template_paths, @project_deployment.merged_file)
     end
 
     private
 
-    def spiff_template_paths
-      spiff_templates = template_paths
-      spiff_templates << stub_file_path
-    end
-
     def template_paths
-      @project_deployment.templates.map { |t| template_path(t) }
+      @template_paths ||= @project_deployment.templates.map do |t|
+        template_path(t)
+      end.push(stub_file_path)
     end
 
     def stub_file_path

data/lib/bosh/workspace/merge_tool.rb

@@ -0,0 +1,73 @@
+module Bosh::Workspace
+  class MergeTool
+    include Bosh::Exec
+
+    attr_accessor :name, :version
+
+    def initialize(merge_tool = nil)
+      @name, @version = case merge_tool
+      when Hash
+        [merge_tool['name'], merge_tool['version']]
+      when String
+        [merge_tool, 'current']
+      else
+        ['spiff', 'current']
+      end
+
+      unless available_tool_names.include?(@name)
+        say("#{@name} is not supported, please specify spiff or spruce instead.".make_red)
+      end
+    end
+
+    def available_tool_names
+      %w(spiff spruce)
+    end
+
+    def merge(templates, target_file)
+      check_tool_version if version != 'current'
+      run_merge_tool(:merge, templates) do |output|
+        File.open(target_file, 'w') { |file| file.write(output) }
+      end
+    end
+
+    private
+
+    def run_merge_tool(verb, params = [])
+      params.map!(&:shellescape)
+      cmd = [name, verb.to_s] + params + ['2>&1']
+      sh(cmd.join(" "), :yield => :on_false) do |result|
+        command_not_found if result.not_found?
+        command_failed(result.command, result.output) if result.failed?
+        yield result.output
+      end
+    end
+
+    def check_tool_version
+      run_merge_tool('-v') do |output|
+        actual_version = output.match(/(\d+\.\d+\.\d+)/).to_a.first
+        if actual_version.nil? || actual_version != version
+          warning "Deployment requires #{name} to have version #{version}. " +
+                  "Your actual #{name} version is #{actual_version}."
+        end
+      end
+    end
+
+    def command_not_found(command)
+      say("Can't find #{name} in $PATH".make_red)
+      say("Go to #{installation_instructions_url} for installation instructions")
+      err("Please make sure #{name} is installed")
+    end
+
+    def installation_instructions_url
+      case name
+      when 'spiff' then 'spiff.cfapps.io'
+      when 'spruce' then 'https://github.com/geofffranks/spruce#installation'
+      end
+    end
+
+    def command_failed(command, output)
+      say("Command failed: '#{command}'")
+      err(output)
+    end    
+  end
+end