bosh-workspace 0.7.0 → 0.8.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NjQwYmNmNWQzMDgxYjhmNTFkOGYxNGJmMTNjOWE1ODZiM2U5ZmJlYg==
-  data.tar.gz: !binary |-
-    Y2Y1M2Y2MGZjNmY5MDFhY2NiN2RhYTk3MzliZDA3ZDg5NzFlZWIxMg==
+SHA1:
+  metadata.gz: 6e7ec012c1379173ecbf91bc2592edf84a13df4e
+  data.tar.gz: f6cd1d81669866d40090afc48dfb3721873fc8da
 SHA512:
-  metadata.gz: !binary |-
-    NGE4ZWQzNjVmY2ZkZDgwOGE3Y2QyNzVlMGUxMGZkMjBkYmE2ZGJlNzFhY2M0
-    YjE2MmY5OThlOWI1MDcxNmJiNTRkNWM0OWM1MWUwMTNlNzYxODcxNjI5OGJh
-    N2M0OGI5OTEwOTM4M2U5ZTkyNTdmMjE3MTAwOTQwNThkYWZmMzk=
-  data.tar.gz: !binary |-
-    NGFhZmE1YmMxNzdiZDZhZGFiMjYzNGZiN2UwNmFkZmYyZTZjMzdjMTNlODAz
-    ZjI5NjIyMzQ3NDQzZTRmZTQzNGNmYmM0YWI4MjI2ODBhMzdiNGE1NTc5Yzgy
-    OTk1YWFlODU5ZGYwNTBiNDNjMTJlMDkzODg5Y2I4YTYxMzQyNDY=
+  metadata.gz: 09373b79183e0bfc9c0b2ba75d9023815290331158c62f540d71a7d51bb7f0cf672138f3a50bf6fb2d9301ad02daa3a61e955c6fa563999596f84aaf57180873
+  data.tar.gz: 936be7053d02d221496fa4133a52ace97fce10d8fbba755a3f3e782e55b18716f98d07a68f069ec6ed59d3b0e320257b7e0c6d66dbf265ee7e3b9711d88c74f8

data/.ruby-version

@@ -1 +1 @@
-1.9.3-p484
+2.0

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.0
+addons:
+  code_climate:
+    repo_token:
+      secure: "AThBxjXIeNoH4i3r5aRxhrcxXOXqKPcQwkRnipFh9wtKYNcRJGfDp1QBn5/N2r3s8sX3FsWJmEat5uJVhibz5BV6/WZFISicyXjSXTqtAi9rmtneA+fb5G/SqcVZmcyhLoOoBxPW+z2aB8zJSX9KHvTv4GaCzD+iuzhbF1SslYk="
+

data/Gemfile

@@ -6,3 +6,7 @@ gemspec
 group :development do
   gem "guard-rspec"
 end
+
+group :test do
+  gem "codeclimate-test-reporter", require: nil
+end

data/Guardfile

@@ -1,7 +1,7 @@
 guard 'rspec', spec_paths: ["spec"] do
   notification :off
   watch(%r{^spec/(.+_spec)\.rb$})
-  watch(%r{^lib/bosh/cli/commands/(.+)\.rb$})    { |m| "spec/plugin_spec.rb" }
-  watch(%r{^lib/bosh/cloudfoundry/(.+)\.rb$})    { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^lib/bosh/cli/commands/(.+)\.rb$})    { |m| "spec/commands/#{m[1]}_spec.rb" }
+  watch(%r{^lib/bosh/workspace/(.+)\.rb$})    { |m| "spec/#{m[1]}_spec.rb" }
   watch('spec/spec_helper.rb')  { "spec" }
 end

data/README.md

@@ -1,65 +1,104 @@
-# Bosh workspace
+# Bosh workspace [![Build Status](https://travis-ci.org/rkoster/bosh-workspace.svg?branch=master)](https://travis-ci.org/rkoster/bosh-workspace) [![Test Coverage](https://codeclimate.com/github/rkoster/bosh-workspace/coverage.png)](https://codeclimate.com/github/rkoster/bosh-workspace) [![Code Climate](https://codeclimate.com/github/rkoster/bosh-workspace.png)](https://codeclimate.com/github/rkoster/bosh-workspace)
 
-## Design Goals
-Enabling BOSH operators to share common configuration between different deployments.
-For example having a `QA` and `production` deployment for which only networking differs.
+This is a `bosh` cli plugin for creating reproducible and upgradable deployments.
 
-## Installation
+## Getting started
 Before you start make sure ruby, bundler and spiff are available on your system.
 Instructions for installing spiff can found [here](https://github.com/cloudfoundry-incubator/spiff#installation).
 
-Add this line to your application's Gemfile:
+### Creating a workspace repository
+First you will have to create a new repo for our company called Foo Group (short FG).
+```
+git init fg-boshworkspace
+cd fg-boshworkspace
+```
 
-    gem 'bosh-workspace'
+Lets create the initial files & directories.
+```
+mkdir deployments templates
+echo 'source "https://rubygems.org"\n\ngem "bosh-workspace"' > Gemfile
+echo "2.0.0" > .ruby-version
+echo '.stemcells*\n.deployments*\n.releases*\n.stubs*\n' > .gitignore
+```
 
-And then execute:
+Now install the gems by running bundler.
+```
+bundle install
+```
 
-    $ bundle
+Lets finish by making an initial commit.
+```
+git add .
+git commit -m "Initial commit"
+```
 
-Or install it yourself as:
+### Creating a first deployment
+For demonstration purposes we will deploy Cloud Foundry on bosh-lite.
+The steps below will show the bosh-workspace equivalent of [bosh-lite manual deploy instructions](https://github.com/cloudfoundry/bosh-lite#manual-deploy).
 
-    $ gem install bosh-workspace
+Before we start make sure you have access to properly [installed bosh-lite](https://github.com/cloudfoundry/bosh-lite#install).
 
+We will start by targetting our bosh-lite.
+```
+bosh target 192.168.50.4
+bosh login admin admin
+```
 
-## Usage
-This BOSH plugin improves the deployments work-flow,
-by extending some of the build in commands bosh commands.
+Now lets create our deployment file.
+```
+cat >deployments/cf-warden.yml <<EOL
+---
+name: cf-warden
+director_uuid: current
+
+releases:
+  - name: cf
+    version: latest
+    git: https://github.com/cloudfoundry/cf-release.git
+
+stemcells:
+  - name: bosh-warden-boshlite-ubuntu-lucid-go_agent
+    version: 60
+
+templates:
+  - cf/cf-deployment.yml
+  - cf/cf-jobs.yml
+  - cf/cf-properties.yml
+  - cf/cf-infrastructure-warden.yml
+  - cf/cf-minimal-dev.yml
+
+meta:
+  default_quota_definitions:
+    default:
+      memory_limit: 102400 # Increased limit for demonstration purposes
+EOL
+```
 
-**Set deployment**
-Sets the current deployment. Will search in `./deployments`.
+Now lets use this deployment and upload it's dependencies.
 ```
 bosh deployment cf-warden
+bosh prepare deployment
 ```
 
-**Prepare deployment**
-Resolves upstream templates (via releases).
-Resolves and uploads releases/stemcells.
+Lets make sure to above template paths exist.
 ```
-bosh prepare deployment
+ln -s ../.releases/cf/templates templates/cf
 ```
 
-**Deploy**
-Merges the specified templates into one deployment manifests using spiff.
-And uses this file to initiate a `deploy`.
+To finish we only have to start the deployment process and commit our changes.
 ```
 bosh deploy
+git add . && git commit -m "Added cf-warden deployment"
 ```
-
-### Deployment file structure
-A deployment file has the following structure:
-
-**name:**
-The name of your deployment
-
-**director_uuid:**
-The director_uuid of the targeted BOSH director
-
-**releases:**
-Array of releases used for resolving upstream templates
-
-**meta:**
-The meta hash is the last file merged into the final deployment file.
-It can be used to define properties deployment specific properties.
+Congratulations you should now have a running Cloud Foundry.
+For further reference on how to start using it go to the [bosh-lite documentation](https://github.com/cloudfoundry/bosh-lite#try-your-cloud-foundry-deployment).
+
+## Experimental
+### dns support
+Dns support can be enabled by adding a `domain_name` property to your deployment.
+For example: `domain_name: microbosh` or if you are using a normal bosh just use `bosh`.
+When enabled, a transformation step will be executed after the spiff merge.
+Which will transform all the static ip references into domain names.
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,5 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+task :default => :spec
+RSpec::Core::RakeTask.new

data/bosh-workspace.gemspec

@@ -21,9 +21,12 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.0.0'
 
   spec.add_runtime_dependency "bosh_cli",  ">= 1.1722.0"
+  spec.add_runtime_dependency "bosh_common",  ">= 1.1722.0"
+  spec.add_runtime_dependency "semi_semantic", "~> 1.1.0"
+  spec.add_runtime_dependency "membrane", "~>0.0.2"
   spec.add_runtime_dependency "git", "~> 1.2.6"
 
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rspec", "~> 2.13.0"
   spec.add_development_dependency "rspec-fire"
   spec.add_development_dependency "rake"

data/lib/bosh/cli/commands/prepare.rb

@@ -0,0 +1,74 @@
+require "bosh/workspace"
+
+module Bosh::Cli::Command
+  class Prepare < Base
+    include Bosh::Cli::Validation
+    include Bosh::Workspace
+    include ProjectDeploymentHelper
+    include ReleaseHelper
+    include StemcellHelper
+
+    usage "prepare deployment"
+    desc "Resolve deployment requirements"
+    def prepare
+      require_project_deployment
+      auth_required
+      nl
+      prepare_release_repos
+      nl
+      prepare_releases
+      nl
+      prepare_stemcells
+    end
+
+    private
+
+    def prepare_release_repos
+      project_deployment_releases.each do |release|
+        say "Cloning release '#{release.name.make_green}' to satisfy template references"
+        release.update_repo
+        say "Version '#{release.version.to_s.make_green}' has been checkout into: #{release.repo_dir}"
+      end
+    end
+
+    def prepare_releases
+      project_deployment_releases.each do |release|
+        prepare_release(release)
+      end
+    end
+
+    def prepare_release(release)
+      if release_uploaded?(release.name, release.version)
+        say "Release '#{release.name_version.make_green}' exists"
+        say "Skipping upload"
+      else
+        say "Uploading '#{release.name_version.make_green}'"
+        release_upload(release.manifest_file)
+      end
+    end
+
+    def prepare_stemcells
+      project_deployment_stemcells.each do |stemcell|
+        prepare_stemcell(stemcell)
+      end
+    end
+
+    def prepare_stemcell(stemcell)
+      if stemcell_uploaded?(stemcell.name, stemcell.version)
+        say "Stemcell '#{stemcell.name_version.make_green}' exists"
+        say "Skipping upload"
+      else
+        cached_stemcell_upload(stemcell)
+      end
+    end
+    
+    def cached_stemcell_upload(stemcell) 
+      unless stemcell.downloaded?
+        say "Downloading '#{stemcell.name_version.make_green}'"
+        stemcell_download(stemcell.file_name) 
+      end
+      say "Uploading '#{stemcell.name_version.make_green}'"
+      stemcell_upload(stemcell.file)
+    end
+  end
+end

data/lib/bosh/cli/commands/{01_make_manifest.rb → project_deployment.rb}

@@ -1,9 +1,9 @@
 require "bosh/workspace"
 
 module Bosh::Cli::Command
-  class Manifests < Base
+  class ProjectDeployment < Base
     include Bosh::Cli::Validation
-    include Bosh::Manifests
+    include Bosh::Workspace
     include ProjectDeploymentHelper
 
     # Hack to unregister original deployment command
@@ -26,20 +26,6 @@ module Bosh::Cli::Command
       deployment_cmd(options).set_current(filename)
     end
 
-    usage "prepare deployment"
-    desc "Resolve deployment requirements"
-    def prepare
-      require_project_deployment
-      auth_required
-
-      release_manager.update_release_repos
-
-      # TODO: Implement
-      # current_deployment.stemcells.each do |stemcell|
-      #   stemcell.upload unless stemcell.exists?
-      # end
-    end
-
     # Hack to unregister original deploy command
     Bosh::Cli::Config.instance_eval("@commands.delete('deploy')")
 
@@ -58,16 +44,8 @@ module Bosh::Cli::Command
     private
 
     def deployment_cmd(options = {})
-      cmd ||= Bosh::Cli::Command::Deployment.new
-      options.each do |key, value|
-        cmd.add_option key.to_sym, value
-      end
-      cmd
-    end
-
-    def release_manager
-      @release_manager ||= begin
-        ReleaseManager.new(project_deployment.releases, work_dir)
+      Bosh::Cli::Command::Deployment.new.tap do |cmd|
+        options.each { |k, v| cmd.add_option k.to_sym, v }
       end
     end
   end

data/lib/bosh/workspace.rb

@@ -1,13 +1,17 @@
-module Bosh; module Manifests; end; end
+module Bosh; module Workspace; end; end
 
 require "cli/core_ext"
 require "cli/validation"
 
 require "bosh/workspace/helpers/spiff_helper"
 require "bosh/workspace/helpers/project_deployment_helper"
+require "bosh/workspace/helpers/release_helper"
+require "bosh/workspace/helpers/stemcell_helper"
+require "bosh/workspace/helpers/dns_helper"
 
 require "bosh/workspace/manifest_builder"
-require "bosh/workspace/release_manager"
 require "bosh/workspace/release"
-require "bosh/workspace/deployment_manifest"
+require "bosh/workspace/stemcell"
+require "bosh/workspace/project_deployment"
+require "bosh/workspace/stub_file"
 require "bosh/workspace/version"

data/lib/bosh/workspace/helpers/dns_helper.rb

@@ -0,0 +1,72 @@
+module Bosh::Workspace
+  class DnsHelper
+    def self.transform(generated_manifest, domain_name)
+      @manifest = YAML.load(IO.read(generated_manifest))
+      @domain_name = domain_name
+      @manual_networks = []
+      @dns = {}
+
+      transform_networks
+
+      if @manual_networks.empty?
+        err "Missing manual network: Only type manual can be transformed to dynamic"
+      end
+      
+      transform_jobs
+      transform_properties
+      transform_job_properties
+
+      IO.write(generated_manifest, @manifest.to_yaml)
+    end
+
+    private
+
+    def self.transform_networks
+      @manifest["networks"].map! do |network|
+        if network["type"] == "manual"
+          @manual_networks << network["name"]
+          { "name" => network["name"], "type" => "dynamic", "cloud_properties" => {} }
+        else
+          network
+        end
+      end
+    end
+   
+    def self.transform_jobs
+      @manifest["jobs"].map! do |job|
+        job["networks"].map! do |network|
+          if @manual_networks.include?(network["name"]) && network["static_ips"]
+            network["static_ips"].each_with_index do |ip, index|
+              @dns[ip] = job_to_dns(job, index, network["name"])
+            end
+            network.delete("static_ips")
+          end
+          network
+        end
+        job
+      end
+    end
+    
+    def self.transform_properties
+      @manifest["properties"] = replace_ips(@manifest["properties"])
+    end
+
+    def self.transform_job_properties
+      @manifest["jobs"].map! do |job|
+        job["properties"] = replace_ips(job["properties"]) if job["properties"]
+        job
+      end
+    end
+
+    def self.job_to_dns(job, index, network_name)
+      job_name = job["name"].gsub('_', '-')
+      [index, job_name, network_name, @manifest["name"], @domain_name].join(".")
+    end
+
+    def self.replace_ips(properties)
+      properties_yaml = properties.to_yaml
+      @dns.each { |ip, domain| properties_yaml.gsub!(ip, domain) }
+      YAML.load properties_yaml
+    end
+  end
+end

data/lib/bosh/workspace/helpers/project_deployment_helper.rb

@@ -1,4 +1,4 @@
-module Bosh::Manifests
+module Bosh::Workspace
   module ProjectDeploymentHelper
 
     def project_deployment?
@@ -7,11 +7,11 @@ module Bosh::Manifests
     end
 
     def project_deployment
-      @project_deployment ||= DeploymentManifest.new(project_deployment_file)
+      @project_deployment ||= ProjectDeployment.new(project_deployment_file)
     end
 
     def project_deployment=(deployment)
-      @project_deployment = DeploymentManifest.new(deployment)
+      @project_deployment = ProjectDeployment.new(deployment)
     end
 
     def project_deployment_file?(deployment)
@@ -46,6 +46,11 @@ module Bosh::Manifests
 
       say("Generating deployment manifest")
       ManifestBuilder.build(project_deployment, work_dir)
+      
+      if domain_name = project_deployment.domain_name
+        say("Transforming to dynamic networking (dns)")
+        DnsHelper.transform(project_deployment.merged_file, domain_name)
+      end
     end
 
     def resolve_director_uuid