bosh-cloudfoundry 0.2.0

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--format progress
+--color

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-19mode
+  - ruby-head

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bosh-cloudfoundry.gemspec
+gemspec
+
+gem "awesome_print"

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/bosh/cli/commands/cf.rb$})    { |m| "spec/unit/cf_command_spec.rb" }
+  watch(%r{^lib/bosh-cloudfoundry/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Dr Nic Williams
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,242 @@
+# Bosh::CloudFoundry
+
+You want CloudFoundry? You can now create, scale, upgrade and patch one or more Cloud Foundry deployments using very simple, easy to learn and remember CLI commands.
+
+Currently supports AWS only. OpenStack support is coming. vSphere and vCloud support will require someone to tell me that they really want it.
+
+## NOTE - currently requires latest edge of nearly everything
+
+This tool currently requires the latest merged patches. The readme below, from "Usage" onward, is written for when there are public stemcells and final releases.
+
+Today, to get everything running:
+
+```
+# on your laptop
+git clone git://github.com/StarkAndWayne/bosh-bootstrap.git
+cd bosh-bootstrap
+bundle
+rake install
+
+bosh-bootstrap deploy --latest-stemcell
+bosh-bootstrap ssh
+
+# now on the inception VM
+cd /var/vcap/store/repos
+git clone git://github.com/StarkAndWayne/bosh-cloudfoundry.git
+cd bosh-cloudfoundry
+bundle
+gem build *.gemspec && mv *.gem pkg/
+sudo gem install pkg/*.gem --no-ri --no-rdoc
+
+export TMPDIR=/var/vcap/store/tmp
+bosh cf upload release --dev
+
+bosh cf prepare system production --release-name appcloud-dev
+# prompts for a DNS host for your CloudFoundry, such as mycompany.com
+# will generate a new IP address
+# now setup your DNS for *.mycompany.com => new IP address
+# the re-run:
+bosh cf prepare system production
+
+bosh cf upload stemcell --latest
+bosh cf merge gerrit 37/13137/4 84/13084/4 83/13583/1
+bosh deploy
+
+# now we can grow our single VM deployment
+
+bosh cf change deas 1
+bosh cf add service postgresql
+bosh deploy
+```
+
+Overtime, as you add more DEAs and other service nodes, your set of VMs might look like:
+
+```
+$ bosh vms
++-----------------------------+---------+---------------------------+-----------------------------+
+| Job/index                   | State   | Resource Pool             | IPs                         |
++-----------------------------+---------+---------------------------+-----------------------------+
+| core/0                      | running | core                      | 10.4.70.116, 54.235.200.165 |
+| dea/0                       | running | dea                       | 10.4.49.7                   |
+| dea/1                       | running | dea                       | 10.111.39.12                |
+| postgresql_m1_medium_free/0 | running | postgresql_m1_medium_free | 10.4.71.164                 |
+| postgresql_m1_small_free/0  | running | postgresql_m1_small_free  | 10.110.83.128               |
+| postgresql_m1_small_free/1  | running | postgresql_m1_small_free  | 10.189.103.26               |
++-----------------------------+---------+---------------------------+-----------------------------+
+```
+
+## Requirements
+
+* Ruby 1.9
+* BOSH running on AWS (other CPIs coming)
+
+## Usage
+
+The tool is very simple to use and to get CloudFoundry deployed on a small set of initial servers.
+
+```
+gem install bosh-cloudfoundry
+bosh cf prepare system demo
+bosh cf deploy
+```
+
+The above can be run the following from your local laptop or a server, such as an `bosh-bootstrap` inception server. 
+
+The latter is preferred. As the Cloud Foundry BOSH release (`cf-release`) is 1.5 Gb, it may be preferable to manage your Cloud Foundry deployments from your inception server, as created/prepared via `bosh-bootstrap`.
+
+```
+bosh-bootstrap deploy --latest-stemcell
+bosh-bootstrap ssh
+
+# now on the inception VM
+gem install bosh-cloudfoundry
+export TMPDIR=/var/vcap/store/tmp
+bosh cf prepare system production
+# prompts for a DNS host for your CloudFoundry, such as mycompany.com
+bosh cf change deas 1
+bosh cf add service postgresql 1
+bosh deploy
+```
+
+During `bosh cf prepare system production`, it will automatically upload the latest release of CloudFoundry (the latest final [BOSH release](http://github.com/cloudfoundry/cf-release)) and the latest stable stemcell (becomes the base AMI for AWS, for example).
+
+NOTE: `export TMPDIR=/var/vcap/store/tmp` tells the upload process to use the larger mounted volume at `/var/vcap/store`. 
+
+You can upload a more recent stemcell or create a new one from source, respectively:
+
+```
+bosh cf upload stemcell --latest
+bosh cf upload stemcell --custom
+```
+
+You can upload a more recent final BOSH release of CloudFoundry, or create a non-final version from the very latest commits to the CloudFoundry BOSH release, respectively:
+
+```
+bosh cf upload release
+bosh cf upload release --dev
+```
+
+### All available commands
+
+Prefix each with `bosh`:
+
+```
+cf prepare system [<name>] [--core-ip ip] [--root-dns dns] 
+                  [--core-server-flavor flavor] [--release-name name] [--release-version 
+                  version] [--stemcell-name name] [--stemcell-version version] 
+                  [--admin-emails email1,email2] [--skip-validations] 
+    create CloudFoundry system 
+    --core-ip ip                 Static IP for CloudController/router, e.g. 1.2.3.4 
+    --root-dns dns               Base DNS for CloudFoundry applications, e.g. vcap.me 
+    --core-server-flavor flavor  Flavor of the CloudFoundry Core server. Default: 'm1.large' 
+    --release-name name          Name of BOSH release within target BOSH. Default: 'appcloud' 
+    --release-version version    Version of target BOSH release within target BOSH. Default: 'latest' 
+    --stemcell-name name         Name of BOSH stemcell within target BOSH. Default: 'bosh-stemcell' 
+    --stemcell-version version   Version of BOSH stemcell within target BOSH. Default: determines latest for stemcell 
+    --admin-emails email1,email2 Admin email accounts in created CloudFoundry 
+    --skip-validations           Skip all validations 
+
+cf change deas [<server_count>] [--flavor flavor] 
+    change the number/flavor of DEA servers (servers that run CF apps) 
+    --flavor flavor Change flavor of all DEA servers 
+
+cf add service <service_name> [<additional_count>] [--flavor flavor] 
+    add additional CloudFoundry service node 
+    --flavor flavor Server flavor for additional service nodes 
+
+cf upload stemcell [--latest] [--custom] 
+    download/create stemcell & upload to BOSH 
+    --latest Use latest stemcell; possibly not tagged stable 
+    --custom Create custom stemcell from BOSH git source 
+
+cf upload release [--dev] 
+    fetch & upload public cloudfoundry release to BOSH 
+    --dev Create development release from very latest cf-release commits 
+
+cf deploy 
+    deploy CloudFoundry system or apply any changes 
+
+cf watch nats 
+    subscribe to all nats messages within CloudFoundry
+```
+
+## Services
+
+By default your CloudFoundry deployment comes with no built-in services. Instead, you easily enable each one and allocate it resources using the `bosh cf add service NAME` command; then deploy again.
+
+```
+$ bosh cf add service postgresql
+$ bosh cf add service redis
+$ bosh deploy
+```
+
+Eventually the new service node servers will be up and running, and then the VMC client will be able to create/bind/delete these services with your CloudFoundry applications.
+
+For example:
+
+``` 
+============== System Services ==============
+ 
++------------+---------+---------------------------------------+
+| Service    | Version | Description                           |
++------------+---------+---------------------------------------+
+| postgresql | 9.0     | PostgreSQL database service (vFabric) |
+| redis      | 2.2     | Redis key-value store service         |
++------------+---------+---------------------------------------+
+ 
+=========== Provisioned Services ============
+ 
++------------------+------------+
+| Name             | Service    |
++------------------+------------+
+| postgresql-6d01e | postgresql |
+| postgresql-ff0c1 | postgresql |
+| redis-d0d3d      | redis      |
++------------------+------------+g
+```
+
+## Orders of easiness vs powerfulness
+
+```
+
++--------------+     +-----------------+       +---------------------+
+|              |     |                 |       |                     |
+|  CLI         |     |                 |       |                     |
+|  commands    +---->|                 |       |                     |
+|              |     |  system         |       |  deployment         |
+|              |     |  config         |       |  manifest           |
++--------------+     |  (yaml)         |       |  (yaml)             |
+                     |                 +------>|                     |
+                     |                 |       |                     |
+                     |                 |       |                     |
+                     |                 |       |                     |
+                     |                 |       |                     |
+                     +-----------------+       |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               |                     |
+                                               +---------------------+
+```
+
+## Development
+
+```
+bundle
+bundle exec rake spec
+bundle exec rake install
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,33 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test, :development)
+
+require "bundler/gem_tasks"
+
+require "rake/dsl_definition"
+require "rake"
+require "rspec/core/rake_task"
+
+
+if defined?(RSpec)
+  namespace :spec do
+    desc "Run Unit Tests"
+    unit_rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
+      t.pattern = "spec/unit/**/*_spec.rb"
+      t.rspec_opts = %w(--format progress --color -d)
+    end
+
+    desc "Run Integration Tests"
+    functional_rspec_task = RSpec::Core::RakeTask.new(:functional) do |t|
+      t.pattern = "spec/functional/**/*_spec.rb"
+      t.rspec_opts = %w(--format progress --color)
+    end
+  end
+
+  desc "Install dependencies and run tests"
+  task :spec => %w(spec:unit spec:functional)
+end

data/bosh-cloudfoundry.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bosh-cloudfoundry/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "bosh-cloudfoundry"
+  gem.version       = Bosh::Cloudfoundry::VERSION
+  gem.authors       = ["Dr Nic Williams"]
+  gem.email         = ["drnicwilliams@gmail.com"]
+  gem.description   = %q{Create and manage your Cloud Foundry deployments}
+  gem.summary       = %q{Create and manage your Cloud Foundry deployments via the BOSH CLI}
+  gem.homepage      = "https://github.com/StarkAndWayne/bosh-cloudfoundry"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "bosh_cli", ">=1.0.3"
+  gem.add_dependency "rake" # file_utils sh helper
+  gem.add_dependency "net-dns"
+  gem.add_dependency "fog", ">= 1.8.0"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "ci_reporter"
+  gem.add_development_dependency "debugger"
+  gem.add_development_dependency "guard-rspec"
+end

data/config/defaults.yml

@@ -0,0 +1,6 @@
+---
+
+logging:
+  level: INFO
+
+base_systems_dir: /var/vcap/store/systems

data/lib/bosh-cloudfoundry.rb

@@ -0,0 +1,34 @@
+require "bosh-cloudfoundry/version"
+
+module Bosh; module CloudFoundry; end; end
+
+require "logger"
+require "common/common"
+require "common/thread_formatter"
+require "cli"
+
+# for generating password
+require 'openssl'
+
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+# for validating DNS -> IP setups
+require 'net/dns'
+
+# for:
+# * validating compute flavors
+# * provisioning IP addresses
+require "fog"
+require 'fog/aws/models/compute/flavors'
+
+# CLI mixins
+require "bosh-cloudfoundry/config_options"
+require "bosh-cloudfoundry/bosh_release_manager"
+require "bosh-cloudfoundry/gerrit_patches_helper"
+
+require "bosh-cloudfoundry/config"
+require "bosh-cloudfoundry/providers"
+
+require "bosh-cloudfoundry/system_deployment_manifest_renderer"

data/lib/bosh-cloudfoundry/bosh_release_manager.rb

@@ -0,0 +1,141 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module CloudFoundry; end; end
+
+# There are two concepts of "latest".
+# * for upload: "latest" is the highest release in cf-release
+# * for manifest creation: "latest" is the highest release already uploaded to the BOSH
+module Bosh::CloudFoundry::BoshReleaseManager
+  
+  # @return [Array] BOSH releases available in target BOSH
+  # [{"name"=>"appcloud", "versions"=>["124", "126"], "in_use"=>[]}]
+  def bosh_releases
+    @bosh_releases ||= releases = director.list_releases
+  end
+
+  # @return [Array] BOSH release names available in target BOSH
+  def bosh_release_names
+    @bosh_release_names ||= bosh_releases.map { |rel| rel["name"] }
+  end
+
+  # @return [Array] BOSH release versions for specific release name in target BOSH
+  def bosh_release_versions(release_name)
+    if release = bosh_releases.find { |rel| rel["name"] == release_name }
+      release["versions"]
+    else
+      []
+    end
+  end
+
+  def release_name_version
+    "#{release_name}/#{release_version}"
+  end
+
+  # @return [Version String] BOSH version number; converts 'latest' into actual version
+  # TODO implement this; map "latest" to highest uploaded release in BOSH
+  # return "unknown" if BOSH has no releases of this name yet
+  def effective_release_version
+    release_version.to_s
+  end
+
+  # for upload, "latest" is the newest release in cf-release
+  def upload_final_release
+    release_number = use_latest_release? ? 
+      latest_uploadable_final_release_number :
+      release_version
+    chdir(cf_release_dir) do
+      bosh_cmd "upload release releases/appcloud-#{release_number}.yml"
+    end
+    @bosh_releases = nil # reset cache
+  end
+
+  # Looks at the last line of releases/index.yml in cf-release 
+  # for the latest release number that could be uploaded
+  # @returns [String] a number such as "126"
+  def latest_uploadable_final_release_number
+    chdir(cf_release_dir) do
+      return `tail -n 1 releases/index.yml | awk '{print $2}'`.strip
+    end
+  end
+
+  # Looks at the last line of releases/index.yml in cf-release 
+  # for the latest release number that could be uploaded
+  # @returns [String] a dev release code such as "126.8-dev"
+  def latest_uploadable_dev_release_number
+    chdir(cf_release_dir) do
+      return `tail -n 1 dev_releases/index.yml | awk '{print $2}'`.strip
+    end
+  end
+
+  # @returns [String] absolute path to latest release to be uploaded
+  def latest_dev_release_filename
+    dev_release_number = latest_uploadable_dev_release_number
+    return nil unless dev_release_number.size > 0
+    File.join(cf_release_dir, "#{release_name}-#{dev_release_number}.yml")
+  end
+
+  def create_and_upload_dev_release(release_name=default_dev_release_name)
+    chdir(cf_release_dir) do
+      write_dev_config_file(release_name)
+      sh "bosh create release --with-tarball --force"
+      sh "bosh -n --color upload release"
+    end
+    @bosh_releases = nil # reset cache
+  end
+
+  def write_dev_config_file(release_name)
+    dev_config_file = "config/dev.yml"
+    if File.exist?(dev_config_file)
+      dev_config = YAML.load_file(dev_config_file)
+    else
+      dev_config = {}
+    end
+    dev_config["dev_name"] = release_name
+    File.open(dev_config_file, "w") { |file| file << dev_config.to_yaml }
+  end
+
+  # assume unchanged config/final.yml
+  def clone_or_update_cf_release
+    cf_release_dirname = File.basename(cf_release_dir)
+    if File.directory?(cf_release_dir)
+      chdir(cf_release_dir) do
+        sh "git pull origin master"
+      end
+    else
+      chdir(releases_dir) do
+        sh "git clone #{cf_release_git_repo} #{cf_release_dirname}"
+        chdir(cf_release_dirname) do
+          sh "git update-index --assume-unchanged config/final.yml 2>/dev/null"
+        end
+      end
+    end
+    chdir(cf_release_dir) do
+      say "Rewriting all git:// & git@ to https:// ..."
+      # Snippet written by Mike Reeves <swampfoxmr@gmail.com> on bosh-users mailing list
+      # Date 2012-12-06
+      sh "sed -i 's#git@github.com:#https://github.com/#g' .gitmodules"
+      sh "sed -i 's#git://github.com#https://github.com#g' .gitmodules"
+      sh "git submodule update --init --recursive"
+    end
+  end
+
+  def default_release_name
+    "appcloud"
+  end
+
+  def default_dev_release_name
+    default_release_name + "-dev"
+  end
+
+  def switch_to_development_release
+    system_config.release_name = default_dev_release_name
+    system_config.release_version = "latest"
+    system_config.save
+  end
+
+  def switch_to_final_release
+    system_config.release_name = default_release_name
+    system_config.release_version = "latest"
+    system_config.save
+  end
+end