bosh-cloudfoundry 0.3.0 → 0.4.0

data/ChangeLog.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## v0.4
+
+Defaults to patched dev release (since a cf-release final release doesn't work).
+
 ## v0.3
 
 Additions:

data/Gemfile

@@ -1,6 +1,8 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in bosh-cloudfoundry.gemspec
 gemspec
 
-gem "awesome_print"
+gem "awesome_print"
+gem "rb-fsevent", "~> 0.9.1"
+gem "guard-rspec"

data/Guardfile

@@ -1,7 +1,7 @@
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
-guard 'rspec', :version => 2 do
+guard 'rspec' do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/bosh/cli/commands/cf.rb$})    { |m| "spec/unit/cf_command_spec.rb" }
   watch(%r{^lib/bosh-cloudfoundry/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" }

data/README.md

@@ -1,53 +1,42 @@
 # Bosh::CloudFoundry
 
-You want CloudFoundry? You can now create, scale, upgrade and patch one or more Cloud Foundry deployments using very simple, easy to learn and remember CLI commands.
+You want Cloud Foundry? You can now create, scale, upgrade and patch one or more Cloud Foundry deployments using very simple, easy to learn and remember CLI commands.
 
-Currently supports AWS only. OpenStack support is coming. vSphere and vCloud support will require someone to tell me that they really want it.
+Currently supports AWS and OpenStack only. vSphere and vCloud support will require someone to tell me that they really want it.
 
-## NOTE - currently requires latest edge of many things
+## Zero to Cloud Foundry in a few hours
 
-This tool currently requires the latest merged patches. The readme below, from "Usage" onward, is written for when there are public stemcells and final releases.
+If you have an AWS or OpenStack account then you can deploy a running Cloud Foundry VM by following this very simple tutorial. It first uses the very powerful [bosh-bootstrap](https://github.com/StarkAndWayne/bosh-bootstrap) tool to deploy a BOSH, and then using this project (a plugin for the BOSH CLI) it will do everything necessary to deploy Cloud Foundry via your shiny new, empty BOSH.
 
-Today, to get everything running:
+**You should not need to know anything before running this tutorial.** If you do this tutorial and you feel it was too complex or you needed to know something additional, please [create a ticket](https://github.com/StarkAndWayne/bosh-cloudfoundry/issues).
 
 ```
 # on your laptop
 gem install bosh-bootstrap
-bosh-bootstrap deploy --latest-stemcell
+bosh-bootstrap deploy --edge-deployer
+# prompts for AWS/OpenStack credentials
+# and some other fields that you can
+# just press Enter for.
 
+# SSH into inception VM
 bosh-bootstrap ssh
 
-# now on the inception VM
-
-# TODO - restore gem installation when rubygem security issue resolved
-# sudo gem install bosh-cloudfoundry
-
-cd /var/vcap/store/repos
-git clone git://github.com/StarkAndWayne/bosh-cloudfoundry.git
-cd bosh-cloudfoundry
-git checkout -b v0.3
-git pull origin v0.3
-rake install
-
+sudo gem install bosh-cloudfoundry
 export TMPDIR=/var/vcap/store/tmp
-bosh cf upload release --dev
-
-bosh cf prepare system production --release-name appcloud-dev
+bosh cf prepare system production
 # prompts for a DNS host for your CloudFoundry, such as mycompany.com
 # will generate a new IP address
 # now setup your DNS for *.mycompany.com => new IP address
 # the re-run:
 bosh cf prepare system production
 
-bosh cf upload stemcell --latest
-bosh cf merge gerrit 37/13137/4 84/13084/4 83/13583/1 09/13609/2
-bosh cf deploy
+bosh deploy
 
 # now we can grow our single VM deployment
 
 bosh cf change deas 1
 bosh cf add service postgresql
-bosh cf deploy
+bosh deploy
 ```
 
 Overtime, as you add more DEAs and other service nodes, your set of VMs might look like:
@@ -70,6 +59,7 @@ $ bosh vms
 
 * Ruby 1.9
 * BOSH running on AWS (other CPIs coming)
+* Unix-based local system (Windows support for bosh-bootstrap is a priority)
 
 ## Usage
 

data/bosh-cloudfoundry.gemspec

@@ -23,7 +23,4 @@ Gem::Specification.new do |gem|
   gem.add_dependency "net-dns"
   gem.add_dependency "fog", ">= 1.8.0"
   gem.add_development_dependency "rspec"
-  gem.add_development_dependency "ci_reporter"
-  gem.add_development_dependency "debugger"
-  gem.add_development_dependency "guard-rspec"
 end

data/lib/bosh-cloudfoundry/bosh_release_manager.rb

@@ -77,7 +77,7 @@ module Bosh::CloudFoundry::BoshReleaseManager
   def create_and_upload_dev_release(release_name=default_dev_release_name)
     chdir(cf_release_dir) do
       write_dev_config_file(release_name)
-      sh "bosh create release --with-tarball --force"
+      sh "bosh -n --color create release --with-tarball --force"
       sh "bosh -n --color upload release"
     end
     @bosh_releases = nil # reset cache

data/lib/bosh-cloudfoundry/config_options.rb

@@ -56,6 +56,10 @@ module Bosh::CloudFoundry::ConfigOptions
     end
   end
 
+  def system_config_setup?
+    system
+  end
+
   # @return [String] CloudFoundry system path
   def system
     options[:system] || common_config.target_system

data/lib/bosh-cloudfoundry/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cloudfoundry
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/bosh/cli/commands/cf.rb

@@ -39,6 +39,10 @@ module Bosh::Cli::Command
     option "--admin-emails email1,email2", Array, "Admin email accounts in created CloudFoundry"
     option "--skip-validations", "Skip all validations"
     def prepare_system(name=nil)
+      # FUTURE remove this when cf-release has a final release with
+      # all patches in https://github.com/StarkAndWayne/bosh-cloudfoundry/issues/42
+      options[:dev] = true
+
       setup_system_dir(name)
       confirm_or_prompt_all_defaults
       confirm_or_prompt_for_system_requirements
@@ -111,24 +115,30 @@ module Bosh::Cli::Command
 
     usage "cf upload stemcell"
     desc "download/create stemcell & upload to BOSH"
-    option "--latest", "Use latest stemcell; possibly not tagged stable"
+    # option "--stable", "Use latest stemcell; possibly not tagged stable"
+    option "--latest", "Use latest stemcell; possibly not tagged stable [default]"
     option "--custom", "Create custom stemcell from BOSH git source"
     def upload_stemcell
-      stemcell_type = "stable"
-      stemcell_type = "latest" if options[:latest]
+      stemcell_type = "stable" if options[:stable]
       stemcell_type = "custom" if options[:custom]
+      stemcell_type ||= "latest"
       create_or_download_stemcell_then_upload(stemcell_type)
     end
 
     usage "cf upload release"
     desc "fetch & upload latest public cloudfoundry release to BOSH"
-    option "--dev", "Create development release from very latest cf-release commits"
+    # option "--dev", "Create development release from very latest cf-release commits & specific patches"
+    option "--final", "Upload latest final release from very latest cf-release commits"
     def upload_release
       clone_or_update_cf_release
-      if options.delete(:dev)
-        create_and_upload_dev_release
-      else
+      if options.delete(:final)
         upload_final_release
+      else
+        # FUTURE once all patches from https://github.com/StarkAndWayne/bosh-cloudfoundry/issues/42
+        # are merged into cf-release, then no more gerrit merging required
+        options[:dev] = true
+        merge_gerrit(*%w[37/13137/4 84/13084/4])
+        # create_and_upload_dev_release
       end
     end
 
@@ -153,7 +163,7 @@ module Bosh::Cli::Command
         end
       end
       apply_gerrit_patches
-      create_and_upload_dev_release
+      create_and_upload_dev_release(release_name)
     end
 
     usage "cf deploy"
@@ -249,32 +259,20 @@ module Bosh::Cli::Command
     end
 
     # @return [String] label for the CPI being used by the target BOSH
-    # * "aws" - AWS
-    # 
-    # Yet to be supported by bosh-cloudfoundry:
-    # * "openstack" - VMWare vSphere
-    # * "vsphere" - VMWare vSphere
-    # * "vcloud" - VMWare vCloud
     def bosh_provider
-      if aws?
-        "aws"
-      elsif openstack?
-        "openstack"
-      else
-        err("Please implement cf.rb's bosh_provider for this IaaS")
-      end
+      system_config_setup? ? system_config.bosh_provider : bosh_cpi
     end
 
     # Deploying CloudFoundry to AWS?
     # Is the target BOSH's IaaS using the AWS CPI?
     def aws?
-      system_config.bosh_provider == "aws"
+      bosh_provider == "aws"
     end
 
     # Deploying CloudFoundry to OpenStack?
     # Is the target BOSH's IaaS using the OpenStack CPI?
     def openstack?
-      system_config.bosh_provider == "openstack"
+      bosh_provider == "openstack"
     end
 
     # User is prompted for common values at the
@@ -550,7 +548,7 @@ module Bosh::Cli::Command
           err("Server flavor '#{flavor}' is not a valid OpenStack compute flavor")
         end
       else
-        err("Please implemenet cf.rb's validate_compute_flavor for this IaaS")
+        err("Please implemenet cf.rb's validate_compute_flavor for #{bosh_provider.inspect}")
       end
     end
 
@@ -569,31 +567,6 @@ module Bosh::Cli::Command
       renderer.perform
     end
 
-    def generate_dea_servers(server_count, server_flavor)
-      director_uuid = "DIRECTOR_UUID"
-      release_name = "appcloud"
-      stemcell_version = "0.6.4"
-      if aws?
-        resource_pool_cloud_properties = "instance_type: #{server_flavor}"
-      else
-        err("Please implemenet cf.rb's generate_dea_servers for this IaaS")
-      end
-      dea_max_memory = 2048 # FIXME a value based on server flavor RAM?
-      nats_password = "mynats1234"
-      system_dir = File.join(base_systems_dir, system_name)
-      mkdir_p(system_dir)
-      chdir system_dir do
-        require 'bosh-cloudfoundry/generators/dea_generator'
-        Bosh::CloudFoundry::Generators::DeaGenerator.start([
-          system_name,
-          server_count, server_flavor,
-          director_uuid, release_name, stemcell_version,
-          resource_pool_cloud_properties,
-          dea_max_memory,
-          nats_password])
-      end
-    end
-
     # Valdiate that +service_name+ is a known, supported service name
     def validate_service_name(service_name)
       return true if skip_validations?
@@ -618,30 +591,6 @@ module Bosh::Cli::Command
       end
     end
 
-    def generate_service_servers(service_name, server_count, server_flavor)
-      director_uuid = "DIRECTOR_UUID"
-      release_name = "appcloud"
-      stemcell_version = "0.6.4"
-      if aws?
-        resource_pool_cloud_properties = "instance_type: #{server_flavor}"
-      else
-        err("Please implemenet cf.rb's generate_service_servers for this IaaS")
-      end
-      persistent_disk = 16192
-      nats_password = "mynats1234"
-      system_dir = File.join(base_systems_dir, system_name)
-      mkdir_p(system_dir)
-      chdir system_dir do
-        require 'bosh-cloudfoundry/generators/service_generator'
-        Bosh::CloudFoundry::Generators::ServiceGenerator.start([
-          system_name,
-          service_name, server_count, server_flavor,
-          director_uuid, release_name, stemcell_version,
-          resource_pool_cloud_properties, persistent_disk,
-          nats_password])
-      end
-    end
-
     def default_core_server_flavor
       if aws?
         "m1.large"

data/spec/unit/cf_command_spec.rb

@@ -41,9 +41,14 @@ describe Bosh::Cli::Command::Base do
     it "downloads stemcell and uploads it" do
       @cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       @cmd.stub!(:bosh_target_uuid).and_return("DIRECTOR_UUID")
+      @cmd.stub!(:bosh_cpi).and_return("aws")
       @cmd.should_receive(:`).
-        with("bosh public stemcells --tags aws,stable | grep ' bosh-stemcell-' | awk '{ print $2 }' | sort -r | head -n 1").
+        with("bosh public stemcells --tags aws | grep ' bosh-stemcell-' | awk '{ print $2 }' | sort -r | head -n 1").
         and_return("bosh-stemcell-aws-0.6.7.tgz")
+      # FIXME default to stable stemcells when 0.8.1 is marked stable
+      # @cmd.should_receive(:`).
+      #   with("bosh public stemcells --tags aws,stable | grep ' bosh-stemcell-' | awk '{ print $2 }' | sort -r | head -n 1").
+      #   and_return("bosh-stemcell-aws-0.6.7.tgz")
       @cmd.should_receive(:sh).
         with("bosh -n --color download public stemcell bosh-stemcell-aws-0.6.7.tgz")
       @cmd.should_receive(:sh).
@@ -58,6 +63,7 @@ describe Bosh::Cli::Command::Base do
       mkdir_p(File.join(@repos_dir, "bosh", "agent"))
       @cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       @cmd.stub!(:bosh_target_uuid).and_return("DIRECTOR_UUID")
+      @cmd.stub!(:bosh_cpi).and_return("aws")
       @cmd.should_receive(:sh).with("git pull origin master")
       @cmd.should_receive(:sh).with("bundle install --without development test")
       @cmd.should_receive(:sh).with("sudo bundle exec rake stemcell:basic['aws']")
@@ -94,34 +100,38 @@ describe Bosh::Cli::Command::Base do
       @cmd.should_receive(:`).with("tail -n 1 releases/index.yml | awk '{print $2}'").
         and_return("126")
       @cmd.should_receive(:sh).with("bosh -n --color upload release releases/appcloud-126.yml")
+      @cmd.add_option(:final, true)
       @cmd.upload_release
     end
 
-    it "updates/creates/uploads development/edge cf-release" do
-      cf_release_dir = File.join(@releases_dir, "cf-release")
-      FileUtils.mkdir_p(cf_release_dir)
-      @cmd.common_config.cf_release_dir = cf_release_dir
-      @cmd.add_option(:dev, true)
-
-      @cmd.should_receive(:sh).with("git pull origin master")
-      script = <<-BASH.gsub(/^      /, '')
-      grep -rI "github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | uniq while read file
-      do
-        echo "changing - $file"
-        sed -i 's#git://github.com#https://github.com#g' $file
-        sed -i 's#git@github.com:#https://github.com:#g' $file
-      done
-      BASH
-      @cmd.should_receive(:sh).with("sed -i 's#git@github.com:#https://github.com/#g' .gitmodules")
-      @cmd.should_receive(:sh).with("sed -i 's#git://github.com#https://github.com#g' .gitmodules")
-      @cmd.should_receive(:sh).with("git submodule update --init --recursive")
-      @cmd.should_receive(:write_dev_config_file).with("appcloud-dev")
-      @cmd.should_receive(:sh).with("bosh create release --with-tarball --force")
-      @cmd.should_receive(:sh).with("bosh -n --color upload release")
-      @cmd.upload_release
-    end
+    it "updates/creates/uploads development/edge cf-release (requires system setup)" 
+    # TODO turn this into a unit test for the specific methods
+    # do
+    #   cf_release_dir = File.join(@releases_dir, "cf-release")
+    #   FileUtils.mkdir_p(cf_release_dir)
+    #   @cmd.common_config.cf_release_dir = cf_release_dir
+    #   @cmd.add_option(:dev, true)
+    # 
+    #   @cmd.should_receive(:sh).with("git pull origin master")
+    #   script = <<-BASH.gsub(/^      /, '')
+    #   grep -rI "github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | uniq while read file
+    #   do
+    #     echo "changing - $file"
+    #     sed -i 's#git://github.com#https://github.com#g' $file
+    #     sed -i 's#git@github.com:#https://github.com:#g' $file
+    #   done
+    #   BASH
+    #   @cmd.should_receive(:sh).with("sed -i 's#git@github.com:#https://github.com/#g' .gitmodules")
+    #   @cmd.should_receive(:sh).with("sed -i 's#git://github.com#https://github.com#g' .gitmodules")
+    #   @cmd.should_receive(:sh).with("git submodule update --init --recursive")
+    #   @cmd.should_receive(:write_dev_config_file).with("appcloud-dev")
+    #   @cmd.should_receive(:sh).with("bosh create release --with-tarball --force")
+    #   @cmd.should_receive(:sh).with("bosh -n --color upload release")
+    #   @cmd.upload_release
+    # end
 
     def generate_new_system(cmd = nil)
+      needs_initial_release_uploaded = true
       cmd ||= begin
         cmd = Bosh::Cli::Command::CloudFoundry.new(nil)
         cmd.add_option(:non_interactive, true)
@@ -134,13 +144,22 @@ describe Bosh::Cli::Command::Base do
 
       cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       cmd.stub!(:bosh_target_uuid).and_return("DIRECTOR_UUID")
+      cmd.stub!(:bosh_cpi).and_return("aws")
       cmd.should_receive(:generate_common_password).and_return('c1oudc0wc1oudc0w')
-      cmd.should_receive(:bosh_releases).exactly(1).times.and_return([
-        {"name"=>"appcloud", "versions"=>["124", "126"], "in_use"=>[]},
-        {"name"=>"appcloud-dev", "versions"=>["124.1-dev", "126.1-dev"], "in_use"=>[]},
-      ])
       cmd.should_receive(:validate_dns_a_record).with("api.mycompany.com", '1.2.3.4').and_return(true)
       cmd.should_receive(:validate_dns_a_record).with("demoapp.mycompany.com", '1.2.3.4').and_return(true)
+
+      if needs_initial_release_uploaded
+        cmd.should_receive(:bosh_releases).exactly(1).times.and_return([])
+        cmd.should_receive(:clone_or_update_cf_release)
+        cmd.should_receive(:merge_gerrit).with(*%w[37/13137/4 84/13084/4 09/13609/2])
+      else
+        cmd.should_receive(:bosh_releases).exactly(1).times.and_return([
+          {"name"=>"appcloud", "versions"=>["124", "126"], "in_use"=>[]},
+          {"name"=>"appcloud-dev", "versions"=>["124.1-dev", "126.1-dev"], "in_use"=>[]},
+        ])
+      end
+
       cmd.should_receive(:bosh_stemcell_versions).exactly(4).times.and_return(['0.6.4'])
       cmd.should_receive(:render_system)
 
@@ -155,7 +174,7 @@ describe Bosh::Cli::Command::Base do
 
       cmd.add_option(:core_ip, '1.2.3.4')
       cmd.add_option(:root_dns, 'mycompany.com')
-      cmd.add_option(:cf_release, 'appcloud')
+      # cmd.add_option(:cf_release, 'appcloud')
       cmd.add_option(:core_server_flavor, 'm1.large')
       cmd.add_option(:admin_emails, ['drnic@starkandwayne.com'])
 
@@ -174,6 +193,12 @@ describe Bosh::Cli::Command::Base do
       File.basename(@cmd.system).should == "production"
     end
 
+    it "temporarily uploads latest stemcell & patched cf-release by default" do
+      generate_new_system(@cmd)
+      File.basename(@cmd.system).should == "production"
+      @cmd.system_config.release_name.should == "appcloud-dev"
+    end
+
     it "new system has common random password" do
       generate_new_system(@cmd)
       @cmd.system_config.common_password.should == "c1oudc0wc1oudc0w"
@@ -184,6 +209,7 @@ describe Bosh::Cli::Command::Base do
 
       @cmd.should_receive(:render_system)
 
+      @cmd.stub!(:bosh_cpi).and_return("aws")
       @cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       @cmd.add_option(:flavor, 'm1.xlarge')
       @cmd.change_deas(3)
@@ -204,6 +230,7 @@ describe Bosh::Cli::Command::Base do
 
       @cmd.should_receive(:render_system)
 
+      @cmd.stub!(:bosh_cpi).and_return("aws")
       @cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       @cmd.add_option(:flavor, 'm1.large')
       @cmd.add_service_node("postgresql", 4)
@@ -219,6 +246,7 @@ describe Bosh::Cli::Command::Base do
     
       @cmd.should_receive(:render_system)
     
+      @cmd.stub!(:bosh_cpi).and_return("aws")
       @cmd.stub!(:bosh_target).and_return("http://9.8.7.6:25555")
       @cmd.add_option(:flavor, 'm1.large')
       @cmd.add_service_node("redis", 2)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh-cloudfoundry
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-02 00:00:00.000000000 Z
+date: 2013-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_cli
@@ -91,54 +91,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: ci_reporter
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: debugger
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard-rspec
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Create and manage your Cloud Foundry deployments
 email:
 - drnicwilliams@gmail.com
@@ -209,7 +161,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -743491069374654137
+      hash: -44016591154602000
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -218,7 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -743491069374654137
+      hash: -44016591154602000
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24