vagrant-bosh 0.0.3 → 0.0.4

data/README.md

@@ -39,16 +39,39 @@ end
 ### Deployment manifest gotchas
 
 - It must specify release source(s) via `url` key in the `releases` section.
+  See [release URL confgurations](docs/release-url.md).
 
 - It must have exactly one deployment job; however, deployment job
   can be made up from multiple job templates that come from multiple releases.
 
-- It does not support `static` network type though `dynamic` network type is supported
+- It does not support `static` network type, though `dynamic` network type is supported
   (Network configuration should be done via standard Vagrant configuration DSL).
 
 - It does not support stemcell specification because guest VM OS is picked via `config.vm.box` directive.
 
 
+### Provisioner options
+
+- `manifest` (String, default: `nil`) 
+  should contain full BOSH deployment manifest
+
+- `full_stemcell_compatibility` (Boolean, default: `false`) 
+  forces provisioner to install all (not just minimum) dependencies usually found on a stemcell
+
+- `agent_infrastructure` (String, default: `warden`)
+  configures BOSH Agent infrastructure (e.g. `aws`, `openstack`)
+
+- `agent_platform` (String, default: `ubuntu`)
+  configured BOSH Agent platform (e.g. `ubuntu`, `centos`)
+
+- `agent_configuration` (Hash, default: '{ ... }')
+
+
+### Using BOSH provisioner to build BOSH stemcells
+
+See [building AWS Stemcell](docs/build-aws-stemcell.md).
+
+
 ### Planned
 
 - Speed up apply step (Monit is sluggish)

data/dev/Vagrantfile

@@ -7,8 +7,11 @@ end
 require "../lib/vagrant-bosh"
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box = "saucy"
+  config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/saucy/current/saucy-server-cloudimg-amd64-vagrant-disk1.box"
+
+  # config.vm.box = "precise64"
+  # config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 
   # Example port forward for example-bosh-manifest.yml
   config.vm.network "forwarded_port", guest: 25555, host: 25555 # BOSH Director API
@@ -24,7 +27,7 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provision "bosh" do |c|
-    # c.manifest = `cat example-bosh-manifest.yml`
-    c.manifest = `cat example-winston-manifest.yml`
+    c.manifest = `cat example-bosh-manifest.yml`
+    # c.manifest = `cat example-winston-manifest.yml`
   end
 end

data/dev/example-winston-manifest.yml

@@ -2,8 +2,8 @@ name: room101
 
 releases:
 - name: room101
-  version: 0+dev.3
-  url: file:///vagrant/room101-0+dev.3.tgz
+  version: latest
+  url: dir+bosh://../../room101
 
 networks:
 - name: room101

data/docs/build-aws-stemcell.md

@@ -0,0 +1,66 @@
+## Building AWS BOSH stemcell with BOSH provisioner
+
+!!! Stemcells produced using this method should NOT be used in production !!!
+
+To quickly ligth BOSH stemcells for AWS:
+
+1. Configure AWS Vagrant provider with your AWS account settings. For example:
+
+```
+# AWS provider is configured to use an AMI
+config.vm.box = "dummy"
+
+config.vm.provider "aws" do |aws, override|
+  aws.access_key_id = "AKIxxx"
+  aws.secret_access_key = "xxx"
+
+  aws.keypair_name = "my-aws-key-pair-name"
+  aws.security_groups = ['my-aws-sec-group']
+
+  aws.tags = { "Name" => "vagrant-bosh-stemcell" }
+
+  aws.block_device_mapping = [{
+    :DeviceName => "/dev/sda1",
+    "Ebs.VolumeSize" => 8
+  }]
+
+  # https://cloud-images.ubuntu.com/trusty/current/
+  aws.ami = "ami-018c9568"
+
+  override.ssh.username = "ubuntu"
+  override.ssh.private_key_path = "/Users/some-user/.ssh/my-aws-private-key"
+end
+```
+
+2. Configure BOSH provisioner with following settings:
+
+```
+config.vm.provision "bosh" do |c|
+  c.manifest = nil
+
+  # Some BOSH releases (e.g. cf-release) depend on more pre-installed packages
+  c.full_stemcell_compatibility = true
+
+  c.agent_infrastructure = "aws"
+  c.agent_platform = "ubuntu"
+  c.agent_configuration = {}
+end
+```
+
+3. Remove `cloud-init` package included by default in Ubuntu AMIs 
+   to avoid conflicts with BOSH Agent auto configuration:
+
+```
+config.vm.provision "shell", inline: "apt-get -y purge --auto-remove cloud-init"
+config.vm.provision "shell", inline: "echo 'LABEL=cloudimg-rootfs /  ext4 defaults  0 0' > /etc/fstab"
+```
+
+4. `vagrant up --provider aws`
+
+5. Once VM is provisioned, in AWS Console right-click on the VM and select `Cerate image` to create an AMI.
+   Optionally make AMI public by changing its permissions if stemcell will be used from a different AWS account.
+
+6. Unpack one of the officially published `ligth-bosh` stemcells and
+   update `stemcell.MF` with new AMI reference, then repack.
+
+7. Upload your new light stemcell to a BOSH Director and use it in your deployment.

data/docs/release-url.md

@@ -0,0 +1,39 @@
+## Release URL configurations
+
+- `https/http`: Download release tar from the URL. Exact version (dev or final) must be specified.
+
+```
+releases:
+- name: bosh
+  version: 88
+  url: https://s3.amazonaws.com/bosh-jenkins-artifacts/release/bosh-2619.tgz
+```
+
+- `dir+bosh`: Use release directory located on the _host_ FS. 
+  Exact dev release version or `latest` must be specified.
+  If version is `latest` new dev release will be created via `bosh create release --force`.
+
+```
+releases:
+- name: bosh
+  version: latest
+  url: dir+bosh://../../bosh
+```
+
+- `dir`: Use release directory located on the _guest_ FS. Exact dev release version must be specified.
+
+```
+releases:
+- name: bosh
+  version: 88+dev.1
+  url: dir:///tmp/bosh
+```
+
+- `file`: Use release tar located on the _guest_ FS. Exact version (dev or final) must be specified.
+
+```
+releases:
+- name: bosh
+  version: 88
+  url: file:///tmp/bosh-2619.tgz
+```

data/go/src/boshprovisioner/downloader/local_fs_downloader.go

@@ -8,7 +8,7 @@ import (
 	boshsys "bosh/system"
 )
 
-const localFSDownloaderLogTag = "Downloader"
+const localFSDownloaderLogTag = "LocalFSDownloader"
 
 type LocalFSDownloader struct {
 	fs     boshsys.FileSystem

data/go/src/boshprovisioner/main/config.go

@@ -10,9 +10,27 @@ import (
 	bpvm "boshprovisioner/vm"
 )
 
-type Config struct {
-	ManifestPath string `json:"manifest_path"`
+var DefaultWardenConfig = Config{
+	VMProvisioner: bpvm.VMProvisionerConfig{
+		AgentProvisioner: bpvm.AgentProvisionerConfig{
+			Infrastructure: "warden",
+			Platform:       "ubuntu",
+			Mbus:           "https://user:password@127.0.0.1:4321/agent",
+		},
+	},
+}
+
+var DefaultWardenAgentConfiguration = map[string]interface{}{
+	"Platform": map[string]interface{}{
+		"Linux": map[string]interface{}{
+			"UseDefaultTmpDir":              true,
+			"UsePreformattedPersistentDisk": true,
+			"BindMountPersistentDisk":       true,
+		},
+	},
+}
 
+type Config struct {
 	// Assets dir is used as a temporary location to transfer files from host to guest.
 	// It will not be created since assets already must be present.
 	AssetsDir string `json:"assets_dir"`
@@ -21,12 +39,11 @@ type Config struct {
 	// It will be created if it does not exist.
 	ReposDir string `json:"repos_dir"`
 
-	// e.g. "https://user:password@127.0.0.1:4321/agent"
-	Mbus string `json:"mbus"`
-
 	Blobstore bpprov.BlobstoreConfig `json:"blobstore"`
 
 	VMProvisioner bpvm.VMProvisionerConfig `json:"vm_provisioner"`
+
+	DeploymentProvisioner bpprov.DeploymentProvisionerConfig `json:"deployment_provisioner"`
 }
 
 func NewConfigFromPath(path string, fs boshsys.FileSystem) (Config, error) {
@@ -37,11 +54,17 @@ func NewConfigFromPath(path string, fs boshsys.FileSystem) (Config, error) {
 		return config, bosherr.WrapError(err, "Reading config %s", path)
 	}
 
+	config = DefaultWardenConfig
+
 	err = json.Unmarshal(bytes, &config)
 	if err != nil {
 		return config, bosherr.WrapError(err, "Unmarshalling config")
 	}
 
+	if config.VMProvisioner.AgentProvisioner.Configuration == nil {
+		config.VMProvisioner.AgentProvisioner.Configuration = DefaultWardenAgentConfiguration
+	}
+
 	err = config.validate()
 	if err != nil {
 		return config, bosherr.WrapError(err, "Validating config")
@@ -51,10 +74,6 @@ func NewConfigFromPath(path string, fs boshsys.FileSystem) (Config, error) {
 }
 
 func (c Config) validate() error {
-	if c.ManifestPath == "" {
-		return bosherr.New("Must provide non-empty manifest_path")
-	}
-
 	if c.AssetsDir == "" {
 		return bosherr.New("Must provide non-empty assets_dir")
 	}
@@ -63,10 +82,6 @@ func (c Config) validate() error {
 		return bosherr.New("Must provide non-empty repos_dir")
 	}
 
-	if c.Mbus == "" {
-		return bosherr.New("Must provide non-empty mbus")
-	}
-
 	if c.Blobstore.Type != bpprov.BlobstoreConfigTypeLocal {
 		return bosherr.New("Blobstore type must be local")
 	}

data/go/src/boshprovisioner/main/config_test.go

@@ -0,0 +1,67 @@
+package main_test
+
+import (
+	fakesys "bosh/system/fakes"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "boshprovisioner/main"
+	bpvm "boshprovisioner/vm"
+)
+
+var _ = Describe("NewConfigFromPath", func() {
+	var (
+		fs *fakesys.FakeFileSystem
+	)
+
+	BeforeEach(func() {
+		fs = fakesys.NewFakeFileSystem()
+	})
+
+	It("defautls null values to agent provisioner config defaults", func() {
+		configJSON := `{
+      "manifest_path": "fake-manifest-path",
+      "assets_dir": "fake-assets-dir",
+      "repos_dir": "fake-repos-dir",
+      "blobstore": {
+        "provider": "local",
+        "options": {
+          "blobstore_path": "fake-blobstore-path"
+        }
+      },
+      "vm_provisioner": {
+        "agent_provisioner": {
+          "infrastructure": null,
+          "platform": null,
+          "configuration": null,
+          "mbus": null
+        }
+      }
+    }`
+
+		err := fs.WriteFileString("/tmp/config", configJSON)
+		Expect(err).ToNot(HaveOccurred())
+
+		config, err := NewConfigFromPath("/tmp/config", fs)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(config.VMProvisioner.AgentProvisioner).To(Equal(
+			bpvm.AgentProvisionerConfig{
+				Infrastructure: "warden",
+				Platform:       "ubuntu",
+
+				Configuration: map[string]interface{}{
+					"Platform": map[string]interface{}{
+						"Linux": map[string]interface{}{
+							"UseDefaultTmpDir":              true,
+							"UsePreformattedPersistentDisk": true,
+							"BindMountPersistentDisk":       true,
+						},
+					},
+				},
+
+				Mbus: "https://user:password@127.0.0.1:4321/agent",
+			},
+		))
+	})
+})

data/go/src/boshprovisioner/main/main.go

@@ -72,7 +72,7 @@ func main() {
 
 	err = fs.MkdirAll(config.ReposDir, os.ModeDir)
 	if err != nil {
-		logger.Error(mainLogTag, "Failed to create repos dir %s", err.Error())
+		logger.Error(mainLogTag, "Failed to create repos dir: %s", err.Error())
 		os.Exit(1)
 	}
 
@@ -86,7 +86,7 @@ func main() {
 
 	err = blobstoreProvisioner.Provision()
 	if err != nil {
-		logger.Error(mainLogTag, "Failed to provision blobstore %s", err.Error())
+		logger.Error(mainLogTag, "Failed to provision blobstore: %s", err.Error())
 		os.Exit(1)
 	}
 
@@ -131,7 +131,6 @@ func main() {
 		fs,
 		runner,
 		config.AssetsDir,
-		config.Mbus,
 		config.Blobstore.AsMap(),
 		config.VMProvisioner,
 		eventLog,
@@ -141,7 +140,6 @@ func main() {
 	vagrantVMProvisioner := vagrantVMProvisionerFactory.NewVMProvisioner()
 
 	releaseCompiler := bpprov.NewReleaseCompiler(
-		reposFactory.NewBlobstoreReleasesRepo(),
 		releaseReaderFactory,
 		packagesCompilerFactory,
 		templatesCompiler,
@@ -155,9 +153,9 @@ func main() {
 		logger,
 	)
 
-	deploymentProvisioner := bpprov.NewDeploymentProvisioner(
-		config.ManifestPath,
+	singleVMProvisionerFactory := bpprov.NewSingleVMProvisionerFactory(
 		deploymentReaderFactory,
+		config.DeploymentProvisioner,
 		vagrantVMProvisioner,
 		releaseCompiler,
 		instanceProvisioner,
@@ -165,9 +163,11 @@ func main() {
 		logger,
 	)
 
+	deploymentProvisioner := singleVMProvisionerFactory.NewSingleVMProvisioner()
+
 	err = deploymentProvisioner.Provision()
 	if err != nil {
-		logger.Error(mainLogTag, "Failed to provision deployment %s", err.Error())
+		logger.Error(mainLogTag, "Failed to provision deployment: %s", err.Error())
 		os.Exit(1)
 	}
 }

data/go/src/boshprovisioner/main/main_suite_test.go

@@ -0,0 +1,13 @@
+package main_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestMain(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Main Suite")
+}

data/go/src/boshprovisioner/main/repos_factory.go

@@ -13,7 +13,6 @@ import (
 	bptplsrepo "boshprovisioner/instance/templatescompiler/templatesrepo"
 	bpcpkgsrepo "boshprovisioner/packagescompiler/compiledpackagesrepo"
 	bppkgsrepo "boshprovisioner/packagescompiler/packagesrepo"
-	bprelrepo "boshprovisioner/releasesrepo"
 )
 
 type ReposFactory struct {
@@ -40,15 +39,6 @@ func NewReposFactory(
 	}
 }
 
-func (f ReposFactory) NewBlobstoreReleasesRepo() bprelrepo.BlobstoreReleasesRepository {
-	return bprelrepo.NewBlobstoreReleasesRepository(
-		f.downloader,
-		f.blobstore,
-		f.newIndex("releases"),
-		f.logger,
-	)
-}
-
 func (f ReposFactory) NewJobsRepo() bpjobsrepo.JobsRepository {
 	return bpjobsrepo.NewConcreteJobsRepository(
 		f.newIndex("jobs"),

data/go/src/boshprovisioner/packagescompiler/concrete_packages_compiler.go

@@ -8,7 +8,7 @@ import (
 	bosherr "bosh/errors"
 	boshlog "bosh/logger"
 
-	bragentclient "boshprovisioner/agent/client"
+	bpagentclient "boshprovisioner/agent/client"
 	bpeventlog "boshprovisioner/eventlog"
 	bpcpkgsrepo "boshprovisioner/packagescompiler/compiledpackagesrepo"
 	bppkgsrepo "boshprovisioner/packagescompiler/packagesrepo"
@@ -18,7 +18,7 @@ import (
 const concretePackagesCompilerLogTag = "ConcretePackagesCompiler"
 
 type ConcretePackagesCompiler struct {
-	agentClient          bragentclient.Client
+	agentClient          bpagentclient.Client
 	packagesRepo         bppkgsrepo.PackagesRepository
 	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository
 	blobstore            boshblob.Blobstore
@@ -28,7 +28,7 @@ type ConcretePackagesCompiler struct {
 }
 
 func NewConcretePackagesCompiler(
-	agentClient bragentclient.Client,
+	agentClient bpagentclient.Client,
 	packagesRepo bppkgsrepo.PackagesRepository,
 	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository,
 	blobstore boshblob.Blobstore,

data/go/src/boshprovisioner/provisioner/deployment_provisioner_interface.go

@@ -0,0 +1,10 @@
+package provisioner
+
+type DeploymentProvisioner interface {
+	Provision() error
+}
+
+type DeploymentProvisionerConfig struct {
+	// If manifest path is empty, release compilation and job provisioning will be skipped
+	ManifestPath string `json:"manifest_path"`
+}

data/go/src/boshprovisioner/provisioner/release_compiler.go

@@ -1,8 +1,6 @@
 package provisioner
 
 import (
-	"fmt"
-
 	bosherr "bosh/errors"
 	boshlog "bosh/logger"
 
@@ -11,14 +9,12 @@ import (
 	bptplcomp "boshprovisioner/instance/templatescompiler"
 	bppkgscomp "boshprovisioner/packagescompiler"
 	bprel "boshprovisioner/release"
-	bprelrepo "boshprovisioner/releasesrepo"
 	bpvm "boshprovisioner/vm"
 )
 
 const releaseCompilerLogTag = "ReleaseCompiler"
 
 type ReleaseCompiler struct {
-	releasesRepo         bprelrepo.ReleasesRepository
 	releaseReaderFactory bprel.ReaderFactory
 
 	packagesCompilerFactory bppkgscomp.ConcretePackagesCompilerFactory
@@ -31,7 +27,6 @@ type ReleaseCompiler struct {
 }
 
 func NewReleaseCompiler(
-	releasesRepo bprelrepo.ReleasesRepository,
 	releaseReaderFactory bprel.ReaderFactory,
 	packagesCompilerFactory bppkgscomp.ConcretePackagesCompilerFactory,
 	templatesCompiler bptplcomp.TemplatesCompiler,
@@ -40,7 +35,6 @@ func NewReleaseCompiler(
 	logger boshlog.Logger,
 ) ReleaseCompiler {
 	return ReleaseCompiler{
-		releasesRepo:         releasesRepo,
 		releaseReaderFactory: releaseReaderFactory,
 
 		packagesCompilerFactory: packagesCompilerFactory,
@@ -53,55 +47,17 @@ func NewReleaseCompiler(
 	}
 }
 
-func (p ReleaseCompiler) Compile(instance bpdep.Instance, releases []bpdep.Release) error {
+func (p ReleaseCompiler) Compile(instance bpdep.Instance, depReleases []bpdep.Release) error {
 	vm, err := p.vmProvisioner.Provision(instance)
 	if err != nil {
-		return bosherr.WrapError(err, "Provisioning agent")
+		return bosherr.WrapError(err, "Provisioning VM")
 	}
 
 	defer vm.Deprovision()
 
-	err = p.uploadReleases(releases)
-	if err != nil {
-		return bosherr.WrapError(err, "Uploading releases")
-	}
-
 	pkgsCompiler := p.packagesCompilerFactory.NewCompiler(vm.AgentClient())
 
-	err = p.compileReleases(pkgsCompiler, releases)
-	if err != nil {
-		return bosherr.WrapError(err, "Compiling releases")
-	}
-
-	return nil
-}
-
-func (p ReleaseCompiler) uploadReleases(releases []bpdep.Release) error {
-	stage := p.eventLog.BeginStage("Uploading releases", len(releases)+1)
-
-	for _, depRelease := range releases {
-		releaseDesc := fmt.Sprintf("%s/%s", depRelease.Name, depRelease.Version)
-
-		task := stage.BeginTask(fmt.Sprintf("Release %s", releaseDesc))
-
-		err := task.End(p.releasesRepo.Pull(depRelease))
-		if err != nil {
-			return bosherr.WrapError(err, "Pulling release %s", depRelease.Name)
-		}
-	}
-
-	task := stage.BeginTask("Deleting old releases")
-
-	err := task.End(p.releasesRepo.KeepOnly(releases))
-	if err != nil {
-		return bosherr.WrapError(err, "Keeping only releases")
-	}
-
-	return nil
-}
-
-func (p ReleaseCompiler) compileReleases(pkgsCompiler bppkgscomp.PackagesCompiler, releases []bpdep.Release) error {
-	for _, depRelease := range releases {
+	for _, depRelease := range depReleases {
 		err := p.compileRelease(pkgsCompiler, depRelease)
 		if err != nil {
 			return bosherr.WrapError(err, "Release %s", depRelease.Name)
@@ -111,15 +67,19 @@ func (p ReleaseCompiler) compileReleases(pkgsCompiler bppkgscomp.PackagesCompile
 	return nil
 }
 
-func (p ReleaseCompiler) compileRelease(pkgsCompiler bppkgscomp.PackagesCompiler, release bpdep.Release) error {
-	reader := p.releaseReaderFactory.NewTarReader(release.URL)
+func (p ReleaseCompiler) compileRelease(pkgsCompiler bppkgscomp.PackagesCompiler, depRelease bpdep.Release) error {
+	relReader := p.releaseReaderFactory.NewReader(
+		depRelease.Name,
+		depRelease.Version,
+		depRelease.URL,
+	)
 
-	relRelease, err := reader.Read()
+	relRelease, err := relReader.Read()
 	if err != nil {
 		return bosherr.WrapError(err, "Reading release")
 	}
 
-	defer reader.Close()
+	defer relReader.Close()
 
 	err = pkgsCompiler.Compile(relRelease)
 	if err != nil {

data/go/src/boshprovisioner/provisioner/{deployment_provisioner.go → single_configured_vm_provisioner.go}

@@ -10,11 +10,9 @@ import (
 	bpvm "boshprovisioner/vm"
 )
 
-const deploymentProvisionerLogTag = "DeploymentProvisioner"
-
-// DeploymentProvisioner interprets deployment manifest and
-// configures system just like regular BOSH VM would be configured.
-type DeploymentProvisioner struct {
+// SingleConfiguredVMProvisioner interprets deployment manifest and
+// configures 1 VM just like regular BOSH VM.
+type SingleConfiguredVMProvisioner struct {
 	manifestPath            string
 	deploymentReaderFactory bpdep.ReaderFactory
 
@@ -26,7 +24,7 @@ type DeploymentProvisioner struct {
 	logger   boshlog.Logger
 }
 
-func NewDeploymentProvisioner(
+func NewSingleConfiguredVMProvisioner(
 	manifestPath string,
 	deploymentReaderFactory bpdep.ReaderFactory,
 	vmProvisioner bpvm.VMProvisioner,
@@ -34,8 +32,8 @@ func NewDeploymentProvisioner(
 	instanceProvisioner bpinstance.InstanceProvisioner,
 	eventLog bpeventlog.Log,
 	logger boshlog.Logger,
-) DeploymentProvisioner {
-	return DeploymentProvisioner{
+) SingleConfiguredVMProvisioner {
+	return SingleConfiguredVMProvisioner{
 		manifestPath:            manifestPath,
 		deploymentReaderFactory: deploymentReaderFactory,
 
@@ -48,7 +46,7 @@ func NewDeploymentProvisioner(
 	}
 }
 
-func (p DeploymentProvisioner) Provision() error {
+func (p SingleConfiguredVMProvisioner) Provision() error {
 	stage := p.eventLog.BeginStage("Setting up instance", 2)
 
 	reader := p.deploymentReaderFactory.NewManifestReader(p.manifestPath)
@@ -106,9 +104,7 @@ func (p DeploymentProvisioner) Provision() error {
 	return nil
 }
 
-func (p DeploymentProvisioner) validateInstance(deployment bpdep.Deployment) (bpdep.Job, bpdep.Instance, error) {
-	p.logger.Debug(deploymentProvisionerLogTag, "Validate instance")
-
+func (p SingleConfiguredVMProvisioner) validateInstance(deployment bpdep.Deployment) (bpdep.Job, bpdep.Instance, error) {
 	var job bpdep.Job
 	var instance bpdep.Instance