vagrant-bosh 0.0.3 → 0.0.4

data/go/src/boshprovisioner/release/tar_reader.go

@@ -17,7 +17,8 @@ const tarReaderLogTag = "TarReader"
 // TarReader reads .tgz release file and returns a Release.
 // See unpacked release directory layout at the end of the file.
 type TarReader struct {
-	path       string
+	url string
+
 	downloader bpdload.Downloader
 	extractor  bptar.Extractor
 	fs         boshsys.FileSystem
@@ -29,14 +30,15 @@ type TarReader struct {
 }
 
 func NewTarReader(
-	path string,
+	url string,
 	downloader bpdload.Downloader,
 	extractor bptar.Extractor,
 	fs boshsys.FileSystem,
 	logger boshlog.Logger,
 ) *TarReader {
 	return &TarReader{
-		path:       path,
+		url: url,
+
 		downloader: downloader,
 		extractor:  extractor,
 		fs:         fs,
@@ -44,61 +46,61 @@ func NewTarReader(
 	}
 }
 
-func (tr *TarReader) Read() (Release, error) {
+func (r *TarReader) Read() (Release, error) {
 	var release Release
 
-	downloadPath, err := tr.downloader.Download(tr.path)
+	downloadPath, err := r.downloader.Download(r.url)
 	if err != nil {
 		return release, bosherr.WrapError(err, "Downloading release")
 	}
 
-	tr.downloadPath = downloadPath
+	r.downloadPath = downloadPath
 
-	extractPath, err := tr.extractor.Extract(tr.downloadPath)
+	extractPath, err := r.extractor.Extract(r.downloadPath)
 	if err != nil {
-		cleanUpErr := tr.downloader.CleanUp(tr.downloadPath)
+		cleanUpErr := r.downloader.CleanUp(r.downloadPath)
 		if cleanUpErr != nil {
-			tr.logger.Debug(tarReaderLogTag,
+			r.logger.Debug(tarReaderLogTag,
 				"Failed to clean up downloaded release %v", cleanUpErr)
 		}
 
 		return release, bosherr.WrapError(err, "Extracting release")
 	}
 
-	tr.extractPath = extractPath
+	r.extractPath = extractPath
 
-	manifestPath := filepath.Join(tr.extractPath, "release.MF")
+	manifestPath := filepath.Join(r.extractPath, "release.MF")
 
-	manifest, err := bprelman.NewManifestFromPath(manifestPath, tr.fs)
+	manifest, err := bprelman.NewManifestFromPath(manifestPath, r.fs)
 	if err != nil {
-		closeErr := tr.Close()
+		closeErr := r.Close()
 		if closeErr != nil {
-			tr.logger.Debug(tarReaderLogTag,
+			r.logger.Debug(tarReaderLogTag,
 				"Failed to close release %v", closeErr)
 		}
 
 		return release, bosherr.WrapError(err, "Building manifest")
 	}
 
-	tr.logger.Debug(tarReaderLogTag, "Done building manifest %#v", manifest)
+	r.logger.Debug(tarReaderLogTag, "Done building manifest %#v", manifest)
 
 	release.populateFromManifest(manifest)
 
-	tr.populateReleaseTarPaths(&release)
+	r.populateReleaseTarPaths(&release)
 
 	return release, nil
 }
 
-func (tr TarReader) Close() error {
-	dlErr := tr.downloader.CleanUp(tr.downloadPath)
+func (r TarReader) Close() error {
+	dlErr := r.downloader.CleanUp(r.downloadPath)
 	if dlErr != nil {
-		tr.logger.Debug(tarReaderLogTag,
+		r.logger.Debug(tarReaderLogTag,
 			"Failed to clean up downloaded release %v", dlErr)
 	}
 
-	exErr := tr.extractor.CleanUp(tr.extractPath)
+	exErr := r.extractor.CleanUp(r.extractPath)
 	if exErr != nil {
-		tr.logger.Debug(tarReaderLogTag,
+		r.logger.Debug(tarReaderLogTag,
 			"Failed to clean up extracted release %v", exErr)
 	}
 
@@ -110,15 +112,15 @@ func (tr TarReader) Close() error {
 }
 
 // populateReleaseTarPaths sets TarPath for each job/package in the release.
-func (tr TarReader) populateReleaseTarPaths(release *Release) {
+func (r TarReader) populateReleaseTarPaths(release *Release) {
 	for i, job := range release.Jobs {
 		fileName := job.Name + ".tgz"
-		release.Jobs[i].TarPath = filepath.Join(tr.extractPath, "jobs", fileName)
+		release.Jobs[i].TarPath = filepath.Join(r.extractPath, "jobs", fileName)
 	}
 
 	for _, pkg := range release.Packages {
 		fileName := pkg.Name + ".tgz"
-		pkg.TarPath = filepath.Join(tr.extractPath, "packages", fileName)
+		pkg.TarPath = filepath.Join(r.extractPath, "packages", fileName)
 	}
 }
 

data/go/src/boshprovisioner/vm/vagrant/agent_provisioner.go

@@ -1,4 +1,4 @@
-package vm
+package vagrant
 
 import (
 	"encoding/json"
@@ -13,6 +13,7 @@ import (
 	bpagclient "boshprovisioner/agent/client"
 	bpdep "boshprovisioner/deployment"
 	bpeventlog "boshprovisioner/eventlog"
+	bpvm "boshprovisioner/vm"
 )
 
 const agentProvisionerLogTag = "AgentProvisioner"
@@ -27,8 +28,8 @@ type AgentProvisioner struct {
 	runitProvisioner RunitProvisioner
 	monitProvisioner MonitProvisioner
 
-	blobstoreConfig map[string]interface{}
-	agentURL        string
+	blobstoreConfig        map[string]interface{}
+	agentProvisionerConfig bpvm.AgentProvisionerConfig
 
 	eventLog bpeventlog.Log
 	logger   boshlog.Logger
@@ -41,7 +42,7 @@ func NewAgentProvisioner(
 	runitProvisioner RunitProvisioner,
 	monitProvisioner MonitProvisioner,
 	blobstoreConfig map[string]interface{},
-	agentURL string,
+	agentProvisionerConfig bpvm.AgentProvisionerConfig,
 	eventLog bpeventlog.Log,
 	logger boshlog.Logger,
 ) AgentProvisioner {
@@ -53,43 +54,56 @@ func NewAgentProvisioner(
 		runitProvisioner: runitProvisioner,
 		monitProvisioner: monitProvisioner,
 
-		blobstoreConfig: blobstoreConfig,
-		agentURL:        agentURL,
+		blobstoreConfig:        blobstoreConfig,
+		agentProvisionerConfig: agentProvisionerConfig,
 
 		eventLog: eventLog,
 		logger:   logger,
 	}
 }
 
-func (p AgentProvisioner) Provision(instance bpdep.Instance) (bpagclient.Client, error) {
-	stage := p.eventLog.BeginStage("Updating BOSH agent", 5)
+func (p AgentProvisioner) Provision() error {
+	stage := p.eventLog.BeginStage("Updating BOSH agent", 4)
 
 	task := stage.BeginTask("Placing binaries")
 
 	err := task.End(p.placeBinaries())
 	if err != nil {
-		return nil, bosherr.WrapError(err, "Placing agent binaries")
+		return bosherr.WrapError(err, "Placing agent binaries")
 	}
 
-	task = stage.BeginTask("Configuring settings")
+	task = stage.BeginTask("Placing configuration files")
 
-	err = task.End(p.configureSettings(instance))
+	err = task.End(p.placeConfFiles())
 	if err != nil {
-		return nil, bosherr.WrapError(err, "Configuring settings")
+		return bosherr.WrapError(err, "Configuring settings")
 	}
 
-	task = stage.BeginTask("Configuring monit")
+	task = stage.BeginTask("Registering monit service")
 
 	err = task.End(p.monitProvisioner.Provision())
 	if err != nil {
-		return nil, bosherr.WrapError(err, "Provisioning monit")
+		return bosherr.WrapError(err, "Provisioning monit")
 	}
 
-	task = stage.BeginTask("Starting")
+	task = stage.BeginTask("Registering agent service")
 
 	err = task.End(p.runitProvisioner.Provision("agent", 10*time.Second))
 	if err != nil {
-		return nil, bosherr.WrapError(err, "Provisioning agent with runit")
+		return bosherr.WrapError(err, "Provisioning agent with runit")
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) Configure(instance bpdep.Instance) (bpagclient.Client, error) {
+	stage := p.eventLog.BeginStage("Configuring BOSH agent", 1)
+
+	task := stage.BeginTask("Configuring infrastructure settings")
+
+	err := task.End(p.placeInfSettings(instance))
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Placing infrastructure settings")
 	}
 
 	agentClient, err := p.buildAgentClient()
@@ -141,20 +155,53 @@ func (p AgentProvisioner) placeBinary(name, path string) error {
 	return nil
 }
 
-func (p AgentProvisioner) configureSettings(instance bpdep.Instance) error {
+func (p AgentProvisioner) placeConfFiles() error {
 	err := p.setUpDataDir()
 	if err != nil {
 		return bosherr.WrapError(err, "Setting up data dir")
 	}
 
-	err = p.placeConfFiles()
+	fileNames := map[string]string{
+		"agent/agent.cert": "agent.cert", // Needed by agent HTTP handler
+		"agent/agent.key":  "agent.key",
+	}
+
+	for assetName, fileName := range fileNames {
+		err := p.assetManager.Place(assetName, filepath.Join("/var/vcap/bosh/", fileName))
+		if err != nil {
+			return bosherr.WrapError(err, "Placing %s", fileName)
+		}
+	}
+
+	err = p.placeAgentConf()
 	if err != nil {
-		return bosherr.WrapError(err, "Placing conf files")
+		return bosherr.WrapError(err, "Placing agent configuration")
 	}
 
-	err = p.placeInfSettings(instance)
+	return nil
+}
+
+func (p AgentProvisioner) placeAgentConf() error {
+	// etc/infrastructure and etc/plaform is loaded by BOSH Agent runit script
+	err := p.fs.WriteFileString("/var/vcap/bosh/etc/infrastructure", p.agentProvisionerConfig.Infrastructure)
 	if err != nil {
-		return bosherr.WrapError(err, "Placing infrastructure settings")
+		return bosherr.WrapError(err, "Writing agent infrastructure")
+	}
+
+	err = p.fs.WriteFileString("/var/vcap/bosh/etc/platform", p.agentProvisionerConfig.Platform)
+	if err != nil {
+		return bosherr.WrapError(err, "Writing agent platform")
+	}
+
+	// Go Agent will can unmarshal 'null' into an empty config
+	bytes, err := json.Marshal(p.agentProvisionerConfig.Configuration)
+	if err != nil {
+		return bosherr.WrapError(err, "Marshalling agent configuration")
+	}
+
+	err = p.fs.WriteFile("/var/vcap/bosh/agent.json", bytes)
+	if err != nil {
+		return bosherr.WrapError(err, "Writing agent configuration")
 	}
 
 	return nil
@@ -180,23 +227,6 @@ func (p AgentProvisioner) setUpDataDir() error {
 	return nil
 }
 
-func (p AgentProvisioner) placeConfFiles() error {
-	fileNames := map[string]string{
-		"agent/agent.cert": "agent.cert", // Needed by agent HTTP handler
-		"agent/agent.key":  "agent.key",
-		"agent/agent.json": "agent.json",
-	}
-
-	for assetName, fileName := range fileNames {
-		err := p.assetManager.Place(assetName, filepath.Join("/var/vcap/bosh/", fileName))
-		if err != nil {
-			return bosherr.WrapError(err, "Placing %s", fileName)
-		}
-	}
-
-	return nil
-}
-
 func (p AgentProvisioner) placeInfSettings(instance bpdep.Instance) error {
 	type h map[string]interface{}
 
@@ -228,7 +258,7 @@ func (p AgentProvisioner) placeInfSettings(instance bpdep.Instance) error {
 		"disks":    h{"persistent": h{}},
 
 		"blobstore": p.blobstoreConfig,
-		"mbus":      p.agentURL, // todo port can conflict with jobs
+		"mbus":      p.agentProvisionerConfig.Mbus, // todo port can conflict with jobs
 
 		"env": h{},
 		"ntp": []string{},
@@ -248,7 +278,7 @@ func (p AgentProvisioner) placeInfSettings(instance bpdep.Instance) error {
 }
 
 func (p AgentProvisioner) buildAgentClient() (bpagclient.Client, error) {
-	agentClient, err := bpagclient.NewInsecureHTTPClientWithURI(p.agentURL, p.logger)
+	agentClient, err := bpagclient.NewInsecureHTTPClientWithURI(p.agentProvisionerConfig.Mbus, p.logger)
 	if err != nil {
 		return nil, bosherr.WrapError(err, "Building agent client")
 	}

data/go/src/boshprovisioner/vm/vagrant/asset_manager.go

@@ -1,4 +1,4 @@
-package vm
+package vagrant
 
 import (
 	"fmt"

data/go/src/boshprovisioner/vm/vagrant/configured_vm.go

@@ -0,0 +1,29 @@
+package vagrant
+
+import (
+	bpagclient "boshprovisioner/agent/client"
+)
+
+// ConfiguredVM represents provisioned Vagrant machine
+// that can be communicated with via an AgentClient.
+type ConfiguredVM struct {
+	vmProvisioner *VMProvisioner
+	agentClient   bpagclient.Client
+}
+
+func NewConfiguredVM(vmProvisioner *VMProvisioner, agentClient bpagclient.Client) ConfiguredVM {
+	return ConfiguredVM{
+		vmProvisioner: vmProvisioner,
+		agentClient:   agentClient,
+	}
+}
+
+func (vm ConfiguredVM) AgentClient() bpagclient.Client {
+	return vm.agentClient
+}
+
+func (vm ConfiguredVM) Deprovision() error {
+	return vm.vmProvisioner.deprovision(vm)
+}
+
+func (vm ConfiguredVM) vagrantVM() {}

data/go/src/boshprovisioner/vm/vagrant/deps_provisioner.go

@@ -1,8 +1,9 @@
-package vm
+package vagrant
 
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	bosherr "bosh/errors"
 	boshlog "bosh/logger"
@@ -52,7 +53,7 @@ func (p DepsProvisioner) Provision() error {
 
 	stage := p.eventLog.BeginStage("Installing dependencies", len(pkgNames))
 
-	installedPkgs, err := p.listInstalledPkgs()
+	installedPkgNames, err := p.listInstalledPkgNames()
 	if err != nil {
 		return bosherr.WrapError(err, "Listing installed packages")
 	}
@@ -60,7 +61,7 @@ func (p DepsProvisioner) Provision() error {
 	for _, pkgName := range pkgNames {
 		task := stage.BeginTask(fmt.Sprintf("Package %s", pkgName))
 
-		if p.isPkgInstalled(pkgName, installedPkgs) {
+		if p.isPkgInstalled(pkgName, installedPkgNames) {
 			p.logger.Debug(depsProvisionerLogTag, "Package %s is already installed", pkgName)
 			task.End(nil)
 			continue
@@ -90,39 +91,50 @@ func (p DepsProvisioner) installPkg(name string) error {
 			return bosherr.WrapError(err, "Updating sources")
 		}
 
-		// Try second time after updating
-		_, _, _, err = p.runner.RunCommand("apt-get", "-y", "install", name)
-		if err != nil {
-			return bosherr.WrapError(err, "Installing %s after updating", name)
+		var lastInstallErr error
+
+		// For some reason libssl-dev was really hard to install on the first try
+		for i := 0; i < 3; i++ {
+			_, _, _, lastInstallErr = p.runner.RunCommand("apt-get", "-y", "install", name)
+			if lastInstallErr == nil {
+				return nil
+			}
+
+			time.Sleep(1 * time.Second)
 		}
+
+		return bosherr.WrapError(lastInstallErr, "Installing %s after updating", name)
 	}
 
-	return nil
+	return err
 }
 
-func (p DepsProvisioner) listInstalledPkgs() ([]string, error) {
-	var installedPkgs []string
+func (p DepsProvisioner) listInstalledPkgNames() ([]string, error) {
+	var installedPkgNames []string
 
 	installedPkgStdout, _, _, err := p.runner.RunCommand("dpkg", "--get-selections")
 	if err != nil {
-		return installedPkgs, bosherr.WrapError(err, "dkpg query")
+		return nil, bosherr.WrapError(err, "dkpg query")
 	}
 
+	// e.g. 'zlib1g:amd64 install'
+	//      'util-linux   install'
 	for _, line := range strings.Split(installedPkgStdout, "\n") {
 		pieces := strings.Fields(line)
 
 		// Last line is empty
 		if len(pieces) == 2 && pieces[1] == "install" {
-			installedPkgs = append(installedPkgs, pieces[0])
+			pkgName := strings.Split(pieces[0], ":")[0]
+			installedPkgNames = append(installedPkgNames, pkgName)
 		}
 	}
 
-	return installedPkgs, nil
+	return installedPkgNames, nil
 }
 
 func (p DepsProvisioner) isPkgInstalled(pkgName string, installedPkgs []string) bool {
 	for _, installedPkgName := range installedPkgs {
-		if pkgName == installedPkgName {
+		if installedPkgName == pkgName {
 			return true
 		}
 	}
@@ -131,12 +143,12 @@ func (p DepsProvisioner) isPkgInstalled(pkgName string, installedPkgs []string)
 }
 
 var depsProvisionerPkgsForMinimumStemcellCompatibility = []string{
-	// For packaging scripts in BOSH releases
+	// Most BOSH releases require it for packaging
 	"build-essential", // 16sec
 	"cmake",           // 6sec
 
 	"libcap2-bin",
-	"libcap2-dev",
+	"libcap-dev",
 
 	"libbz2-1.0",   // noop on precise64 Vagrant box
 	"libbz2-dev",   // 2sec
@@ -148,6 +160,16 @@ var depsProvisionerPkgsForMinimumStemcellCompatibility = []string{
 
 	// For warden
 	"quota", // 1sec
+
+	// Started needing that in saucy for building BOSH
+	"libssl-dev",
+
+	"bison",
+	"flex",
+
+	"gettext",
+	"libreadline6-dev",
+	"libncurses5-dev",
 }
 
 // Taken from base_apt stemcell builder stage
@@ -159,11 +181,6 @@ var depsProvisionerPkgsForFullStemcellCompatibility = []string{
 	"apparmor-utils",
 	"openssh-server",
 
-	"gettext",
-	"libreadline6-dev",
-	"libncurses5-dev",
-
-	"libssl-dev",
 	"libgcrypt-dev",
 	"ca-certificates",
 
@@ -177,9 +194,6 @@ var depsProvisionerPkgsForFullStemcellCompatibility = []string{
 	"libxslt1.1",
 	"libxslt1-dev",
 
-	"bison",
-	"flex",
-
 	// Utils
 	"bind9-host",
 	"dnsutils",

data/go/src/boshprovisioner/vm/vagrant/monit_provisioner.go

@@ -1,4 +1,4 @@
-package vm
+package vagrant
 
 import (
 	"time"

data/go/src/boshprovisioner/vm/vagrant/non_configured_vm.go

@@ -0,0 +1,26 @@
+package vagrant
+
+import (
+	bpagclient "boshprovisioner/agent/client"
+)
+
+// NonConfiguredVM represents provisioned Vagrant machine
+// that CANNOT be communicated with via an AgentClient.
+type NonConfiguredVM struct {
+	vmProvisioner *VMProvisioner
+}
+
+func NewNonConfiguredVM(vmProvisioner *VMProvisioner) NonConfiguredVM {
+	return NonConfiguredVM{vmProvisioner: vmProvisioner}
+}
+
+func (vm NonConfiguredVM) AgentClient() bpagclient.Client {
+	// Programmer error
+	panic("Must not ask for AgentClient from a non-configured VM")
+}
+
+func (vm NonConfiguredVM) Deprovision() error {
+	return vm.vmProvisioner.deprovision(vm)
+}
+
+func (vm NonConfiguredVM) vagrantVM() {}

data/go/src/boshprovisioner/vm/vagrant/runit_provisioner.go

@@ -1,4 +1,4 @@
-package vm
+package vagrant
 
 import (
 	"fmt"

data/go/src/boshprovisioner/vm/vagrant/simple_cmds.go

@@ -1,4 +1,4 @@
-package vm
+package vagrant
 
 import (
 	boshlog "bosh/logger"