vagrant-bosh 0.0.2 → 0.0.3

data/go/src/boshprovisioner/vm/vagrant/deps_provisioner.go

@@ -0,0 +1,200 @@
+package vm
+
+import (
+	"fmt"
+	"strings"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpeventlog "boshprovisioner/eventlog"
+)
+
+const (
+	depsProvisionerLogTag          = "DepsProvisioner"
+	depsProvisionerAptGetUpdateMsg = "E: Unable to fetch some archives, maybe run apt-get update"
+)
+
+// DepsProvisioner installs basic dependencies for running
+// packaging scripts from BOSH packages. It also installs
+// non-captured dependencies by few common BOSH releases.
+// (e.g. cmake, quota)
+type DepsProvisioner struct {
+	fullStemcellCompatibility bool
+
+	runner   boshsys.CmdRunner
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewDepsProvisioner(
+	fullStemcellCompatibility bool,
+	runner boshsys.CmdRunner,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) DepsProvisioner {
+	return DepsProvisioner{
+		fullStemcellCompatibility: fullStemcellCompatibility,
+
+		runner:   runner,
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (p DepsProvisioner) Provision() error {
+	pkgNames := depsProvisionerPkgsForMinimumStemcellCompatibility
+
+	if p.fullStemcellCompatibility {
+		pkgNames = append(pkgNames, depsProvisionerPkgsForFullStemcellCompatibility...)
+	}
+
+	stage := p.eventLog.BeginStage("Installing dependencies", len(pkgNames))
+
+	installedPkgs, err := p.listInstalledPkgs()
+	if err != nil {
+		return bosherr.WrapError(err, "Listing installed packages")
+	}
+
+	for _, pkgName := range pkgNames {
+		task := stage.BeginTask(fmt.Sprintf("Package %s", pkgName))
+
+		if p.isPkgInstalled(pkgName, installedPkgs) {
+			p.logger.Debug(depsProvisionerLogTag, "Package %s is already installed", pkgName)
+			task.End(nil)
+			continue
+		}
+
+		err := task.End(p.installPkg(pkgName))
+		if err != nil {
+			return bosherr.WrapError(err, "Installing %s", pkgName)
+		}
+	}
+
+	return nil
+}
+
+func (p DepsProvisioner) installPkg(name string) error {
+	p.logger.Debug(depsProvisionerLogTag, "Installing package %s", name)
+
+	_, _, _, err := p.runner.RunCommand("apt-get", "-y", "install", name)
+	if err == nil {
+		return nil
+	}
+
+	// Avoid running 'apt-get update' since it usually takes 30sec
+	if strings.Contains(err.Error(), depsProvisionerAptGetUpdateMsg) {
+		_, _, _, err := p.runner.RunCommand("apt-get", "-y", "update")
+		if err != nil {
+			return bosherr.WrapError(err, "Updating sources")
+		}
+
+		// Try second time after updating
+		_, _, _, err = p.runner.RunCommand("apt-get", "-y", "install", name)
+		if err != nil {
+			return bosherr.WrapError(err, "Installing %s after updating", name)
+		}
+	}
+
+	return nil
+}
+
+func (p DepsProvisioner) listInstalledPkgs() ([]string, error) {
+	var installedPkgs []string
+
+	installedPkgStdout, _, _, err := p.runner.RunCommand("dpkg", "--get-selections")
+	if err != nil {
+		return installedPkgs, bosherr.WrapError(err, "dkpg query")
+	}
+
+	for _, line := range strings.Split(installedPkgStdout, "\n") {
+		pieces := strings.Fields(line)
+
+		// Last line is empty
+		if len(pieces) == 2 && pieces[1] == "install" {
+			installedPkgs = append(installedPkgs, pieces[0])
+		}
+	}
+
+	return installedPkgs, nil
+}
+
+func (p DepsProvisioner) isPkgInstalled(pkgName string, installedPkgs []string) bool {
+	for _, installedPkgName := range installedPkgs {
+		if pkgName == installedPkgName {
+			return true
+		}
+	}
+
+	return false
+}
+
+var depsProvisionerPkgsForMinimumStemcellCompatibility = []string{
+	// For packaging scripts in BOSH releases
+	"build-essential", // 16sec
+	"cmake",           // 6sec
+
+	"libcap2-bin",
+	"libcap2-dev",
+
+	"libbz2-1.0",   // noop on precise64 Vagrant box
+	"libbz2-dev",   // 2sec
+	"libxslt1-dev", // 2sec
+	"libxml2-dev",  // 2sec
+
+	// Used by BOSH Agent
+	"iputils-arping",
+
+	// For warden
+	"quota", // 1sec
+}
+
+// Taken from base_apt stemcell builder stage
+var depsProvisionerPkgsForFullStemcellCompatibility = []string{
+	"libaio1",
+	"uuid-dev",
+	"nfs-common",
+	"zlib1g-dev",
+	"apparmor-utils",
+	"openssh-server",
+
+	"gettext",
+	"libreadline6-dev",
+	"libncurses5-dev",
+
+	"libssl-dev",
+	"libgcrypt-dev",
+	"ca-certificates",
+
+	// CURL
+	"libcurl3",
+	"libcurl3-dev",
+
+	// XML
+	"libxml2",
+	"libxml2-dev",
+	"libxslt1.1",
+	"libxslt1-dev",
+
+	"bison",
+	"flex",
+
+	// Utils
+	"bind9-host",
+	"dnsutils",
+	"zip",
+	"unzip",
+	"psmisc",
+	"lsof",
+	"strace",
+	"curl",
+	"wget",
+	"gdb",
+	"sysstat",
+	"rsync",
+
+	"iptables",
+	"tcpdump",
+	"traceroute",
+}

data/go/src/boshprovisioner/vm/{monit_provisioner.go → vagrant/monit_provisioner.go}

@@ -1,6 +1,8 @@
 package vm
 
 import (
+	"time"
+
 	bosherr "bosh/errors"
 	boshlog "bosh/logger"
 )
@@ -42,7 +44,7 @@ func (p MonitProvisioner) Provision() error {
 		return bosherr.WrapError(err, "Configuring monitrc")
 	}
 
-	err = p.runitProvisioner.Provision("monit")
+	err = p.runitProvisioner.Provision("monit", 1*time.Minute)
 	if err != nil {
 		return bosherr.WrapError(err, "Provisioning monit with runit")
 	}

data/go/src/boshprovisioner/vm/{runit_provisioner.go → vagrant/runit_provisioner.go}

@@ -11,7 +11,10 @@ import (
 	boshsys "bosh/system"
 )
 
-const runitProvisionerLogTag = "RunitProvisioner"
+const (
+	runitProvisionerLogTag = "RunitProvisioner"
+	runitStopStepDuration  = 1 * time.Second
+)
 
 var (
 	// Matches 'svlogd -tt /var/vcap/bosh/log'
@@ -47,13 +50,13 @@ func NewRunitProvisioner(
 	}
 }
 
-func (p RunitProvisioner) Provision(name string) error {
+func (p RunitProvisioner) Provision(name string, stopTimeout time.Duration) error {
 	err := p.installRunit()
 	if err != nil {
 		return bosherr.WrapError(err, "Installing runit")
 	}
 
-	err = p.setUpService(name)
+	err = p.setUpService(name, stopTimeout)
 	if err != nil {
 		return bosherr.WrapError(err, "Setting up service")
 	}
@@ -83,13 +86,13 @@ func (p RunitProvisioner) installRunit() error {
 	return nil
 }
 
-func (p RunitProvisioner) setUpService(name string) error {
+func (p RunitProvisioner) setUpService(name string, stopTimeout time.Duration) error {
 	p.logger.Info(runitProvisionerLogTag, "Setting up %s service", name)
 
 	servicePath := fmt.Sprintf("/etc/sv/%s", name)
 	enableServicePath := fmt.Sprintf("/etc/service/%s", name)
 
-	err := p.stopRunAndLog(servicePath, enableServicePath, name)
+	err := p.stopRunAndLog(servicePath, enableServicePath, name, stopTimeout)
 	if err != nil {
 		return bosherr.WrapError(err, "Stopping run and log")
 	}
@@ -169,13 +172,13 @@ func (p RunitProvisioner) setUpLog(servicePath, name string) error {
 	return nil
 }
 
-func (p RunitProvisioner) stopRunAndLog(servicePath, enableServicePath, name string) error {
-	err := p.stopRunsv(name)
+func (p RunitProvisioner) stopRunAndLog(servicePath, enableServicePath, name string, stopTimeout time.Duration) error {
+	err := p.stopRunsv(name, stopTimeout)
 	if err != nil {
 		return bosherr.WrapError(err, "Stopping service")
 	}
 
-	err = p.stopRunsv(fmt.Sprintf("%s/log", name))
+	err = p.stopRunsv(fmt.Sprintf("%s/log", name), stopTimeout)
 	if err != nil {
 		return bosherr.WrapError(err, "Stopping log service")
 	}
@@ -196,7 +199,7 @@ func (p RunitProvisioner) startRunAndLog(servicePath, enableServicePath, name st
 	return err
 }
 
-func (p RunitProvisioner) stopRunsv(name string) error {
+func (p RunitProvisioner) stopRunsv(name string, stopTimeout time.Duration) error {
 	p.logger.Info(runitProvisionerLogTag, "Stopping runsv")
 
 	downStdout, _, _, err := p.runner.RunCommand("sv", "down", name)
@@ -210,15 +213,16 @@ func (p RunitProvisioner) stopRunsv(name string) error {
 	}
 
 	var lastStatusStdout string
+	var passedDuration time.Duration
 
-	for i := 0; i < 20; i++ {
+	for ; passedDuration < stopTimeout; passedDuration += runitStopStepDuration {
 		lastStatusStdout, _, _, _ = p.runner.RunCommand("sv", "status", name)
 
 		if runitStatusDownRegex.MatchString(lastStatusStdout) {
 			return nil
 		}
 
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(runitStopStepDuration)
 	}
 
 	return bosherr.New("Failed to stop runsv for %s. Output: %s", name, lastStatusStdout)

data/go/src/boshprovisioner/vm/vagrant/vm.go

@@ -0,0 +1,27 @@
+package vm
+
+import (
+	bpagclient "boshprovisioner/agent/client"
+)
+
+// VM represents already provisioned Vagrant machine
+// that can be communicated with via an AgentClient.
+type VM struct {
+	vmProvisioner *VMProvisioner
+	agentClient   bpagclient.Client
+}
+
+func NewVM(vmProvisioner *VMProvisioner, agentClient bpagclient.Client) VM {
+	return VM{
+		vmProvisioner: vmProvisioner,
+		agentClient:   agentClient,
+	}
+}
+
+func (vm VM) AgentClient() bpagclient.Client {
+	return vm.agentClient
+}
+
+func (vm VM) Deprovision() error {
+	return vm.vmProvisioner.Deprovision(vm)
+}

data/go/src/boshprovisioner/vm/vagrant/vm_provisioner.go

@@ -0,0 +1,81 @@
+package vm
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpvm "boshprovisioner/vm"
+)
+
+// VMProvisioner installs system dependencies that are usually
+// found on a stemcell, adds vcap user and finally install Agent and Monit.
+type VMProvisioner struct {
+	vcapUserProvisioner VCAPUserProvisioner
+	depsProvisioner     DepsProvisioner
+	agentProvisioner    AgentProvisioner
+
+	// Cannot provision more than 1 VM at a time until previous VM is deprovisioned.
+	vmProvisioned bool
+
+	// Remember if we have recently run provisioning for a VM to skip it next time
+	vmPreviouslyProvisioned bool
+
+	logger boshlog.Logger
+}
+
+func NewVMProvisioner(
+	vcapUserProvisioner VCAPUserProvisioner,
+	depsProvisioner DepsProvisioner,
+	agentProvisioner AgentProvisioner,
+	logger boshlog.Logger,
+) *VMProvisioner {
+	return &VMProvisioner{
+		vcapUserProvisioner: vcapUserProvisioner,
+		depsProvisioner:     depsProvisioner,
+		agentProvisioner:    agentProvisioner,
+
+		logger: logger,
+	}
+}
+
+func (p *VMProvisioner) Provision(instance bpdep.Instance) (bpvm.VM, error) {
+	var vm VM
+
+	if p.vmProvisioned {
+		return vm, bosherr.New("Vagrant VM is already provisioned")
+	}
+
+	if !p.vmPreviouslyProvisioned {
+		err := p.vcapUserProvisioner.Provision()
+		if err != nil {
+			return vm, bosherr.WrapError(err, "Provisioning vcap user")
+		}
+
+		err = p.depsProvisioner.Provision()
+		if err != nil {
+			return vm, bosherr.WrapError(err, "Provisioning dependencies")
+		}
+	}
+
+	p.vmPreviouslyProvisioned = true
+
+	agentClient, err := p.agentProvisioner.Provision(instance)
+	if err != nil {
+		return vm, bosherr.WrapError(err, "Provisioning agent")
+	}
+
+	p.vmProvisioned = true
+
+	return NewVM(p, agentClient), nil
+}
+
+func (p *VMProvisioner) Deprovision(vm VM) error {
+	if !p.vmProvisioned {
+		return bosherr.New("Vagrant VM is not provisioned")
+	}
+
+	p.vmProvisioned = false
+
+	return nil
+}

data/go/src/boshprovisioner/vm/{vm_provisioner_factory.go → vagrant/vm_provisioner_factory.go}

@@ -5,15 +5,18 @@ import (
 	boshsys "bosh/system"
 
 	bpeventlog "boshprovisioner/eventlog"
+	bpvm "boshprovisioner/vm"
 )
 
 type VMProvisionerFactory struct {
 	fs     boshsys.FileSystem
 	runner boshsys.CmdRunner
 
-	assetsDir       string
-	mbus            string
-	blobstoreConfig map[string]interface{}
+	assetsDir string
+	mbus      string
+
+	blobstoreConfig     map[string]interface{}
+	vmProvisionerConfig bpvm.VMProvisionerConfig
 
 	eventLog bpeventlog.Log
 	logger   boshlog.Logger
@@ -25,6 +28,7 @@ func NewVMProvisionerFactory(
 	assetsDir string,
 	mbus string,
 	blobstoreConfig map[string]interface{},
+	vmProvisionerConfig bpvm.VMProvisionerConfig,
 	eventLog bpeventlog.Log,
 	logger boshlog.Logger,
 ) VMProvisionerFactory {
@@ -32,16 +36,18 @@ func NewVMProvisionerFactory(
 		fs:     fs,
 		runner: runner,
 
-		assetsDir:       assetsDir,
-		mbus:            mbus,
-		blobstoreConfig: blobstoreConfig,
+		assetsDir: assetsDir,
+		mbus:      mbus,
+
+		blobstoreConfig:     blobstoreConfig,
+		vmProvisionerConfig: vmProvisionerConfig,
 
 		eventLog: eventLog,
 		logger:   logger,
 	}
 }
 
-func (f VMProvisionerFactory) NewVMProvisioner() VMProvisioner {
+func (f VMProvisionerFactory) NewVMProvisioner() *VMProvisioner {
 	cmds := NewSimpleCmds(f.runner, f.logger)
 
 	vcapUserProvisioner := NewVCAPUserProvisioner(
@@ -69,6 +75,7 @@ func (f VMProvisionerFactory) NewVMProvisioner() VMProvisioner {
 	)
 
 	depsProvisioner := NewDepsProvisioner(
+		f.vmProvisionerConfig.FullStemcellCompatibility,
 		f.runner,
 		f.eventLog,
 		f.logger,