vagrant-bosh 0.0.1

data/go/src/boshprovisioner/tar/extractor_interface.go

@@ -0,0 +1,6 @@
+package tar
+
+type Extractor interface {
+	Extract(string) (string, error)
+	CleanUp(string) error
+}

data/go/src/boshprovisioner/util/string_keyed.go

@@ -0,0 +1,70 @@
+package util
+
+import (
+	"reflect"
+
+	bosherr "bosh/errors"
+)
+
+type stringKeyed struct{}
+
+func NewStringKeyed() stringKeyed { return stringKeyed{} }
+
+// convertMapToStringKeyMap converts map to string keyed map.
+func (sk stringKeyed) ConvertMap(m map[interface{}]interface{}) (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+
+	for name, val := range m {
+		nameStr, ok := name.(string)
+		if !ok {
+			return result, bosherr.New("Map contains non-string key %v", name)
+		}
+
+		convertedVal, err := sk.ConvertInterface(val)
+		if err != nil {
+			return result, err
+		}
+
+		result[nameStr] = convertedVal
+	}
+
+	return result, nil
+}
+
+func (sk stringKeyed) ConvertInterface(val interface{}) (interface{}, error) {
+	if val == nil {
+		return nil, nil
+	}
+
+	switch reflect.TypeOf(val).Kind() {
+	case reflect.Map:
+		valMap, ok := val.(map[interface{}]interface{})
+		if !ok {
+			return nil, bosherr.New("Converting map %v", val)
+		}
+
+		return sk.ConvertMap(valMap)
+
+	case reflect.Slice:
+		valSlice, ok := val.([]interface{})
+		if !ok {
+			return nil, bosherr.New("Converting slice %v", val)
+		}
+
+		slice := make([]interface{}, len(valSlice))
+
+		for i, v := range valSlice {
+			convertedVal, err := sk.ConvertInterface(v)
+			if err != nil {
+				return nil, err
+			}
+
+			slice[i] = convertedVal
+		}
+
+		return slice, nil
+
+	default:
+		return val, nil
+	}
+}

data/go/src/boshprovisioner/vm/agent_provisioner.go

@@ -0,0 +1,266 @@
+package vm
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpagclient "boshprovisioner/agent/client"
+	bpdep "boshprovisioner/deployment"
+	bpeventlog "boshprovisioner/eventlog"
+)
+
+const agentProvisionerLogTag = "AgentProvisioner"
+
+// AgentProvisioner places BOSH Agent and Monit onto machine
+// installing needed dependencies beforehand.
+type AgentProvisioner struct {
+	fs           boshsys.FileSystem
+	cmds         SimpleCmds
+	assetManager AssetManager
+
+	runitProvisioner RunitProvisioner
+	monitProvisioner MonitProvisioner
+
+	blobstoreConfig map[string]interface{}
+	agentURL        string
+
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewAgentProvisioner(
+	fs boshsys.FileSystem,
+	cmds SimpleCmds,
+	assetManager AssetManager,
+	runitProvisioner RunitProvisioner,
+	monitProvisioner MonitProvisioner,
+	blobstoreConfig map[string]interface{},
+	agentURL string,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) AgentProvisioner {
+	return AgentProvisioner{
+		fs:           fs,
+		cmds:         cmds,
+		assetManager: assetManager,
+
+		runitProvisioner: runitProvisioner,
+		monitProvisioner: monitProvisioner,
+
+		blobstoreConfig: blobstoreConfig,
+		agentURL:        agentURL,
+
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (p AgentProvisioner) Provision(instance bpdep.Instance) (bpagclient.Client, error) {
+	stage := p.eventLog.BeginStage("Updating BOSH agent", 5)
+
+	task := stage.BeginTask("Placing binaries")
+
+	err := task.End(p.placeBinaries())
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Placing agent binaries")
+	}
+
+	task = stage.BeginTask("Configuring settings")
+
+	err = task.End(p.configureSettings(instance))
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Configuring settings")
+	}
+
+	task = stage.BeginTask("Configuring monit")
+
+	err = task.End(p.monitProvisioner.Provision())
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Provisioning monit")
+	}
+
+	task = stage.BeginTask("Starting")
+
+	err = task.End(p.runitProvisioner.Provision("agent"))
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Provisioning agent with runit")
+	}
+
+	agentClient, err := p.buildAgentClient()
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Building agent client")
+	}
+
+	return agentClient, nil
+}
+
+// placeBinaries places agent/monit binaries into /var/vcap/bosh/bin
+func (p AgentProvisioner) placeBinaries() error {
+	binPath := "/var/vcap/bosh/bin"
+
+	// Implicitly creates /var/vcap/bosh
+	err := p.cmds.MkdirP(binPath)
+	if err != nil {
+		return err
+	}
+
+	binNames := map[string]string{
+		"agent/bosh-agent":         "bosh-agent",
+		"agent/bosh-agent-rc":      "bosh-agent-rc",
+		"agent/bosh-blobstore-dav": "bosh-blobstore-dav",
+		"monit/monit":              "monit",
+	}
+
+	for assetName, binName := range binNames {
+		err = p.placeBinary(assetName, filepath.Join(binPath, binName))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) placeBinary(name, path string) error {
+	err := p.assetManager.Place(name, path)
+	if err != nil {
+		return bosherr.WrapError(err, "Placing %s binary", name)
+	}
+
+	err = p.cmds.ChmodX(path)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) configureSettings(instance bpdep.Instance) error {
+	err := p.setUpDataDir()
+	if err != nil {
+		return bosherr.WrapError(err, "Setting up data dir")
+	}
+
+	err = p.placeConfFiles()
+	if err != nil {
+		return bosherr.WrapError(err, "Placing conf files")
+	}
+
+	err = p.placeInfSettings(instance)
+	if err != nil {
+		return bosherr.WrapError(err, "Placing infrastructure settings")
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) setUpDataDir() error {
+	err := p.cmds.Bash("ln -nsf data/sys /var/vcap/sys")
+	if err != nil {
+		return err
+	}
+
+	// todo hacky data dir
+	err = p.cmds.MkdirP("/var/vcap/data")
+	if err != nil {
+		return err
+	}
+
+	err = p.cmds.Chmod("777", "/var/vcap/data")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) placeConfFiles() error {
+	fileNames := map[string]string{
+		"agent/agent.cert": "agent.cert", // Needed by agent HTTP handler
+		"agent/agent.key":  "agent.key",
+		"agent/agent.json": "agent.json",
+	}
+
+	for assetName, fileName := range fileNames {
+		err := p.assetManager.Place(assetName, filepath.Join("/var/vcap/bosh/", fileName))
+		if err != nil {
+			return bosherr.WrapError(err, "Placing %s", fileName)
+		}
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) placeInfSettings(instance bpdep.Instance) error {
+	type h map[string]interface{}
+
+	netSettings := map[string]h{}
+
+	for _, netAssoc := range instance.NetworkAssociations {
+		netConfig := instance.NetworkConfigurationForNetworkAssociation(netAssoc)
+
+		netSettings[netAssoc.Network.Name] = h{
+			"type":    netAssoc.Network.Type,
+			"ip":      netConfig.IP,
+			"netmask": netConfig.Netmask,
+			"gateway": netConfig.Gateway,
+
+			"dns_record_name":  instance.DNDRecordName(netAssoc),
+			"cloud_properties": h{},
+		}
+	}
+
+	settings := h{
+		"agent_id": fmt.Sprintf("agent-id-%s-%d", instance.JobName, instance.Index),
+
+		"vm": h{
+			"name": fmt.Sprintf("vm-name-%s-%d", instance.JobName, instance.Index),
+			"id":   fmt.Sprintf("vm-id-%s-%d", instance.JobName, instance.Index),
+		},
+
+		"networks": netSettings,
+		"disks":    h{"persistent": h{}},
+
+		"blobstore": p.blobstoreConfig,
+		"mbus":      p.agentURL, // todo port can conflict with jobs
+
+		"env": h{},
+		"ntp": []string{},
+	}
+
+	settingsJSON, err := json.Marshal(settings)
+	if err != nil {
+		return bosherr.WrapError(err, "Marshalling agent infrastructure settings")
+	}
+
+	err = p.fs.WriteFile("/var/vcap/bosh/warden-cpi-agent-env.json", settingsJSON)
+	if err != nil {
+		return bosherr.WrapError(err, "Writing agent infrastructure settings")
+	}
+
+	return nil
+}
+
+func (p AgentProvisioner) buildAgentClient() (bpagclient.Client, error) {
+	agentClient, err := bpagclient.NewInsecureHTTPClientWithURI(p.agentURL, p.logger)
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Building agent client")
+	}
+
+	for i := 0; i < 10; i++ {
+		_, err = agentClient.Ping()
+		if err == nil {
+			return agentClient, nil
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	return nil, err
+}

data/go/src/boshprovisioner/vm/asset_manager.go

@@ -0,0 +1,61 @@
+package vm
+
+import (
+	"fmt"
+	"path/filepath"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+type AssetManager struct {
+	rootDir string
+	fs      boshsys.FileSystem
+	runner  boshsys.CmdRunner
+	logger  boshlog.Logger
+}
+
+func NewAssetManager(
+	rootDir string,
+	fs boshsys.FileSystem,
+	runner boshsys.CmdRunner,
+	logger boshlog.Logger,
+) AssetManager {
+	return AssetManager{
+		rootDir: rootDir,
+		fs:      fs,
+		runner:  runner,
+		logger:  logger,
+	}
+}
+
+func (m AssetManager) Place(name, dstPath string) error {
+	srcPath := filepath.Join(m.rootDir, name)
+
+	if !m.fs.FileExists(srcPath) {
+		return bosherr.New("Missing asset %s at %s", name, srcPath)
+	}
+
+	tempDir, err := m.fs.TempDir("vm-AssetManager")
+	if err != nil {
+		return bosherr.WrapError(err, "Creating temp dir")
+	}
+
+	defer m.fs.RemoveAll(tempDir)
+
+	srcCopyPath := fmt.Sprintf("%s/copy", tempDir)
+
+	// todo fs.CopyFile leaks fds
+	_, _, _, err = m.runner.RunCommand("cp", srcPath, srcCopyPath)
+	if err != nil {
+		return bosherr.WrapError(err, "Copying asset")
+	}
+
+	err = m.fs.Rename(srcCopyPath, dstPath)
+	if err != nil {
+		return bosherr.WrapError(err, "Renaming asset %s to %s", name, dstPath)
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/vm/deps_provisioner.go

@@ -0,0 +1,92 @@
+package vm
+
+import (
+	"fmt"
+	"strings"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpeventlog "boshprovisioner/eventlog"
+)
+
+const (
+	depsProvisionerLogTag = "DepsProvisioner"
+	runAptGetUpdateMsg    = "E: Unable to fetch some archives, maybe run apt-get update"
+)
+
+// DepsProvisioner installs basic dependencies for running
+// packaging scripts from BOSH packages. It also installs
+// non-captured dependencies by few common BOSH releases.
+// (e.g. cmake, quota)
+type DepsProvisioner struct {
+	runner   boshsys.CmdRunner
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewDepsProvisioner(
+	runner boshsys.CmdRunner,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) DepsProvisioner {
+	return DepsProvisioner{
+		runner:   runner,
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (p DepsProvisioner) Provision() error {
+	pkgNames := []string{
+		// For packaging scripts in BOSH releases
+		"build-essential", // 16sec
+		"cmake",           // 6sec
+		"libcap-dev",      // 3sec
+
+		"libbz2-1.0",  // noop on precise64 Vagrant box
+		"libbz2-dev",  // 2sec
+		"libxslt-dev", // 2sec
+		"libxml2-dev", // 2sec
+
+		// For warden
+		"quota", // 1sec
+	}
+
+	stage := p.eventLog.BeginStage("Installing dependencies", len(pkgNames))
+
+	for _, pkgName := range pkgNames {
+		task := stage.BeginTask(fmt.Sprintf("Package %s", pkgName))
+
+		err := task.End(p.installPkg(pkgName))
+		if err != nil {
+			return bosherr.WrapError(err, "Installing %s", pkgName)
+		}
+	}
+
+	return nil
+}
+
+func (p DepsProvisioner) installPkg(name string) error {
+	_, _, _, err := p.runner.RunCommand("apt-get", "-y", "install", name)
+	if err == nil {
+		return nil
+	}
+
+	// Avoid running 'apt-get update' since it usually takes 30sec
+	if strings.Contains(err.Error(), runAptGetUpdateMsg) {
+		_, _, _, err := p.runner.RunCommand("apt-get", "-y", "update")
+		if err != nil {
+			return bosherr.WrapError(err, "Updating sources")
+		}
+
+		// Try second time after updating
+		_, _, _, err = p.runner.RunCommand("apt-get", "-y", "install", name)
+		if err != nil {
+			return bosherr.WrapError(err, "Installing %s after updating", name)
+		}
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/vm/monit_provisioner.go

@@ -0,0 +1,83 @@
+package vm
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+)
+
+const monitProvisionerLogTag = "MonitProvisioner"
+
+// MonitProvisioner installs Monit binary.
+type MonitProvisioner struct {
+	cmds             SimpleCmds
+	assetManager     AssetManager
+	runitProvisioner RunitProvisioner
+	logger           boshlog.Logger
+}
+
+func NewMonitProvisioner(
+	cmds SimpleCmds,
+	assetManager AssetManager,
+	runitProvisioner RunitProvisioner,
+	logger boshlog.Logger,
+) MonitProvisioner {
+	return MonitProvisioner{
+		cmds:             cmds,
+		assetManager:     assetManager,
+		runitProvisioner: runitProvisioner,
+		logger:           logger,
+	}
+}
+
+func (p MonitProvisioner) Provision() error {
+	path := "/var/vcap/monit"
+
+	err := p.cmds.MkdirP(path)
+	if err != nil {
+		return err
+	}
+
+	err = p.configureMonitrc()
+	if err != nil {
+		return bosherr.WrapError(err, "Configuring monitrc")
+	}
+
+	err = p.runitProvisioner.Provision("monit")
+	if err != nil {
+		return bosherr.WrapError(err, "Provisioning monit with runit")
+	}
+
+	return nil
+}
+
+func (p MonitProvisioner) configureMonitrc() error {
+	p.logger.Info(monitProvisionerLogTag, "Configuring monitc")
+
+	err := p.cmds.MkdirP("/var/vcap/bosh/etc")
+	if err != nil {
+		return err
+	}
+
+	err = p.assetManager.Place("monit/monitrc", "/var/vcap/bosh/etc/monitrc")
+	if err != nil {
+		return bosherr.WrapError(err, "Placing monitrc")
+	}
+
+	err = p.cmds.Chmod("700", "/var/vcap/bosh/etc/monitrc")
+	if err != nil {
+		return err
+	}
+
+	// monit refuses to start without an include file present
+	err = p.cmds.MkdirP("/var/vcap/monit")
+	if err != nil {
+		return err
+	}
+
+	err = p.cmds.Touch("/var/vcap/monit/empty.monitrc")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}