vagrant-bosh 0.0.1

data/go/src/boshprovisioner/packagescompiler/concrete_packages_compiler.go

@@ -0,0 +1,179 @@
+package packagescompiler
+
+import (
+	"fmt"
+
+	boshcomp "bosh/agent/compiler"
+	boshblob "bosh/blobstore"
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bragentclient "boshprovisioner/agent/client"
+	bpeventlog "boshprovisioner/eventlog"
+	bpcpkgsrepo "boshprovisioner/packagescompiler/compiledpackagesrepo"
+	bppkgsrepo "boshprovisioner/packagescompiler/packagesrepo"
+	bprel "boshprovisioner/release"
+)
+
+const concretePackagesCompilerLogTag = "ConcretePackagesCompiler"
+
+type ConcretePackagesCompiler struct {
+	agentClient          bragentclient.Client
+	packagesRepo         bppkgsrepo.PackagesRepository
+	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository
+	blobstore            boshblob.Blobstore
+
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewConcretePackagesCompiler(
+	agentClient bragentclient.Client,
+	packagesRepo bppkgsrepo.PackagesRepository,
+	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository,
+	blobstore boshblob.Blobstore,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) ConcretePackagesCompiler {
+	return ConcretePackagesCompiler{
+		agentClient:          agentClient,
+		packagesRepo:         packagesRepo,
+		compiledPackagesRepo: compiledPackagesRepo,
+		blobstore:            blobstore,
+
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+// Compile populates blobstore with compiled packages for a given release packages.
+// All packages are compiled regardless if they will be later used or not.
+// Currently Compile does not account for stemcell differences.
+func (pc ConcretePackagesCompiler) Compile(release bprel.Release) error {
+	packages := release.ResolvedPackageDependencies()
+
+	releaseDesc := fmt.Sprintf("Compiling release %s/%s", release.Name, release.Version)
+
+	stage := pc.eventLog.BeginStage(releaseDesc, len(packages))
+
+	for _, pkg := range packages {
+		pkgDesc := fmt.Sprintf("%s/%s", pkg.Name, pkg.Version)
+
+		task := stage.BeginTask(fmt.Sprintf("Package %s", pkgDesc))
+
+		_, found, err := pc.compiledPackagesRepo.Find(*pkg)
+		if err != nil {
+			return task.End(bosherr.WrapError(err, "Finding compiled package %s", pkg.Name))
+		} else if found {
+			task.End(nil)
+			continue
+		}
+
+		err = task.End(pc.compilePkg(*pkg))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// FindCompiledPackage returns previously compiled package for a given template.
+// If such compiled package is not found, error is returned.
+func (pc ConcretePackagesCompiler) FindCompiledPackage(pkg bprel.Package) (CompiledPackageRecord, error) {
+	var compiledPkgRec CompiledPackageRecord
+
+	rec, found, err := pc.compiledPackagesRepo.Find(pkg)
+	if err != nil {
+		return compiledPkgRec, bosherr.WrapError(err, "Finding compiled package %s", pkg.Name)
+	} else if !found {
+		return compiledPkgRec, bosherr.New("Expected to find compiled package %s", pkg.Name)
+	}
+
+	compiledPkgRec.SHA1 = rec.SHA1
+	compiledPkgRec.BlobID = rec.BlobID
+
+	return compiledPkgRec, nil
+}
+
+// compilePackage populates blobstore with a compiled package for a
+// given package. Assumes that dependencies of given package have
+// already been compiled and are in the blobstore.
+func (pc ConcretePackagesCompiler) compilePkg(pkg bprel.Package) error {
+	pc.logger.Debug(concretePackagesCompilerLogTag,
+		"Preparing to compile package %v", pkg)
+
+	pkgRec, found, err := pc.packagesRepo.Find(pkg)
+	if err != nil {
+		return bosherr.WrapError(err, "Finding package source blob %s", pkg.Name)
+	}
+
+	if !found {
+		blobID, fingerprint, err := pc.blobstore.Create(pkg.TarPath)
+		if err != nil {
+			return bosherr.WrapError(err, "Creating package source blob %s", pkg.Name)
+		}
+
+		pkgRec = bppkgsrepo.PackageRecord{
+			BlobID: blobID,
+			SHA1:   fingerprint,
+		}
+
+		err = pc.packagesRepo.Save(pkg, pkgRec)
+		if err != nil {
+			return bosherr.WrapError(err, "Saving package record %s", pkg.Name)
+		}
+	}
+
+	deps, err := pc.buildPkgDeps(pkg)
+	if err != nil {
+		return err
+	}
+
+	compiledPkgRes, err := pc.agentClient.CompilePackage(
+		pkgRec.BlobID, // source tar
+		pkgRec.SHA1,   // source tar
+		pkg.Name,
+		pkg.Version,
+		deps,
+	)
+	if err != nil {
+		return bosherr.WrapError(err, "Compiling package %s", pkg.Name)
+	}
+
+	compiledPkgRec := bpcpkgsrepo.CompiledPackageRecord{
+		BlobID: compiledPkgRes.BlobID,
+		SHA1:   compiledPkgRes.SHA1,
+	}
+
+	err = pc.compiledPackagesRepo.Save(pkg, compiledPkgRec)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving compiled package %s", pkg.Name)
+	}
+
+	return nil
+}
+
+// buildPkgDeps prepares dependencies for agent's compile_package.
+// Assumes that all package dependencies were already compiled.
+func (pc ConcretePackagesCompiler) buildPkgDeps(pkg bprel.Package) (boshcomp.Dependencies, error) {
+	deps := boshcomp.Dependencies{}
+
+	for _, depPkg := range pkg.Dependencies {
+		compiledPkgRec, found, err := pc.compiledPackagesRepo.Find(*depPkg)
+		if err != nil {
+			return deps, bosherr.WrapError(err, "Finding compiled package %s", depPkg.Name)
+		} else if !found {
+			return deps, bosherr.New("Expected to find compiled package %s", depPkg.Name)
+		}
+
+		deps[depPkg.Name] = boshcomp.Package{
+			Name:        depPkg.Name,
+			Version:     depPkg.Version,
+			BlobstoreID: compiledPkgRec.BlobID, // compiled tar
+			Sha1:        compiledPkgRec.SHA1,   // compiled tar
+		}
+	}
+
+	return deps, nil
+}

data/go/src/boshprovisioner/packagescompiler/concrete_packages_compiler_factory.go

@@ -0,0 +1,48 @@
+package packagescompiler
+
+import (
+	boshblob "bosh/blobstore"
+	boshlog "bosh/logger"
+
+	bpagclient "boshprovisioner/agent/client"
+	bpeventlog "boshprovisioner/eventlog"
+	bpcpkgsrepo "boshprovisioner/packagescompiler/compiledpackagesrepo"
+	bppkgsrepo "boshprovisioner/packagescompiler/packagesrepo"
+)
+
+type ConcretePackagesCompilerFactory struct {
+	packagesRepo         bppkgsrepo.PackagesRepository
+	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository
+	blobstore            boshblob.Blobstore
+
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewConcretePackagesCompilerFactory(
+	packagesRepo bppkgsrepo.PackagesRepository,
+	compiledPackagesRepo bpcpkgsrepo.CompiledPackagesRepository,
+	blobstore boshblob.Blobstore,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) ConcretePackagesCompilerFactory {
+	return ConcretePackagesCompilerFactory{
+		packagesRepo:         packagesRepo,
+		compiledPackagesRepo: compiledPackagesRepo,
+		blobstore:            blobstore,
+
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (f ConcretePackagesCompilerFactory) NewCompiler(agentClient bpagclient.Client) PackagesCompiler {
+	return NewConcretePackagesCompiler(
+		agentClient,
+		f.packagesRepo,
+		f.compiledPackagesRepo,
+		f.blobstore,
+		f.eventLog,
+		f.logger,
+	)
+}

data/go/src/boshprovisioner/packagescompiler/packages_compiler_interface.go

@@ -0,0 +1,20 @@
+package packagescompiler
+
+import (
+	bprel "boshprovisioner/release"
+)
+
+type CompiledPackageRecord struct {
+	SHA1   string
+	BlobID string
+}
+
+// PackagesCompiler takes each release package and compiles it.
+// Compiled packages are used as:
+//   (1) compile dependencies for other packages
+//   (2) runtime dependencies for jobs
+// todo account for stemcells
+type PackagesCompiler interface {
+	Compile(bprel.Release) error
+	FindCompiledPackage(bprel.Package) (CompiledPackageRecord, error)
+}

data/go/src/boshprovisioner/packagescompiler/packagesrepo/concrete_packages_repository.go

@@ -0,0 +1,65 @@
+package packagesrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpindex "boshprovisioner/index"
+	bprel "boshprovisioner/release"
+)
+
+type CPRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+type pkgToPkgRecKey struct {
+	// Mostly for ease of debugging
+	PackageName    string
+	PackageVersion string
+
+	// Fingerprint of a package captures its dependenices
+	PackageFingerprint string
+}
+
+func NewConcretePackagesRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CPRepository {
+	return CPRepository{
+		index:  index,
+		logger: logger,
+	}
+}
+
+func (r CPRepository) Find(pkg bprel.Package) (PackageRecord, bool, error) {
+	var record PackageRecord
+
+	err := r.index.Find(r.pkgKey(pkg), &record)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return record, false, nil
+		}
+
+		return record, false, bosherr.WrapError(err, "Finding package record")
+	}
+
+	return record, true, nil
+}
+
+func (r CPRepository) Save(pkg bprel.Package, record PackageRecord) error {
+	err := r.index.Save(r.pkgKey(pkg), record)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving package record")
+	}
+
+	return nil
+}
+
+func (r CPRepository) pkgKey(pkg bprel.Package) pkgToPkgRecKey {
+	return pkgToPkgRecKey{
+		PackageName:        pkg.Name,
+		PackageVersion:     pkg.Version,
+		PackageFingerprint: pkg.Fingerprint,
+	}
+}

data/go/src/boshprovisioner/packagescompiler/packagesrepo/packages_repository_interface.go

@@ -0,0 +1,16 @@
+package packagesrepo
+
+import (
+	bprel "boshprovisioner/release"
+)
+
+type PackageRecord struct {
+	BlobID string
+	SHA1   string
+}
+
+// PackagesRepository maintains list of package source code as blobs.
+type PackagesRepository interface {
+	Find(bprel.Package) (PackageRecord, bool, error)
+	Save(bprel.Package, PackageRecord) error
+}

data/go/src/boshprovisioner/provisioner/blobstore_config.go

@@ -0,0 +1,65 @@
+package provisioner
+
+import (
+	bosherr "bosh/errors"
+)
+
+const (
+	BlobstoreConfigTypeLocal = "local"
+)
+
+type BlobstoreConfig struct {
+	Type    string                 `json:"provider"`
+	Options map[string]interface{} `json:"options"`
+}
+
+func (c BlobstoreConfig) Validate() error {
+	// Only local blobstore provides options[blobstore_path]
+	if c.Type == BlobstoreConfigTypeLocal {
+		_, err := c.extractLocalPath()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c BlobstoreConfig) LocalPath() string {
+	if c.Type != BlobstoreConfigTypeLocal {
+		return ""
+	}
+
+	path, err := c.extractLocalPath()
+	if err != nil {
+		return ""
+	}
+
+	return path
+}
+
+func (c BlobstoreConfig) extractLocalPath() (string, error) {
+	path, ok := c.Options["blobstore_path"]
+	if !ok {
+		return "", bosherr.New("Missing blobstore_path in options")
+	}
+
+	pathStr, ok := path.(string)
+	if !ok {
+		return "", bosherr.New("Must provide blobstore_path as a string")
+	}
+
+	if pathStr == "" {
+		return "", bosherr.New("Must provide non-empty blobstore_path in options")
+	}
+
+	return pathStr, nil
+}
+
+// AsMap is used to populate agent infrastructure configuration
+func (c BlobstoreConfig) AsMap() map[string]interface{} {
+	return map[string]interface{}{
+		"provider": c.Type,
+		"options":  c.Options,
+	}
+}

data/go/src/boshprovisioner/provisioner/blobstore_provisioner.go

@@ -0,0 +1,38 @@
+package provisioner
+
+import (
+	"os"
+
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+type BlobstoreProvisioner struct {
+	fs              boshsys.FileSystem
+	blobstoreConfig BlobstoreConfig
+	logger          boshlog.Logger
+}
+
+func NewBlobstoreProvisioner(
+	fs boshsys.FileSystem,
+	blobstoreConfig BlobstoreConfig,
+	logger boshlog.Logger,
+) BlobstoreProvisioner {
+	return BlobstoreProvisioner{
+		fs:              fs,
+		blobstoreConfig: blobstoreConfig,
+		logger:          logger,
+	}
+}
+
+func (p BlobstoreProvisioner) Provision() error {
+	blobstorePath := p.blobstoreConfig.LocalPath()
+	if blobstorePath != "" {
+		err := p.fs.MkdirAll(blobstorePath, os.ModeDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/provisioner/deployment_provisioner.go

@@ -0,0 +1,97 @@
+package provisioner
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpeventlog "boshprovisioner/eventlog"
+)
+
+const deploymentProvisionerLogTag = "DeploymentProvisioner"
+
+// DeploymentProvisioner interprets deployment manifest and
+// configures system just like regular BOSH VM would be configured.
+type DeploymentProvisioner struct {
+	manifestPath            string
+	deploymentReaderFactory bpdep.ReaderFactory
+
+	releaseCompiler     ReleaseCompiler
+	instanceProvisioner InstanceProvisioner
+
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewDeploymentProvisioner(
+	manifestPath string,
+	deploymentReaderFactory bpdep.ReaderFactory,
+	releaseCompiler ReleaseCompiler,
+	instanceProvisioner InstanceProvisioner,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) DeploymentProvisioner {
+	return DeploymentProvisioner{
+		manifestPath:            manifestPath,
+		deploymentReaderFactory: deploymentReaderFactory,
+
+		releaseCompiler:     releaseCompiler,
+		instanceProvisioner: instanceProvisioner,
+
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (p DeploymentProvisioner) Provision() error {
+	stage := p.eventLog.BeginStage("Setting up instance", 2)
+
+	reader := p.deploymentReaderFactory.NewManifestReader(p.manifestPath)
+
+	task := stage.BeginTask("Reading deployment manifest")
+
+	deployment, err := reader.Read()
+	if task.End(err) != nil {
+		return bosherr.WrapError(err, "Reading deployment")
+	}
+
+	task = stage.BeginTask("Validating instance")
+
+	job, instance, err := p.validateInstance(deployment)
+	if task.End(err) != nil {
+		return bosherr.WrapError(err, "Validating instance")
+	}
+
+	err = p.releaseCompiler.Compile(deployment.CompilationInstance, deployment.Releases)
+	if err != nil {
+		return bosherr.WrapError(err, "Compiling releases")
+	}
+
+	err = p.instanceProvisioner.Provision(job, instance)
+	if err != nil {
+		return bosherr.WrapError(err, "Provisioning instance")
+	}
+
+	return nil
+}
+
+func (p DeploymentProvisioner) validateInstance(deployment bpdep.Deployment) (bpdep.Job, bpdep.Instance, error) {
+	p.logger.Debug(deploymentProvisionerLogTag, "Validate instance")
+
+	var job bpdep.Job
+	var instance bpdep.Instance
+
+	if len(deployment.Jobs) > 1 {
+		return job, instance, bosherr.New("Must have exactly 1 job")
+	}
+
+	job = deployment.Jobs[0]
+
+	if len(job.Instances) != 1 {
+		return job, instance, bosherr.New("Must have exactly 1 instance")
+	}
+
+	instance = job.Instances[0]
+
+	return job, instance, nil
+}

data/go/src/boshprovisioner/provisioner/instance_provisioner.go

@@ -0,0 +1,48 @@
+package provisioner
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpinstupd "boshprovisioner/instance/updater"
+	bpvm "boshprovisioner/vm"
+)
+
+const instanceProvisionerLogTag = "InstanceProvisioner"
+
+type InstanceProvisioner struct {
+	instanceUpdaterFactory bpinstupd.UpdaterFactory
+	vmProvisioner          bpvm.VMProvisioner
+	logger                 boshlog.Logger
+}
+
+func NewInstanceProvisioner(
+	instanceUpdaterFactory bpinstupd.UpdaterFactory,
+	vmProvisioner bpvm.VMProvisioner,
+	logger boshlog.Logger,
+) InstanceProvisioner {
+	return InstanceProvisioner{
+		instanceUpdaterFactory: instanceUpdaterFactory,
+		vmProvisioner:          vmProvisioner,
+		logger:                 logger,
+	}
+}
+
+func (p InstanceProvisioner) Provision(job bpdep.Job, instance bpdep.Instance) error {
+	p.logger.Debug(instanceProvisionerLogTag, "Updating instance")
+
+	vm, err := p.vmProvisioner.Provision(instance)
+	if err != nil {
+		return bosherr.WrapError(err, "Provisioning agent")
+	}
+
+	updater := p.instanceUpdaterFactory.NewUpdater(vm.AgentClient(), job, instance)
+
+	err = updater.Update()
+	if err != nil {
+		return bosherr.WrapError(err, "Updating instance %d", instance.Index)
+	}
+
+	return nil
+}