vagrant-bosh 0.0.1

data/go/src/boshprovisioner/instance/updater/waiter_test.go

@@ -0,0 +1,103 @@
+package updater_test
+
+import (
+	"errors"
+	"time"
+
+	boshaction "bosh/agent/action"
+	boshlog "bosh/logger"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	fakebpagclient "boshprovisioner/agent/client/fakes"
+	. "boshprovisioner/instance/updater"
+)
+
+var _ = Describe("Waiter", func() {
+	var (
+		sleptTimes  []time.Duration
+		agentClient *fakebpagclient.FakeClient
+		waiter      Waiter
+	)
+
+	const (
+		firstTimeGap      = 5000 * time.Millisecond
+		subsequentTimeGap = 1000 * time.Millisecond
+	)
+
+	BeforeEach(func() {
+		sleptTimes = []time.Duration{}
+		sleepFunc := func(d time.Duration) { sleptTimes = append(sleptTimes, d) }
+
+		agentClient = &fakebpagclient.FakeClient{}
+		logger := boshlog.NewLogger(boshlog.LevelNone)
+		waiter = NewWaiter(5000, 14000, sleepFunc, agentClient, logger)
+	})
+
+	Describe("Wait", func() {
+		Context("when agent reports its state as 'running' after some time", func() {
+			BeforeEach(func() {
+				agentClient.GetStateStates = []boshaction.GetStateV1ApplySpec{
+					boshaction.GetStateV1ApplySpec{JobState: "not-running"},
+					boshaction.GetStateV1ApplySpec{JobState: "not-running"},
+					boshaction.GetStateV1ApplySpec{JobState: "not-running"},
+					boshaction.GetStateV1ApplySpec{JobState: "not-running"},
+					boshaction.GetStateV1ApplySpec{JobState: "not-running"}, // 4
+					boshaction.GetStateV1ApplySpec{JobState: "running"},
+				}
+			})
+
+			It("waits for instance to become running checking every interval", func() {
+				err := waiter.Wait()
+				Expect(err).ToNot(HaveOccurred())
+
+				Expect(sleptTimes).To(Equal([]time.Duration{
+					firstTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap, // 4
+					subsequentTimeGap,
+				}))
+			})
+		})
+
+		Context("when agent does not report its state as 'running' after some time", func() {
+			BeforeEach(func() {
+				agentClient.GetStateState = boshaction.GetStateV1ApplySpec{JobState: "not-running"}
+			})
+
+			It("return error after trying as many times as possible", func() {
+				err := waiter.Wait()
+				Expect(err).To(Equal(ErrNotRunning))
+
+				Expect(sleptTimes).To(Equal([]time.Duration{
+					firstTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap, // 4
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap,
+					subsequentTimeGap, // 8
+					subsequentTimeGap,
+				}))
+			})
+		})
+
+		Context("when agent state cannot be retrieved", func() {
+			BeforeEach(func() {
+				agentClient.GetStateErr = errors.New("fake-get-state-err")
+			})
+
+			It("returns error indicated failure to retrieve state", func() {
+				err := waiter.Wait()
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("fake-get-state-err"))
+
+				Expect(sleptTimes).To(Equal([]time.Duration{firstTimeGap}))
+			})
+		})
+	})
+})

data/go/src/boshprovisioner/main/config.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+
+	bosherr "bosh/errors"
+	boshsys "bosh/system"
+
+	bpprov "boshprovisioner/provisioner"
+)
+
+type Config struct {
+	ManifestPath string `json:"manifest_path"`
+
+	// Assets dir is used as a temporary location to transfer files from host to guest.
+	// It will not be created since assets already must be present.
+	AssetsDir string `json:"assets_dir"`
+
+	// Repos dir is mainly used to record what was placed in the blobstore.
+	// It will be created if it does not exist.
+	ReposDir string `json:"repos_dir"`
+
+	// e.g. "https://user:password@127.0.0.1:4321/agent"
+	Mbus string `json:"mbus"`
+
+	Blobstore bpprov.BlobstoreConfig `json:"blobstore"`
+}
+
+func NewConfigFromPath(path string, fs boshsys.FileSystem) (Config, error) {
+	var config Config
+
+	bytes, err := fs.ReadFile(path)
+	if err != nil {
+		return config, bosherr.WrapError(err, "Reading config %s", path)
+	}
+
+	err = json.Unmarshal(bytes, &config)
+	if err != nil {
+		return config, bosherr.WrapError(err, "Unmarshalling config")
+	}
+
+	err = config.validate()
+	if err != nil {
+		return config, bosherr.WrapError(err, "Validating config")
+	}
+
+	return config, nil
+}
+
+func (c Config) validate() error {
+	if c.ManifestPath == "" {
+		return bosherr.New("Must provide non-empty manifest_path")
+	}
+
+	if c.AssetsDir == "" {
+		return bosherr.New("Must provide non-empty assets_dir")
+	}
+
+	if c.ReposDir == "" {
+		return bosherr.New("Must provide non-empty repos_dir")
+	}
+
+	if c.Mbus == "" {
+		return bosherr.New("Must provide non-empty mbus")
+	}
+
+	if c.Blobstore.Type != bpprov.BlobstoreConfigTypeLocal {
+		return bosherr.New("Blobstore type must be local")
+	}
+
+	err := c.Blobstore.Validate()
+	if err != nil {
+		return bosherr.WrapError(err, "Validating blobstore configuration")
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/main/main.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	boshblob "bosh/blobstore"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+	boshuuid "bosh/uuid"
+
+	bpdep "boshprovisioner/deployment"
+	bpdload "boshprovisioner/downloader"
+	bpeventlog "boshprovisioner/eventlog"
+	bptplcomp "boshprovisioner/instance/templatescompiler"
+	bpinstupd "boshprovisioner/instance/updater"
+	bppkgscomp "boshprovisioner/packagescompiler"
+	bpprov "boshprovisioner/provisioner"
+	bprel "boshprovisioner/release"
+	bpreljob "boshprovisioner/release/job"
+	bptar "boshprovisioner/tar"
+	bpvm "boshprovisioner/vm"
+)
+
+const mainLogTag = "main"
+
+var (
+	configPathOpt = flag.String("configPath", "", "Path to configuration file")
+)
+
+func main() {
+	logger, fs, runner, uuidGen := basicDeps()
+
+	defer logger.HandlePanic("Main")
+
+	flag.Parse()
+
+	config, err := NewConfigFromPath(*configPathOpt, fs)
+	if err != nil {
+		logger.Error(mainLogTag, "Loading config %s", err.Error())
+		os.Exit(1)
+	}
+
+	localBlobstore := boshblob.NewLocalBlobstore(
+		fs,
+		uuidGen,
+		config.Blobstore.Options,
+	)
+
+	blobstore := boshblob.NewSHA1VerifiableBlobstore(localBlobstore)
+
+	downloader := bpdload.NewDefaultMuxDownloader(blobstore, fs, logger)
+
+	extractor := bptar.NewCmdExtractor(runner, fs, logger)
+
+	compressor := bptar.NewCmdCompressor(runner, fs, logger)
+
+	renderedArchivesCompiler := bptplcomp.NewRenderedArchivesCompiler(
+		fs,
+		runner,
+		compressor,
+		logger,
+	)
+
+	jobReaderFactory := bpreljob.NewReaderFactory(
+		downloader,
+		extractor,
+		fs,
+		logger,
+	)
+
+	err = fs.MkdirAll(config.ReposDir, os.ModeDir)
+	if err != nil {
+		logger.Error(mainLogTag, "Failed to create repos dir %s", err.Error())
+		os.Exit(1)
+	}
+
+	reposFactory := NewReposFactory(config.ReposDir, fs, downloader, blobstore, logger)
+
+	blobstoreProvisioner := bpprov.NewBlobstoreProvisioner(
+		fs,
+		config.Blobstore,
+		logger,
+	)
+
+	err = blobstoreProvisioner.Provision()
+	if err != nil {
+		logger.Error(mainLogTag, "Failed to provision blobstore %s", err.Error())
+		os.Exit(1)
+	}
+
+	templatesCompiler := bptplcomp.NewConcreteTemplatesCompiler(
+		renderedArchivesCompiler,
+		jobReaderFactory,
+		reposFactory.NewJobsRepo(),
+		reposFactory.NewTemplateToJobRepo(),
+		reposFactory.NewRuntimePackagesRepo(),
+		reposFactory.NewTemplatesRepo(),
+		blobstore,
+		logger,
+	)
+
+	eventLog := bpeventlog.NewLog(logger)
+
+	packagesCompilerFactory := bppkgscomp.NewConcretePackagesCompilerFactory(
+		reposFactory.NewPackagesRepo(),
+		reposFactory.NewCompiledPackagesRepo(),
+		blobstore,
+		eventLog,
+		logger,
+	)
+
+	updaterFactory := bpinstupd.NewUpdaterFactory(
+		templatesCompiler,
+		packagesCompilerFactory,
+		eventLog,
+		logger,
+	)
+
+	releaseReaderFactory := bprel.NewReaderFactory(
+		downloader,
+		extractor,
+		fs,
+		logger,
+	)
+
+	deploymentReaderFactory := bpdep.NewReaderFactory(fs, logger)
+
+	vmProvisionerFactory := bpvm.NewVMProvisionerFactory(
+		fs,
+		runner,
+		config.AssetsDir,
+		config.Mbus,
+		config.Blobstore.AsMap(),
+		eventLog,
+		logger,
+	)
+
+	vmProvisioner := vmProvisionerFactory.NewVMProvisioner()
+
+	releaseCompiler := bpprov.NewReleaseCompiler(
+		reposFactory.NewBlobstoreReleasesRepo(),
+		releaseReaderFactory,
+		packagesCompilerFactory,
+		templatesCompiler,
+		vmProvisioner,
+		eventLog,
+		logger,
+	)
+
+	instanceProvisioner := bpprov.NewInstanceProvisioner(
+		updaterFactory,
+		vmProvisioner,
+		logger,
+	)
+
+	deploymentProvisioner := bpprov.NewDeploymentProvisioner(
+		config.ManifestPath,
+		deploymentReaderFactory,
+		releaseCompiler,
+		instanceProvisioner,
+		eventLog,
+		logger,
+	)
+
+	err = deploymentProvisioner.Provision()
+	if err != nil {
+		logger.Error(mainLogTag, "Failed to provision deployment %s", err.Error())
+		os.Exit(1)
+	}
+}
+
+func basicDeps() (boshlog.Logger, boshsys.FileSystem, boshsys.CmdRunner, boshuuid.Generator) {
+	logger := boshlog.NewWriterLogger(boshlog.LevelDebug, os.Stderr, os.Stderr)
+
+	fs := boshsys.NewOsFileSystem(logger)
+
+	runner := boshsys.NewExecCmdRunner(logger)
+
+	uuidGen := boshuuid.NewGenerator()
+
+	return logger, fs, runner, uuidGen
+}

data/go/src/boshprovisioner/main/repos_factory.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"path/filepath"
+
+	boshblob "bosh/blobstore"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdload "boshprovisioner/downloader"
+	bpindex "boshprovisioner/index"
+	bpjobsrepo "boshprovisioner/instance/templatescompiler/jobsrepo"
+	bptplsrepo "boshprovisioner/instance/templatescompiler/templatesrepo"
+	bpcpkgsrepo "boshprovisioner/packagescompiler/compiledpackagesrepo"
+	bppkgsrepo "boshprovisioner/packagescompiler/packagesrepo"
+	bprelrepo "boshprovisioner/releasesrepo"
+)
+
+type ReposFactory struct {
+	dirPath    string
+	fs         boshsys.FileSystem
+	downloader bpdload.Downloader
+	blobstore  boshblob.Blobstore
+	logger     boshlog.Logger
+}
+
+func NewReposFactory(
+	dirPath string,
+	fs boshsys.FileSystem,
+	downloader bpdload.Downloader,
+	blobstore boshblob.Blobstore,
+	logger boshlog.Logger,
+) ReposFactory {
+	return ReposFactory{
+		dirPath:    dirPath,
+		fs:         fs,
+		downloader: downloader,
+		blobstore:  blobstore,
+		logger:     logger,
+	}
+}
+
+func (f ReposFactory) NewBlobstoreReleasesRepo() bprelrepo.BlobstoreReleasesRepository {
+	return bprelrepo.NewBlobstoreReleasesRepository(
+		f.downloader,
+		f.blobstore,
+		f.newIndex("releases"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewJobsRepo() bpjobsrepo.JobsRepository {
+	return bpjobsrepo.NewConcreteJobsRepository(
+		f.newIndex("jobs"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewTemplateToJobRepo() bpjobsrepo.TemplateToJobRepository {
+	return bpjobsrepo.NewConcreteTemplateToJobRepository(
+		f.newIndex("templates_to_job"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewRuntimePackagesRepo() bpjobsrepo.RuntimePackagesRepository {
+	return bpjobsrepo.NewConcreteRuntimePackagesRepository(
+		f.newIndex("runtime_packages"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewTemplatesRepo() bptplsrepo.TemplatesRepository {
+	return bptplsrepo.NewConcreteTemplatesRepository(
+		f.newIndex("templates"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewPackagesRepo() bppkgsrepo.PackagesRepository {
+	return bppkgsrepo.NewConcretePackagesRepository(
+		f.newIndex("packages"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) NewCompiledPackagesRepo() bpcpkgsrepo.CompiledPackagesRepository {
+	return bpcpkgsrepo.NewConcreteCompiledPackagesRepository(
+		f.newIndex("compiled_packages"),
+		f.logger,
+	)
+}
+
+func (f ReposFactory) newIndex(name string) bpindex.Index {
+	return bpindex.NewFileIndex(filepath.Join(f.dirPath, name+".json"), f.fs)
+}

data/go/src/boshprovisioner/packagescompiler/compiledpackagesrepo/compiled_packages_repository_interface.go

@@ -0,0 +1,17 @@
+package compiledpackagesrepo
+
+import (
+	bprel "boshprovisioner/release"
+)
+
+type CompiledPackageRecord struct {
+	BlobID string
+	SHA1   string
+}
+
+// CompiledPackagesRepository maintains list of compiled packages as blobs
+// todo account for stemcell
+type CompiledPackagesRepository interface {
+	Find(bprel.Package) (CompiledPackageRecord, bool, error)
+	Save(bprel.Package, CompiledPackageRecord) error
+}

data/go/src/boshprovisioner/packagescompiler/compiledpackagesrepo/concrete_compiled_packages_repository.go

@@ -0,0 +1,61 @@
+package compiledpackagesrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpindex "boshprovisioner/index"
+	bprel "boshprovisioner/release"
+)
+
+type CCPRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+type packageToCompiledPackageKey struct {
+	PackageName    string
+	PackageVersion string
+
+	// Fingerprint of a package captures its dependenices
+	PackageFingerprint string
+}
+
+func NewConcreteCompiledPackagesRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CCPRepository {
+	return CCPRepository{index: index, logger: logger}
+}
+
+func (r CCPRepository) Find(pkg bprel.Package) (CompiledPackageRecord, bool, error) {
+	var record CompiledPackageRecord
+
+	err := r.index.Find(r.pkgKey(pkg), &record)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return record, false, nil
+		}
+
+		return record, false, bosherr.WrapError(err, "Finding compiled package")
+	}
+
+	return record, true, nil
+}
+
+func (r CCPRepository) Save(pkg bprel.Package, record CompiledPackageRecord) error {
+	err := r.index.Save(r.pkgKey(pkg), record)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving compiled package")
+	}
+
+	return nil
+}
+
+func (r CCPRepository) pkgKey(pkg bprel.Package) packageToCompiledPackageKey {
+	return packageToCompiledPackageKey{
+		PackageName:        pkg.Name,
+		PackageVersion:     pkg.Version,
+		PackageFingerprint: pkg.Fingerprint,
+	}
+}