vagrant-bosh 0.0.1

data/go/src/boshprovisioner/release/reader_factory.go

@@ -0,0 +1,34 @@
+package release
+
+import (
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdload "boshprovisioner/downloader"
+	bptar "boshprovisioner/tar"
+)
+
+type ReaderFactory struct {
+	downloader bpdload.Downloader
+	extractor  bptar.Extractor
+	fs         boshsys.FileSystem
+	logger     boshlog.Logger
+}
+
+func NewReaderFactory(
+	downloader bpdload.Downloader,
+	extractor bptar.Extractor,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) ReaderFactory {
+	return ReaderFactory{
+		downloader: downloader,
+		extractor:  extractor,
+		fs:         fs,
+		logger:     logger,
+	}
+}
+
+func (rf ReaderFactory) NewTarReader(path string) *TarReader {
+	return NewTarReader(path, rf.downloader, rf.extractor, rf.fs, rf.logger)
+}

data/go/src/boshprovisioner/release/release.go

@@ -0,0 +1,144 @@
+package release
+
+import (
+	bprelman "boshprovisioner/release/manifest"
+)
+
+type Release struct {
+	Manifest bprelman.Manifest
+
+	Name    string
+	Version string
+
+	Jobs []Job
+
+	Packages []*Package
+}
+
+type Job struct {
+	Name    string
+	Version string
+
+	Fingerprint string
+	SHA1        string
+
+	TarPath string
+}
+
+type Package struct {
+	Name    string
+	Version string
+
+	Fingerprint string
+	SHA1        string
+
+	TarPath string
+
+	// Package dependencies used at compilation of this package
+	Dependencies []*Package
+}
+
+// ResolvedPackageDependencies returns list of packages
+// in order such that each package at a higher index
+// only depends on packages at lower indecies.
+func (r Release) ResolvedPackageDependencies() []*Package {
+	var resolvedPkgs []*Package
+	var pendingPkgs []*Package
+
+	for _, pkg := range r.Packages {
+		pkgRef := pkg
+		pendingPkgs = append(pendingPkgs, pkgRef)
+	}
+
+	iterations := 0
+
+	for len(pendingPkgs) > 0 {
+		pkg := pendingPkgs[0]
+
+		depsSatisfied := true
+
+		for _, depPkg := range pkg.Dependencies {
+			depSatisfied := false
+
+			for _, resPkg := range resolvedPkgs {
+				if depPkg.Name == resPkg.Name {
+					depSatisfied = true
+					break
+				}
+			}
+
+			if !depSatisfied {
+				depsSatisfied = false
+				break
+			}
+		}
+
+		pendingPkgs = pendingPkgs[1:]
+		if depsSatisfied {
+			resolvedPkgs = append(resolvedPkgs, pkg)
+		} else {
+			pendingPkgs = append(pendingPkgs, pkg)
+		}
+
+		if iterations > 100 {
+			panic("Failed to resolve package dependenices witin 100 iterations")
+		} else {
+			iterations++
+		}
+	}
+
+	return resolvedPkgs
+}
+
+// populateFromManifest populates release information
+// interpreted from release manifest.
+func (r *Release) populateFromManifest(manifest bprelman.Manifest) {
+	r.populateRelease(manifest.Release)
+	r.populatePackages(manifest.Release.Packages)
+	r.populateJobs(manifest.Release.Jobs)
+	r.Manifest = manifest
+}
+
+func (r *Release) populateRelease(manRelease bprelman.Release) {
+	r.Name = manRelease.Name
+	r.Version = manRelease.Version
+}
+
+func (r *Release) populatePackages(manPkgs []bprelman.Package) {
+	nameToPkg := map[bprelman.DependencyName]*Package{}
+
+	for _, manPkg := range manPkgs {
+		pkg := Package{
+			Name:    manPkg.Name,
+			Version: manPkg.Version,
+
+			Fingerprint: manPkg.Fingerprint,
+			SHA1:        manPkg.SHA1,
+		}
+
+		r.Packages = append(r.Packages, &pkg)
+
+		nameToPkg[bprelman.DependencyName(pkg.Name)] = &pkg
+	}
+
+	// Connect compile time dependencies for packages
+	for i, manPkg := range manPkgs {
+		for _, depName := range manPkg.DependencyNames {
+			r.Packages[i].Dependencies = append(r.Packages[i].Dependencies, nameToPkg[depName])
+		}
+	}
+}
+
+func (r *Release) populateJobs(manJobs []bprelman.Job) {
+	for _, manJob := range manJobs {
+		job := Job{
+			Name:    manJob.Name,
+			Version: manJob.Version,
+
+			Fingerprint: manJob.Fingerprint,
+			SHA1:        manJob.SHA1,
+		}
+
+		r.Jobs = append(r.Jobs, job)
+	}
+}

data/go/src/boshprovisioner/release/release_suite_test.go

@@ -0,0 +1,13 @@
+package release_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestRelease(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Release Suite")
+}

data/go/src/boshprovisioner/release/release_test.go

@@ -0,0 +1,129 @@
+package release_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "boshprovisioner/release"
+)
+
+var _ = Describe("Release", func() {
+	var (
+		release Release
+	)
+
+	BeforeEach(func() {
+		release = Release{}
+	})
+
+	Describe("ResolvedPackageDependencies", func() {
+		It("returns packages in correct order with transitive deps", func() {
+			pkg1 := Package{Name: "fake-package-name-1"}
+
+			pkg2 := Package{
+				Name:         "fake-package-name-2",
+				Dependencies: []*Package{&pkg1},
+			}
+
+			pkg3 := Package{
+				Name:         "fake-package-name-3",
+				Dependencies: []*Package{&pkg2},
+			}
+
+			release.Packages = []*Package{&pkg3, &pkg2, &pkg1}
+
+			pkgs := release.ResolvedPackageDependencies()
+			Expect(pkgs).To(Equal([]*Package{&pkg1, &pkg2, &pkg3}))
+		})
+
+		It("returns packages in resolved order with multiple deps", func() {
+			pkg1 := Package{Name: "fake-package-name-1"}
+			pkg2 := Package{Name: "fake-package-name-2"}
+
+			pkg3 := Package{
+				Name:         "fake-package-name-3",
+				Dependencies: []*Package{&pkg1, &pkg2},
+			}
+
+			release.Packages = []*Package{&pkg3, &pkg2, &pkg1}
+
+			pkgs := release.ResolvedPackageDependencies()
+			Expect(pkgs).To(Equal([]*Package{&pkg2, &pkg1, &pkg3})) // or pkg1, pkg2, pkg3
+		})
+
+		It("compiles BOSH release packages (example)", func() {
+			nginx := Package{Name: "nginx"}
+			genisoimage := Package{Name: "genisoimage"}
+			powerdns := Package{Name: "powerdns"}
+			ruby := Package{Name: "ruby"}
+
+			blobstore := Package{
+				Name:         "blobstore",
+				Dependencies: []*Package{&ruby},
+			}
+
+			mysql := Package{Name: "mysql"}
+
+			nats := Package{
+				Name:         "nats",
+				Dependencies: []*Package{&ruby},
+			}
+
+			common := Package{Name: "common"}
+			redis := Package{Name: "redis"}
+			libpq := Package{Name: "libpq"}
+			postgres := Package{Name: "postgres"}
+
+			registry := Package{
+				Name:         "registry",
+				Dependencies: []*Package{&libpq, &mysql, &ruby},
+			}
+
+			director := Package{
+				Name:         "director",
+				Dependencies: []*Package{&libpq, &mysql, &ruby},
+			}
+
+			healthMonitor := Package{
+				Name:         "health_monitor",
+				Dependencies: []*Package{&ruby},
+			}
+
+			release.Packages = []*Package{
+				&nginx,
+				&genisoimage,
+				&powerdns,
+				&blobstore, // before ruby
+				&ruby,
+				&mysql,
+				&nats,
+				&common,
+				&director, // before libpq, postgres; after ruby
+				&redis,
+				&registry, // before libpq, postgres; after ruby
+				&libpq,
+				&postgres,
+				&healthMonitor, // after ruby, libpq, postgres
+			}
+
+			pkgs := release.ResolvedPackageDependencies()
+
+			Expect(pkgs).To(Equal([]*Package{
+				&nginx,
+				&genisoimage,
+				&powerdns,
+				&ruby,
+				&mysql,
+				&nats,
+				&common,
+				&redis,
+				&libpq,
+				&postgres,
+				&healthMonitor, // currently always appended to the end
+				&blobstore,
+				&director,
+				&registry,
+			}))
+		})
+	})
+})

data/go/src/boshprovisioner/release/tar_reader.go

@@ -0,0 +1,139 @@
+package release
+
+import (
+	"path/filepath"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdload "boshprovisioner/downloader"
+	bprelman "boshprovisioner/release/manifest"
+	bptar "boshprovisioner/tar"
+)
+
+const tarReaderLogTag = "TarReader"
+
+// TarReader reads .tgz release file and returns a Release.
+// See unpacked release directory layout at the end of the file.
+type TarReader struct {
+	path       string
+	downloader bpdload.Downloader
+	extractor  bptar.Extractor
+	fs         boshsys.FileSystem
+	logger     boshlog.Logger
+
+	// location to clean if successfully downloaded/extracted
+	downloadPath string
+	extractPath  string
+}
+
+func NewTarReader(
+	path string,
+	downloader bpdload.Downloader,
+	extractor bptar.Extractor,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) *TarReader {
+	return &TarReader{
+		path:       path,
+		downloader: downloader,
+		extractor:  extractor,
+		fs:         fs,
+		logger:     logger,
+	}
+}
+
+func (tr *TarReader) Read() (Release, error) {
+	var release Release
+
+	downloadPath, err := tr.downloader.Download(tr.path)
+	if err != nil {
+		return release, bosherr.WrapError(err, "Downloading release")
+	}
+
+	tr.downloadPath = downloadPath
+
+	extractPath, err := tr.extractor.Extract(tr.downloadPath)
+	if err != nil {
+		cleanUpErr := tr.downloader.CleanUp(tr.downloadPath)
+		if cleanUpErr != nil {
+			tr.logger.Debug(tarReaderLogTag,
+				"Failed to clean up downloaded release %v", cleanUpErr)
+		}
+
+		return release, bosherr.WrapError(err, "Extracting release")
+	}
+
+	tr.extractPath = extractPath
+
+	manifestPath := filepath.Join(tr.extractPath, "release.MF")
+
+	manifest, err := bprelman.NewManifestFromPath(manifestPath, tr.fs)
+	if err != nil {
+		closeErr := tr.Close()
+		if closeErr != nil {
+			tr.logger.Debug(tarReaderLogTag,
+				"Failed to close release %v", closeErr)
+		}
+
+		return release, bosherr.WrapError(err, "Building manifest")
+	}
+
+	tr.logger.Debug(tarReaderLogTag, "Done building manifest %#v", manifest)
+
+	release.populateFromManifest(manifest)
+
+	tr.populateReleaseTarPaths(&release)
+
+	return release, nil
+}
+
+func (tr TarReader) Close() error {
+	dlErr := tr.downloader.CleanUp(tr.downloadPath)
+	if dlErr != nil {
+		tr.logger.Debug(tarReaderLogTag,
+			"Failed to clean up downloaded release %v", dlErr)
+	}
+
+	exErr := tr.extractor.CleanUp(tr.extractPath)
+	if exErr != nil {
+		tr.logger.Debug(tarReaderLogTag,
+			"Failed to clean up extracted release %v", exErr)
+	}
+
+	if dlErr != nil {
+		return dlErr
+	}
+
+	return exErr
+}
+
+// populateReleaseTarPaths sets TarPath for each job/package in the release.
+func (tr TarReader) populateReleaseTarPaths(release *Release) {
+	for i, job := range release.Jobs {
+		fileName := job.Name + ".tgz"
+		release.Jobs[i].TarPath = filepath.Join(tr.extractPath, "jobs", fileName)
+	}
+
+	for _, pkg := range release.Packages {
+		fileName := pkg.Name + ".tgz"
+		pkg.TarPath = filepath.Join(tr.extractPath, "packages", fileName)
+	}
+}
+
+/*
+Example layout of an unpackaged release tar:
+
+$ tree ~/Downloads/dummy-release
+~/Downloads/dummy-release
+├── jobs
+│   ├── dummy.tgz
+│   ├── dummy_with_bad_package.tgz
+│   ├── dummy_with_package.tgz
+│   └── dummy_with_properties.tgz
+├── packages
+│   ├── bad_package.tgz
+│   └── dummy_package.tgz
+└── release.MF
+*/

data/go/src/boshprovisioner/releasesrepo/blobstore_releases_repository.go

@@ -0,0 +1,114 @@
+package releasesrepo
+
+import (
+	boshblob "bosh/blobstore"
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpdload "boshprovisioner/downloader"
+	bpindex "boshprovisioner/index"
+)
+
+type BlobstoreReleasesRepository struct {
+	downloader bpdload.Downloader
+	blobstore  boshblob.Blobstore
+	index      bpindex.Index
+	logger     boshlog.Logger
+}
+
+type blobstoreRecord struct {
+	BlobID      string
+	Fingerprint string
+}
+
+func NewBlobstoreReleasesRepository(
+	downloader bpdload.Downloader,
+	blobstore boshblob.Blobstore,
+	index bpindex.Index,
+	logger boshlog.Logger,
+) BlobstoreReleasesRepository {
+	return BlobstoreReleasesRepository{
+		downloader: downloader,
+		blobstore:  blobstore,
+		index:      index,
+		logger:     logger,
+	}
+}
+
+func (rr BlobstoreReleasesRepository) Pull(release bpdep.Release) error {
+	var record blobstoreRecord
+
+	err := rr.index.Find(release, &record)
+	if err == nil {
+		return nil // nothing to do if already exists
+	} else if err != bpindex.ErrNotFound {
+		return bosherr.WrapError(err, "Finding release in index")
+	}
+
+	path, err := rr.downloader.Download(release.URL)
+	if err != nil {
+		return bosherr.WrapError(err, "Downloading release")
+	}
+
+	defer rr.downloader.CleanUp(path)
+
+	blobID, fingerprint, err := rr.blobstore.Create(path)
+	if err != nil {
+		return bosherr.WrapError(err, "Creating release blob")
+	}
+
+	record = blobstoreRecord{
+		BlobID:      blobID,
+		Fingerprint: fingerprint,
+	}
+
+	err = rr.index.Save(release, record)
+	if err != nil {
+		// todo delete from blobstore
+
+		return bosherr.WrapError(err, "Saving release to index")
+	}
+
+	return nil
+}
+
+func (rr BlobstoreReleasesRepository) KeepOnly(releasesToKeep []bpdep.Release) error {
+	var allReleases []bpdep.Release
+
+	err := rr.index.ListKeys(&allReleases)
+	if err != nil {
+		return bosherr.WrapError(err, "Listing releases in index")
+	}
+
+	for _, foundRelease := range allReleases {
+		var keep bool
+
+		for _, releaseToKeep := range releasesToKeep {
+			if foundRelease == releaseToKeep {
+				keep = true
+				break
+			}
+		}
+
+		if keep {
+			continue
+		}
+
+		var record blobstoreRecord
+
+		err := rr.index.Find(foundRelease, &record)
+		if err != nil {
+			return bosherr.WrapError(err, "Finding release to delete")
+		}
+
+		// todo delete from blobstore
+
+		err = rr.index.Remove(foundRelease)
+		if err != nil {
+			return bosherr.WrapError(err, "Deleting release from index")
+		}
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/releasesrepo/releases_repository_interface.go

@@ -0,0 +1,15 @@
+package releasesrepo
+
+import (
+	bpdep "boshprovisioner/deployment"
+)
+
+// ReleasesRepository manages collection of releases
+// available to use for provisioning
+type ReleasesRepository interface {
+	// Pull downloads/copies/retrieves a release
+	Pull(bpdep.Release) error
+
+	// KeepOnly deletes all releases but the provided ones
+	KeepOnly([]bpdep.Release) error
+}

data/go/src/boshprovisioner/tar/cmd_compressor.go

@@ -0,0 +1,68 @@
+package tar
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+const cmdCompressorLogTag = "CmdCompressor"
+
+type CmdCompressor struct {
+	runner boshsys.CmdRunner
+	fs     boshsys.FileSystem
+	logger boshlog.Logger
+}
+
+func NewCmdCompressor(
+	runner boshsys.CmdRunner,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) CmdCompressor {
+	return CmdCompressor{
+		runner: runner,
+		fs:     fs,
+		logger: logger,
+	}
+}
+
+func (c CmdCompressor) Compress(path string) (string, error) {
+	compressPath, err := c.tmpPath()
+	if err != nil {
+		return "", err
+	}
+
+	c.logger.Debug(cmdCompressorLogTag, "Compressing %s to %s", path, compressPath)
+
+	_, _, _, err = c.runner.RunCommand("tar", "-C", path, "-czf", compressPath, ".")
+	if err != nil {
+		return "", bosherr.WrapError(err, "Running tar")
+	}
+
+	return compressPath, nil
+}
+
+func (c CmdCompressor) CleanUp(path string) error {
+	return c.fs.RemoveAll(path)
+}
+
+func (c CmdCompressor) tmpPath() (string, error) {
+	file, err := c.fs.TempFile("tar-CmdCompressor")
+	if err != nil {
+		return "", bosherr.WrapError(err, "Creating extract destination")
+	}
+
+	compressPath := file.Name()
+
+	err = file.Close()
+	if err != nil {
+		return "", bosherr.WrapError(err, "Closing temp file")
+	}
+
+	err = c.fs.RemoveAll(compressPath)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Remove temp file")
+	}
+
+	return compressPath, nil
+}

data/go/src/boshprovisioner/tar/cmd_extractor.go

@@ -0,0 +1,47 @@
+package tar
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+const cmdExtractorLogTag = "CmdExtractor"
+
+type CmdExtractor struct {
+	runner boshsys.CmdRunner
+	fs     boshsys.FileSystem
+	logger boshlog.Logger
+}
+
+func NewCmdExtractor(
+	runner boshsys.CmdRunner,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) CmdExtractor {
+	return CmdExtractor{
+		runner: runner,
+		fs:     fs,
+		logger: logger,
+	}
+}
+
+func (e CmdExtractor) Extract(path string) (string, error) {
+	extractPath, err := e.fs.TempDir("tar-CmdExtractor")
+	if err != nil {
+		return "", bosherr.WrapError(err, "Creating extract destination")
+	}
+
+	e.logger.Debug(cmdExtractorLogTag, "Extract tar %s to %s", path, extractPath)
+
+	_, _, _, err = e.runner.RunCommand("tar", "-C", extractPath, "-xzf", path)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Running tar")
+	}
+
+	return extractPath, nil
+}
+
+func (e CmdExtractor) CleanUp(path string) error {
+	return e.fs.RemoveAll(path)
+}

data/go/src/boshprovisioner/tar/compressor_interface.go

@@ -0,0 +1,6 @@
+package tar
+
+type Compressor interface {
+	Compress(string) (string, error)
+	CleanUp(string) error
+}