vagrant-bosh 0.0.1

data/go/src/boshprovisioner/provisioner/release_compiler.go

@@ -0,0 +1,133 @@
+package provisioner
+
+import (
+	"fmt"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpeventlog "boshprovisioner/eventlog"
+	bptplcomp "boshprovisioner/instance/templatescompiler"
+	bppkgscomp "boshprovisioner/packagescompiler"
+	bprel "boshprovisioner/release"
+	bprelrepo "boshprovisioner/releasesrepo"
+	bpvm "boshprovisioner/vm"
+)
+
+const releaseCompilerLogTag = "ReleaseCompiler"
+
+type ReleaseCompiler struct {
+	releasesRepo         bprelrepo.ReleasesRepository
+	releaseReaderFactory bprel.ReaderFactory
+
+	packagesCompilerFactory bppkgscomp.ConcretePackagesCompilerFactory
+	templatesCompiler       bptplcomp.TemplatesCompiler
+
+	vmProvisioner bpvm.VMProvisioner
+
+	eventLog bpeventlog.Log
+	logger   boshlog.Logger
+}
+
+func NewReleaseCompiler(
+	releasesRepo bprelrepo.ReleasesRepository,
+	releaseReaderFactory bprel.ReaderFactory,
+	packagesCompilerFactory bppkgscomp.ConcretePackagesCompilerFactory,
+	templatesCompiler bptplcomp.TemplatesCompiler,
+	vmProvisioner bpvm.VMProvisioner,
+	eventLog bpeventlog.Log,
+	logger boshlog.Logger,
+) ReleaseCompiler {
+	return ReleaseCompiler{
+		releasesRepo:         releasesRepo,
+		releaseReaderFactory: releaseReaderFactory,
+
+		packagesCompilerFactory: packagesCompilerFactory,
+		templatesCompiler:       templatesCompiler,
+
+		vmProvisioner: vmProvisioner,
+
+		eventLog: eventLog,
+		logger:   logger,
+	}
+}
+
+func (p ReleaseCompiler) Compile(instance bpdep.Instance, releases []bpdep.Release) error {
+	vm, err := p.vmProvisioner.Provision(instance)
+	if err != nil {
+		return bosherr.WrapError(err, "Provisioning agent")
+	}
+
+	err = p.uploadReleases(releases)
+	if err != nil {
+		return bosherr.WrapError(err, "Uploading releases")
+	}
+
+	pkgsCompiler := p.packagesCompilerFactory.NewCompiler(vm.AgentClient())
+
+	err = p.compileReleases(pkgsCompiler, releases)
+	if err != nil {
+		return bosherr.WrapError(err, "Compiling releases")
+	}
+
+	return nil
+}
+
+func (p ReleaseCompiler) uploadReleases(releases []bpdep.Release) error {
+	stage := p.eventLog.BeginStage("Uploading releases", len(releases)+1)
+
+	for _, depRelease := range releases {
+		releaseDesc := fmt.Sprintf("%s/%s", depRelease.Name, depRelease.Version)
+
+		task := stage.BeginTask(fmt.Sprintf("Release %s", releaseDesc))
+
+		err := task.End(p.releasesRepo.Pull(depRelease))
+		if err != nil {
+			return bosherr.WrapError(err, "Pulling release %s", depRelease.Name)
+		}
+	}
+
+	task := stage.BeginTask("Deleting old releases")
+
+	err := task.End(p.releasesRepo.KeepOnly(releases))
+	if err != nil {
+		return bosherr.WrapError(err, "Keeping only releases")
+	}
+
+	return nil
+}
+
+func (p ReleaseCompiler) compileReleases(pkgsCompiler bppkgscomp.PackagesCompiler, releases []bpdep.Release) error {
+	for _, depRelease := range releases {
+		err := p.compileRelease(pkgsCompiler, depRelease)
+		if err != nil {
+			return bosherr.WrapError(err, "Release %s", depRelease.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p ReleaseCompiler) compileRelease(pkgsCompiler bppkgscomp.PackagesCompiler, release bpdep.Release) error {
+	reader := p.releaseReaderFactory.NewTarReader(release.URL)
+
+	relRelease, err := reader.Read()
+	if err != nil {
+		return bosherr.WrapError(err, "Reading release")
+	}
+
+	defer reader.Close()
+
+	err = pkgsCompiler.Compile(relRelease)
+	if err != nil {
+		return bosherr.WrapError(err, "Compiling release packages")
+	}
+
+	err = p.templatesCompiler.Precompile(relRelease)
+	if err != nil {
+		return bosherr.WrapError(err, "Precompiling release job templates")
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/release/job/job.go

@@ -0,0 +1,86 @@
+package job
+
+import (
+	bpreljobman "boshprovisioner/release/job/manifest"
+)
+
+type Job struct {
+	Manifest bpreljobman.Manifest
+
+	Name string
+
+	MonitTemplate Template
+
+	Templates []Template
+
+	// Runtime package dependencies for this job
+	Packages []Package
+
+	Properties []Property
+}
+
+type Template struct {
+	SrcPathEnd string
+	DstPathEnd string // End of the path on the VM
+
+	Path string
+}
+
+type Package struct {
+	Name string
+}
+
+type Property struct {
+	Name        string
+	Description string
+
+	Default interface{}
+}
+
+// populateFromManifest populates job information interpreted from job manifest.
+func (j *Job) populateFromManifest(manifest bpreljobman.Manifest) {
+	j.populateJob(manifest.Job)
+	j.populateTemplates(manifest.Job.TemplateNames)
+	j.populatePackages(manifest.Job.PackageNames)
+	j.populateProperties(manifest.Job.PropertyMappings)
+	j.Manifest = manifest
+}
+
+func (j *Job) populateJob(manJob bpreljobman.Job) {
+	j.Name = manJob.Name
+}
+
+func (j *Job) populateTemplates(manTemplateNames bpreljobman.TemplateNames) {
+	j.MonitTemplate = Template{
+		SrcPathEnd: "monit",
+		DstPathEnd: "monit",
+	}
+
+	for srcPathEnd, dstPathEnd := range manTemplateNames {
+		template := Template{
+			SrcPathEnd: srcPathEnd,
+			DstPathEnd: dstPathEnd,
+		}
+
+		j.Templates = append(j.Templates, template)
+	}
+}
+
+func (j *Job) populatePackages(manPackageNames []string) {
+	for _, name := range manPackageNames {
+		j.Packages = append(j.Packages, Package{Name: name})
+	}
+}
+
+func (j *Job) populateProperties(manPropMappings bpreljobman.PropertyMappings) {
+	for propName, propDef := range manPropMappings {
+		property := Property{
+			Name:        propName,
+			Description: propDef.Description,
+
+			Default: propDef.Default,
+		}
+
+		j.Properties = append(j.Properties, property)
+	}
+}

data/go/src/boshprovisioner/release/job/manifest/manifest.go

@@ -0,0 +1,79 @@
+// Package manifest represents internal structure of a release job.
+package manifest
+
+import (
+	bosherr "bosh/errors"
+	boshsys "bosh/system"
+	"github.com/fraenkel/candiedyaml"
+)
+
+type Manifest struct {
+	Job Job
+}
+
+type Job struct {
+	Name string `yaml:"name"`
+
+	TemplateNames TemplateNames `yaml:"templates"`
+
+	PackageNames []string `yaml:"packages"`
+
+	PropertyMappings PropertyMappings `yaml:"properties"`
+}
+
+type TemplateNames map[string]string
+
+type PropertyMappings map[string]PropertyDefinition
+
+type PropertyDefinition struct {
+	Description string `yaml:"description"`
+
+	// Non-raw field is populated by the validator.
+	DefaultRaw interface{} `yaml:"default"`
+	Default    interface{}
+}
+
+func NewManifestFromPath(path string, fs boshsys.FileSystem) (Manifest, error) {
+	bytes, err := fs.ReadFile(path)
+	if err != nil {
+		return Manifest{}, bosherr.WrapError(err, "Reading manifest %s", path)
+	}
+
+	return NewManifestFromBytes(bytes)
+}
+
+func NewManifestFromBytes(bytes []byte) (Manifest, error) {
+	var manifest Manifest
+	var job Job
+
+	err := candiedyaml.Unmarshal(bytes, &job)
+	if err != nil {
+		return manifest, bosherr.WrapError(err, "Parsing job")
+	}
+
+	manifest.Job = job
+
+	err = NewSyntaxValidator(&manifest).Validate()
+	if err != nil {
+		return Manifest{}, bosherr.WrapError(err, "Validating manifest syntactically")
+	}
+
+	return manifest, nil
+}
+
+/*
+# Example for job.MF
+name: dummy
+
+templates:
+  dummy_ctl: bin/dummy_ctl
+
+packages:
+- dummy_package
+- dummy_package2
+
+properties:
+  dummy_value:
+    description: Some value for the dummy job
+    default: 300
+*/

data/go/src/boshprovisioner/release/job/manifest/manifest_suite_test.go

@@ -0,0 +1,13 @@
+package manifest_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestManifest(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Manifest Suite")
+}

data/go/src/boshprovisioner/release/job/manifest/manifest_test.go

@@ -0,0 +1,42 @@
+package manifest_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "boshprovisioner/release/job/manifest"
+)
+
+var _ = Describe("Manifest", func() {
+	Describe("NewManifestFromBytes", func() {
+		It("returns manifest with property definiton default that have string keys", func() {
+			manifestBytes := []byte(`
+properties:
+  key:
+    default:
+      prop:
+        nest-prop: instance-val
+      props:
+      - name: nest-prop
+`)
+
+			manifest, err := NewManifestFromBytes(manifestBytes)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(manifest.Job.PropertyMappings).To(HaveLen(1))
+
+			for _, propDef := range manifest.Job.PropertyMappings {
+				// candiedyaml unmarshals manifest to map[interface{}]interface{}
+				// (encoding/json unmarshals manifest to map[string]interface{})
+				Expect(propDef.Default).To(Equal(map[string]interface{}{
+					"prop": map[string]interface{}{
+						"nest-prop": "instance-val",
+					},
+					"props": []interface{}{
+						map[string]interface{}{"name": "nest-prop"},
+					},
+				}))
+			}
+		})
+	})
+})

data/go/src/boshprovisioner/release/job/manifest/syntax_validator.go

@@ -0,0 +1,43 @@
+package manifest
+
+import (
+	bosherr "bosh/errors"
+
+	bputil "boshprovisioner/util"
+)
+
+type SyntaxValidator struct {
+	job *Job
+}
+
+func NewSyntaxValidator(manifest *Manifest) SyntaxValidator {
+	if manifest == nil {
+		panic("Expected manifest to not be nil")
+	}
+
+	return SyntaxValidator{job: &manifest.Job}
+}
+
+func (v SyntaxValidator) Validate() error {
+	for name, propDef := range v.job.PropertyMappings {
+		propDef, err := v.validatePropDef(propDef)
+		if err != nil {
+			return bosherr.WrapError(err, "Property %s", name)
+		}
+
+		v.job.PropertyMappings[name] = propDef
+	}
+
+	return nil
+}
+
+func (v SyntaxValidator) validatePropDef(propDef PropertyDefinition) (PropertyDefinition, error) {
+	def, err := bputil.NewStringKeyed().ConvertInterface(propDef.DefaultRaw)
+	if err != nil {
+		return propDef, bosherr.WrapError(err, "Default")
+	}
+
+	propDef.Default = def
+
+	return propDef, nil
+}

data/go/src/boshprovisioner/release/job/reader_factory.go

@@ -0,0 +1,34 @@
+package job
+
+import (
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdload "boshprovisioner/downloader"
+	bptar "boshprovisioner/tar"
+)
+
+type ReaderFactory struct {
+	downloader bpdload.Downloader
+	extractor  bptar.Extractor
+	fs         boshsys.FileSystem
+	logger     boshlog.Logger
+}
+
+func NewReaderFactory(
+	downloader bpdload.Downloader,
+	extractor bptar.Extractor,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) ReaderFactory {
+	return ReaderFactory{
+		downloader: downloader,
+		extractor:  extractor,
+		fs:         fs,
+		logger:     logger,
+	}
+}
+
+func (rf ReaderFactory) NewTarReader(path string) *TarReader {
+	return NewTarReader(path, rf.downloader, rf.extractor, rf.fs, rf.logger)
+}

data/go/src/boshprovisioner/release/job/tar_reader.go

@@ -0,0 +1,133 @@
+package job
+
+import (
+	"path/filepath"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdload "boshprovisioner/downloader"
+	bpreljobman "boshprovisioner/release/job/manifest"
+	bptar "boshprovisioner/tar"
+)
+
+const tarReaderLogTag = "TarReader"
+
+// TarReader reads .tgz job file and returns a Job.
+// See unpacked job directory layout at the end of the file.
+type TarReader struct {
+	path       string
+	downloader bpdload.Downloader
+	extractor  bptar.Extractor
+	fs         boshsys.FileSystem
+	logger     boshlog.Logger
+
+	// location to clean if successfully downloaded/extracted
+	downloadPath string
+	extractPath  string
+}
+
+func NewTarReader(
+	path string,
+	downloader bpdload.Downloader,
+	extractor bptar.Extractor,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) *TarReader {
+	return &TarReader{
+		path:       path,
+		downloader: downloader,
+		extractor:  extractor,
+		fs:         fs,
+		logger:     logger,
+	}
+}
+
+func (tr *TarReader) Read() (Job, error) {
+	var job Job
+
+	downloadPath, err := tr.downloader.Download(tr.path)
+	if err != nil {
+		return job, bosherr.WrapError(err, "Downloading job")
+	}
+
+	tr.downloadPath = downloadPath
+
+	extractPath, err := tr.extractor.Extract(tr.downloadPath)
+	if err != nil {
+		cleanUpErr := tr.downloader.CleanUp(tr.downloadPath)
+		if cleanUpErr != nil {
+			tr.logger.Debug(tarReaderLogTag,
+				"Failed to clean up downloaded job %v", cleanUpErr)
+		}
+
+		return job, bosherr.WrapError(err, "Extracting job")
+	}
+
+	tr.extractPath = extractPath
+
+	manifestPath := filepath.Join(tr.extractPath, "job.MF")
+
+	manifest, err := bpreljobman.NewManifestFromPath(manifestPath, tr.fs)
+	if err != nil {
+		closeErr := tr.Close()
+		if closeErr != nil {
+			tr.logger.Debug(tarReaderLogTag,
+				"Failed to close job %v", closeErr)
+		}
+
+		return job, bosherr.WrapError(err, "Building manifest")
+	}
+
+	tr.logger.Debug(tarReaderLogTag, "Done building manifest %#v", manifest)
+
+	job.populateFromManifest(manifest)
+
+	tr.populateJobPaths(&job)
+
+	return job, nil
+}
+
+func (tr TarReader) Close() error {
+	dlErr := tr.downloader.CleanUp(tr.downloadPath)
+	if dlErr != nil {
+		tr.logger.Debug(tarReaderLogTag,
+			"Failed to clean up downloaded job %v", dlErr)
+	}
+
+	exErr := tr.extractor.CleanUp(tr.extractPath)
+	if exErr != nil {
+		tr.logger.Debug(tarReaderLogTag,
+			"Failed to clean up extracted job %v", exErr)
+	}
+
+	if dlErr != nil {
+		return dlErr
+	}
+
+	return exErr
+}
+
+// populateJobPaths sets Path for each template and monit in the job.
+func (tr TarReader) populateJobPaths(job *Job) {
+	// monit file is outside of templates/ directory
+	job.MonitTemplate.Path = filepath.Join(
+		tr.extractPath, job.MonitTemplate.SrcPathEnd)
+
+	for i, template := range job.Templates {
+		job.Templates[i].Path = filepath.Join(
+			tr.extractPath, "templates", template.SrcPathEnd)
+	}
+}
+
+/*
+Example layout of an unpackaged job tar:
+
+$ tree ~/Downloads/dummy-job
+~/Downloads/dummy-job
+├── job.MF
+├── monit
+└── templates
+    └── dummy_ctl
+*/

data/go/src/boshprovisioner/release/manifest/manifest.go

@@ -0,0 +1,96 @@
+// Package manifest represents internal structure of a release.
+package manifest
+
+import (
+	bosherr "bosh/errors"
+	boshsys "bosh/system"
+	"github.com/fraenkel/candiedyaml"
+)
+
+type Manifest struct {
+	Release Release
+}
+
+type Release struct {
+	Name    string `yaml:"name"`
+	Version string `yaml:"version"`
+
+	Jobs     []Job     `yaml:"jobs"`
+	Packages []Package `yaml:"packages"`
+
+	CommitHash         string `yaml:"commit_hash"`
+	UncommittedChanges bool   `yaml:"uncommitted_changes"`
+}
+
+type Job struct {
+	Name    string `yaml:"name"`
+	Version string `yaml:"version"`
+
+	Fingerprint string `yaml:"fingerprint"`
+	SHA1        string `yaml:"sha1"`
+
+	DependencyNames []DependencyName `yaml:"dependencies"`
+}
+
+type Package struct {
+	Name    string `yaml:"name"`
+	Version string `yaml:"version"`
+
+	Fingerprint string `yaml:"fingerprint"`
+	SHA1        string `yaml:"sha1"`
+
+	DependencyNames []DependencyName `yaml:"dependencies"`
+}
+
+func (p Package) DependencyName() DependencyName {
+	return DependencyName(p.Name)
+}
+
+type DependencyName string
+
+func NewManifestFromPath(path string, fs boshsys.FileSystem) (Manifest, error) {
+	bytes, err := fs.ReadFile(path)
+	if err != nil {
+		return Manifest{}, bosherr.WrapError(err, "Reading manifest %s", path)
+	}
+
+	var release Release
+
+	err = candiedyaml.Unmarshal(bytes, &release)
+	if err != nil {
+		return Manifest{}, bosherr.WrapError(err, "Parsing release")
+	}
+
+	return Manifest{Release: release}, nil
+}
+
+/*
+# Example for release.MF
+name: bosh
+version: 77
+
+commit_hash: bbe5476c
+uncommitted_changes: true
+
+packages:
+- name: registry
+  version: !binary |-
+    ZGQxYmEzMzBiYzQ0YjMxODFiMjYzMzgzYjhlNDI1MmQ3MDUxZGVjYQ==
+  sha1: !binary |-
+    NmVhYTZjOTYxZWFjN2JkOTk0ZDE2NDRhZDQwNWIzMzk1NDIwZWNhZg==
+  fingerprint: !binary |-
+    ZGQxYmEzMzBiYzQ0YjMxODFiMjYzMzgzYjhlNDI1MmQ3MDUxZGVjYQ==
+  dependencies:
+  - libpq
+  - mysql
+  - ruby
+
+jobs:
+- name: powerdns
+  version: !binary |-
+    MGI4MGIzYzE5OGJmN2FiYzZjODEyNjIwMTNkZTQ5NDM2OWZkMjViNg==
+  fingerprint: !binary |-
+    MGI4MGIzYzE5OGJmN2FiYzZjODEyNjIwMTNkZTQ5NDM2OWZkMjViNg==
+  sha1: !binary |-
+    YWI5NzA5YmVhYjViZTBmYjYyYTJkMWYzYzg4ZDA2YzliNGJkZWM2NQ==
+*/

data/go/src/boshprovisioner/release/manifest_reader.go

@@ -0,0 +1,29 @@
+package release
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+type ManifestReader struct {
+	path   string
+	fs     boshsys.FileSystem
+	logger boshlog.Logger
+}
+
+func NewManifestReader(
+	path string,
+	fs boshsys.FileSystem,
+	logger boshlog.Logger,
+) ManifestReader {
+	return ManifestReader{path: path, fs: fs, logger: logger}
+}
+
+func (r ManifestReader) Read() (Release, error) {
+	return Release{}, bosherr.New("Not implemented")
+}
+
+func (r ManifestReader) Close() error {
+	return nil
+}