vagrant-bosh 0.0.1

data/go/src/boshprovisioner/instance/templatescompiler/concrete_templates_compiler.go

@@ -0,0 +1,273 @@
+package templatescompiler
+
+import (
+	"fmt"
+
+	boshblob "bosh/blobstore"
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpjobsrepo "boshprovisioner/instance/templatescompiler/jobsrepo"
+	bptplsrepo "boshprovisioner/instance/templatescompiler/templatesrepo"
+	bprel "boshprovisioner/release"
+	bpreljob "boshprovisioner/release/job"
+)
+
+type ConcreteTemplatesCompiler struct {
+	renderedArchivesCompiler RenderedArchivesCompiler
+	jobReaderFactory         bpreljob.ReaderFactory
+
+	jobsRepo      bpjobsrepo.JobsRepository
+	tplToJobRepo  bpjobsrepo.TemplateToJobRepository
+	runPkgsRepo   bpjobsrepo.RuntimePackagesRepository
+	templatesRepo bptplsrepo.TemplatesRepository
+
+	blobstore boshblob.Blobstore
+	logger    boshlog.Logger
+}
+
+func NewConcreteTemplatesCompiler(
+	renderedArchivesCompiler RenderedArchivesCompiler,
+	jobReaderFactory bpreljob.ReaderFactory,
+	jobsRepo bpjobsrepo.JobsRepository,
+	tplToJobRepo bpjobsrepo.TemplateToJobRepository,
+	runPkgsRepo bpjobsrepo.RuntimePackagesRepository,
+	templatesRepo bptplsrepo.TemplatesRepository,
+	blobstore boshblob.Blobstore,
+	logger boshlog.Logger,
+) ConcreteTemplatesCompiler {
+	return ConcreteTemplatesCompiler{
+		renderedArchivesCompiler: renderedArchivesCompiler,
+		jobReaderFactory:         jobReaderFactory,
+
+		jobsRepo:      jobsRepo,
+		tplToJobRepo:  tplToJobRepo,
+		runPkgsRepo:   runPkgsRepo,
+		templatesRepo: templatesRepo,
+
+		blobstore: blobstore,
+		logger:    logger,
+	}
+}
+
+// Precompile prepares release jobs to be later combined with instance properties
+func (tc ConcreteTemplatesCompiler) Precompile(release bprel.Release) error {
+	var allPkgs []bprel.Package
+
+	for _, pkg := range release.Packages {
+		if pkg == nil {
+			// todo panic or should not be here?
+			return bosherr.New("Expected release to not have nil package")
+		}
+
+		allPkgs = append(allPkgs, *pkg)
+	}
+
+	for _, job := range release.Jobs {
+		jobRec, found, err := tc.jobsRepo.Find(job)
+		if err != nil {
+			return bosherr.WrapError(err, "Finding job source blob %s", job.Name)
+		}
+
+		if !found {
+			blobID, fingerprint, err := tc.blobstore.Create(job.TarPath)
+			if err != nil {
+				return bosherr.WrapError(err, "Creating job source blob %s", job.Name)
+			}
+
+			jobRec = bpjobsrepo.JobRecord{
+				BlobID: blobID,
+				SHA1:   fingerprint,
+			}
+
+			err = tc.jobsRepo.Save(job, jobRec)
+			if err != nil {
+				return bosherr.WrapError(err, "Saving job record %s", job.Name)
+			}
+		}
+
+		err = tc.tplToJobRepo.SaveForJob(release, job)
+		if err != nil {
+			return bosherr.WrapError(err, "Saving release job %s", job.Name)
+		}
+
+		// todo associate to release instead
+		err = tc.runPkgsRepo.SaveAllForReleaseJob(job, allPkgs)
+		if err != nil {
+			return bosherr.WrapError(err, "Saving release job %s", job.Name)
+		}
+	}
+
+	return nil
+}
+
+// Compile populates blobstore with rendered jobs for a given deployment instance.
+func (tc ConcreteTemplatesCompiler) Compile(job bpdep.Job, instance bpdep.Instance) error {
+	relJobReaders, err := tc.buildJobReaders(job)
+	if err != nil {
+		return err
+	}
+
+	blobID, fingerprint, err := tc.compileJob(relJobReaders, instance)
+	if err != nil {
+		return err
+	}
+
+	templateRec := bptplsrepo.TemplateRecord{
+		BlobID: blobID,
+		SHA1:   fingerprint,
+	}
+
+	err = tc.templatesRepo.Save(job, instance, templateRec)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving compiled templates record %s", job.Name)
+	}
+
+	return nil
+}
+
+// FindPackages returns list of packages required to run job template.
+// List of packages is usually specified in release job metadata.
+func (tc ConcreteTemplatesCompiler) FindPackages(template bpdep.Template) ([]bprel.Package, error) {
+	var pkgs []bprel.Package
+
+	job, found, err := tc.tplToJobRepo.FindByTemplate(template)
+	if err != nil {
+		return pkgs, bosherr.WrapError(err, "Finding job by template %s", template.Name)
+	} else if !found {
+		return pkgs, bosherr.New("Expected to find job by template %s", template.Name)
+	}
+
+	pkgs, found, err = tc.runPkgsRepo.FindByReleaseJob(job)
+	if err != nil {
+		return pkgs, bosherr.WrapError(err, "Finding packages by job %s", job.Name)
+	} else if !found {
+		return pkgs, bosherr.New("Expected to find packages by job %s", job.Name)
+	}
+
+	return pkgs, nil
+}
+
+// FindRenderedArchive returns previously compiled template for a given instance.
+// If such compiled template is not found, error is returned.
+func (tc ConcreteTemplatesCompiler) FindRenderedArchive(job bpdep.Job, instance bpdep.Instance) (RenderedArchiveRecord, error) {
+	var renderedArchiveRec RenderedArchiveRecord
+
+	rec, found, err := tc.templatesRepo.Find(job, instance)
+	if err != nil {
+		return renderedArchiveRec, bosherr.WrapError(err, "Finding compiled templates %s", job.Name)
+	} else if !found {
+		return renderedArchiveRec, bosherr.New("Expected to find compiled templates %s", job.Name)
+	}
+
+	renderedArchiveRec.SHA1 = rec.SHA1
+	renderedArchiveRec.BlobID = rec.BlobID
+
+	return renderedArchiveRec, nil
+}
+
+type jobReader struct {
+	relJob    bprel.Job
+	tarReader *bpreljob.TarReader
+}
+
+func (tc ConcreteTemplatesCompiler) buildJobReaders(job bpdep.Job) ([]jobReader, error) {
+	var readers []jobReader
+
+	for _, template := range job.Templates {
+		relJob, found, err := tc.tplToJobRepo.FindByTemplate(template)
+		if err != nil {
+			return readers, bosherr.WrapError(err, "Finding dep-template -> rel-job %s", template.Name)
+		} else if !found {
+			return readers, bosherr.New("Expected to find dep-template -> rel-job %s", template.Name)
+		}
+
+		jobRec, found, err := tc.jobsRepo.Find(relJob)
+		if err != nil {
+			return readers, bosherr.WrapError(err, "Finding job source blob %s", template.Name)
+		} else if !found {
+			return readers, bosherr.New("Expected to find job source blob %s", template.Name)
+		}
+
+		jobURL := fmt.Sprintf("blobstore:///%s?fingerprint=%s", jobRec.BlobID, jobRec.SHA1)
+
+		reader := jobReader{
+			relJob:    relJob,
+			tarReader: tc.jobReaderFactory.NewTarReader(jobURL),
+		}
+
+		readers = append(readers, reader)
+	}
+
+	return readers, nil
+}
+
+// compileJob produces and saves rendered templates archive to a blobstore.
+func (tc ConcreteTemplatesCompiler) compileJob(jobReaders []jobReader, instance bpdep.Instance) (string, string, error) {
+	var relJobs []bpreljob.Job
+
+	for _, jobReader := range jobReaders {
+		relJob, err := jobReader.tarReader.Read()
+		if err != nil {
+			return "", "", bosherr.WrapError(err, "Reading job")
+		}
+
+		defer jobReader.tarReader.Close()
+
+		err = tc.associatePackages(jobReader.relJob, relJob)
+		if err != nil {
+			return "", "", bosherr.WrapError(err, "Preparing runtime dep packages")
+		}
+
+		relJobs = append(relJobs, relJob)
+	}
+
+	renderedArchivePath, err := tc.renderedArchivesCompiler.Compile(relJobs, instance)
+	if err != nil {
+		return "", "", bosherr.WrapError(err, "Compiling templates")
+	}
+
+	defer tc.renderedArchivesCompiler.CleanUp(renderedArchivePath)
+
+	blobID, fingerprint, err := tc.blobstore.Create(renderedArchivePath)
+	if err != nil {
+		return "", "", bosherr.WrapError(err, "Creating compiled templates")
+	}
+
+	return blobID, fingerprint, nil
+}
+
+func (tc ConcreteTemplatesCompiler) associatePackages(rJob bprel.Job, relJob bpreljob.Job) error {
+	_, found, err := tc.runPkgsRepo.FindByReleaseJob(rJob)
+	if err != nil {
+		return bosherr.WrapError(err, "Finding runtime deps for %s", rJob.Name)
+	}
+
+	if !found {
+		allPkgs, found, err := tc.runPkgsRepo.FindAllByReleaseJob(rJob)
+		if err != nil {
+			return bosherr.WrapError(err, "Finding rel-job -> rel-pkgs %s", rJob.Name)
+		} else if !found {
+			return bosherr.New("Expected to find rel-job -> rel-pkgs %s", rJob.Name)
+		}
+
+		var pkgs []bprel.Package
+
+		for _, pkg := range allPkgs {
+			for _, p := range relJob.Packages {
+				if pkg.Name == p.Name {
+					pkgs = append(pkgs, pkg)
+					break
+				}
+			}
+		}
+
+		err = tc.runPkgsRepo.SaveForReleaseJob(rJob, pkgs)
+		if err != nil {
+			return bosherr.WrapError(err, "Saving job packages %s", rJob.Name)
+		}
+	}
+
+	return nil
+}

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/erb_renderer.go

@@ -0,0 +1,117 @@
+package erbrenderer
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+)
+
+const erbRendererLogTag = "ERBRenderer"
+
+type ERBRenderer struct {
+	fs      boshsys.FileSystem
+	runner  boshsys.CmdRunner
+	context TemplateEvaluationContext
+	logger  boshlog.Logger
+
+	rendererScript string
+}
+
+func NewERBRenderer(
+	fs boshsys.FileSystem,
+	runner boshsys.CmdRunner,
+	context TemplateEvaluationContext,
+	logger boshlog.Logger,
+) ERBRenderer {
+	return ERBRenderer{
+		fs:      fs,
+		runner:  runner,
+		context: context,
+		logger:  logger,
+
+		rendererScript: templateEvaluationContextRb,
+	}
+}
+
+func (r ERBRenderer) Render(srcPath, dstPath string) error {
+	r.logger.Debug(erbRendererLogTag, "Rendering template %s", dstPath)
+
+	dirPath := filepath.Dir(dstPath)
+
+	err := r.fs.MkdirAll(dirPath, os.FileMode(0755))
+	if err != nil {
+		return bosherr.WrapError(err, "Creating directory %s", dirPath)
+	}
+
+	rendererScriptPath, err := r.writeRendererScript()
+	if err != nil {
+		return err
+	}
+
+	contextPath, err := r.writeContext()
+	if err != nil {
+		return err
+	}
+
+	// Use ruby to compile job templates
+	command := boshsys.Command{
+		Name: r.determineRubyExePath(),
+		Args: []string{rendererScriptPath, contextPath, srcPath, dstPath},
+	}
+
+	_, _, _, err = r.runner.RunComplexCommand(command)
+	if err != nil {
+		return bosherr.WrapError(err, "Running ruby")
+	}
+
+	return nil
+}
+
+func (r ERBRenderer) writeRendererScript() (string, error) {
+	// todo use temp path; it's same everytime?
+	path := "/tmp/erb-render.rb"
+
+	err := r.fs.WriteFileString(path, r.rendererScript)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Writing renderer script")
+	}
+
+	return path, nil
+}
+
+func (r ERBRenderer) writeContext() (string, error) {
+	contextBytes, err := json.Marshal(r.context)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Marshalling context")
+	}
+
+	// todo use temp path?
+	path := "/tmp/erb-context.json"
+
+	err = r.fs.WriteFileString(path, string(contextBytes))
+	if err != nil {
+		return "", bosherr.WrapError(err, "Writing context")
+	}
+
+	return path, nil
+}
+
+func (r ERBRenderer) determineRubyExePath() string {
+	// Prefer ruby executable on the PATH
+	if r.runner.CommandExists("ruby") {
+		return "ruby"
+	}
+
+	// Fallback to chef-solo ruby usually found in vagrant boxes
+	vagrantRuby := "/opt/vagrant_ruby/bin/ruby"
+
+	if r.fs.FileExists(vagrantRuby) {
+		return vagrantRuby
+	}
+
+	return "ruby"
+}

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/erbrenderer_suite_test.go

@@ -0,0 +1,13 @@
+package erbrenderer_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestErbrenderer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Erbrenderer Suite")
+}

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/render_properties.go

@@ -0,0 +1,77 @@
+package erbrenderer
+
+import (
+	"encoding/json"
+	"strings"
+
+	bosherr "bosh/errors"
+
+	bpdep "boshprovisioner/deployment"
+	bpreljob "boshprovisioner/release/job"
+)
+
+type RenderProperties struct {
+	relJob   bpreljob.Job
+	instance bpdep.Instance
+}
+
+func NewRenderProperties(relJob bpreljob.Job, instance bpdep.Instance) RenderProperties {
+	return RenderProperties{relJob: relJob, instance: instance}
+}
+
+// AsMap returns job and instance properties merged together.
+func (p RenderProperties) AsMap() (map[string]interface{}, error) {
+	result, err := p.deepCopyInstanceProperties()
+	if err != nil {
+		return result, err
+	}
+
+	for _, prop := range p.relJob.Properties {
+		p.copyProperty(prop.Name, prop.Default, result)
+	}
+
+	return result, nil
+}
+
+// copyProperty fills in property in dst with value if it's not already set.
+func (p RenderProperties) copyProperty(path string, value interface{}, dst map[string]interface{}) {
+	pathParts := strings.Split(path, ".")
+
+	for i, part := range pathParts {
+		if dstNested, ok := dst[part]; ok { // Found section; check if modified
+			if dstNestedMap, ok := dstNested.(map[string]interface{}); ok {
+				dst = dstNestedMap
+			} else {
+				break // Not a section; cannot modify
+			}
+		} else if len(pathParts)-1 == i { // Last property path part
+			dst[part] = value
+		} else {
+			m := map[string]interface{}{}
+			dst[part] = m
+			dst = m
+		}
+	}
+}
+
+// deepCopyInstanceProperties makes a deep copy of instance properties.
+// Always returns an initialized map even if instance properties are nil.
+func (p RenderProperties) deepCopyInstanceProperties() (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+
+	if p.instance.Properties == nil {
+		return result, nil
+	}
+
+	bytes, err := json.Marshal(p.instance.Properties)
+	if err != nil {
+		return result, bosherr.WrapError(err, "Marshalling instance properties")
+	}
+
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		return result, bosherr.WrapError(err, "Unmarshalling instance properties")
+	}
+
+	return result, nil
+}

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/render_properties_test.go

@@ -0,0 +1,142 @@
+package erbrenderer_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	bpdep "boshprovisioner/deployment"
+	. "boshprovisioner/instance/templatescompiler/erbrenderer"
+	bpreljob "boshprovisioner/release/job"
+)
+
+var _ = Describe("RenderProperties", func() {
+	var (
+		job      bpreljob.Job
+		instance bpdep.Instance
+		props    RenderProperties
+	)
+
+	JustBeforeEach(func() {
+		props = NewRenderProperties(job, instance)
+	})
+
+	Describe("AsMap", func() {
+		Context("when job specifies a default for a nested property", func() {
+			BeforeEach(func() {
+				job = bpreljob.Job{
+					Properties: []bpreljob.Property{
+						bpreljob.Property{
+							Name:    "prop.nest-prop",
+							Default: "job-val",
+						},
+					},
+				}
+			})
+
+			Context("when instance specifies nested property", func() {
+				BeforeEach(func() {
+					instance = bpdep.Instance{
+						Properties: map[string]interface{}{
+							"prop": map[string]interface{}{
+								"nest-prop": "instance-val",
+							},
+						},
+					}
+				})
+
+				It("returns map with property value from job", func() {
+					Expect(props.AsMap()).To(Equal(map[string]interface{}{
+						"prop": map[string]interface{}{
+							"nest-prop": "instance-val",
+						},
+					}))
+				})
+			})
+
+			Context("when instance does not specify nested property", func() {
+				Context("when first nesting level is specified", func() {
+					BeforeEach(func() {
+						instance = bpdep.Instance{
+							Properties: map[string]interface{}{
+								"prop": map[string]interface{}{},
+							},
+						}
+					})
+
+					It("returns map with default property from job", func() {
+						Expect(props.AsMap()).To(Equal(map[string]interface{}{
+							"prop": map[string]interface{}{
+								"nest-prop": "job-val",
+							},
+						}))
+					})
+				})
+
+				Context("when first nesting level is not specified", func() {
+					BeforeEach(func() {
+						instance = bpdep.Instance{}
+					})
+
+					It("returns map with default property from job", func() {
+						Expect(props.AsMap()).To(Equal(map[string]interface{}{
+							"prop": map[string]interface{}{
+								"nest-prop": "job-val",
+							},
+						}))
+					})
+				})
+			})
+		})
+
+		Context("when job does not specify a default (nil) for a nested property", func() {
+			BeforeEach(func() {
+				job = bpreljob.Job{
+					Properties: []bpreljob.Property{
+						bpreljob.Property{
+							Name:    "prop.nest-prop",
+							Default: nil,
+						},
+					},
+				}
+			})
+
+			Context("when instance specifies nested property", func() {
+				BeforeEach(func() {
+					instance = bpdep.Instance{
+						Properties: map[string]interface{}{
+							"prop": map[string]interface{}{
+								"nest-prop": "instance-val",
+							},
+						},
+					}
+				})
+
+				It("returns map with property value from instance", func() {
+					Expect(props.AsMap()).To(Equal(map[string]interface{}{
+						"prop": map[string]interface{}{
+							"nest-prop": "instance-val",
+						},
+					}))
+				})
+			})
+
+			Context("when instance does not specify nested property", func() {
+				BeforeEach(func() {
+					instance = bpdep.Instance{
+						Properties: map[string]interface{}{
+							"prop": map[string]interface{}{},
+						},
+					}
+				})
+
+				It("returns map with default property from job", func() {
+					Expect(props.AsMap()).To(Equal(map[string]interface{}{
+						"prop": map[string]interface{}{
+							"nest-prop": nil,
+						},
+					}))
+				})
+			})
+		}) //~
+	})
+})

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/template_evaluation_context.go

@@ -0,0 +1,85 @@
+package erbrenderer
+
+import (
+	"encoding/json"
+
+	bosherr "bosh/errors"
+
+	bpdep "boshprovisioner/deployment"
+	bpreljob "boshprovisioner/release/job"
+)
+
+type TemplateEvaluationContext struct {
+	relJob   bpreljob.Job
+	instance bpdep.Instance
+}
+
+// rootContext is exposed as an open struct in ERB templates.
+// It must stay same to provide backwards compatible API.
+type rootContext struct {
+	Index int `json:"index"`
+
+	JobContext jobContext `json:"job"`
+
+	Deployment string `json:"deployment"`
+
+	// Usually is accessed with <%= spec.networks.default.ip %>
+	NetworkContexts map[string]networkContext `json:"networks"`
+
+	Properties map[string]interface{} `json:"properties"`
+}
+
+type jobContext struct {
+	Name string `json:"name"`
+}
+
+// networkContext is not fully backwards compatible.
+type networkContext struct {
+	IP      string `json:"ip"`
+	Netmask string `json:"netmask"`
+	Gateway string `json:"gateway"`
+}
+
+func NewTemplateEvaluationContext(
+	relJob bpreljob.Job,
+	instance bpdep.Instance,
+) TemplateEvaluationContext {
+	return TemplateEvaluationContext{
+		relJob:   relJob,
+		instance: instance,
+	}
+}
+
+func (c TemplateEvaluationContext) MarshalJSON() ([]byte, error) {
+	properties, err := NewRenderProperties(c.relJob, c.instance).AsMap()
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Rendering properties")
+	}
+
+	context := rootContext{
+		Index:      c.instance.Index,
+		JobContext: jobContext{Name: c.instance.JobName},
+		Deployment: c.instance.DeploymentName,
+
+		NetworkContexts: c.buildNetworkContexts(),
+		Properties:      properties,
+	}
+
+	return json.Marshal(context)
+}
+
+func (c TemplateEvaluationContext) buildNetworkContexts() map[string]networkContext {
+	networkContexts := map[string]networkContext{}
+
+	for _, na := range c.instance.NetworkAssociations {
+		netConfig := c.instance.NetworkConfigurationForNetworkAssociation(na)
+
+		networkContexts[na.Network.Name] = networkContext{
+			IP:      netConfig.IP,
+			Netmask: netConfig.Netmask,
+			Gateway: netConfig.Gateway,
+		}
+	}
+
+	return networkContexts
+}