vagrant-bosh 0.0.1

data/go/src/boshprovisioner/instance/templatescompiler/erbrenderer/template_evaluation_context_rb.go

@@ -0,0 +1,155 @@
+package erbrenderer
+
+const templateEvaluationContextRb = `
+# Based on common/properties/template_evaluation_context.rb
+require "rubygems"
+require "ostruct"
+require "json"
+require "erb"
+
+class TemplateEvaluationContext
+  attr_reader :name, :index
+  attr_reader :properties, :raw_properties
+  attr_reader :spec
+
+  def initialize(spec)
+    @name = spec["job"]["name"] if spec["job"].is_a?(Hash)
+    @index = spec["index"]
+    @properties = openstruct(spec["properties"] || {})
+    @raw_properties = spec["properties"] || {}
+    @spec = openstruct(spec)
+  end
+
+  def get_binding
+    binding.taint
+  end
+
+  def p(*args)
+    names = Array(args[0])
+
+    names.each do |name|
+      result = lookup_property(@raw_properties, name)
+      return result unless result.nil?
+    end
+
+    return args[1] if args.length == 2
+    raise UnknownProperty.new(names)
+  end
+
+  def if_p(*names)
+    values = names.map do |name|
+      value = lookup_property(@raw_properties, name)
+      return ActiveElseBlock.new(self) if value.nil?
+      value
+    end
+
+    yield *values
+    InactiveElseBlock.new
+  end
+
+  private
+  
+  def openstruct(object)
+    case object
+      when Hash
+        mapped = object.inject({}) { |h, (k,v)| h[k] = openstruct(v); h }
+        OpenStruct.new(mapped)
+      when Array
+        object.map { |item| openstruct(item) }
+      else
+        object
+    end
+  end
+
+  def lookup_property(collection, name)
+    keys = name.split(".")
+    ref = collection
+
+    keys.each do |key|
+      ref = ref[key]
+      return nil if ref.nil?
+    end
+
+    ref
+  end
+
+  class UnknownProperty < StandardError
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+      super("Can't find property '#{name}'")
+    end
+  end
+
+  class ActiveElseBlock
+    def initialize(template)
+      @context = template
+    end
+
+    def else
+      yield
+    end
+
+    def else_if_p(*names, &block)
+      @context.if_p(*names, &block)
+    end
+  end
+
+  class InactiveElseBlock
+    def else; end
+
+    def else_if_p(*names)
+      InactiveElseBlock.new
+    end
+  end
+end
+
+# todo do not use JSON in releases
+class << JSON
+  alias dump_array_or_hash dump
+
+  def dump(*args)
+    arg = args[0]
+    if arg.is_a?(String) || arg.is_a?(Numeric)
+      arg.inspect
+    else
+      dump_array_or_hash(*args)
+    end
+  end
+end
+
+class ERBRenderer
+  def initialize(context)
+    @context = context
+  end
+
+  def render(src_path, dst_path)
+    erb = ERB.new(File.read(src_path))
+    erb.filename = src_path
+
+    File.open(dst_path, "w") do |f| 
+      f.write(erb.result(@context.get_binding))
+    end
+
+  rescue Exception => e
+    name = "#{@context.name}/#{@context.index}"
+
+    line_i = e.backtrace.index { |l| l.include?(erb.filename) }
+    line_num = line_i ? e.backtrace[line_i].split(':')[1] : "unknown"
+    location = "(line #{line_num}: #{e.inspect})"
+
+    raise("Error filling in template '#{src_path}' for #{name} #{location}")
+  end
+end
+
+if $0 == __FILE__
+  context_path, src_path, dst_path = *ARGV
+
+  context_hash = JSON.load(File.read(context_path))
+  context = TemplateEvaluationContext.new(context_hash)
+
+  renderer = ERBRenderer.new(context)
+  renderer.render(src_path, dst_path)
+end
+`

data/go/src/boshprovisioner/instance/templatescompiler/jobsrepo/concrete_jobs_repository.go

@@ -0,0 +1,64 @@
+package jobsrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpindex "boshprovisioner/index"
+	bprel "boshprovisioner/release"
+)
+
+type CJRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+type jobToJobKey struct {
+	JobName    string
+	JobVersion string
+
+	// Fingerprint of a package captures its dependenices
+	JobFingerprint string
+}
+
+func NewConcreteJobsRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CJRepository {
+	return CJRepository{
+		index:  index,
+		logger: logger,
+	}
+}
+
+func (r CJRepository) Find(job bprel.Job) (JobRecord, bool, error) {
+	var record JobRecord
+
+	err := r.index.Find(r.jobKey(job), &record)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return record, false, nil
+		}
+
+		return record, false, bosherr.WrapError(err, "Finding package record")
+	}
+
+	return record, true, nil
+}
+
+func (r CJRepository) Save(job bprel.Job, record JobRecord) error {
+	err := r.index.Save(r.jobKey(job), record)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving package record")
+	}
+
+	return nil
+}
+
+func (r CJRepository) jobKey(job bprel.Job) jobToJobKey {
+	return jobToJobKey{
+		JobName:        job.Name,
+		JobVersion:     job.Version,
+		JobFingerprint: job.Fingerprint,
+	}
+}

data/go/src/boshprovisioner/instance/templatescompiler/jobsrepo/concrete_runtime_packages_repository.go

@@ -0,0 +1,105 @@
+package jobsrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpindex "boshprovisioner/index"
+	bprel "boshprovisioner/release"
+)
+
+type CRPRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+func NewConcreteRuntimePackagesRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CRPRepository {
+	return CRPRepository{
+		index:  index,
+		logger: logger,
+	}
+}
+
+type jobToPkgsKey struct {
+	// Mostly for ease of debugging
+	JobName    string
+	JobVersion string
+
+	// Fingerprint of a job captures its dependenices
+	JobFingerprint string
+
+	// Indicates if contains all packages in a job's release
+	ContainsAllPackages bool
+}
+
+func (r CRPRepository) FindByReleaseJob(job bprel.Job) ([]bprel.Package, bool, error) {
+	var pkgs []bprel.Package
+
+	err := r.index.Find(r.jobSpecificKey(job), &pkgs)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return pkgs, false, nil
+		}
+
+		return pkgs, false, bosherr.WrapError(err, "Finding rel-job -> specific rel-pkgs record")
+	}
+
+	return pkgs, true, nil
+}
+
+func (r CRPRepository) SaveForReleaseJob(job bprel.Job, pkgs []bprel.Package) error {
+	err := r.index.Save(r.jobSpecificKey(job), pkgs)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving rel-job -> specific rel-pkgs record")
+	}
+
+	return nil
+}
+
+// FindAllByReleaseJob keeps association between all possible packages for a job
+func (r CRPRepository) FindAllByReleaseJob(job bprel.Job) ([]bprel.Package, bool, error) {
+	var pkgs []bprel.Package
+
+	err := r.index.Find(r.jobAllKey(job), &pkgs)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return pkgs, false, nil
+		}
+
+		return pkgs, false, bosherr.WrapError(err, "Finding rel-job -> all rel-pkgs record")
+	}
+
+	return pkgs, true, nil
+}
+
+func (r CRPRepository) SaveAllForReleaseJob(job bprel.Job, pkgs []bprel.Package) error {
+	err := r.index.Save(r.jobAllKey(job), pkgs)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving rel-job -> all rel-pkgs record")
+	}
+
+	return nil
+}
+
+func (r CRPRepository) jobSpecificKey(job bprel.Job) jobToPkgsKey {
+	return jobToPkgsKey{
+		JobName:        job.Name,
+		JobVersion:     job.Version,
+		JobFingerprint: job.Fingerprint,
+
+		ContainsAllPackages: false,
+	}
+}
+
+func (r CRPRepository) jobAllKey(job bprel.Job) jobToPkgsKey {
+	return jobToPkgsKey{
+		JobName:        job.Name,
+		JobVersion:     job.Version,
+		JobFingerprint: job.Fingerprint,
+
+		ContainsAllPackages: true,
+	}
+}

data/go/src/boshprovisioner/instance/templatescompiler/jobsrepo/concrete_template_to_job_repository.go

@@ -0,0 +1,76 @@
+package jobsrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpindex "boshprovisioner/index"
+	bprel "boshprovisioner/release"
+)
+
+type CTTJRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+type templateToJobKey struct {
+	JobName        string
+	ReleaseName    string
+	ReleaseVersion string
+}
+
+func NewConcreteTemplateToJobRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CTTJRepository {
+	return CTTJRepository{
+		index:  index,
+		logger: logger,
+	}
+}
+
+func (r CTTJRepository) FindByTemplate(template bpdep.Template) (bprel.Job, bool, error) {
+	var job bprel.Job
+
+	err := r.index.Find(r.templateKey(template), &job)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return job, false, nil
+		}
+
+		return job, false, bosherr.WrapError(err, "Finding dep-template -> rel-job record")
+	}
+
+	return job, true, nil
+}
+
+func (r CTTJRepository) SaveForJob(release bprel.Release, job bprel.Job) error {
+	err := r.index.Save(r.jobKey(release, job), job)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving dep-template -> rel-job record")
+	}
+
+	return nil
+}
+
+func (r CTTJRepository) templateKey(template bpdep.Template) templateToJobKey {
+	if template.Release == nil {
+		panic("Expected template.Release to not be nil")
+	}
+
+	return templateToJobKey{
+		JobName:        template.Name,
+		ReleaseName:    template.Release.Name,
+		ReleaseVersion: template.Release.Version,
+	}
+}
+
+// todo should job point back to release
+func (r CTTJRepository) jobKey(release bprel.Release, job bprel.Job) templateToJobKey {
+	return templateToJobKey{
+		JobName:        job.Name,
+		ReleaseName:    release.Name,
+		ReleaseVersion: release.Version,
+	}
+}

data/go/src/boshprovisioner/instance/templatescompiler/jobsrepo/jobs_repository_interface.go

@@ -0,0 +1,31 @@
+package jobsrepo
+
+import (
+	bpdep "boshprovisioner/deployment"
+	bprel "boshprovisioner/release"
+)
+
+type JobRecord struct {
+	BlobID string
+	SHA1   string
+}
+
+// JobsRepository maintains list of job source code as blobs
+type JobsRepository interface {
+	Find(bprel.Job) (JobRecord, bool, error)
+	Save(bprel.Job, JobRecord) error
+}
+
+type TemplateToJobRepository interface {
+	FindByTemplate(bpdep.Template) (bprel.Job, bool, error)
+	SaveForJob(bprel.Release, bprel.Job) error
+}
+
+type RuntimePackagesRepository interface {
+	FindByReleaseJob(bprel.Job) ([]bprel.Package, bool, error)
+	SaveForReleaseJob(bprel.Job, []bprel.Package) error
+
+	// Keeps association between all possible packages for a job
+	FindAllByReleaseJob(bprel.Job) ([]bprel.Package, bool, error)
+	SaveAllForReleaseJob(bprel.Job, []bprel.Package) error
+}

data/go/src/boshprovisioner/instance/templatescompiler/rendered_archives_compiler.go

@@ -0,0 +1,81 @@
+package templatescompiler
+
+import (
+	"path/filepath"
+
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+	boshsys "bosh/system"
+
+	bpdep "boshprovisioner/deployment"
+	bperb "boshprovisioner/instance/templatescompiler/erbrenderer"
+	bpreljob "boshprovisioner/release/job"
+	bptar "boshprovisioner/tar"
+)
+
+type RenderedArchivesCompiler struct {
+	fs         boshsys.FileSystem
+	runner     boshsys.CmdRunner
+	compressor bptar.Compressor
+	logger     boshlog.Logger
+}
+
+func NewRenderedArchivesCompiler(
+	fs boshsys.FileSystem,
+	runner boshsys.CmdRunner,
+	compressor bptar.Compressor,
+	logger boshlog.Logger,
+) RenderedArchivesCompiler {
+	return RenderedArchivesCompiler{
+		fs:         fs,
+		runner:     runner,
+		compressor: compressor,
+		logger:     logger,
+	}
+}
+
+// Compile takes release jobs and instance and produces rendered templates archive.
+// Rendered templates archive contains rendered job templates
+// that can be unpacked by a GoAgent to populate a VM.
+func (rac RenderedArchivesCompiler) Compile(relJobs []bpreljob.Job, instance bpdep.Instance) (string, error) {
+	path, err := rac.fs.TempDir("instance-templatescompiler-RenderedArchivesCompiler")
+	if err != nil {
+		return "", bosherr.WrapError(err, "Creating compiled templates directory")
+	}
+
+	defer rac.fs.RemoveAll(path)
+
+	for _, relJob := range relJobs {
+		context := bperb.NewTemplateEvaluationContext(relJob, instance)
+
+		renderer := bperb.NewERBRenderer(rac.fs, rac.runner, context, rac.logger)
+
+		dstPath := filepath.Join(path, relJob.Name, "monit")
+
+		err := renderer.Render(relJob.MonitTemplate.Path, dstPath)
+		if err != nil {
+			return "", bosherr.WrapError(err, "Rendering monit ERB")
+		}
+
+		for _, template := range relJob.Templates {
+			dstPath := filepath.Join(path, relJob.Name, template.DstPathEnd)
+
+			err := renderer.Render(template.Path, dstPath)
+			if err != nil {
+				return "", bosherr.WrapError(err, "Rendering %s ERB", template.DstPathEnd)
+			}
+		}
+	}
+
+	renderedArchivePath, err := rac.compressor.Compress(path)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Compressing templates")
+	}
+
+	return renderedArchivePath, nil
+}
+
+// CleanUp deletes previously produced rendered templates archive.
+func (rac RenderedArchivesCompiler) CleanUp(path string) error {
+	return rac.fs.RemoveAll(path)
+}

data/go/src/boshprovisioner/instance/templatescompiler/templates_compiler_interface.go

@@ -0,0 +1,20 @@
+package templatescompiler
+
+import (
+	bpdep "boshprovisioner/deployment"
+	bprel "boshprovisioner/release"
+)
+
+type RenderedArchiveRecord struct {
+	SHA1   string
+	BlobID string
+}
+
+type TemplatesCompiler interface {
+	Precompile(bprel.Release) error
+	Compile(bpdep.Job, bpdep.Instance) error
+	FindRenderedArchive(bpdep.Job, bpdep.Instance) (RenderedArchiveRecord, error)
+
+	// todo does it belong here?
+	FindPackages(template bpdep.Template) ([]bprel.Package, error)
+}

data/go/src/boshprovisioner/instance/templatescompiler/templatesrepo/ct_repository.go

@@ -0,0 +1,54 @@
+package templatesrepo
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpdep "boshprovisioner/deployment"
+	bpindex "boshprovisioner/index"
+)
+
+type CTRepository struct {
+	index  bpindex.Index
+	logger boshlog.Logger
+}
+
+// todo fingerprint property changes
+type jobToTemplateKey struct {
+	JobName string
+}
+
+func NewConcreteTemplatesRepository(
+	index bpindex.Index,
+	logger boshlog.Logger,
+) CTRepository {
+	return CTRepository{index: index, logger: logger}
+}
+
+func (tr CTRepository) Find(job bpdep.Job, instance bpdep.Instance) (TemplateRecord, bool, error) {
+	var record TemplateRecord
+
+	err := tr.index.Find(tr.templateKey(job), &record)
+	if err != nil {
+		if err == bpindex.ErrNotFound {
+			return record, false, nil
+		}
+
+		return record, false, bosherr.WrapError(err, "Finding tempate")
+	}
+
+	return record, true, nil
+}
+
+func (tr CTRepository) Save(job bpdep.Job, instance bpdep.Instance, record TemplateRecord) error {
+	err := tr.index.Save(tr.templateKey(job), record)
+	if err != nil {
+		return bosherr.WrapError(err, "Saving template")
+	}
+
+	return nil
+}
+
+func (tr CTRepository) templateKey(job bpdep.Job) jobToTemplateKey {
+	return jobToTemplateKey{JobName: job.Name}
+}

data/go/src/boshprovisioner/instance/templatescompiler/templatesrepo/templates_repository_interface.go

@@ -0,0 +1,16 @@
+package templatesrepo
+
+import (
+	bpdep "boshprovisioner/deployment"
+)
+
+type TemplateRecord struct {
+	BlobID string
+	SHA1   string
+}
+
+// TemplatesRepository maintains list of rendered templates as blobs
+type TemplatesRepository interface {
+	Find(bpdep.Job, bpdep.Instance) (TemplateRecord, bool, error)
+	Save(bpdep.Job, bpdep.Instance, TemplateRecord) error
+}

data/go/src/boshprovisioner/instance/updater/applier/applier.go

@@ -0,0 +1,93 @@
+package applier
+
+import (
+	bosherr "bosh/errors"
+	boshlog "bosh/logger"
+
+	bpagclient "boshprovisioner/agent/client"
+	bpdep "boshprovisioner/deployment"
+	bptplcomp "boshprovisioner/instance/templatescompiler"
+	bppkgscomp "boshprovisioner/packagescompiler"
+)
+
+const applierLogTag = "Applier"
+
+type Applier struct {
+	depJob   bpdep.Job
+	instance bpdep.Instance
+
+	templatesCompiler bptplcomp.TemplatesCompiler
+	packagesCompiler  bppkgscomp.PackagesCompiler
+
+	agentClient bpagclient.Client
+	logger      boshlog.Logger
+}
+
+func NewApplier(
+	depJob bpdep.Job,
+	instance bpdep.Instance,
+	templatesCompiler bptplcomp.TemplatesCompiler,
+	packagesCompiler bppkgscomp.PackagesCompiler,
+	agentClient bpagclient.Client,
+	logger boshlog.Logger,
+) Applier {
+	return Applier{
+		depJob:   depJob,
+		instance: instance,
+
+		templatesCompiler: templatesCompiler,
+		packagesCompiler:  packagesCompiler,
+
+		agentClient: agentClient,
+		logger:      logger,
+	}
+}
+
+func (a Applier) Apply() error {
+	a.logger.Debug(applierLogTag, "Applying empty state")
+
+	emptyState := NewEmptyState(a.instance)
+
+	_, err := a.agentClient.Apply(emptyState.AsApplySpec())
+	if err != nil {
+		return bosherr.WrapError(err, "Applying empty spec")
+	}
+
+	// Changes local copy of an instance
+	a.instance.CurrentState, err = a.agentClient.GetState()
+	if err != nil {
+		return bosherr.WrapError(err, "Getting state")
+	}
+
+	a.logger.Debug(applierLogTag, "Finished applying empty state")
+
+	// Recompile job templates since current instance state might have changed.
+	// e.g. dynamic IP could now be set
+	err = a.templatesCompiler.Compile(a.depJob, a.instance)
+	if err != nil {
+		return bosherr.WrapError(err, "Compiling templates %s", a.depJob.Name)
+	}
+
+	a.logger.Debug(applierLogTag, "Applying job state")
+
+	jobState := NewJobState(
+		a.depJob,
+		a.instance,
+		a.templatesCompiler,
+		a.packagesCompiler,
+	)
+
+	jobApplySpec, err := jobState.AsApplySpec()
+	if err != nil {
+		return err
+	}
+
+	_, err = a.agentClient.Apply(jobApplySpec)
+	if err != nil {
+		return bosherr.WrapError(err, "Applying job spec")
+	}
+
+	a.logger.Debug(applierLogTag, "Finished applying job state")
+
+	return nil
+}