bosh_cli 0.19.6 → 1.0.rc1

data/lib/cli/errors.rb

@@ -16,6 +16,8 @@ module Bosh::Cli
     def self.exit_code(code = nil)
       define_method(:exit_code) { code }
     end
+
+    error_code(42)
   end
 
   class UnknownCommand       < CliError; error_code(100); end
@@ -45,4 +47,8 @@ module Bosh::Cli
   class UndefinedProperty    < CliError; error_code(509); end
   class MalformedManifest    < CliError; error_code(511); end
   class MissingTarget        < CliError; error_code(512); end
+  class InvalidProperty      < CliError; error_code(513); end
+  class InvalidManifest      < CliError; error_code(514); end
+  class PropertyMismatch     < CliError; error_code(515); end
+  class InvalidPropertyMapping < CliError; error_code(516); end
 end

data/lib/cli/job_builder.rb

@@ -7,6 +7,9 @@ module Bosh::Cli
     attr_reader :name, :version, :packages, :templates,
                 :release_dir, :built_packages, :tarball_path
 
+    # @return [Hash] Properties defined in this job
+    attr_reader :properties
+
     def self.run_prepare_script(script_path)
       unless File.exists?(script_path)
         raise InvalidJob, "Prepare script at `#{script_path}' doesn't exist"
@@ -27,42 +30,88 @@ module Bosh::Cli
         # with CLI itself
         %w{ BUNDLE_GEMFILE RUBYOPT }.each { |key| ENV.delete(key) }
 
+        output = nil
         Dir.chdir(script_dir) do
           cmd = "./#{script_name} 2>&1"
-          say("Running #{cmd}...")
-          script_output = `#{cmd}`
-          script_output.split("\n").each do |line|
-            say("> #{line}")
-          end
+          output = `#{cmd}`
         end
 
         unless $?.exitstatus == 0
           raise InvalidJob, "`#{script_path}' script failed"
         end
+
+        output
       ensure
         ENV.each_pair { |k, v| ENV[k] = old_env[k] }
       end
     end
 
+    # @param [String] directory Release directory
+    # @param [Hash] options Build options
+    def self.discover(directory, options = {})
+      builders = []
+
+      Dir[File.join(directory, "jobs", "*")].each do |job_dir|
+        next unless File.directory?(job_dir)
+        job_dirname = File.basename(job_dir)
+
+        prepare_script = File.join(job_dir, "prepare")
+        if File.exists?(prepare_script)
+          run_prepare_script(prepare_script)
+        end
+
+        job_spec = load_yaml_file(File.join(job_dir, "spec"))
+        if job_spec["name"] != job_dirname
+          raise InvalidJob,
+                "Found `#{job_spec["name"]}' job in " +
+                "`#{job_dirname}' directory, please fix it"
+        end
+
+        final = options[:final]
+        dry_run = options[:dry_run]
+        blobstore = options[:blobstore]
+        package_names = options[:package_names]
+
+        builder = new(job_spec, directory, final, blobstore, package_names)
+        builder.dry_run = true if dry_run
+        builders << builder
+      end
+
+      builders
+    end
+
     def initialize(spec, release_dir, final, blobstore, built_packages = [])
       spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
 
-      @name           = spec["name"]
-      @packages       = spec["packages"].to_a
+      @name = spec["name"]
+      @version = nil
+      @tarball_path = nil
+      @packages = spec["packages"].to_a
       @built_packages = built_packages.to_a
-      @release_dir    = release_dir
-      @templates_dir  = File.join(job_dir, "templates")
-      @tarballs_dir   = File.join(release_dir, "tmp", "jobs")
-      @final          = final
-      @blobstore      = blobstore
-      @artefact_type  = "job"
+      @release_dir = release_dir
+      @templates_dir = File.join(job_dir, "templates")
+      @tarballs_dir = File.join(release_dir, "tmp", "jobs")
+      @final = final
+      @blobstore = blobstore
+      @artefact_type = "job"
 
       case spec["templates"]
       when Hash
         @templates = spec["templates"].keys
       else
         raise InvalidJob, "Incorrect templates section in `#{@name}' " +
-            "job spec (should resolve to a hash)"
+          "job spec (Hash expected, #{spec["properties"].class} given)"
+      end
+
+      if spec.has_key?("properties")
+        if spec["properties"].is_a?(Hash)
+          @properties = spec["properties"]
+        else
+          raise InvalidJob, "Incorrect properties section in `#{@name}' " +
+            "job spec (Hash expected, #{spec["properties"].class} given)"
+        end
+      else
+        @properties = {}
       end
 
       if @name.blank?
@@ -109,8 +158,6 @@ module Bosh::Cli
       FileUtils.mkdir_p(@dev_builds_dir)
       FileUtils.mkdir_p(@final_builds_dir)
 
-      at_exit { FileUtils.rm_rf(build_dir) }
-
       init_indices
     end
 
@@ -139,7 +186,7 @@ module Bosh::Cli
     end
 
     def prepare_files
-      preparation_script = File.join(job_dir, "prepare")
+      File.join(job_dir, "prepare")
     end
 
     def build_dir
@@ -176,17 +223,22 @@ module Bosh::Cli
       self
     end
 
+    # @return [Array<String>] Returns full paths of all templates in the job
+    #   (regular job templates and monit)
+    def all_templates
+      regular_templates = @templates.map do |template|
+        File.join(@templates_dir, template)
+      end
+
+      regular_templates.sort + monit_files
+    end
+
     private
 
     def make_fingerprint
       contents = ""
 
-      # templates, monit, spec
-      files = templates.map do |template|
-        File.join(@templates_dir, template)
-      end.sort
-
-      files += monit_files
+      files = all_templates
       files << File.join(job_dir, "spec")
 
       files.each do |filename|

data/lib/cli/job_property_collection.rb

@@ -0,0 +1,87 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  class JobPropertyCollection
+    include Enumerable
+    include Bosh::Common::PropertyHelper
+
+    # @param [JobBuilder] job_builder Which job this property collection is for
+    # @param [Hash] global_properties Globally defined properties
+    # @param [Hash] job_properties Properties defined for this job only
+    # @param [Hash] mappings Property mappings for this job
+    def initialize(job_builder, global_properties, job_properties, mappings)
+      @job_builder = job_builder
+
+      @job_properties = deep_copy(job_properties || {})
+      merge(@job_properties, deep_copy(global_properties))
+
+      @mappings = mappings || {}
+      @properties = []
+
+      resolve_mappings
+      filter_properties
+    end
+
+    def each
+      @properties.each { |property| yield property }
+    end
+
+    # @return [Hash] Property hash (keys are property name components)
+    def to_hash
+      @properties
+    end
+
+    private
+
+    def resolve_mappings
+      @mappings.each_pair do |to, from|
+        resolved = lookup_property(@job_properties, from)
+
+        if resolved.nil?
+          raise InvalidPropertyMapping,
+                "Cannot satisfy property mapping `#{to}: #{from}', " +
+                "as `#{from}' is not in deployment properties"
+        end
+
+        @job_properties[to] = resolved
+      end
+    end
+
+    # @return [void] Modifies @properties
+    def filter_properties
+      if @job_builder.properties.empty?
+        # If at least one template doesn't have properties defined, we
+        # need all properties to be available to job (backward-compatibility)
+        @properties = @job_properties
+        return
+      end
+
+      @properties = {}
+
+      @job_builder.properties.each_pair do |name, definition|
+        copy_property(
+          @properties, @job_properties, name, definition["default"])
+      end
+    end
+
+    # @param [Object] object Serializable object
+    # @return [Object] Deep copy of the object
+    def deep_copy(object)
+      Marshal.load(Marshal.dump(object))
+    end
+
+    # @param [Hash] base
+    # @param [Hash] extras
+    # @return [void] Modifies base
+    def merge(base, extras)
+      base.merge!(extras) do |_, old_value, new_value|
+        if old_value.is_a?(Hash) && new_value.is_a?(Hash)
+          merge(old_value, new_value)
+        end
+
+        old_value
+      end
+    end
+
+  end
+end

data/lib/cli/job_property_validator.rb

@@ -0,0 +1,130 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  class JobPropertyValidator
+    # TODO: tests
+
+    attr_reader :template_errors
+    attr_reader :jobs_without_properties
+
+    # @param [Array<JobBuilder>] built_jobs Built job templates
+    # @param [Hash] manifest Deployment manifest
+    def initialize(built_jobs, manifest)
+      @built_jobs = {}
+      @manifest = manifest
+
+      @jobs_without_properties = []
+
+      built_jobs.each do |job|
+        @built_jobs[job.name] = job
+        if job.properties.empty?
+          @jobs_without_properties << job
+        end
+      end
+
+      unless @manifest["properties"].is_a?(Hash)
+        bad_manifest("Invalid properties format in deployment " +
+          "manifest, Hash expected, #{@manifest["properties"].class} given")
+      end
+
+      unless @manifest["jobs"].is_a?(Array)
+        bad_manifest("Invalid jobs format in deployment " +
+          "manifest, Array expected, #{@manifest["jobs"].class} given")
+      end
+
+      @manifest["jobs"].each do |job|
+        unless job.is_a?(Hash)
+          bad_manifest("Invalid job spec in the manifest " +
+                       "Hash expected, #{job.class} given")
+        end
+
+        job_name = job["name"]
+        if job_name.nil?
+          bad_manifest("Manifest contains at least one job without name")
+        end
+
+        if job["template"].nil?
+          bad_manifest("Job `#{job_name}' doesn't have a template")
+        end
+      end
+
+      @template_errors = []
+      # TODO: track missing props and show the list to user (super helpful!)
+    end
+
+    def validate
+      @manifest["jobs"].each do |job_spec|
+        validate_templates(job_spec)
+      end
+    end
+
+    # Tries to fill in each job template with job properties, collects errors
+    # @param [Hash] job_spec Job spec from the manifest
+    def validate_templates(job_spec)
+      built_job = @built_jobs[job_spec["template"]]
+
+      if built_job.nil?
+        raise CliError, "Job `#{job_spec["template"]}' has not been built"
+      end
+
+      collection = JobPropertyCollection.new(
+        built_job, @manifest["properties"],
+        job_spec["properties"], job_spec["property_mappings"])
+
+      # Spec is usually more than that but jobs rarely use anything but
+      # networks and properties.
+      # TODO: provide all keys in the spec?
+      spec = {
+        "job" => {
+          "name" => job_spec["name"]
+        },
+        "networks" => {
+          "default" => {"ip" => "10.0.0.1"}
+        },
+        "properties" => collection.to_hash,
+        "index" => 0
+      }
+
+      built_job.all_templates.each do |template_path|
+        # TODO: add progress bar?
+        evaluate_template(built_job, template_path, spec)
+      end
+    end
+
+    private
+
+    # @param [JobBuilder] job Job builder
+    # @param [String] template_path Template path
+    # @param [Hash] spec Fake instance spec
+    def evaluate_template(job, template_path, spec)
+      erb = ERB.new(File.read(template_path))
+      context = Bosh::Common::TemplateEvaluationContext.new(spec)
+      begin
+        erb.result(context.get_binding)
+      rescue Exception => e
+        @template_errors << TemplateError.new(job, template_path, e)
+      end
+    end
+
+    def bad_manifest(message)
+      raise InvalidManifest, message
+    end
+
+    class TemplateError
+      attr_reader :job
+      attr_reader :template_path
+      attr_reader :exception
+      attr_reader :line
+
+      # @param [JobBuilder] job
+      # @param [String] template_path
+      # @param [Exception] exception
+      def initialize(job, template_path, exception)
+        @job = job
+        @template_path = template_path
+        @exception = exception
+        @line = exception.backtrace.first.split(":")[1]
+      end
+    end
+  end
+end

data/lib/cli/package_builder.rb

@@ -43,6 +43,35 @@ module Bosh::Cli
     # final build tarballs should be ignored as well
     # final builds metadata should be checked in
 
+    # @param [String] directory Release directory
+    # @param [Hash] options Package build options
+    def self.discover(directory, options = {})
+      builders = []
+
+      Dir[File.join(directory, "packages", "*")].each do |package_dir|
+        next unless File.directory?(package_dir)
+        package_dirname = File.basename(package_dir)
+        package_spec = load_yaml_file(File.join(package_dir, "spec"))
+
+        if package_spec["name"] != package_dirname
+          raise InvalidPackage,
+                "Found `#{package_spec["name"]}' package in " +
+                "`#{package_dirname}' directory, please fix it"
+        end
+
+        is_final = options[:final]
+        blobstore = options[:blobstore]
+        dry_run = options[:dry_run]
+
+        builder = new(package_spec, directory, is_final, blobstore)
+        builder.dry_run = true if dry_run
+
+        builders << builder
+      end
+
+      builders
+    end
+
     def initialize(spec, release_dir, final, blobstore,
         sources_dir = nil, blobs_dir = nil, alt_src_dir = nil)
       spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
@@ -87,8 +116,6 @@ module Bosh::Cli
       FileUtils.mkdir_p(@dev_builds_dir)
       FileUtils.mkdir_p(@final_builds_dir)
 
-      at_exit { FileUtils.rm_rf(build_dir) }
-
       init_indices
     end
 
@@ -166,10 +193,10 @@ module Bosh::Cli
           ENV["RELEASE_DIR"] = @release_dir
           in_build_dir do
             pre_packaging_out = `bash -x pre_packaging 2>&1`
-            pre_packaging_out.split("\n").each do |line|
-              say("> #{line}")
-            end
             unless $?.exitstatus == 0
+              pre_packaging_out.split("\n").each do |line|
+                say("> #{line}")
+              end
               raise InvalidPackage, "`#{name}' pre-packaging failed"
             end
           end