bosh_cli 1.2831.0 → 1.2839.0

data/lib/cli/glob_match.rb

@@ -0,0 +1,32 @@
+module Bosh::Cli
+  class GlobMatch
+    # Helper class encapsulating the data we know about the glob. We need
+    # both directory and file path, as we match the same path in several
+    # directories (src, src_alt, blobs)
+    attr_reader :dir
+    attr_reader :path
+
+    def initialize(dir, path)
+      @dir = dir
+      @path = path
+    end
+
+    def full_path
+      File.join(dir, path)
+    end
+
+    def <=>(other)
+      @path <=> other.path
+    end
+
+    # GlobMatch will be used as Hash key (as implied by using Set),
+    # hence we need to define both eql? and hash
+    def eql?(other)
+      @path == other.path
+    end
+
+    def hash
+      @path.hash
+    end
+  end
+end

data/lib/cli/job_property_collection.rb

@@ -8,12 +8,12 @@ module Bosh::Cli
     include Enumerable
     include Bosh::Template::PropertyHelper
 
-    # @param [JobBuilder] job_builder Which job this property collection is for
+    # @param [Bosh::Cli::Resources::Job] job Which job this property collection is for
     # @param [Hash] global_properties Globally defined properties
     # @param [Hash] job_properties Properties defined for this job only
     # @param [Hash] mappings Property mappings for this job
-    def initialize(job_builder, global_properties, job_properties, mappings)
-      @job_builder = job_builder
+    def initialize(job, global_properties, job_properties, mappings)
+      @job = job
 
       @job_properties = Bosh::Common::DeepCopy.copy(job_properties || {})
       merge(@job_properties, Bosh::Common::DeepCopy.copy(global_properties))
@@ -52,7 +52,7 @@ module Bosh::Cli
 
     # @return [void] Modifies @properties
     def filter_properties
-      if @job_builder.properties.empty?
+      if @job.properties.empty?
         # If at least one template doesn't have properties defined, we
         # need all properties to be available to job (backward-compatibility)
         @properties = @job_properties
@@ -61,7 +61,7 @@ module Bosh::Cli
 
       @properties = {}
 
-      @job_builder.properties.each_pair do |name, definition|
+      @job.properties.each_pair do |name, definition|
         copy_property(
           @properties, @job_properties, name, definition["default"])
       end

data/lib/cli/job_property_validator.rb

@@ -7,7 +7,7 @@ module Bosh::Cli
     attr_reader :template_errors
     attr_reader :jobs_without_properties
 
-    # @param [Array<JobBuilder>] built_jobs Built job templates
+    # @param [Array<Bosh::Cli::Resources::Job>] built_jobs Built job templates
     # @param [Hash] manifest Deployment manifest
     def initialize(built_jobs, manifest)
       @built_jobs = {}
@@ -44,7 +44,7 @@ module Bosh::Cli
         end
 
         if job["template"].nil?
-          bad_manifest("Job `#{job_name}' doesn't have a template")
+          bad_manifest("Job '#{job_name}' doesn't have a template")
         end
       end
 
@@ -68,7 +68,7 @@ module Bosh::Cli
       built_job = @built_jobs[job_spec["template"]]
 
       if built_job.nil?
-        raise CliError, "Job `#{job_spec["template"]}' has not been built"
+        raise CliError, "Job '#{job_spec["template"]}' has not been built"
       end
 
       collection = JobPropertyCollection.new(
@@ -86,8 +86,8 @@ module Bosh::Cli
         'properties' => collection.to_hash
       }
 
-      built_job.all_templates.each do |template_path|
-        evaluate_template(built_job, template_path, spec)
+      built_job.files.each do |file_tuple|
+        evaluate_template(built_job, file_tuple.first, spec)
       end
     end
 
@@ -104,7 +104,7 @@ module Bosh::Cli
 
     private
 
-    # @param [JobBuilder] job Job builder
+    # @param [Bosh::Cli::Resources::Job] job Job builder
     # @param [String] template_path Template path
     # @param [Hash] spec Fake instance spec
     def evaluate_template(job, template_path, spec)
@@ -127,7 +127,7 @@ module Bosh::Cli
       attr_reader :exception
       attr_reader :line
 
-      # @param [JobBuilder] job
+      # @param [Bosh::Cli::Resources::Job] job
       # @param [String] template_path
       # @param [Exception] exception
       def initialize(job, template_path, exception)
@@ -138,4 +138,4 @@ module Bosh::Cli
       end
     end
   end
-end
+end

data/lib/cli/release_builder.rb

@@ -5,16 +5,16 @@ module Bosh::Cli
     attr_reader :release, :packages, :jobs, :name, :version, :build_dir, :commit_hash, :uncommitted_changes
 
     # @param [Bosh::Cli::Release] release Current release
-    # @param [Array<Bosh::Cli::PackageBuilder>] packages Built packages
-    # @param [Array<Bosh::Cli::JobBuilder>] jobs Built jobs
+    # @param [Array<Bosh::Cli::BuildArtifact>] package_artifacts Built packages
+    # @param [Array<Bosh::Cli::BuildArtifact>] job_artifacts Built jobs
     # @param [Hash] options Release build options
-    def initialize(release, packages, jobs, name, options = { })
+    def initialize(release, package_artifacts, job_artifacts, name, options = { })
       @release = release
       @final = options.has_key?(:final) ? !!options[:final] : false
       @commit_hash = options.fetch(:commit_hash, '00000000')
       @uncommitted_changes = options.fetch(:uncommitted_changes, true)
-      @packages = packages
-      @jobs = jobs
+      @packages = package_artifacts # todo
+      @jobs = job_artifacts # todo
       @name = name
       raise 'Release name is blank' if name.blank?
 
@@ -49,19 +49,18 @@ module Bosh::Cli
       @final
     end
 
-    # @return [Array] List of jobs affected by this release compared
-    #   to the previous one.
+    # @return [Array<Bosh::Cli::BuildArtifact>] List of job artifacts
+    #   affected by this release compared to the previous one.
     def affected_jobs
-      result = Set.new(@jobs.select { |job| job.new_version? })
-      return result if @packages.empty?
+      result = Set.new(@jobs.select { |job_artifact| job_artifact.new_version? })
+      return result.to_a if @packages.empty?
 
-      new_package_names = @packages.inject([]) do |list, package|
-        list << package.name if package.new_version?
-        list
-      end
+      new_package_names = @packages.map do |package_artifact|
+        package_artifact.name if package_artifact.new_version?
+      end.compact
 
       @jobs.each do |job|
-        result << job if (new_package_names & job.packages).size > 0
+        result << job if (new_package_names & job.dependencies).size > 0
       end
 
       result.to_a
@@ -83,22 +82,25 @@ module Bosh::Cli
 
     # Copies packages into release
     def copy_packages
-      packages.each do |package|
-        say("%-40s %s" % [package.name.make_green,
-                           pretty_size(package.tarball_path)])
-        FileUtils.cp(package.tarball_path,
-                     File.join(build_dir, "packages", "#{package.name}.tgz"),
+      packages.each do |package_artifact|
+        name = package_artifact.name
+        tarball_path = package_artifact.tarball_path
+        say("%-40s %s" % [name.make_green, pretty_size(tarball_path)])
+        FileUtils.cp(tarball_path,
+                     File.join(build_dir, "packages", "#{name}.tgz"),
                      :preserve => true)
       end
       @packages_copied = true
     end
 
-    # Copies jobs into release
+    # Copies jobs into release todo DRY vs copy_packages
     def copy_jobs
-      jobs.each do |job|
-        say("%-40s %s" % [job.name.make_green, pretty_size(job.tarball_path)])
-        FileUtils.cp(job.tarball_path,
-                     File.join(build_dir, "jobs", "#{job.name}.tgz"),
+      jobs.each do |job_artifact|
+        name = job_artifact.name
+        tarball_path = job_artifact.tarball_path
+        say("%-40s %s" % [name.make_green, pretty_size(tarball_path)])
+        FileUtils.cp(tarball_path,
+                     File.join(build_dir, "jobs", "#{name}.tgz"),
                      :preserve => true)
       end
       @jobs_copied = true
@@ -107,40 +109,37 @@ module Bosh::Cli
     # Generates release manifest
     def generate_manifest
       manifest = {}
-      manifest["packages"] = []
-
-      manifest["packages"] = packages.map do |package|
+      manifest['packages'] = packages.map do |build_artifact|
         {
-          "name" => package.name,
-          "version" => package.version,
-          "sha1" => package.checksum,
-          "fingerprint" => package.fingerprint,
-          "dependencies" => package.dependencies
+          'name' => build_artifact.name,
+          'version' => build_artifact.version,
+          'fingerprint' => build_artifact.fingerprint,
+          'sha1' => build_artifact.sha1,
+          'dependencies' => build_artifact.dependencies,
         }
       end
-
-      manifest["jobs"] = jobs.map do |job|
+      manifest['jobs'] = jobs.map do |build_artifact|
         {
-          "name" => job.name,
-          "version" => job.version,
-          "fingerprint" => job.fingerprint,
-          "sha1" => job.checksum,
+          'name' => build_artifact.name,
+          'version' => build_artifact.version,
+          'fingerprint' => build_artifact.fingerprint,
+          'sha1' => build_artifact.sha1,
         }
       end
 
-      manifest["commit_hash"] = commit_hash
-      manifest["uncommitted_changes"] = uncommitted_changes
+      manifest['commit_hash'] = commit_hash
+      manifest['uncommitted_changes'] = uncommitted_changes
 
       unless @name.bosh_valid_id?
-        raise InvalidRelease, "Release name `#{@name}' is not a valid BOSH identifier"
+        raise InvalidRelease, "Release name '#{@name}' is not a valid BOSH identifier"
       end
-      manifest["name"] = @name
+      manifest['name'] = @name
 
       # New release versions are allowed to have the same fingerprint as old versions.
       # For reverse compatibility, random uuids are stored instead.
-      @index.add_version(SecureRandom.uuid, { "version" => version })
+      @index.add_version(SecureRandom.uuid, { 'version' => version })
 
-      manifest["version"] = version
+      manifest['version'] = version
       manifest_yaml = Psych.dump(manifest)
 
       say("Writing manifest...")

data/lib/cli/release_compiler.rb

@@ -14,13 +14,13 @@ module Bosh::Cli
     # @param [Bosh::Blobstore::Client] blobstore Blobstore client
     # @param [Array] package_matches List of package checksums that director
     #   can match
-    # @param [String] release_dir Release directory
+    # @param [String] release_source Release directory
     def initialize(manifest_file, blobstore,
-                   package_matches = [], release_dir = nil)
+                   package_matches = [], release_source = nil)
 
       @blobstore = blobstore
-      @release_dir = release_dir || Dir.pwd
-      @manifest_file = File.expand_path(manifest_file, @release_dir)
+      @release_source = release_source || Dir.pwd
+      @manifest_file = File.expand_path(manifest_file, @release_source)
       @tarball_path = nil
 
       @build_dir = Dir.mktmpdir
@@ -95,18 +95,18 @@ module Bosh::Cli
 
     def find_package(package)
       name = package.name
-      final_package_dir = File.join(@release_dir, '.final_builds', 'packages', name)
+      final_package_dir = File.join(@release_source, '.final_builds', 'packages', name)
       final_index = Versions::VersionsIndex.new(final_package_dir)
-      dev_package_dir = File.join(@release_dir, '.dev_builds', 'packages', name)
+      dev_package_dir = File.join(@release_source, '.dev_builds', 'packages', name)
       dev_index = Versions::VersionsIndex.new(dev_package_dir)
       find_in_indices(final_index, dev_index, package, 'package')
     end
 
     def find_job(job)
       name = job.name
-      final_jobs_dir = File.join(@release_dir, '.final_builds', 'jobs', name)
+      final_jobs_dir = File.join(@release_source, '.final_builds', 'jobs', name)
       final_index = Versions::VersionsIndex.new(final_jobs_dir)
-      dev_jobs_dir = File.join(@release_dir, '.dev_builds', 'jobs', name)
+      dev_jobs_dir = File.join(@release_source, '.dev_builds', 'jobs', name)
       dev_index = Versions::VersionsIndex.new(dev_jobs_dir)
       find_in_indices(final_index, dev_index, job, 'job')
     end
@@ -148,9 +148,9 @@ module Bosh::Cli
     # @return [Boolean]
     def remote_package_exists?(local_package)
       # If checksum is known to director we can always match it
-      @package_matches.include?(local_package.sha1) ||
-        (local_package.fingerprint &&
-         @package_matches.include?(local_package.fingerprint))
+      @package_matches.include?(local_package.sha1) ||                     # !!! Needs test coverage
+        (local_package.fingerprint &&                                      # !!! Needs test coverage
+         @package_matches.include?(local_package.fingerprint))             # !!! Needs test coverage
     end
 
     # Checks if local job is already known remotely

data/lib/cli/resources/job.rb

@@ -0,0 +1,190 @@
+module Bosh::Cli::Resources
+  class Job
+    BUILD_HOOK_FILES = ['prepare']
+
+    # @param [String] directory base Release directory
+    def self.discover(release_base, packages)
+      Dir[File.join(release_base, 'jobs', '*')].inject([]) do |jobs, job_base|
+        next unless File.directory?(job_base)
+        jobs << new(job_base, release_base, packages)
+      end
+    end
+
+    attr_reader :job_base, :release_base, :package_dependencies
+
+    def initialize(job_base, release_base, packages)
+      @release_base = Pathname.new(release_base)
+      @job_base = Pathname.new(job_base)
+      @package_dependencies = packages
+    end
+
+    def spec
+      @spec ||= load_yaml_file(job_base.join('spec'))
+    rescue
+      raise Bosh::Cli::InvalidJob, 'Job spec is missing'
+    end
+
+    def name
+      spec['name']
+    end
+
+    def dependencies
+      package_dependencies #TODO: should this be packages or package_dependencies?
+    end
+
+    def singular_type
+      'job'
+    end
+
+    def plural_type
+      'jobs'
+    end
+
+    def files
+      validate!
+
+      files = (templates_files + monit_files).map { |absolute_path| [absolute_path, relative_path(absolute_path)] }
+      files << [File.join(job_base, 'spec'), 'job.MF']
+      files
+    end
+
+    # TODO: check dependency packages
+    def validate!
+      if name.blank?
+        raise Bosh::Cli::InvalidJob, 'Job name is missing'
+      end
+
+      unless name.bosh_valid_id?
+        raise Bosh::Cli::InvalidJob, "'#{name}' is not a valid BOSH identifier"
+      end
+
+      unless spec['templates'].is_a?(Hash)
+        raise Bosh::Cli::InvalidJob, "Incorrect templates section in '#{name}' job spec (Hash expected, #{spec['templates'].class} given)"
+      end
+
+      if extra_templates.size > 0
+        raise Bosh::Cli::InvalidJob, "There are unused template files for job '#{name}': #{extra_templates.join(", ")}"
+      end
+
+      if missing_templates.size > 0
+        raise Bosh::Cli::InvalidJob, "Some template files required by '#{name}' job are missing: #{missing_templates.join(", ")}"
+      end
+
+      if missing_packages.size > 0
+        raise Bosh::Cli::InvalidJob, "Some packages required by '#{name}' job are missing: #{missing_packages.join(", ")}"
+      end
+
+      if spec.has_key?('properties')
+        unless spec['properties'].is_a?(Hash)
+          raise Bosh::Cli::InvalidJob, "Incorrect properties section in '#{name}' job spec (Hash expected, #{spec['properties'].class} given)"
+        end
+      end
+
+      unless monit_files.size > 0
+        raise Bosh::Cli::InvalidJob, "Cannot find monit file for '#{name}'"
+      end
+    end
+
+    def additional_fingerprints
+      []
+    end
+
+    def format_fingerprint(digest, filename, name, file_mode)
+      "%s%s%s" % [File.basename(filename), digest, file_mode]
+    end
+
+    def run_script(script_name, *args)
+      if BUILD_HOOK_FILES.include?(script_name.to_s)
+        send(:"run_script_#{script_name}", *args)
+      end
+    end
+
+    def properties
+      spec['properties'] || {}
+    end
+
+    private
+
+    def extra_templates
+      return [] if !File.directory?(templates_dir)
+
+      Dir.chdir(templates_dir) do
+        Dir["**/*"].reject do |file|
+          File.directory?(file) || templates.include?(file)
+        end
+      end
+    end
+
+    def missing_packages
+      @missing_packages ||= (packages - package_dependencies)
+    end
+
+    def missing_templates
+      templates.select do |template|
+        !File.exists?(File.join(templates_dir, template))
+      end
+    end
+
+    def monit_files
+      monit = File.join(job_base, 'monit')
+      files = Dir.glob(File.join(job_base, '*.monit'))
+      files << monit if File.exist?(monit)
+      files
+    end
+
+    def packages
+      spec['packages'] || []
+    end
+
+    def relative_path(path)
+      Pathname.new(path).relative_path_from(job_base).to_s
+    end
+
+    def templates
+      spec['templates'].keys
+    end
+
+    def templates_dir
+      @templates_dir ||= job_base.join('templates')
+    end
+
+    def templates_files
+      templates.map { |file| File.join(templates_dir, file) }
+    end
+
+    def run_script_prepare
+      script_path = File.join(job_base, 'prepare')
+
+      return nil unless File.exists?(script_path)
+
+      unless File.executable?(script_path)
+        raise Bosh::Cli::InvalidJob, "Prepare script at '#{script_path}' is not executable"
+      end
+
+      old_env = ENV
+      script_dir = File.dirname(script_path)
+      script_name = File.basename(script_path)
+
+      begin
+        # We need to temporarily delete some rubygems related artefacts
+        # because preparation scripts shouldn't share any assumptions
+        # with CLI itself
+        %w{ BUNDLE_GEMFILE RUBYOPT }.each { |key| ENV.delete(key) }
+
+        output = nil
+        Dir.chdir(script_dir) do
+          cmd = "./#{script_name} 2>&1"
+          output = `#{cmd}`
+        end
+
+        unless $?.exitstatus == 0
+          raise Bosh::Cli::InvalidJob, "'#{script_path}' script failed: #{output}"
+        end
+
+        output
+      ensure
+        ENV.each_pair { |k, v| ENV[k] = old_env[k] }
+      end
+    end
+  end
+end