bosh_cli 0.16

data/lib/cli/job_builder.rb

@@ -0,0 +1,226 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  class JobBuilder
+    include PackagingHelper
+
+    attr_reader :name, :version, :packages, :templates,
+                :release_dir, :built_packages, :tarball_path
+
+    def self.run_prepare_script(script_path)
+      unless File.exists?(script_path)
+        raise InvalidJob, "Prepare script at `#{script_path}' doesn't exist"
+      end
+
+      unless File.executable?(script_path)
+        raise InvalidJob, "Prepare script at `#{script_path}' is not executable"
+      end
+
+      old_env = ENV
+
+      script_dir = File.dirname(script_path)
+      script_name = File.basename(script_path)
+
+      begin
+        # We need to temporarily delete some rubygems related artefacts
+        # because preparation scripts shouldn't share any assumptions
+        # with CLI itself
+        %w{ BUNDLE_GEMFILE RUBYOPT }.each { |key| ENV.delete(key) }
+
+        Dir.chdir(script_dir) do
+          cmd = "./#{script_name} 2>&1"
+          say("Running #{cmd}...")
+          script_output = `#{cmd}`
+          script_output.split("\n").each do |line|
+            say("> #{line}")
+          end
+        end
+
+        unless $?.exitstatus == 0
+          raise InvalidJob, "`#{script_path}' script failed"
+        end
+      ensure
+        ENV.each_pair { |k, v| ENV[k] = old_env[k] }
+      end
+    end
+
+    def initialize(spec, release_dir, final, blobstore, built_packages = [])
+      spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
+
+      @name           = spec["name"]
+      @packages       = spec["packages"].to_a
+      @built_packages = built_packages.to_a
+      @release_dir    = release_dir
+      @templates_dir  = File.join(job_dir, "templates")
+      @tarballs_dir   = File.join(release_dir, "tmp", "jobs")
+      @final          = final
+      @blobstore      = blobstore
+      @artefact_type  = "job"
+
+      case spec["templates"]
+      when Hash
+        @templates = spec["templates"].keys
+      else
+        raise InvalidJob, "Incorrect templates section in `#{@name}' " +
+            "job spec (should resolve to a hash)"
+      end
+
+      if @name.blank?
+        raise InvalidJob, "Job name is missing"
+      end
+
+      if @templates.nil?
+        raise InvalidJob, "Please include templates section with at least 1 " +
+            "(possibly dummy) file into `#{@name}' job spec"
+      end
+
+      unless @name.bosh_valid_id?
+        raise InvalidJob, "`#{@name}' is not a valid BOSH identifier"
+      end
+
+      unless File.exists?(File.join(job_dir, "spec"))
+        raise InvalidJob, "Cannot find spec file for '#{name}'"
+      end
+
+      if missing_packages.size > 0
+        raise InvalidJob, "Some packages required by '#{name}' job " +
+            "are missing: %s" % [missing_packages.join(", ")]
+      end
+
+      if missing_templates.size > 0
+        raise InvalidJob, "Some template files required by '#{name}' job " +
+            "are missing: %s" % [missing_templates.join(", ")]
+      end
+
+      if extra_templates.size > 0
+        raise InvalidJob, "There are unused template files for job " +
+            "'#{name}': %s" % [extra_templates.join(", ")]
+      end
+
+      unless monit_files.size > 0
+        raise InvalidJob, "Cannot find monit file for '#{name}'"
+      end
+
+      @dev_builds_dir = File.join(@release_dir, ".dev_builds", "jobs", @name)
+      @final_builds_dir = File.join(@release_dir, ".final_builds",
+                                    "jobs", @name)
+
+      FileUtils.mkdir_p(job_dir)
+      FileUtils.mkdir_p(@dev_builds_dir)
+      FileUtils.mkdir_p(@final_builds_dir)
+
+      at_exit { FileUtils.rm_rf(build_dir) }
+
+      init_indices
+    end
+
+    def copy_files
+      FileUtils.mkdir_p(File.join(build_dir, "templates"))
+      copied = 0
+
+      templates.each do |template|
+        src = File.join(@templates_dir, template)
+        dst = File.join(build_dir, "templates", template)
+        FileUtils.mkdir_p(File.dirname(dst))
+
+        FileUtils.cp(src, dst, :preserve => true)
+        copied += 1
+      end
+
+      monit_files.each do |file|
+        FileUtils.cp(file, build_dir, :preserve => true)
+        copied += 1
+      end
+
+      FileUtils.cp(File.join(job_dir, "spec"), File.join(build_dir, "job.MF"),
+                   :preserve => true)
+      copied += 1
+      copied
+    end
+
+    def prepare_files
+      preparation_script = File.join(job_dir, "prepare")
+    end
+
+    def build_dir
+      @build_dir ||= Dir.mktmpdir
+    end
+
+    def job_dir
+      File.join(@release_dir, "jobs", @name)
+    end
+
+    def dev_builds_dir
+      File.join(@release_dir, ".dev_builds", "jobs", name)
+    end
+
+    def final_builds_dir
+      File.join(@release_dir, ".final_builds", "jobs", name)
+    end
+
+    def fingerprint
+      @fingerprint ||= make_fingerprint
+    end
+
+    def monit_files
+      glob = File.join(job_dir, '*.monit')
+      files = Dir.glob(glob)
+      monit = File.join(job_dir, "monit")
+      files << monit if File.exist?(monit)
+      files
+    end
+
+    def reload
+      @fingerprint = nil
+      @build_dir   = nil
+      self
+    end
+
+    private
+
+    def make_fingerprint
+      contents = ""
+
+      # templates, monit, spec
+      files = templates.map do |template|
+        File.join(@templates_dir, template)
+      end.sort
+
+      files += monit_files
+      files << File.join(job_dir, "spec")
+
+      files.each do |filename|
+        path = File.basename(filename)
+        digest = Digest::SHA1.file(filename).hexdigest
+        contents << "%s%s%s" % [path, digest, tracked_permissions(filename)]
+      end
+
+      Digest::SHA1.hexdigest(contents)
+    end
+
+    def missing_packages
+      @missing_packages ||= packages - built_packages
+    end
+
+    def missing_templates
+      templates.select do |template|
+        !File.exists?(File.join(@templates_dir, template))
+      end
+    end
+
+    def extra_templates
+      return [] if !File.directory?(@templates_dir)
+
+      Dir.chdir(@templates_dir) do
+        Dir["**/*"].reject do |file|
+          File.directory?(file) || templates.include?(file)
+        end
+      end
+    end
+
+    def in_build_dir(&block)
+      Dir.chdir(build_dir) { yield }
+    end
+
+  end
+end

data/lib/cli/package_builder.rb

@@ -0,0 +1,254 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  class PackageBuilder
+    include PackagingHelper
+
+    attr_reader :name, :globs, :version, :dependencies, :tarball_path
+    # We have two ways of getting/storing a package:
+    # development versions of packages, kept in release directory
+    # final versions of packages, kept in blobstore
+    # development packages and their metadata should always be gitignored
+    # final build tarballs should be ignored as well
+    # final builds metadata should be checked in
+
+    def initialize(spec, release_dir, final, blobstore,
+        sources_dir = nil, blobs_dir = nil)
+      spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
+
+      @name = spec["name"]
+      @globs = spec["files"]
+      @dependencies = Array(spec["dependencies"])
+      @release_dir = release_dir
+      @sources_dir = sources_dir || File.join(@release_dir, "src")
+      @blobs_dir = blobs_dir || File.join(@release_dir, "blobs")
+      @final = final
+      @blobstore = blobstore
+      @artefact_type = "package"
+
+      @metadata_files = %w(packaging pre_packaging)
+
+      if @name.blank?
+        raise InvalidPackage, "Package name is missing"
+      end
+
+      unless @name.bosh_valid_id?
+        raise InvalidPackage, "Package name should be a valid BOSH identifier"
+      end
+
+      unless @globs.is_a?(Array) && @globs.size > 0
+        raise InvalidPackage, "Package '#{@name}' doesn't include any files"
+      end
+
+      @dev_builds_dir = File.join(@release_dir, ".dev_builds",
+                                  "packages", @name)
+      @final_builds_dir = File.join(@release_dir, ".final_builds",
+                                    "packages", @name)
+
+      FileUtils.mkdir_p(package_dir)
+      FileUtils.mkdir_p(@dev_builds_dir)
+      FileUtils.mkdir_p(@final_builds_dir)
+
+      at_exit { FileUtils.rm_rf(build_dir) }
+
+      init_indices
+    end
+
+    def reload # Mostly for tests
+      @fingerprint    = nil
+      @resolved_globs = nil
+      init_indices
+      self
+    end
+
+    def fingerprint
+      @fingerprint ||= make_fingerprint
+    end
+
+    def resolved_globs
+      @resolved_globs ||= resolve_globs
+    end
+
+    def source_files
+      resolved_globs[:source]
+    end
+
+    def blob_files
+      resolved_globs[:blob]
+    end
+
+    def files
+      (resolved_globs[:blob] + resolved_globs[:source]).sort
+    end
+
+    def build_dir
+      @build_dir ||= Dir.mktmpdir
+    end
+
+    def package_dir
+      File.join(@release_dir, "packages", name)
+    end
+
+    def copy_files
+      copied = 0
+
+      files.each do |filename|
+        file_path = get_file_path(filename)
+        destination = File.join(build_dir, filename)
+
+        if File.directory?(file_path)
+          FileUtils.mkdir_p(destination)
+        else
+          FileUtils.mkdir_p(File.dirname(destination))
+          FileUtils.cp(file_path, destination, :preserve => true)
+          copied += 1
+        end
+      end
+
+      in_package_dir do
+        @metadata_files.each do |filename|
+          destination = File.join(build_dir, filename)
+          next unless File.exists?(filename)
+          if File.exists?(destination)
+            raise InvalidPackage, "Package '#{name}' has '#{filename}' file " +
+              "which conflicts with BOSH packaging"
+          end
+          FileUtils.cp(filename, destination, :preserve => true)
+          copied += 1
+        end
+      end
+
+      pre_package
+      copied
+    end
+
+    def pre_package
+      pre_packaging_script = File.join(package_dir, "pre_packaging")
+
+      if File.exists?(pre_packaging_script)
+
+        say("Pre-packaging...")
+        FileUtils.cp(pre_packaging_script, build_dir, :preserve => true)
+
+        old_env = ENV
+
+        begin
+          ENV.delete_if { |key, _| key[0, 7] == "BUNDLE_" }
+          if ENV["RUBYOPT"]
+            ENV["RUBYOPT"] = ENV["RUBYOPT"].sub("-rbundler/setup", "")
+          end
+          ENV["BUILD_DIR"] = build_dir
+          in_build_dir do
+            pre_packaging_out = `bash -x pre_packaging 2>&1`
+            pre_packaging_out.split("\n").each do |line|
+              say("> #{line}")
+            end
+            unless $?.exitstatus == 0
+              raise InvalidPackage, "`#{name}' pre-packaging failed"
+            end
+          end
+
+        ensure
+          ENV.delete("BUILD_DIR")
+          old_env.each { |k, v| ENV[k] = old_env[k] }
+        end
+
+        FileUtils.rm(File.join(build_dir, "pre_packaging"))
+      end
+    end
+
+    private
+
+    def get_file_path(file)
+      file_path = File.join(@sources_dir, file)
+      if !File.exists?(file_path)
+        file_path = File.join(@blobs_dir, file)
+        unless File.exists?(file_path)
+          raise InvalidPackage, "#{file} cannot be found"
+        end
+      end
+      file_path
+    end
+
+    def make_fingerprint
+      contents = ""
+
+      signatures = files.map do |file|
+        path = get_file_path(file)
+        unless File.directory?(path)
+          file_digest = Digest::SHA1.file(path).hexdigest
+        end
+
+        "%s%s%s" % [file, file_digest, tracked_permissions(path)]
+      end
+      contents << signatures.join("")
+
+      in_package_dir do
+        @metadata_files.each do |file|
+          if File.file?(file)
+            file_digest = Digest::SHA1.file(file).hexdigest
+            contents << "%s%s" % [file, file_digest]
+          end
+        end
+      end
+
+      contents << @dependencies.sort.join(",")
+
+      Digest::SHA1.hexdigest(contents)
+    end
+
+    def resolve_globs
+      glob_map = { :blob => [], :source => [] }
+      blob_list = []
+      source_list = []
+
+      @globs.each do |glob|
+        matching_source = []
+        matching_blob = []
+
+        in_sources_dir do
+          matching_source = Dir.glob(glob, File::FNM_DOTMATCH).reject do |fn|
+            [".", ".."].include?(File.basename(fn))
+          end
+        end
+
+        in_blobs_dir do
+          matching_blob = Dir.glob(glob, File::FNM_DOTMATCH).reject do |fn|
+            [".", ".."].include?(File.basename(fn))
+          end
+        end
+
+        if matching_blob.size == 0 && matching_source.size ==0
+          raise InvalidPackage, "`#{name}' has a glob that " +
+              "resolves to an empty file list: #{glob}"
+        end
+
+        blob_list << matching_blob
+        source_list << matching_source
+      end
+      glob_map[:blob] = blob_list.flatten.sort
+      glob_map[:source] = source_list.flatten.sort
+      glob_map
+    end
+
+    def in_blobs_dir(&block)
+      # old release does not have 'blob'
+      if File.directory?(@blobs_dir)
+        Dir.chdir(@blobs_dir) { yield }
+      end
+    end
+
+    def in_sources_dir(&block)
+      Dir.chdir(@sources_dir) { yield }
+    end
+
+    def in_build_dir(&block)
+      Dir.chdir(build_dir) { yield }
+    end
+
+    def in_package_dir(&block)
+      Dir.chdir(package_dir) { yield }
+    end
+
+  end
+end