bosh_cli 1.2831.0 → 1.2839.0

data/lib/cli/resources/package.rb

@@ -0,0 +1,210 @@
+module Bosh::Cli::Resources
+  class Package
+    BUILD_HOOK_FILES = ['packaging', 'pre_packaging']
+
+    # We have two ways of getting/storing a package:
+    # development versions of packages, kept in release directory
+    # final versions of packages, kept in blobstore
+    # development packages and their metadata should always be gitignored
+    # final build tarballs should be ignored as well
+    # final builds metadata should be checked in
+
+    # @param [String] directory base Release directory
+    def self.discover(release_base)
+      Dir[File.join(release_base, 'packages', '*')].inject([]) do |packages, package_base|
+        next unless File.directory?(package_base)
+        packages << new(package_base, release_base)
+      end
+    end
+
+    attr_reader :package_base, :release_base
+
+    def initialize(package_base, release_base)
+      @release_base = Pathname.new(release_base)
+      @package_base = Pathname.new(package_base)
+    end
+
+    def spec
+      @spec ||= load_yaml_file(package_base.join('spec'))
+    rescue
+      raise Bosh::Cli::InvalidPackage, 'Package spec is missing'
+    end
+
+    def name
+      spec['name']
+    end
+
+    def dependencies
+      @dependencies ||= Array(spec['dependencies']).sort
+    end
+
+    def singular_type
+      'package'
+    end
+
+    def plural_type
+      'packages'
+    end
+
+    def files
+      validate!
+      known_files = {}
+
+      files = []
+      files += resolved_globs.map do |match|
+        known_files[match.path] = true
+        [match.full_path, match.path]
+      end
+
+      BUILD_HOOK_FILES.each do |build_hook_file|
+        source_file = package_base.join(build_hook_file)
+        if source_file.exist?
+          if known_files.has_key?(build_hook_file)
+            raise Bosh::Cli::InvalidPackage, "Package '#{name}' has '#{build_hook_file}' file " +
+                "which conflicts with BOSH packaging"
+          end
+
+          files << [source_file.to_s, build_hook_file]
+        end
+      end
+
+      files
+    end
+
+    def validate!
+      basename = File.basename(package_base.to_s)
+
+      unless name == basename
+        raise Bosh::Cli::InvalidPackage, "Found '#{name}' package in '#{basename}' directory, please fix it"
+      end
+
+      unless name.bosh_valid_id?
+        raise Bosh::Cli::InvalidPackage, "Package name, '#{name}', should be a valid BOSH identifier"
+      end
+
+      unless spec['files'].is_a?(Array) && spec['files'].size > 0
+        raise Bosh::Cli::InvalidPackage, "Package '#{name}' doesn't include any files"
+      end
+
+      resolve_globs
+    end
+
+    def additional_fingerprints
+      dependencies
+    end
+
+    def format_fingerprint(digest, filename, name, file_mode)
+      is_hook = BUILD_HOOK_FILES.include?(name)
+      "%s%s%s" % [name, digest, is_hook ? '' : file_mode]
+    end
+
+    def run_script(script_name, *args)
+      if BUILD_HOOK_FILES.include?(script_name.to_s)
+        send(:"run_script_#{script_name}", *args)
+      end
+    end
+
+    # ---
+
+    private
+
+    def excluded_files
+      @excluded_files ||= Array(spec['excluded_files']).sort
+    end
+
+    # @return Array<Bosh::Cli::GlobMatch>
+    def resolve_globs
+      @resolved_globs ||= begin
+        all_matches = Set.new
+
+        spec['files'].each do |glob|
+          glob_matches = Set.new
+          src_matches = resolve_glob_in_dir(glob, release_src)
+          glob_matches += src_matches.map { |path| Bosh::Cli::GlobMatch.new(release_src, path) }
+
+          # Blobs directory is a little bit different: whatever matches a blob
+          # will complement already found matches, unless this particular path
+          # has already been matched. The GlobMatch class defines <=> to compare
+          # path, thereby rejecting blobs if the file exists in src.
+          if File.directory?(File.join(release_blobs))
+            blob_matches = resolve_glob_in_dir(glob, release_blobs)
+            glob_matches += blob_matches.map { |path| Bosh::Cli::GlobMatch.new(release_blobs, path) }
+          end
+
+          if glob_matches.empty?
+            raise Bosh::Cli::InvalidPackage, "Package '#{name}' has a glob that resolves to an empty file list: #{glob}"
+          end
+
+          all_matches += glob_matches
+        end
+
+        all_matches.reject! do |match|
+          excluded_files.detect { |excluded_glob| File.fnmatch(excluded_glob, match.path) }
+        end
+
+        all_matches.sort
+      end
+    end
+
+    def resolve_glob_in_dir(glob, dir)
+      Dir.chdir(dir) do
+        Dir.glob(glob, File::FNM_DOTMATCH).reject do |fn|
+          %w(. ..).include?(File.basename(fn))
+        end
+      end
+    end
+
+    def resolved_globs
+      @resolved_globs
+    end
+
+    def release_src
+      release_base.join('src')
+    end
+
+    def release_alt
+      release_base.join('src_alt')
+    end
+
+    def release_blobs
+      release_base.join('blobs')
+    end
+
+    def run_script_pre_packaging(staging_dir)
+      pre_packaging_script = package_base.join('pre_packaging')
+
+      if File.exists?(pre_packaging_script)
+        say('Pre-packaging...')
+        FileUtils.cp(pre_packaging_script, staging_dir, :preserve => true)
+
+        old_env = ENV
+
+        begin
+          ENV.delete_if { |key, _| key[0, 7] == 'BUNDLE_' }
+          if ENV['RUBYOPT']
+            ENV['RUBYOPT'] = ENV['RUBYOPT'].sub('-rbundler/setup', '')
+          end
+          # todo: test these
+          ENV['BUILD_DIR'] = staging_dir
+          ENV['RELEASE_DIR'] = release_base.to_s
+          Dir.chdir(staging_dir) do
+            output = `bash -x pre_packaging 2>&1`
+
+            unless $?.exitstatus == 0
+              output.split("\n").each do |line|
+                say("> #{line}")
+              end
+              raise Bosh::Cli::InvalidPackage, "'#{name}' pre-packaging failed"
+            end
+          end
+
+        ensure
+          ENV.delete('BUILD_DIR')
+          old_env.each { |k, v| ENV[k] = old_env[k] }
+        end
+
+        FileUtils.rm(File.join(staging_dir, 'pre_packaging'))
+      end
+    end
+  end
+end

data/lib/cli/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cli
-    VERSION = '1.2831.0'
+    VERSION = '1.2839.0'
   end
 end

data/lib/cli/versions/local_version_storage.rb

@@ -10,12 +10,12 @@ module Bosh::Cli::Versions
       @name_prefix = name_prefix
     end
 
-    def put_file(version, src_file_path)
+    def put_file(version, origin_file_path)
       destination = file_path(version)
-      unless File.exist?(src_file_path)
-        raise "Trying to store non-existant file `#{src_file_path}' for version `#{version}'"
+      unless File.exist?(origin_file_path)
+        raise "Trying to store non-existant file `#{origin_file_path}' for version `#{version}'"
       end
-      FileUtils.cp(src_file_path, destination, :preserve => true)
+      FileUtils.cp(origin_file_path, destination, :preserve => true)
 
       File.expand_path(destination)
     end

data/lib/cli/versions/version_file_resolver.rb

@@ -9,7 +9,6 @@ module Bosh::Cli::Versions
 
     def find_file(blobstore_id, sha1, version, desc)
       if @storage.has_file?(version)
-        say('FOUND LOCAL'.make_green)
         file_path = @storage.get_file(version)
         file_sha1 = Digest::SHA1.file(file_path).hexdigest
         if file_sha1 == sha1
@@ -22,8 +21,7 @@ module Bosh::Cli::Versions
         err("Cannot find #{desc}")
       end
 
-      say('FOUND REMOTE'.make_yellow)
-      say("Downloading #{desc} from blobstore (id=#{blobstore_id})...".make_green)
+      say("Downloading from blobstore (id=#{blobstore_id})...".make_green)
 
       tmp_file_path = File.join(@tmpdir, "bosh-tmp-file-#{SecureRandom.uuid}")
       begin

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_cli
 version: !ruby/object:Gem::Version
-  version: 1.2831.0
+  version: 1.2839.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-04 00:00:00.000000000 Z
+date: 2015-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2831.0
+        version: 1.2839.0
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -194,7 +194,7 @@ dependencies:
         version: 0.5.4
 description: |-
   BOSH CLI
-  4cf37f
+  41b440
 email: support@cloudfoundry.com
 executables:
 - bosh
@@ -203,9 +203,13 @@ extra_rdoc_files: []
 files:
 - bin/bosh
 - lib/cli.rb
+- lib/cli/archive_builder.rb
+- lib/cli/archive_repository.rb
+- lib/cli/archive_repository_provider.rb
 - lib/cli/backup_destination_path.rb
 - lib/cli/base_command.rb
 - lib/cli/blob_manager.rb
+- lib/cli/build_artifact.rb
 - lib/cli/changeset_helper.rb
 - lib/cli/client/compiled_packages_client.rb
 - lib/cli/client/director.rb
@@ -255,8 +259,8 @@ files:
 - lib/cli/download_with_progress.rb
 - lib/cli/errors.rb
 - lib/cli/file_with_progress_bar.rb
+- lib/cli/glob_match.rb
 - lib/cli/interactive_progress_renderer.rb
-- lib/cli/job_builder.rb
 - lib/cli/job_command_args.rb
 - lib/cli/job_property_collection.rb
 - lib/cli/job_property_validator.rb
@@ -266,8 +270,6 @@ files:
 - lib/cli/manifest_warnings.rb
 - lib/cli/name_version_pair.rb
 - lib/cli/non_interactive_progress_renderer.rb
-- lib/cli/package_builder.rb
-- lib/cli/packaging_helper.rb
 - lib/cli/public_stemcell.rb
 - lib/cli/public_stemcell_presenter.rb
 - lib/cli/public_stemcells.rb
@@ -275,6 +277,8 @@ files:
 - lib/cli/release_builder.rb
 - lib/cli/release_compiler.rb
 - lib/cli/release_tarball.rb
+- lib/cli/resources/job.rb
+- lib/cli/resources/package.rb
 - lib/cli/resurrection.rb
 - lib/cli/runner.rb
 - lib/cli/source_control/git_ignore.rb

data/lib/cli/job_builder.rb

@@ -1,277 +0,0 @@
-module Bosh::Cli
-  class JobBuilder
-    include PackagingHelper
-
-    attr_reader :name, :version, :packages, :templates,
-                :release_dir, :built_packages, :tarball_path
-
-    # @return [Hash] Properties defined in this job
-    attr_reader :properties
-
-    def self.run_prepare_script(script_path)
-      unless File.exists?(script_path)
-        raise InvalidJob, "Prepare script at `#{script_path}' doesn't exist"
-      end
-
-      unless File.executable?(script_path)
-        raise InvalidJob, "Prepare script at `#{script_path}' is not executable"
-      end
-
-      old_env = ENV
-
-      script_dir = File.dirname(script_path)
-      script_name = File.basename(script_path)
-
-      begin
-        # We need to temporarily delete some rubygems related artefacts
-        # because preparation scripts shouldn't share any assumptions
-        # with CLI itself
-        %w{ BUNDLE_GEMFILE RUBYOPT }.each { |key| ENV.delete(key) }
-
-        output = nil
-        Dir.chdir(script_dir) do
-          cmd = "./#{script_name} 2>&1"
-          output = `#{cmd}`
-        end
-
-        unless $?.exitstatus == 0
-          raise InvalidJob, "`#{script_path}' script failed: #{output}"
-        end
-
-        output
-      ensure
-        ENV.each_pair { |k, v| ENV[k] = old_env[k] }
-      end
-    end
-
-    # @param [String] directory Release directory
-    # @param [Hash] options Build options
-    def self.discover(directory, options = {})
-      builders = []
-
-      Dir[File.join(directory, "jobs", "*")].each do |job_dir|
-        next unless File.directory?(job_dir)
-        job_dirname = File.basename(job_dir)
-
-        prepare_script = File.join(job_dir, "prepare")
-        if File.exists?(prepare_script)
-          run_prepare_script(prepare_script)
-        end
-
-        job_spec = load_yaml_file(File.join(job_dir, "spec"))
-        if job_spec["name"] != job_dirname
-          raise InvalidJob,
-                "Found `#{job_spec["name"]}' job in " +
-                "`#{job_dirname}' directory, please fix it"
-        end
-
-        final = options[:final]
-        dry_run = options[:dry_run]
-        blobstore = options[:blobstore]
-        package_names = options[:package_names]
-
-        builder = new(job_spec, directory, final, blobstore, package_names)
-        builder.dry_run = true if dry_run
-        builders << builder
-      end
-
-      builders
-    end
-
-    def initialize(spec, release_dir, final, blobstore, built_packages = [])
-      spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
-
-      @name = spec["name"]
-      @version = nil
-      @tarball_path = nil
-      @packages = spec["packages"].to_a
-      @built_packages = built_packages.to_a
-      @release_dir = release_dir
-      @templates_dir = File.join(job_dir, "templates")
-      @tarballs_dir = File.join(release_dir, "tmp", "jobs")
-      @final = final
-      @blobstore = blobstore
-      @artefact_type = "job"
-
-      case spec["templates"]
-      when Hash
-        @templates = spec["templates"].keys
-      else
-        raise InvalidJob, "Incorrect templates section in `#{@name}' " +
-          "job spec (Hash expected, #{spec["properties"].class} given)"
-      end
-
-      if spec.has_key?("properties")
-        if spec["properties"].is_a?(Hash)
-          @properties = spec["properties"]
-        else
-          raise InvalidJob, "Incorrect properties section in `#{@name}' " +
-            "job spec (Hash expected, #{spec["properties"].class} given)"
-        end
-      else
-        @properties = {}
-      end
-
-      if @name.blank?
-        raise InvalidJob, "Job name is missing"
-      end
-
-      if @templates.nil?
-        raise InvalidJob, "Please include templates section with at least 1 " +
-            "(possibly dummy) file into `#{@name}' job spec"
-      end
-
-      unless @name.bosh_valid_id?
-        raise InvalidJob, "`#{@name}' is not a valid BOSH identifier"
-      end
-
-      unless File.exists?(File.join(job_dir, "spec"))
-        raise InvalidJob, "Cannot find spec file for '#{name}'"
-      end
-
-      if missing_packages.size > 0
-        raise InvalidJob, "Some packages required by '#{name}' job " +
-            "are missing: %s" % [missing_packages.join(", ")]
-      end
-
-      if missing_templates.size > 0
-        raise InvalidJob, "Some template files required by '#{name}' job " +
-            "are missing: %s" % [missing_templates.join(", ")]
-      end
-
-      if extra_templates.size > 0
-        raise InvalidJob, "There are unused template files for job " +
-            "'#{name}': %s" % [extra_templates.join(", ")]
-      end
-
-      unless monit_files.size > 0
-        raise InvalidJob, "Cannot find monit file for '#{name}'"
-      end
-
-      @dev_builds_dir = File.join(@release_dir, ".dev_builds", "jobs", @name)
-      @final_builds_dir = File.join(@release_dir, ".final_builds",
-                                    "jobs", @name)
-
-      FileUtils.mkdir_p(job_dir)
-      FileUtils.mkdir_p(@dev_builds_dir)
-      FileUtils.mkdir_p(@final_builds_dir)
-
-      init_indices
-    end
-
-    def copy_files
-      FileUtils.mkdir_p(File.join(build_dir, "templates"))
-      copied = 0
-
-      templates.each do |template|
-        src = File.join(@templates_dir, template)
-        dst = File.join(build_dir, "templates", template)
-        FileUtils.mkdir_p(File.dirname(dst))
-
-        FileUtils.cp(src, dst, :preserve => true)
-        copied += 1
-      end
-
-      monit_files.each do |file|
-        FileUtils.cp(file, build_dir, :preserve => true)
-        copied += 1
-      end
-
-      FileUtils.cp(File.join(job_dir, "spec"), File.join(build_dir, "job.MF"),
-                   :preserve => true)
-      copied += 1
-      copied
-    end
-
-    def prepare_files
-      File.join(job_dir, "prepare")
-    end
-
-    def build_dir
-      @build_dir ||= Dir.mktmpdir
-    end
-
-    def job_dir
-      File.join(@release_dir, "jobs", @name)
-    end
-
-    def dev_builds_dir
-      File.join(@release_dir, ".dev_builds", "jobs", name)
-    end
-
-    def final_builds_dir
-      File.join(@release_dir, ".final_builds", "jobs", name)
-    end
-
-    def fingerprint
-      @fingerprint ||= make_fingerprint
-    end
-
-    def monit_files
-      glob = File.join(job_dir, '*.monit')
-      files = Dir.glob(glob)
-      monit = File.join(job_dir, "monit")
-      files << monit if File.exist?(monit)
-      files
-    end
-
-    def reload
-      @fingerprint = nil
-      @build_dir   = nil
-      self
-    end
-
-    # @return [Array<String>] Returns full paths of all templates in the job
-    #   (regular job templates and monit)
-    def all_templates
-      regular_templates = @templates.map do |template|
-        File.join(@templates_dir, template)
-      end
-
-      regular_templates.sort + monit_files
-    end
-
-    private
-
-    def make_fingerprint
-      versioning_scheme = 2
-      contents = "v#{versioning_scheme}"
-
-      files = all_templates
-      files << File.join(job_dir, "spec")
-
-      files.each do |filename|
-        path = File.basename(filename)
-        digest = Digest::SHA1.file(filename).hexdigest
-        contents << "%s%s%s" % [path, digest, tracked_permissions(filename)]
-      end
-
-      Digest::SHA1.hexdigest(contents)
-    end
-
-    def missing_packages
-      @missing_packages ||= packages - built_packages
-    end
-
-    def missing_templates
-      templates.select do |template|
-        !File.exists?(File.join(@templates_dir, template))
-      end
-    end
-
-    def extra_templates
-      return [] if !File.directory?(@templates_dir)
-
-      Dir.chdir(@templates_dir) do
-        Dir["**/*"].reject do |file|
-          File.directory?(file) || templates.include?(file)
-        end
-      end
-    end
-
-    def in_build_dir(&block)
-      Dir.chdir(build_dir) { yield }
-    end
-
-  end
-end