buildpack-packager 2.3.4

data/buildpack-packager.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'buildpack/packager/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'buildpack-packager'
+  spec.version       = Buildpack::Packager::VERSION
+  spec.authors       = ['Cloud Foundry Buildpacks Team']
+  spec.email         = ['cf-buildpacks-eng@pivotal.io']
+  spec.summary       = 'Tool that packages your buildpacks based on a manifest'
+  spec.description   = 'Tool that packages your Cloud Foundry buildpacks based on a manifest'
+  spec.homepage      = 'https://github.com/cloudfoundry/buildpack-packager'
+  spec.license       = 'Apache-2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '~> 2.2'
+
+  spec.add_dependency 'activesupport', '~> 4.1'
+  spec.add_dependency 'kwalify', '~> 0'
+  spec.add_dependency 'terminal-table', '~> 1.4'
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rubyzip', '~> 1.2'
+  spec.add_development_dependency 'rubocop', '~> 0'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.5'
+end

data/doc/disconnected_environments.md

@@ -0,0 +1,50 @@
+# Disconnected environments
+
+### Deploying Apps on disconnected environments
+Cached buildpacks only ensure that a the buildpacks dependencies are cached, not your applications.
+
+When you work with a disconnected environment, it's important to use your package manager
+to 'vendor' your applications dependencies.
+
+The specific mechanism varies between platforms. See your buildpack's documentation for 'vendoring' advice.
+
+## Building a cached buildpack
+1. Make sure you have fetched submodules
+
+  ```shell
+  git submodule update --init
+  ```
+
+1. Get the latest buildpack dependencies
+
+  ```shell
+  BUNDLE_GEMFILE=cf.Gemfile bundle
+  ```
+
+1. Build the buildpack
+
+  ```shell
+  BUNDLE_GEMFILE=cf.Gemfile bundle exec buildpack-packager [ uncached | cached ]
+  ```
+  
+  This produces a buildpack for use on Cloud Foundry.
+  
+  'cached' generates a zip with all the dependencies cached.
+  
+  'uncached' does not include the dependencies, however it excludes some files as specified 
+  in manifest.yml. 
+
+1. Use in Cloud Foundry
+
+  Currently, you can only specify cached buildpacks by creating Cloud Foundry Admin Buildpacks.
+  
+  This means you need admin rights. See the 
+  [Open source admin documentation](http://docs.cloudfoundry.org/adminguide/buildpacks.html)
+  for more information.
+  
+  Upload the buildpack to your Cloud Foundry and specify it by name:
+
+  ```shell
+  cf create-buildpack custom_ruby_buildpack ruby_buildpack-cached-custom.zip 1
+  ```
+

data/lib/buildpack/manifest_dependency.rb

@@ -0,0 +1,14 @@
+module Buildpack
+  class ManifestDependency
+    attr_reader :name, :version
+
+    def initialize(name, version)
+      @name = name
+      @version = version
+    end
+
+    def ==(other_object)
+      name == other_object.name && version == other_object.version
+    end
+  end
+end

data/lib/buildpack/manifest_validator.rb

@@ -0,0 +1,88 @@
+require 'kwalify'
+require 'kwalify/parser/yaml-patcher'
+require 'uri'
+
+require 'buildpack/manifest_dependency'
+
+module Buildpack
+  class ManifestValidator
+    class ManifestValidationError < StandardError; end
+
+    SCHEMA_FILE = File.join(File.dirname(__FILE__), 'packager', 'manifest_schema.yml')
+
+    attr_reader :errors
+
+    def initialize(manifest_path)
+      @manifest_path = manifest_path
+    end
+
+    def valid?
+      validate
+      errors.empty?
+    end
+
+    private
+
+    def validate
+      schema = Kwalify::Yaml.load_file(SCHEMA_FILE)
+      validator = Kwalify::Validator.new(schema)
+      parser = Kwalify::Yaml::Parser.new(validator)
+      manifest_data = parser.parse_file(@manifest_path)
+
+      @errors = {}
+      @errors[:manifest_parser_errors] = parser.errors unless parser.errors.empty?
+
+      if manifest_data['default_versions'] && !@errors[:manifest_parser_errors]
+        default_version_errors = validate_default_versions(manifest_data)
+        @errors[:default_version] = default_version_errors unless default_version_errors.empty?
+      end
+    end
+
+    def validate_default_versions(manifest_data)
+      error_messages = []
+
+      default_versions = create_manifest_dependencies(manifest_data['default_versions'])
+      dependency_versions = create_manifest_dependencies(manifest_data['dependencies'])
+
+      error_messages += validate_no_duplicate_names(default_versions)
+      error_messages += validate_defaults_in_dependencies(default_versions, dependency_versions)
+      wrap_errors_with_common_text(error_messages)
+
+      error_messages
+    end
+
+    def create_manifest_dependencies(dependency_entries)
+      dependency_entries.map do |dependency|
+        ManifestDependency.new(dependency['name'], dependency['version'])
+      end
+    end
+
+    def validate_no_duplicate_names(default_versions)
+      default_versions_names = default_versions.map { |default_version| default_version.name }
+      duplicate_names = default_versions_names.find_all { |dep| default_versions_names.count(dep) > 1 }.uniq
+
+      duplicate_names.map do |name|
+        "- #{name} had more than one 'default_versions' entry in the buildpack manifest."
+      end
+    end
+
+    def validate_defaults_in_dependencies(default_versions, dependency_versions)
+      unmatched_dependencies = default_versions.select { |d| !dependency_versions.include?(d) }
+
+      unmatched_dependencies.map do |dependency|
+        name_version = "#{dependency.name} #{dependency.version}"
+
+        "- a 'default_versions' entry for #{name_version} was specified by the buildpack manifest, " +
+          "but no 'dependencies' entry for #{name_version} was found in the buildpack manifest."
+      end
+    end
+
+    def wrap_errors_with_common_text(error_messages)
+      if error_messages.any?
+        error_messages.unshift('The buildpack manifest is malformed:')
+        error_messages<< 'For more information, see https://docs.cloudfoundry.org/buildpacks/custom.html#specifying-default-versions'
+      end
+    end
+  end
+end
+

data/lib/buildpack/packager.rb

@@ -0,0 +1,55 @@
+require 'buildpack/packager/version'
+require 'buildpack/packager/table_presentation'
+require 'buildpack/packager/dependencies_presenter'
+require 'buildpack/packager/default_versions_presenter'
+require 'buildpack/packager/package'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'open3'
+require 'fileutils'
+require 'tmpdir'
+require 'yaml'
+require 'shellwords'
+
+module Buildpack
+  module Packager
+    class CheckSumError < StandardError; end
+
+    def self.package(options)
+      check_for_zip
+
+      package = Package.new(options)
+
+      Dir.mktmpdir do |temp_dir|
+        package.copy_buildpack_to_temp_dir(temp_dir)
+
+        package.build_dependencies(temp_dir) if options[:mode] == :cached
+
+        package.build_zip_file(temp_dir)
+      end
+
+      buildpack_type = options[:mode] == :cached ? "Cached" : "Uncached"
+      human_readable_size = `du -h #{package.zip_file_path} | cut -f1`
+      puts "#{buildpack_type} buildpack created and saved as #{package.zip_file_path} with a size of #{human_readable_size.strip}"
+
+      package
+    end
+
+    def self.list(options)
+      package = Package.new(options)
+      package.list
+    end
+
+    def self.defaults(options)
+      package = Package.new(options)
+      package.defaults
+    end
+
+    def self.check_for_zip
+      _, _, status = Open3.capture3('which zip')
+
+      if status.to_s.include?('exit 1')
+        raise "Zip is not installed\nTry: apt-get install zip\nAnd then rerun"
+      end
+    end
+  end
+end

data/lib/buildpack/packager/default_versions_presenter.rb

@@ -0,0 +1,38 @@
+require 'terminal-table'
+
+module Buildpack
+  module Packager
+    class DefaultVersionsPresenter < Struct.new(:default_versions)
+      include TablePresentation
+
+      attr_reader :default_versions
+
+      def initialize(default_versions)
+        default_versions = [] if default_versions.nil?
+        @default_versions = default_versions
+      end
+
+      def inspect
+        table = Terminal::Table.new do |table|
+          default_versions.sort_by do |dependency|
+            sort_string_for dependency
+          end.each do |dependency|
+            columns = [
+              dependency['name'],
+              sanitize_version_string(dependency['version'])
+            ]
+            table.add_row columns
+          end
+        end
+
+        table.headings = %w(name version)
+
+        table.to_s
+      end
+
+      def present
+        to_markdown(inspect)
+      end
+    end
+  end
+end

data/lib/buildpack/packager/dependencies_presenter.rb

@@ -0,0 +1,41 @@
+require 'terminal-table'
+
+module Buildpack
+  module Packager
+    class DependenciesPresenter < Struct.new(:dependencies)
+      include TablePresentation
+
+      def inspect
+        has_modules = dependencies.any? { |dependency| dependency['modules'] }
+
+        table = Terminal::Table.new do |table|
+          dependencies.sort_by do |dependency|
+            sort_string_for dependency
+          end.each do |dependency|
+            columns = [
+              dependency['name'],
+              sanitize_version_string(dependency['version']),
+              dependency['cf_stacks'].sort.join(',')
+            ]
+            if has_modules
+              columns += [dependency.fetch('modules', []).sort.join(', ')]
+            end
+            table.add_row columns
+          end
+        end
+
+        table.headings = if has_modules
+                           %w(name version cf_stacks modules)
+                         else
+                           %w(name version cf_stacks)
+                         end
+
+        table.to_s
+      end
+
+      def present
+        to_markdown(inspect)
+      end
+    end
+  end
+end

data/lib/buildpack/packager/manifest_schema.yml

@@ -0,0 +1,67 @@
+type:       map
+mapping:
+  "language":
+    type:      str
+    required:  yes
+  "url_to_dependency_map":
+    type:      seq
+    required:  yes
+    sequence:
+      - type: map
+        mapping:
+         "match":
+           type: str
+           required:  yes
+         "name":
+           type: str
+           required:  yes
+         "version":
+           type: text
+           required:  yes
+  "dependencies":
+    type:      seq
+    required:  yes
+    sequence:
+      - type: map
+        mapping:
+         "name":
+           type: str
+           required:  yes
+         "version":
+           type: text
+           required:  yes
+         "uri":
+           type: str
+           required:  yes
+         "modules":
+           type: seq
+           required:  no
+           sequence:
+             - type: str
+         "md5":
+           type: str
+           required:  yes
+         "cf_stacks":
+           type: seq
+           required:  yes
+           sequence:
+             - type: str
+               enum: [ lucid64, cflinuxfs2 ]
+  "exclude_files":
+    type:      seq
+    required:  yes
+    sequence:
+      - type: str
+  "default_versions":
+    type: seq
+    required: no
+    sequence:
+      - type: map
+        required: yes
+        mapping:
+         "name":
+           type: str
+           required:  yes
+         "version":
+           type: text
+           required:  yes

data/lib/buildpack/packager/package.rb

@@ -0,0 +1,139 @@
+require 'active_support/core_ext/hash/indifferent_access'
+require 'open3'
+require 'fileutils'
+require 'tmpdir'
+require 'yaml'
+require 'shellwords'
+require 'buildpack/packager/zip_file_excluder'
+
+module Buildpack
+  module Packager
+    class Package < Struct.new(:options)
+      def copy_buildpack_to_temp_dir(temp_dir)
+        FileUtils.cp_r(File.join(options[:root_dir], '.'), temp_dir)
+        FileUtils.cp(options[:manifest_path], File.join(temp_dir, 'manifest.yml'))
+      end
+
+      def build_dependencies(temp_dir)
+        local_cache_directory = options[:cache_dir] || "#{ENV['HOME']}/.buildpack-packager/cache"
+        FileUtils.mkdir_p(local_cache_directory)
+
+        dependency_dir = File.join(temp_dir, "dependencies")
+        FileUtils.mkdir_p(dependency_dir)
+
+        download_dependencies(manifest[:dependencies], local_cache_directory, dependency_dir)
+      end
+
+      def download_dependencies(dependencies, local_cache_directory, dependency_dir)
+        dependencies.each do |dependency|
+          safe_uri = uri_without_credentials(dependency['uri'])
+          translated_filename = uri_cache_path(safe_uri)
+          local_cached_file = File.expand_path(File.join(local_cache_directory, translated_filename))
+
+          if options[:force_download] || !File.exist?(local_cached_file)
+	    puts "Downloading #{dependency['name']} version #{dependency['version']} from: #{safe_uri}"
+            download_file(dependency['uri'], local_cached_file)
+            human_readable_size = `du -h #{local_cached_file} | cut -f1`.strip
+            puts "  Using #{dependency['name']} version #{dependency['version']} with size #{human_readable_size}"
+
+            from_local_cache = false
+          else
+            human_readable_size = `du -h #{local_cached_file} | cut -f1`.strip
+            puts "Using #{dependency['name']} version #{dependency['version']} from local cache at: #{local_cached_file} with size #{human_readable_size}"
+            from_local_cache = true
+	  end
+
+          ensure_correct_dependency_checksum({
+            local_cached_file: local_cached_file,
+            dependency: dependency,
+            from_local_cache: from_local_cache
+          })
+
+          FileUtils.cp(local_cached_file, dependency_dir)
+        end
+      end
+
+      def build_zip_file(temp_dir)
+        FileUtils.rm_rf(zip_file_path)
+        zip_files(temp_dir, zip_file_path, manifest[:exclude_files])
+      end
+
+      def list
+        DependenciesPresenter.new(manifest['dependencies']).present
+      end
+
+      def defaults
+        DefaultVersionsPresenter.new(manifest['default_versions']).present
+      end
+
+      def zip_file_path
+        Shellwords.escape(File.join(options[:root_dir], zip_file_name))
+      end
+
+      private
+
+      def uri_without_credentials(uri_string)
+        uri = URI(uri_string)
+        if uri.userinfo
+          uri.user = "-redacted-" if uri.user
+          uri.password = "-redacted-" if uri.password
+        end
+        uri.to_s.sub("file:", "file://")
+      end
+
+      def uri_cache_path uri
+        uri.gsub(/[:\/\?&]/, '_')
+      end
+
+      def manifest
+        @manifest ||= YAML.load_file(options[:manifest_path]).with_indifferent_access
+      end
+
+      def zip_file_name
+        "#{manifest[:language]}_buildpack#{cached_identifier}-v#{buildpack_version}.zip"
+      end
+
+      def buildpack_version
+        File.read("#{options[:root_dir]}/VERSION").chomp
+      end
+
+      def cached_identifier
+        return '' unless options[:mode] == :cached
+        '-cached'
+      end
+
+      def ensure_correct_dependency_checksum(local_cached_file:, dependency:, from_local_cache:)
+        if dependency['md5'] != Digest::MD5.file(local_cached_file).hexdigest
+          if from_local_cache
+            FileUtils.rm_rf(local_cached_file)
+
+            download_file(dependency['uri'], local_cached_file)
+            ensure_correct_dependency_checksum({
+              local_cached_file: local_cached_file,
+              dependency: dependency,
+              from_local_cache: false
+            })
+          else
+            raise CheckSumError,
+              "File: #{dependency['name']}, version: #{dependency['version']} downloaded at location #{dependency['uri']}\n\tis reporting a different checksum than the one specified in the manifest."
+          end
+        else
+          puts "  #{dependency['name']} version #{dependency['version']} matches the manifest provided md5 checksum of #{dependency['md5']}\n\n"
+        end
+      end
+
+      def download_file(url, file)
+        raise "Failed to download file from #{url}" unless system("curl #{url} -o #{file} -L --fail -f")
+      end
+
+      def zip_files(source_dir, zip_file_path, excluded_files)
+        excluder = ZipFileExcluder.new
+        manifest_exclusions = excluder.generate_manifest_exclusions excluded_files
+        gitfile_exclusions = excluder.generate_exclusions_from_git_files source_dir
+        all_exclusions = manifest_exclusions + ' ' + gitfile_exclusions
+        `cd #{source_dir} && zip -r #{zip_file_path} ./ #{all_exclusions}`
+      end
+    end
+  end
+end
+