buildpack-packager 2.3.4

data/spec/integration/default_versions_spec.rb

@@ -0,0 +1,170 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe 'Buildpack packager default_versions validation' do
+  let(:flags)          { '--uncached' }
+  let(:buildpack_dir)  { Dir.mktmpdir }
+  let(:cache_dir)      { Dir.mktmpdir }
+  let(:base_manifest_contents) { <<-BASE
+language: python
+url_to_dependency_map: []
+exclude_files: []
+  BASE
+  }
+
+  before do
+    Dir.chdir(buildpack_dir) do
+      File.write('manifest.yml', manifest)
+      File.write('VERSION', '1.7.8')
+    end
+  end
+
+  after do
+    FileUtils.rm_rf(buildpack_dir)
+    FileUtils.rm_rf(cache_dir)
+  end
+
+  shared_examples_for "general output that helps with the error is produced" do
+    it "outputs a link to the Cloud Foundry custom buildpacks page" do
+      output, status = run_packager_binary(buildpack_dir, flags)
+      expect(output).to include("For more information, see https://docs.cloudfoundry.org/buildpacks/custom.html#specifying-default-versions")
+    end
+
+    it "states the buildpack manifest is malformed" do
+      output, status = run_packager_binary(buildpack_dir, flags)
+      expect(output).to include("The buildpack manifest is malformed:")
+    end
+  end
+
+  context 'defaults and dependencies are in agreement' do
+    let(:manifest) {<<-MANIFEST
+#{base_manifest_contents}
+default_versions:
+  - name: python
+    version: 3.3.5
+dependencies:
+  - name: python
+    version: 3.3.5
+    uri: http://example.com/
+    md5: 68901bbf8a04e71e0b30aa19c3946b21
+    cf_stacks:
+      - cflinuxfs2
+    MANIFEST
+    }
+
+    it 'emits no errors' do
+      _, status = run_packager_binary(buildpack_dir, flags)
+
+      expect(status).to be_success
+    end
+  end
+
+  context 'multiple default versions for a dependency' do
+    let(:manifest) {<<-MANIFEST
+#{base_manifest_contents}
+default_versions:
+  - name: python
+    version: 3.3.5
+  - name: python
+    version: 3.3.2
+  - name: ruby
+    version: 4.3.5
+  - name: ruby
+    version: 4.3.2
+dependencies:
+  - name: python
+    uri: https://a.org
+    version: 3.3.5
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+  - name: python
+    uri: https://a.org
+    version: 3.3.2
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+  - name: ruby
+    uri: https://a.org
+    version: 4.3.5
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+  - name: ruby
+    uri: https://a.org
+    version: 4.3.2
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+    MANIFEST
+    }
+
+    it_behaves_like "general output that helps with the error is produced"
+
+    it 'fails and errors stating the context' do
+      output, status = run_packager_binary(buildpack_dir, flags)
+
+      expect(output).to include("python had more " +
+                           "than one 'default_versions' entry in the buildpack manifest.")
+      expect(output).to include("ruby had more " +
+                                  "than one 'default_versions' entry in the buildpack manifest.")
+      expect(status).to_not be_success
+    end
+  end
+
+  context 'no dependency with name found for default in manifest' do
+    let(:manifest) {<<-MANIFEST
+#{base_manifest_contents}
+default_versions:
+  - name: python
+    version: 3.3.5
+  - name: ruby
+    version: 4.3.5
+dependencies: []
+    MANIFEST
+    }
+
+    it_behaves_like "general output that helps with the error is produced"
+
+    it 'fails and errors stating the context' do
+      output, status = run_packager_binary(buildpack_dir, flags)
+
+      expect(output).to include("a 'default_versions' entry for python 3.3.5 was specified by the buildpack manifest, " +
+                                  "but no 'dependencies' entry for python 3.3.5 was found in the buildpack manifest.")
+      expect(output).to include("a 'default_versions' entry for ruby 4.3.5 was specified by the buildpack manifest, " +
+                                  "but no 'dependencies' entry for ruby 4.3.5 was found in the buildpack manifest.")
+      expect(status).to_not be_success
+    end
+  end
+
+  context 'no dependency with version found for default in manifest' do
+    let(:manifest) {<<-MANIFEST
+#{base_manifest_contents}
+default_versions:
+  - name: ruby
+    version: 1.1.1
+  - name: python
+    version: 3.3.5
+dependencies:
+  - name: ruby
+    uri: https://a.org
+    version: 9.9.9
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+  - name: python
+    uri: https://a.org
+    version: 9.9.9
+    md5: 3a2
+    cf_stacks: [cflinuxfs2]
+    MANIFEST
+    }
+
+    it_behaves_like "general output that helps with the error is produced"
+
+    it 'fails and errors stating the context' do
+      output, status = run_packager_binary(buildpack_dir, flags)
+
+      expect(output).to include("a 'default_versions' entry for python 3.3.5 was specified by the buildpack manifest, " +
+                                  "but no 'dependencies' entry for python 3.3.5 was found in the buildpack manifest.")
+      expect(output).to include("a 'default_versions' entry for ruby 1.1.1 was specified by the buildpack manifest, " +
+                                  "but no 'dependencies' entry for ruby 1.1.1 was found in the buildpack manifest.")
+      expect(status).to_not be_success
+    end
+  end
+end

data/spec/integration/output_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe 'Buildpack packager output' do
+  let(:buildpack_type)    { '--uncached' }
+  let(:fixture_name)      { 'buildpack-without-uri-credentials' }
+  let(:buildpack_fixture) { File.join(File.dirname(__FILE__), '..', 'fixtures', fixture_name) }
+  let(:tmpdir)            { Dir.mktmpdir }
+
+  before do
+    FileUtils.rm_rf(tmpdir)
+  end
+
+  def buildpack_packager_execute(buildpack_dir, home_dir)
+    Dir.chdir(buildpack_dir) do
+      `HOME=#{home_dir} buildpack-packager #{buildpack_type}`
+    end
+  end
+
+  subject { buildpack_packager_execute(buildpack_fixture, tmpdir) }
+
+  context 'building the uncached buildpack' do
+    it 'outputs the type of buildpack created, where and its human readable size' do
+      expect(subject).to include("Uncached buildpack created and saved as")
+      expect(subject).to include("spec/fixtures/#{fixture_name}/go_buildpack-v1.7.8.zip")
+      expect(subject).to match(/with a size of 4\.0K$/)
+    end
+  end
+
+  context 'building the cached buildpack' do
+    let(:buildpack_type) { '--cached' }
+
+    it 'outputs the dependencies downloaded, their versions, and download source url' do
+      expect(subject).to include("Downloading go version 1.6.3 from: https://buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz")
+      expect(subject).to include("Using go version 1.6.3 with size")
+      expect(subject).to include("go version 1.6.3 matches the manifest provided md5 checksum of 5f7bf9d61d2b0dd75c9e2cd7a87272cc")
+
+      expect(subject).to include("Downloading godep version v74 from: https://pivotal-buildpacks.s3.amazonaws.com/concourse-binaries/godep/godep-v74-linux-x64.tgz")
+      expect(subject).to include("Using godep version v74 with size 2.8M")
+      expect(subject).to include("godep version v74 matches the manifest provided md5 checksum of 70220eee9f9e654e0b85887f696b6add")
+    end
+
+    it 'outputs the type of buildpack created, where and its human readable size' do
+      expect(subject).to include("Cached buildpack created and saved as")
+      expect(subject).to include("spec/fixtures/#{fixture_name}/go_buildpack-cached-v1.7.8.zip")
+      expect(subject).to match(/with a size of 68M$/)
+    end
+
+    context 'with a buildpack packager dependency cache intact' do
+      before { buildpack_packager_execute(buildpack_fixture, tmpdir) }
+
+      it 'outputs the dependencies downloaded, their versions, and cache location' do
+        expect(subject).to include("Using go version 1.6.3 from local cache at: #{tmpdir}/.buildpack-packager/cache/https___buildpacks.cloudfoundry.org_concourse-binaries_go_go1.6.3.linux-amd64.tar.gz with size")
+        expect(subject).to include("go version 1.6.3 matches the manifest provided md5 checksum of 5f7bf9d61d2b0dd75c9e2cd7a87272cc")
+
+        expect(subject).to include("Using godep version v74 from local cache at: #{tmpdir}/.buildpack-packager/cache/https___pivotal-buildpacks.s3.amazonaws.com_concourse-binaries_godep_godep-v74-linux-x64.tgz with size")
+        expect(subject).to include("godep version v74 matches the manifest provided md5 checksum of 70220eee9f9e654e0b85887f696b6add")
+      end
+    end
+
+    context 'with auth credentials in the dependency uri' do
+      let(:fixture_name) { 'buildpack-with-uri-credentials' }
+
+      it 'outputs the dependencies download source url without the credentials' do
+        expect(subject).to include('Downloading go version 1.6.3 from: https://-redacted-:-redacted-@buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz')
+        expect(subject).to include('Downloading godep version v74 from: https://-redacted-:-redacted-@buildpacks.cloudfoundry.org/concourse-binaries/godep/godep-v74-linux-x64.tgz')
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+$LOAD_PATH << File.expand_path('../../spec/helpers', __FILE__)
+require 'buildpack/packager'
+require 'file_system_helpers'
+require 'cache_directory_helpers'
+require 'fake_binary_hosting_helpers'
+
+RSpec.configure do |config|
+  config.include FileSystemHelpers
+  config.include CacheDirectoryHelpers
+  config.include FakeBinaryHostingHelpers
+end

data/spec/unit/buildpack/packager/zip_file_excluder_spec.rb

@@ -0,0 +1,68 @@
+require 'buildpack/packager/zip_file_excluder'
+require 'spec_helper'
+require 'tmpdir'
+require 'securerandom'
+
+module Buildpack
+  module Packager
+    describe ZipFileExcluder do
+      describe '#generate_exclusions_from_manifest' do
+        let(:excluder) { ZipFileExcluder.new }
+        let(:file1) { SecureRandom.uuid }
+        let(:file2) { SecureRandom.uuid }
+        let(:dir) { "#{SecureRandom.uuid}/" }
+        let(:excluded_files) { [file1, file2] }
+        subject do
+          excluder.generate_manifest_exclusions excluded_files
+        end
+
+        context 'does not include directories' do
+          it do
+            is_expected.to eq "-x #{file1} -x \\*/#{file1} -x #{file2} -x \\*/#{file2}"
+          end
+        end
+
+        context 'includes directories' do
+          let(:excluded_files) { [file1, dir] }
+          it do
+            is_expected.to eq "-x #{file1} -x \\*/#{file1} -x #{dir}\\* -x \\*/#{dir}\\*"
+          end
+        end
+
+        it 'uses short flags to support zip 2.3.*, see [#107948062]' do
+          is_expected.to include('-x')
+          is_expected.to_not include('--exclude')
+        end
+      end
+    end
+
+    describe '#generate_exclusions_from_git_files' do
+      let(:excluder) { ZipFileExcluder.new }
+
+      context 'git files exist' do
+        let (:files) { ['.gitignore', '.git/', '.gitmodules', 'lib/.git'] }
+
+        it 'returns an exclusion string with all the git files' do
+          Dir.mktmpdir do |dir|
+            Dir.mkdir "#{dir}/lib"
+
+            files.each do |gitfilename|
+              if gitfilename =~ /.*\/$/
+                Dir.mkdir "#{dir}/#{gitfilename}"
+              else
+                File.new "#{dir}/#{gitfilename}", 'w'
+              end
+            end
+
+            git_exclusions = excluder.generate_exclusions_from_git_files dir
+
+            expect(git_exclusions).to include '-x .gitignore -x \\*/.gitignore'
+            expect(git_exclusions).to include '-x lib/.git -x \\*/lib/.git'
+            expect(git_exclusions).to include '-x .gitmodules -x \\*/.gitmodules'
+            expect(git_exclusions).to include '-x .git/\\* -x \\*/.git/\\*'
+          end
+        end
+      end
+    end
+  end
+end

data/spec/unit/manifest_dependency_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require 'buildpack/manifest_dependency'
+
+describe Buildpack::ManifestDependency do
+  describe '#==' do
+    context 'two objects with the same name and same version' do
+      let(:first_dependency) { described_class.new('name', 'version') }
+      let(:second_dependency) { described_class.new('name', 'version') }
+
+      it 'returns true' do
+        expect(first_dependency == second_dependency).to be true
+      end
+    end
+    context 'two objects with the same name only' do
+      let(:first_dependency) { described_class.new('name', 'version 11111111') }
+      let(:second_dependency) { described_class.new('name', 'version 22222222') }
+
+      it 'returns false' do
+        expect(first_dependency == second_dependency).to be false
+      end
+    end
+
+    context 'two objects with the same version only' do
+      let(:first_dependency) { described_class.new('name 111', 'version') }
+      let(:second_dependency) { described_class.new('name 222', 'version') }
+
+      it 'returns false' do
+        expect(first_dependency == second_dependency).to be false
+      end
+    end
+
+    context 'two objects with different names and different versions' do
+      let(:first_dependency) { described_class.new('name 111111', 'version 11111111') }
+      let(:second_dependency) { described_class.new('name 222222', 'version 22222222') }
+
+      it 'returns false' do
+        expect(first_dependency == second_dependency).to be false
+      end
+    end
+  end
+end

data/spec/unit/manifest_validator_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'buildpack/manifest_validator'
+
+describe Buildpack::ManifestValidator do
+  let(:manifest_path) { "#{File.dirname(__FILE__)}/../fixtures/manifests/#{manifest_file_name}" }
+  let(:validator) { Buildpack::ManifestValidator.new(manifest_path) }
+
+  context 'with a valid manifest' do
+    let(:manifest_file_name) { 'manifest_valid.yml' }
+
+    it 'reports valid manifests correctly' do
+      expect(validator.valid?).to be(true)
+      expect(validator.errors).to be_empty
+    end
+  end
+
+  context 'with a manifest with an invalid md5 key' do
+    let(:manifest_file_name) { 'manifest_invalid-md6.yml' }
+
+    it 'reports invalid manifests correctly' do
+      expect(validator.valid?).to be(false)
+      expect(validator.errors[:manifest_parser_errors]).not_to be_empty
+    end
+
+    context 'and incorrect defaults' do
+      let(:manifest_file_name) { 'manifest_invalid-md6_and_defaults.yml' }
+
+      it 'reports manifest parser errors only' do
+        expect(validator).to_not receive(:validate_default_versions)
+        expect(validator.valid?).to be(false)
+        expect(validator.errors[:manifest_parser_errors]).not_to be_empty
+      end
+    end
+  end
+end

data/spec/unit/packager/package_spec.rb

@@ -0,0 +1,196 @@
+require 'spec_helper'
+require 'tmpdir'
+
+module Buildpack
+  module Packager
+    describe Package do
+      let(:packager) { Buildpack::Packager::Package.new(options) }
+      let(:manifest_path) { 'manifest.yml' }
+      let(:dependency) { double(:dependency) }
+      let(:mode) { :uncached }
+      let(:local_cache_dir) { nil }
+      let(:force_download) { false }
+      let(:options) do
+        {
+          root_dir: 'root_dir',
+          mode: mode,
+          cache_dir: local_cache_dir,
+          manifest_path: manifest_path,
+          force_download: force_download
+        }
+      end
+
+      let(:manifest) do
+        {
+          language: 'fake_language',
+          dependencies: [dependency],
+          exclude_files: ['.DS_Store', '.gitignore']
+        }
+      end
+
+      before do
+        allow(YAML).to receive(:load_file).and_return(manifest)
+      end
+
+      describe '#copy_buildpack_to_temp_dir' do
+        context 'with full manifest specified' do
+          let(:manifest_path) { 'manifest-including-unsupported.yml' }
+
+          before do
+            allow(FileUtils).to receive(:mv)
+            allow(FileUtils).to receive(:cp_r)
+            allow(FileUtils).to receive(:cp)
+            allow(FileUtils).to receive(:rm)
+          end
+
+          it 'replaces the default manifest with the full manifest' do
+            expect(FileUtils).to receive(:cp).with('manifest-including-unsupported.yml', File.join('hello_dir', 'manifest.yml'))
+            packager.copy_buildpack_to_temp_dir('hello_dir')
+          end
+        end
+      end
+
+      describe '#build_dependencies' do
+        let(:mode) { :cached }
+
+        before do
+          allow(FileUtils).to receive(:mkdir_p)
+          allow(packager).to receive(:download_dependencies)
+        end
+
+        context 'when cache_dir is provided' do
+          let(:local_cache_dir) { 'local_cache_dir' }
+
+          it 'creates the provided cache dir' do
+            expect(FileUtils).to receive(:mkdir_p).with(local_cache_dir)
+            packager.build_dependencies('hello_dir')
+          end
+
+          it 'creates the dependency dir' do
+            expect(FileUtils).to receive(:mkdir_p).with(File.join('hello_dir', 'dependencies'))
+            packager.build_dependencies('hello_dir')
+          end
+
+          it 'calls download_dependencies with right arguments' do
+            expect(packager).to receive(:download_dependencies).with([dependency], local_cache_dir, File.join('hello_dir', 'dependencies'))
+            packager.build_dependencies('hello_dir')
+          end
+        end
+
+        context 'when cache_dir is NOT provided' do
+          it 'creates the default cache dir' do
+            expect(FileUtils).to receive(:mkdir_p).with(File.join(ENV['HOME'], '.buildpack-packager', 'cache'))
+            packager.build_dependencies('hello_dir')
+          end
+        end
+      end
+
+      describe '#download_dependencies' do
+        let(:local_cache_dir) { 'local_cache_dir' }
+        let(:dependency_dir) { File.join('hello_dir', 'dependencies') }
+        let(:url_with_parameters) { 'http://some.cdn/with?parameters=true&secondParameter=present' }
+
+        before do
+          allow(dependency).to receive(:[])
+          allow(dependency).to receive(:[]).with('uri').and_return('file:///fake_uri.tgz')
+          allow(packager).to receive(:ensure_correct_dependency_checksum)
+          allow(FileUtils).to receive(:cp)
+          allow(packager).to receive(:download_file)
+        end
+
+        context 'before dependency has been cached locally' do
+          before do
+            allow(File).to receive(:exist?).and_return(false)
+          end
+
+          it 'downloads the dependency to the local cache' do
+            expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz'))
+            expect(packager).to receive(:download_file).with('file:///fake_uri.tgz', expanded_local_file_location)
+            packager.download_dependencies([dependency], local_cache_dir, dependency_dir)
+          end
+
+          it 'copies the dependency from the local cache to the dependency_dir' do
+            expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz'))
+            expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir)
+            packager.download_dependencies([dependency], local_cache_dir, dependency_dir)
+          end
+        end
+
+        context 'after dependency has been cached locally' do
+          before do
+            allow(File).to receive(:exist?).and_return(true)
+          end
+
+          it 'does not re-download the dependency' do
+            expect(packager).not_to receive(:download_file)
+            packager.download_dependencies([dependency], local_cache_dir, dependency_dir)
+          end
+
+          it 'copies the dependency from the local cache to the dependency_dir' do
+            expanded_local_file_location = File.expand_path(File.join('local_cache_dir', 'file____fake_uri.tgz'))
+            expect(FileUtils).to receive(:cp).with(expanded_local_file_location, dependency_dir)
+            packager.download_dependencies([dependency], local_cache_dir, dependency_dir)
+          end
+        end
+
+        context 'with :force_download option active and a locally cached dependency' do
+          let(:force_download) { true }
+
+          before do
+            allow(File).to receive(:exist?).and_return(true)
+          end
+
+          it 're-downloads the dependency anyway' do
+            expect(packager).to receive(:download_file)
+            packager.download_dependencies([dependency], local_cache_dir, dependency_dir)
+          end
+        end
+
+        it 'translates ? and & characters in the url to underscores' do
+          package = Package.new
+          expect(package.send(:uri_cache_path, url_with_parameters)).to eq("http___some.cdn_with_parameters=true_secondParameter=present")
+        end
+
+        context 'url has login and password authentication credentials' do
+          let(:url_with_credentials) { 'http://log!i213:pas!9sword@some.cdn/with' }
+
+          it 'redacts the credentials in the resulting file path' do
+            package = Package.new
+            expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-:-redacted-@some.cdn/with")
+          end
+        end
+
+        context 'url has a login authentication credential' do
+          let(:url_with_credentials) { 'http://log!i213@some.cdn/with' }
+
+          it 'redacts the credential in the resulting file path' do
+            package = Package.new
+            expect(package.send(:uri_without_credentials, url_with_credentials)).to eq("http://-redacted-@some.cdn/with")
+          end
+        end
+      end
+
+      describe '#build_zip_file' do
+        before do
+          allow(packager).to receive(:buildpack_version).and_return('1.0.0')
+          allow(FileUtils).to receive(:rm_rf)
+          allow(packager).to receive(:zip_files)
+          allow(packager).to receive(:zip_file_path)
+            .and_return(File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip'))
+        end
+
+        it 'removes the file at the zip file path' do
+          zip_file_path = File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip')
+          expect(FileUtils).to receive(:rm_rf).with(zip_file_path)
+          packager.build_zip_file('hello_dir')
+        end
+
+        it 'zips up the temp directory to the zip file path without the excluded files' do
+          zip_file_path = File.join('root_dir', 'fake_language_buildpack-v1.0.0.zip')
+          expect(packager).to receive(:zip_files).with('hello_dir', zip_file_path, ['.DS_Store', '.gitignore'])
+          packager.build_zip_file('hello_dir')
+        end
+      end
+    end
+  end
+end