buildpack-packager 2.3.4

data/lib/buildpack/packager/table_presentation.rb

@@ -0,0 +1,21 @@
+require 'terminal-table'
+
+module Buildpack
+  module Packager
+    module TablePresentation
+      def to_markdown(table_contents)
+        table_contents.split("\n")[1...-1].tap { |lines| lines[1].tr!('+', '|') }.join("\n")
+      end
+
+      def sanitize_version_string(version)
+        version == 0 ? '-' : version
+      end
+
+      def sort_string_for(dependency)
+        interpreter_names = %w(ruby jruby php hhvm python go node)
+        sort_index = interpreter_names.index(dependency['name']) || 9999
+        sprintf '%s-%s-%s', sort_index, dependency['name'], dependency['version']
+      end
+    end
+  end
+end

data/lib/buildpack/packager/version.rb

@@ -0,0 +1,5 @@
+module Buildpack
+  module Packager
+    VERSION = '2.3.4'.freeze
+  end
+end

data/lib/buildpack/packager/zip_file_excluder.rb

@@ -0,0 +1,31 @@
+module Buildpack
+  module Packager
+    class ZipFileExcluder
+      def generate_manifest_exclusions(excluded_files)
+        generate_exclusion_string excluded_files
+      end
+
+      def generate_exclusions_from_git_files(dir)
+        Dir.chdir dir do
+          git_files = Dir.glob('**/.git*').map do |elt|
+            File.directory?(elt) ? "#{elt}/" : elt
+          end
+
+          generate_exclusion_string git_files
+        end
+      end
+
+      private
+
+      def generate_exclusion_string(file_list)
+        file_list.map do |file|
+          if file.chars.last == '/'
+            "-x #{file}\\* -x \\*/#{file}\\*"
+          else
+            "-x #{file} -x \\*/#{file}"
+          end
+        end.join(' ')
+      end
+    end
+  end
+end

data/lib/kwalify/parser/yaml-patcher.rb

@@ -0,0 +1,70 @@
+class Kwalify::Yaml::Parser < Kwalify::BaseParser
+  def parse_block_value(level, rule, path, uniq_table, container)
+    skip_spaces_and_comments
+    ## nil
+    return nil if @column < level || (@column == level && !match?(/-\s+/)) || eos?
+    ## anchor and alias
+    name = nil
+    if scan(/\&([-\w]+)/)
+      name = parse_anchor(rule, path, uniq_table, container)
+    elsif scan(/\*([-\w]+)/)
+      return parse_alias(rule, path, uniq_table, container)
+    end
+    ## type
+    skip_spaces_and_comments if scan(/!!?\w+/)
+    ## sequence
+    if match?(/-\s+/)
+      if rule && !rule.sequence
+        # _validate_error("sequence is not expected.", path)
+        rule = nil
+      end
+      seq = create_sequence(rule, @linenum, @column)
+      @anchors[name] = seq if name
+      parse_block_seq(seq, rule, path, uniq_table)
+      return seq
+    end
+    ## mapping
+    if match?(MAPKEY_PATTERN)
+      if rule && !rule.mapping
+        # _validate_error("mapping is not expected.", path)
+        rule = nil
+      end
+      map = create_mapping(rule, @linenum, @column)
+      @anchors[name] = map if name
+      parse_block_map(map, rule, path, uniq_table)
+      return map
+    end
+    ## sequence (flow-style)
+    if match?(/\[/)
+      if rule && !rule.sequence
+        # _validate_error("sequence is not expected.", path)
+        rule = nil
+      end
+      seq = create_sequence(rule, @linenum, @column)
+      @anchors[name] = seq if name
+      parse_flow_seq(seq, rule, path, uniq_table)
+      return seq
+    end
+    ## mapping (flow-style)
+    if match?(/\{/)
+      if rule && !rule.mapping
+        # _validate_error("mapping is not expected.", path)
+        rule = nil
+      end
+      map = create_mapping(rule, @linenum, @column)
+      @anchors[name] = map if name
+      parse_flow_map(map, rule, path, uniq_table)
+      return map
+    end
+    ## block text
+    if match?(/[|>]/)
+      text = parse_block_text(level, rule, path, uniq_table)
+      @anchors[name] = text if name
+      return text
+    end
+    ## scalar
+    scalar = parse_block_scalar(rule, path, uniq_table)
+    @anchors[name] = scalar if name
+    scalar
+  end
+end

data/spec/buildpack/packager_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Buildpack::Packager do
+  it 'has a version number' do
+    expect(Buildpack::Packager::VERSION).not_to be nil
+  end
+end

data/spec/fixtures/buildpack-with-uri-credentials/VERSION

@@ -0,0 +1 @@
+1.7.8

data/spec/fixtures/buildpack-with-uri-credentials/manifest.yml

@@ -0,0 +1,36 @@
+---
+language: go
+url_to_dependency_map:
+- match: go(\d+\.\d+(.*))
+  name: go
+  version: "$1"
+- match: godep
+  name: godep
+  version: v74
+dependencies:
+- name: go
+  version: 1.6.3
+  uri: https://login:password@buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz
+  md5: 5f7bf9d61d2b0dd75c9e2cd7a87272cc
+  cf_stacks:
+  - cflinuxfs2
+- name: godep
+  version: v74
+  uri: https://api-token:x-oauth-basic@buildpacks.cloudfoundry.org/concourse-binaries/godep/godep-v74-linux-x64.tgz
+  md5: 70220eee9f9e654e0b85887f696b6add
+  cf_stacks:
+  - cflinuxfs2
+exclude_files:
+- ".git/"
+- ".gitignore"
+- ".gitmodules"
+- cf_spec/
+- log/
+- bin/package
+- buildpack-packager/
+- test-godir/
+- test/
+- Makefile
+- PULL_REQUEST_TEMPLATE
+- ISSUE_TEMPLATE
+- go_buildpack-*v*.zip

data/spec/fixtures/buildpack-without-uri-credentials/VERSION

@@ -0,0 +1 @@
+1.7.8

data/spec/fixtures/buildpack-without-uri-credentials/manifest.yml

@@ -0,0 +1,36 @@
+---
+language: go
+url_to_dependency_map:
+- match: go(\d+\.\d+(.*))
+  name: go
+  version: "$1"
+- match: godep
+  name: godep
+  version: v74
+dependencies:
+- name: go
+  version: 1.6.3
+  uri: https://buildpacks.cloudfoundry.org/concourse-binaries/go/go1.6.3.linux-amd64.tar.gz
+  md5: 5f7bf9d61d2b0dd75c9e2cd7a87272cc
+  cf_stacks:
+  - cflinuxfs2
+- name: godep
+  version: v74
+  uri: https://pivotal-buildpacks.s3.amazonaws.com/concourse-binaries/godep/godep-v74-linux-x64.tgz
+  md5: 70220eee9f9e654e0b85887f696b6add
+  cf_stacks:
+  - cflinuxfs2
+exclude_files:
+- ".git/"
+- ".gitignore"
+- ".gitmodules"
+- cf_spec/
+- log/
+- bin/package
+- buildpack-packager/
+- test-godir/
+- test/
+- Makefile
+- PULL_REQUEST_TEMPLATE
+- ISSUE_TEMPLATE
+- go_buildpack-*v*.zip

data/spec/fixtures/manifests/manifest_invalid-md6.yml

@@ -0,0 +1,18 @@
+---
+language: go
+
+url_to_dependency_map:
+  - match: go(\d+\.\d+(\.\d+)?)
+    name: go
+    version: $1
+dependencies:
+  - name: go
+    version: 1.2
+    uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz
+    md6: 68901bbf8a04e71e0b30aa19c3946b21
+    cf_stacks:
+      - lucid64
+      - cflinuxfs2
+
+exclude_files:
+  - .git/

data/spec/fixtures/manifests/manifest_invalid-md6_and_defaults.yml

@@ -0,0 +1,21 @@
+---
+language: go
+
+url_to_dependency_map:
+  - match: go(\d+\.\d+(\.\d+)?)
+    name: go
+    version: $1
+default_versions:
+  - name: go
+    version: 1.3
+dependencies:
+  - name: go
+    version: 1.2
+    uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz
+    md6: 68901bbf8a04e71e0b30aa19c3946b21
+    cf_stacks:
+      - lucid64
+      - cflinuxfs2
+
+exclude_files:
+  - .git/

data/spec/fixtures/manifests/manifest_valid.yml

@@ -0,0 +1,19 @@
+---
+language: go
+
+url_to_dependency_map:
+  - match: go(\d+\.\d+(\.\d+)?)
+    name: go
+    version: $1
+
+dependencies:
+  - name: go
+    version: 1.2
+    uri: http://go.googlecode.com/files/go1.2.linux-amd64.tar.gz
+    md5: 68901bbf8a04e71e0b30aa19c3946b21
+    cf_stacks:
+      - lucid64
+      - cflinuxfs2
+
+exclude_files:
+  - .git/

data/spec/helpers/cache_directory_helpers.rb

@@ -0,0 +1,15 @@
+module CacheDirectoryHelpers
+  BUILDPACK_PACKAGER_CACHE_DIR = File.join(ENV['HOME'], '.buildpack-packager', 'cache')
+
+  def uri_to_cache_filename(uri)
+    uri.gsub(/[\/:]/, '_')
+  end
+
+  def uri_to_cache_path(uri)
+    File.join(BUILDPACK_PACKAGER_CACHE_DIR, uri_to_cache_filename(uri))
+  end
+
+  def remove_from_cache_dir(uri)
+    FileUtils.rm_f(uri_to_cache_path(uri))
+  end
+end

data/spec/helpers/fake_binary_hosting_helpers.rb

@@ -0,0 +1,21 @@
+module FakeBinaryHostingHelpers
+  def upstream_host_dir
+    @upstream_host_dir ||= Dir.mktmpdir('upstream_host_')
+  end
+
+  def create_upstream_file(file_name, file_content)
+    file_path = generate_upstream_file_path(file_name)
+    File.write(file_path, file_content)
+    file_path
+  end
+
+  def remove_upstream_file(file_name)
+    FileUtils.rm_f(generate_upstream_file_path(file_name))
+  end
+
+  private
+
+  def generate_upstream_file_path(file_name)
+    File.join(upstream_host_dir, file_name)
+  end
+end

data/spec/helpers/file_system_helpers.rb

@@ -0,0 +1,35 @@
+require 'zip'
+require 'fileutils'
+
+module FileSystemHelpers
+  def run_packager_binary(buildpack_dir, flags)
+    packager_binary_file = File.join(`pwd`.chomp, 'bin', 'buildpack-packager')
+    Open3.capture2e("cd #{buildpack_dir} && #{packager_binary_file} #{flags}")
+  end
+
+  def make_fake_files(root, file_list)
+    file_list.each do |file|
+      full_path = File.join(root, file)
+      FileUtils.mkdir_p(File.dirname(full_path))
+      File.write(full_path, 'a')
+    end
+  end
+
+  def all_files(root)
+    Dir["#{root}/*"].map do |filename|
+      filename.gsub(root, '').gsub(/^\//, '')
+    end
+  end
+
+  def get_zip_contents(zip_path)
+    Zip::File.open(zip_path) do |zip_file|
+      zip_file
+        .map(&:name)
+        .select { |name| name[/\/$/].nil? }
+    end
+  end
+
+  def get_md5_of_file(path)
+    Digest::MD5.file(path).hexdigest
+  end
+end

data/spec/integration/bin/buildpack_packager/download_caching_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+# Special note
+# =============
+#
+# There are two uses of the term 'caching' in buildpack packager.
+# 1. A 'cached' buildpack is a buildpack zipball that contains the buildpacks dependencies
+# 2. 'Download caching' is where the packager keeps a copy of downloaded dependencies for
+#    the next time it is run.
+#
+# This integration test is concerned with 'Download caching'
+
+describe 'reusing previously downloaded files' do
+  let(:upstream_file_path) do
+    create_upstream_file('sample_download.ignore_me',
+                         'sample_download original text')
+  end
+
+  let(:upstream_file_uri) { "file://#{upstream_file_path}" }
+
+  let(:buildpack_dir) { Dir.mktmpdir('buildpack_') }
+
+  let(:md5) { get_md5_of_file(upstream_file_path) }
+  let(:manifest) do
+    {
+      'exclude_files' => [],
+      'language' => 'sample',
+      'url_to_dependency_map' => [],
+      'dependencies' => [{
+        'version' => '1.0',
+        'name' => 'sample_download.ignore_me',
+        'cf_stacks' => [],
+        'md5' => md5,
+        'uri' => upstream_file_uri
+      }]
+    }
+  end
+
+  before do
+    File.write(File.join(buildpack_dir, 'manifest.yml'), manifest.to_yaml)
+
+    `echo "1.2.3" > #{File.join(buildpack_dir, 'VERSION')}`
+  end
+
+  context 'the file is not in the cache' do
+    specify 'the file should be kept in a cache when it is downloaded' do
+      _, status = run_packager_binary(buildpack_dir, '--cached')
+
+      expect(status).to be_success
+      expect(File).to exist(uri_to_cache_path(upstream_file_uri))
+
+      expect(File.read(uri_to_cache_path(upstream_file_uri))).to include('sample_download original text')
+    end
+  end
+
+  context 'the file has been downloaded before' do
+    specify 'the file in the cache should be used instead of downloading' do
+      run_packager_binary(buildpack_dir, '--cached')
+
+      remove_upstream_file('sample_download.ignore_me') # taking this away means packager must use the cache
+
+      _, status = run_packager_binary(buildpack_dir, '--cached')
+
+      expect(status).to be_success
+    end
+
+    context 'however the file has changed, and the manifest is updated to reflect the new md5' do
+      before do
+        run_packager_binary(buildpack_dir, '--cached')
+
+        create_upstream_file('sample_download.ignore_me', 'sample_download updated text')
+        new_md5 = get_md5_of_file(upstream_file_path)
+        manifest['dependencies'].first['md5'] = new_md5
+        File.write(File.join(buildpack_dir, 'manifest.yml'), manifest.to_yaml)
+      end
+
+      specify 'the cache should now contain the new upstream file' do
+        output, status = run_packager_binary(buildpack_dir, '--cached')
+
+        expect(status).to be_success
+        expect(File.read(uri_to_cache_path(upstream_file_uri))).to include('sample_download updated text')
+      end
+    end
+  end
+end