slug-compiler 2.0.0

data/.gitignore

@@ -0,0 +1 @@
+/slug-compiler-*.gem

data/COPYING

@@ -0,0 +1,20 @@
+Copyright © 2013 Heroku
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Readme.md

@@ -0,0 +1,49 @@
+# Slug compiler
+
+## Overview
+
+A slug is a unit of deployment which includes everything an
+application needs to run. The slug compiler is a program which uses a
+buildpack to transform application source into a slug, typically by
+gathering all dependencies and compiling source to binaries, if
+applicable.
+
+As inputs it takes a source directory, cache directory, buildpack URL,
+and output directory. It places a tarball of the slug as well as a
+JSON file of process types in the output directory if compilation is
+successful.
+
+Note that this is a trimmed down version of the slug compiler that
+currently runs in production on Heroku. It's intended for next-gen
+services but is not currently in production use.
+
+## Usage
+
+In the typical git-based Heroku deploy, the slug compiler is invoked
+via a Git pre-recieve hook. However, this code has been extracted so
+that the same process can be used elsewhere.
+
+Note that it will delete .slugignore patterns and other files, so you
+shouldn't run this straight out of a checkout. Create a fresh copy of
+the application source before compiling it.
+
+## Requirements
+
+* Ruby 1.9
+* tar
+* du
+* git (optional, used for fetching non-HTTP buildpacks)
+
+## Responsibilities
+
+The new slug compiler does much less than the old one. In particular,
+these operations are now the responsibility of the caller:
+
+* checking size of the slug
+* stack migration
+* detecting among set of default buildpacks
+* git submodule handling
+* special-cased bamboo operations
+* calling releases API
+* storing slug file in S3
+* deploy hooks

data/bin/slug-compiler

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require "slug_compiler"
+
+build_dir, buildpack_url, cache_dir, output_dir = ARGV
+abort "USAGE: #{$0} BUILD_DIR BUILDPACK_DIR CACHE_DIR OUTPUT_DIR" unless ARGV.size == 4 and
+  File.exists?(build_dir) and File.exists?(cache_dir) and
+  URI.parse(buildpack_url) rescue false and File.directory?(output_dir)
+
+begin
+  start = Time.now
+  SlugCompiler.run(*ARGV)
+  # TODO: unify this with existing logging
+rescue SlugCompiler::CompileFail => e
+  puts(" !     Heroku push rejected, #{e.message}\n")
+rescue SlugCompiler::CompileError => e
+  puts(" !     Heroku push rejected, #{e.message}\n")
+  SlugCompiler.log("measure=slugc.fail elapsed=#{Time.now - start} " \
+                   "message='#{e.message}'")
+rescue => e
+  puts
+  puts(" !     Heroku push rejected due to an unrecognized error.")
+  puts(" !     We've been notified, see http://support.heroku.com if the problem persists.")
+  puts("\n")
+  SlugCompiler.log_error("measure=slugc.error elapsed=#{Time.now - start}", e)
+  raise if ENV["DEBUG"]
+end

data/lib/slug_compiler.rb

@@ -0,0 +1,243 @@
+require "fileutils"
+require "find"
+require "json"
+require "open-uri"
+require "shellwords"
+require "syslog"
+require "timeout"
+require "uri"
+
+$stdout.sync = true
+Syslog.open("slug-compiler", Syslog::LOG_CONS, Syslog::LOG_USER)
+
+module SlugCompiler
+  class CompileError < RuntimeError; end
+  class CompileFail < RuntimeError; end
+
+  module_function
+
+  def run(build_dir, buildpack_url, cache_dir, output_dir)
+    @compile_id = rand(2**64).to_s(36) # because UUIDs need a gem
+
+    buildpack_dir = fetch_buildpack(buildpack_url)
+    config = buildpack_config(buildpack_url)
+    buildpack_name = detect(build_dir, buildpack_dir)
+    compile(build_dir, buildpack_dir, cache_dir, config)
+
+    prune(build_dir)
+    process_types = parse_procfile(build_dir, output_dir)
+    slug = archive(build_dir, output_dir)
+    log_size(build_dir, cache_dir, slug)
+
+    [slug, process_types]
+  ensure
+    FileUtils.rm_rf(buildpack_dir) if buildpack_dir
+  end
+
+  def fetch_buildpack(buildpack_url)
+    buildpack_dir = "/tmp/buildpack_#{@compile_id}"
+
+    log("fetch_buildpack") do
+      Timeout.timeout((ENV["BUILDPACK_FETCH_TIMEOUT"] || 90).to_i) do
+        FileUtils.mkdir_p(buildpack_dir)
+        if buildpack_url =~ /^https?:\/\/.*\.(tgz|tar\.gz)($|\?)/
+          print("-----> Fetching buildpack... ")
+          retrying(3) do
+            IO.popen("tar xz -C #{buildpack_dir}", "w") do |tar|
+              IO.copy_stream(open(buildpack_url), tar)
+            end
+          end
+        elsif File.directory?(buildpack_url)
+          print("-----> Copying buildpack... ")
+          FileUtils.cp_r(buildpack_url + "/.", buildpack_dir)
+        else
+          print("-----> Cloning buildpack... ")
+          url, sha = buildpack_url.split("#")
+          clear_var("GIT_DIR") do
+            system("git", "clone", Shellwords.escape(url), buildpack_dir,
+                   [:out, :err] => "/dev/null") # or raise("Couldn't clone")
+            system("git", "checkout", Shellwords.escape(treeish),
+                   [:out, :err] => "/dev/null", :chdir => buildpack_dir) if sha
+          end
+        end
+      end
+
+      FileUtils.chmod_R(0755, File.join(buildpack_dir, "bin"))
+
+      puts("done")
+    end
+
+    buildpack_dir
+  rescue StandardError, Timeout::Error => e
+    puts("failed")
+    log_error("error fetching buildpack", e)
+    raise(CompileError, "error fetching buildpack")
+  end
+
+  def buildpack_config(buildpack_url)
+    config = {}
+    if query = (buildpack_url && URI.parse(buildpack_url).query)
+      query.split("&").each do |kv|
+        next if kv.empty?
+        k, v = kv.split("=", 2)
+        config[k] = URI.unescape(v || "")
+      end
+    end
+    config
+  end
+
+  def detect(build_dir, buildpack_dir)
+    name = `#{File.join(buildpack_dir, "bin", "detect")} #{build_dir}`.strip
+    puts("-----> #{name} app detected")
+    return name
+  rescue
+    raise(CompileFail, "no compatible app detected")
+  end
+
+  def compile(build_dir, buildpack_dir, cache_dir, config)
+    bin_compile = File.join(buildpack_dir, 'bin', 'compile')
+    timeout = (ENV["COMPILE_TIMEOUT"] || 900).to_i
+    Timeout.timeout(timeout) do
+      pid = spawn(config, bin_compile, build_dir, cache_dir,
+                  unsetenv_others: true, err: :out)
+      Process.wait(pid)
+      raise(CompileFail, "build failed") unless $?.exitstatus.zero?
+    end
+  rescue Timeout::Error
+    raise(CompileFail, "timed out; must complete in #{timeout} seconds")
+  end
+
+  def prune(build_dir)
+    FileUtils.rm_rf(File.join(build_dir, ".git"))
+    FileUtils.rm_rf(File.join(build_dir, "tmp"))
+
+    Find.find(build_dir) do |path|
+      File.delete(path) if File.basename(path) == ".DS_Store"
+    end
+
+    prune_slugignore(build_dir)
+  end
+
+  def prune_slugignore(build_dir)
+    # general pattern format follows .gitignore:
+    # http://www.kernel.org/pub/software/scm/git/docs/gitignore.html
+    # blank => nothing; leading # => comment
+    # everything else is more or less a shell glob
+    slugignore_path = File.join(build_dir, ".slugignore")
+    return if !File.exists?(slugignore_path)
+
+    log("process_slugignore") do
+      lines = File.read(slugignore_path).split
+      total = lines.inject(0) do |total, line|
+        line = (line.split(/#/).first || "").strip
+        if line.empty?
+          total
+        else
+          globs = if line =~ /\//
+                    [File.join(build_dir, line)]
+                  else
+                    # 1.8.7 and 1.9.2 handle expanding ** differently,
+                    # where in 1.9.2 ** doesn't match the empty case. So
+                    # try empty ** explicitly
+                    ["", "**"].map { |g| File.join(build_dir, g, line) }
+                  end
+
+          to_delete = Dir[*globs].uniq.map { |p| File.expand_path(p) }
+          to_delete = to_delete.select { |p| p.match(/^#{build_dir}/) }
+          to_delete.each { |p| FileUtils.rm_rf(p) }
+          total + to_delete.size
+        end
+      end
+      puts("-----> Deleting #{total} files matching .slugignore patterns.")
+    end
+  end
+
+  def parse_procfile(build_dir, output_dir)
+    path = File.join(build_dir, "Procfile")
+    return unless File.exists?(path)
+
+    process_types = File.read(path).split("\n").inject({}) do |ps, line|
+      if m = line.match(/^([a-zA-Z0-9_]+):?\s+(.*)/)
+        ps[m[1]] = m[2]
+      end
+      ps
+    end
+
+    FileUtils.mkdir_p(output_dir)
+    File.write(File.join(output_dir, "processes.json"),
+               JSON.unparse(process_types))
+    process_types
+  end
+
+  def archive(build_dir, output_dir)
+    FileUtils.mkdir_p(output_dir)
+    slug = File.join(output_dir, "slug.tgz")
+    log("create_tar_slug") do
+      system("tar", "czf", slug, "--xform", "s,^./,./app/,", "--owner=root",
+             "--hard-dereference", "-C", build_dir, ".",
+             [:out, :err] => "/dev/null") or raise("couldn't tar")
+    end
+    return slug
+  rescue => e
+    log_error("could not archive slug", e)
+    raise(CompileError, "could not archive slug")
+  end
+
+  def log_size(build_dir, cache_dir, slug)
+    log("check_sizes") do
+      raw_size = size_of_dir(build_dir)
+      cache_size = size_of_dir(cache_dir) if File.exists? cache_dir
+      slug_size = File.size(slug)
+      log("check_sizes raw=#{raw_size} slug=#{slug_size} cache=#{cache_size}")
+      puts(sprintf("-----> Compiled slug size: %1.0fK", slug_size / 1024))
+    end
+  end
+
+  # utils
+
+  def retrying(n)
+    begin
+      yield
+    rescue => e
+      n = n - 1
+      log_error("retrying", e) && retry if n > 0
+      raise e
+    end
+  end
+
+  def clear_var(k)
+    v = ENV.delete(k)
+    begin
+      yield
+    ensure
+      ENV[k] = v
+    end
+  end
+
+  def log(msg, &block)
+    if block
+      start = Time.now
+      res = nil
+      log("slugc #{msg} at=start")
+      begin
+        res = yield
+      rescue => e
+        log_error("#{msg} elapsed=#{Time.now - start}", e)
+        raise(e)
+      end
+      log("slugc #{msg} at=finish elapsed=#{Time.now - start}")
+      res
+    else
+      Syslog.log(Syslog::LOG_INFO, msg)
+    end
+  end
+
+  def log_error(line, e)
+    Syslog.log(Syslog::LOG_ERR, "slugc #{line} at=error " \
+               "class='#{e.class}' message='#{e.message}'")
+  end
+
+  def size_of_dir(dir)
+    `du -s -x #{dir}`.split(" ").first.to_i*1024
+  end
+end

data/slug-compiler.gemspec

@@ -0,0 +1,18 @@
+# -*- ruby -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Noah Zoschke", "Phil Hagelberg"]
+  gem.email         = ["phil.hagelberg@heroku.com"]
+  gem.description   = %q{Turn application source into deployable slugs}
+  gem.summary       = %q{Turn application source into deployable slugs}
+  gem.homepage      = "https://github.com/heroku/slug-compiler"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "slug-compiler"
+  gem.require_paths = ["lib"]
+  gem.version       = "2.0.0"
+
+  gem.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: slug-compiler
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Noah Zoschke
+- Phil Hagelberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Turn application source into deployable slugs
+email:
+- phil.hagelberg@heroku.com
+executables:
+- slug-compiler
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- COPYING
+- Readme.md
+- bin/slug-compiler
+- lib/slug_compiler.rb
+- slug-compiler.gemspec
+homepage: https://github.com/heroku/slug-compiler
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Turn application source into deployable slugs
+test_files: []