pkgr 0.3.4 → 1.0.1.pre

data/bin/pkgr

@@ -1,56 +1,8 @@
 #!/usr/bin/env ruby
 
-require 'optparse'
-require 'pkgr/cli'
+require "pkgr/cli"
 
-OPTIONS = {:dir => "./", :ref => "master", :reuse => false}
+STDERR.sync = true
+STDOUT.sync = true
 
-option_parser = OptionParser.new do |opts|
-  opts.banner = <<BANNER
-* Description
-  pkgr #{Pkgr::VERSION} - Package Rails apps effortlessly.
-* Usage
-  pkgr --uri GIT_REPOSITORY --config database.yml:http://path/to/database.yml --config ...
-BANNER
-
-  opts.separator ""
-  opts.separator "* Common options"
-  opts.on("--uri=", "Sets the Git repository URI (FILE, HTTP, SSH, GIT, etc.) [required]") do |v|
-    OPTIONS[:uri] = v
-  end
-  opts.on("-c=", "--config=", "Download a configuration file into the config/ folder of the app (HTTP or FILE URIs)") do |v|
-    OPTIONS[:config_files] ||= []
-    OPTIONS[:config_files] << v
-  end
-  opts.on("-b=", "--bump=", "Sets the app version [required]") do |v|
-    OPTIONS[:version] = v
-  end
-  opts.on("-n=", "--name=", "Sets the app name [optional]") do |v|
-    OPTIONS[:name] = v
-  end
-  opts.on("--ref=", "Sets the git reference to checkout [default=#{OPTIONS[:ref]}]") do |v|
-    OPTIONS[:ref] = v
-  end
-  opts.on("--host=", "Sets the build machine hostname. If none, the process will stop just before building the package.") do |v|
-    OPTIONS[:host] = v
-  end
-
-  opts.separator ""
-  opts.separator "* Other"
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-  opts.on_tail("--version", "Show version") do
-    puts Pkgr::VERSION
-    exit
-  end
-end.parse!
-
-begin
-  cli = Pkgr::CLI.new(OPTIONS)
-  cli.run
-rescue Pkgr::CLI::Error, StandardError => e
-  puts e.message
-  puts e.backtrace if $DEBUG
-end
+Pkgr::CLI.start

data/lib/pkgr.rb

@@ -1,35 +1,19 @@
-require 'pkgr/app'
-require 'pkgr/railtie' if defined?(Rails)
+require 'pkgr/version'
+require 'pkgr/cli'
+require 'pkgr/dispatcher'
+require 'mixlib/log'
 
 module Pkgr
-  DEBIAN_DIR = "debian"
+  extend Mixlib::Log
 
-  def self.setup(root)
-    setup_config(root)
+  module Errors
+    class Base < StandardError; end
+    class UnknownAppType < Base; end
+    class ConfigurationInvalid < Base; end
   end
 
-  protected
-
-  def self.setup_config(root)
-    puts "Setting up configuration file..."
-    target = File.join(root, "config", "pkgr.yml")
-    FileUtils.mkdir_p(File.dirname(target))
-    if File.exist?(target)
-      puts "'#{target}' already exists. Skipped."
-    else
-      FileUtils.cp(File.expand_path("../pkgr/data/pkgr.yml", __FILE__), target, :verbose => true)
-      puts "Edit '#{target}' and fill in the required information, then enter 'rake pkgr:generate' to generate the debian files."
-    end
-  end
-
-  
-  def self.mkdir(target)
-    if File.directory?(target)
-      puts "#{target} directory already exists. Skipped."
-    elsif File.file?(target)
-      raise "#{target} already exists and is a file. Aborting."
-    else
-      FileUtils.mkdir_p target, :verbose => true
-    end
+  def data_dir
+    File.expand_path("../pkgr/data", __FILE__)
   end
-end
+  module_function :data_dir
+end

data/lib/pkgr/app.rb

@@ -22,6 +22,7 @@ module Pkgr
       @config = YAML::load_file(path)
       raise ArgumentError, "The given configuration file at '#{path}' is not a well-formed YAML file. Please fix it or remove it and run 'rake pkgr:setup'" unless @config.kind_of?(Hash)
       @config['_path'] = path
+      normalize_name!
     end
     
     def write_config
@@ -215,8 +216,8 @@ module Pkgr
             #{debian_steps.join(" &&\n")}'
         }
         sh cmd
-        # Fetch the .deb, and put it in the `pkg` directory
-        sh "scp #{host}:/tmp/#{name}_#{version}*.deb pkg/"
+        # Fetch all package files, and put it in the `pkg` directory
+        sh "scp #{host}:/tmp/#{name}_#{version}* pkg/"
       end
     end
 
@@ -252,7 +253,7 @@ module Pkgr
 
     private
     def bundler_version
-      @config.fetch('bundler_version') { '1.1.3' }
+      @config.fetch('bundler_version') { '1.3.5' }
     end
 
     def debian_file(filename)
@@ -260,5 +261,19 @@ module Pkgr
       raise "The debian/changelog file does not exist. Please generate it first." unless File.exist?(file)
       file
     end
+
+    def normalize_name!
+      # debian packages can not contain capitalized letters nor underscores
+      if (raw_name = @config.fetch('name', '')) =~ /[A-Z_-]/
+        normalized_name = raw_name.
+                            gsub(/[A-Z]/) { |m| "_#{m.downcase}" }.  # underscore each word
+                            gsub(/[_-]+/, '-').                      # normalize underscores/dashes
+                            sub(/^-/, '')                            # strip leading dash
+        puts "Normalized application name to %s (original: %s)." % [
+          normalized_name.inspect, raw_name.inspect
+        ] unless normalized_name == raw_name
+        @config['name'] = normalized_name
+      end
+    end
   end
-end
+end

data/lib/pkgr/builder.rb

@@ -0,0 +1,207 @@
+require 'tempfile'
+require 'fileutils'
+require 'pkgr/config'
+require 'pkgr/distributions'
+require 'pkgr/process'
+
+module Pkgr
+  class Builder
+    attr_reader :tarball, :config
+
+    # Accepts a path to a tarball (gzipped or not), or you can pass '-' to read from stdin.
+    def initialize(tarball, config)
+      @tarball = tarball
+      @config = config
+      Pkgr.debug "Initializing builder with the following config: #{config.inspect}"
+    end
+
+    # Launch the full packaging procedure
+    def call
+      check
+      setup
+      extract
+      compile
+      write_env
+      write_init
+      package
+    end
+
+    # Check configuration, and verifies that the current distribution's requirements are satisfied
+    def check
+      raise Errors::ConfigurationInvalid, config.errors.join("; ") unless config.valid?
+      distribution.check(config)
+    end
+
+    # Setup the build directory structure
+    def setup
+      Dir.chdir(build_dir) do
+        distribution.templates(config.name).each do |template|
+          template.install(config.sesame)
+        end
+      end
+    end
+
+    # Extract the given tarball to the target directory
+    def extract
+      raise "#{source_dir} does not exist" unless File.directory?(source_dir)
+
+      opts = {}
+      if tarball == "-"
+        # FIXME: not really happy with reading everything in memory
+        opts[:input] = $stdin.read
+      end
+
+      tarball_extract = Mixlib::ShellOut.new("tar xzf #{tarball} -C #{source_dir}", opts)
+      tarball_extract.run_command
+      tarball_extract.error!
+    end
+
+    # Pass the app through the buildpack
+    def compile
+      if buildpack_for_app
+        puts "-----> #{buildpack_for_app.banner} app"
+
+        FileUtils.mkdir_p(compile_cache_dir)
+
+        run_hook config.before_precompile
+        buildpack_for_app.compile(source_dir, compile_cache_dir)
+        buildpack_for_app.release(source_dir, compile_cache_dir)
+      else
+        raise Errors::UnknownAppType, "Can't find a buildpack for your app"
+      end
+    end
+
+    # Parses the output of buildpack/bin/release executable to find out its default Procfile commands.
+    # Then merges those with the ones from the app's Procfile (if any).
+    # Finally, generates a binstub in vendor/pkgr/processes/ so that these commands can be called using the app's executable.
+    def write_env
+      FileUtils.mkdir_p proc_dir
+
+      procfile_entries.each do |process|
+        process_file = File.join(proc_dir, process.name)
+
+        File.open(process_file, "w+") do |f|
+          f << process.command
+          f << " $@"
+        end
+
+        FileUtils.chmod 0755, process_file
+      end
+    end
+
+    # Write startup scripts.
+    def write_init
+      FileUtils.mkdir_p scaling_dir
+      Dir.chdir(scaling_dir) do
+        distribution.initializers_for(config.name, procfile_entries).each do |(process, file)|
+          process_config = config.dup
+          process_config.process_name = process.name
+          process_config.process_command = process.command
+          file.install(process_config.sesame)
+        end
+      end
+    end
+
+    # Launch the FPM command that will generate the package.
+    def package
+      Pkgr.info "Running command: #{fpm_command}"
+      app_package = Mixlib::ShellOut.new(fpm_command)
+      app_package.run_command
+      app_package.error!
+    end
+
+    # Make sure to get rid of the build directory
+    def teardown
+      FileUtils.rm_rf(build_dir)
+    end
+
+    def procfile_entries
+      @procfile_entries ||= begin
+        default_process_types = YAML.load_file(release_file)["default_process_types"]
+
+        entries = if File.exist?(procfile)
+          File.read(procfile).gsub("\r\n","\n").split("\n").map do |line|
+            if line =~ /^([A-Za-z0-9_]+):\s*(.+)$/
+              [$1, $2]
+            end
+          end.compact
+        else
+          []
+        end
+
+        default_process_types.merge(Hash[entries]).map{|name, command| Process.new(name, command)}
+      end
+    end
+
+    # Path to the release file generated after the buildpack compilation.
+    def release_file
+      File.join(source_dir, ".release")
+    end
+
+    # Path to the directory containing the main app files.
+    def source_dir
+      File.join(build_dir, "opt/#{config.name}")
+    end
+
+    # Build directory. Will be used by fpm to make the package.
+    def build_dir
+      @build_dir ||= Dir.mktmpdir
+    end
+
+    def vendor_dir
+      File.join(source_dir, "vendor", "pkgr")
+    end
+
+    # Directory where binstubs will be created for the corresponding Procfile commands.
+    def proc_dir
+      File.join(vendor_dir, "processes")
+    end
+
+    def scaling_dir
+      File.join(vendor_dir, "scaling")
+    end
+
+    # Returns the path to the app's (supposedly present) Procfile.
+    def procfile
+      File.join(source_dir, "Procfile")
+    end
+
+    # Directory where the buildpacks can store stuff.
+    def compile_cache_dir
+      config.compile_cache_dir || File.join(source_dir, ".git/cache")
+    end
+
+    # Returns the current distribution we're packaging for.
+    def distribution
+      @distribution ||= Distributions.current
+    end
+
+    # List of available buildpacks for the current distribution.
+    def buildpacks
+      distribution.buildpacks
+    end
+
+    # Buildpack detected for the app, if any.
+    def buildpack_for_app
+      raise "#{source_dir} does not exist" unless File.directory?(source_dir)
+      @buildpack_for_app ||= buildpacks.find do |buildpack|
+        buildpack.setup
+        buildpack.detect(source_dir)
+      end
+    end
+
+    def fpm_command
+      distribution.fpm_command(build_dir, config)
+    end
+
+    protected
+    def run_hook(file)
+      return true if file.nil?
+      Dir.chdir(source_dir) do
+        app_package = Mixlib::ShellOut.new("bash '#{file}'")
+        app_package.run_command
+        app_package.error!
+      end
+    end
+  end
+end

data/lib/pkgr/buildpack.rb

@@ -0,0 +1,80 @@
+require 'fileutils'
+
+module Pkgr
+  class Buildpack
+    class << self
+      attr_writer :buildpack_cache_dir
+
+      def buildpack_cache_dir
+        @buildpack_cache_dir ||= File.expand_path("~/.pkgr/buildpacks").tap do |dir|
+          FileUtils.mkdir_p(dir)
+        end
+      end
+    end
+
+    attr_reader :url, :banner
+
+    def initialize(url)
+      @url = url
+    end
+
+    def buildpack_cache_dir
+      self.class.buildpack_cache_dir
+    end
+
+    def detect(path)
+      buildpack_detect = Mixlib::ShellOut.new("#{dir}/bin/detect \"#{path}\"")
+      buildpack_detect.run_command
+      @banner = buildpack_detect.stdout.chomp
+      buildpack_detect.exitstatus == 0
+    end
+
+    def compile(path, compile_cache_dir)
+      Dir.chdir(path) do
+        IO.popen(%{ env -i PATH="$PATH" #{dir}/bin/compile "#{path}" "#{compile_cache_dir}" }) do |io|
+          until io.eof?
+            data = io.gets
+            print data
+          end
+        end
+        raise "compile failed" unless $?.exitstatus.zero?
+      end
+
+      true
+    end
+
+    def release(path, compile_cache_dir)
+      buildpack_release = Mixlib::ShellOut.new("#{dir}/bin/release \"#{path}\" \"#{compile_cache_dir}\" > #{path}/.release")
+      buildpack_release.run_command
+      buildpack_release.exitstatus == 0
+    end
+
+    def dir
+      File.join(buildpack_cache_dir, File.basename(url, ".git"))
+    end
+
+    def exists?
+      File.directory?(dir)
+    end
+
+    def setup
+      exists? ? refresh : install
+    end
+
+    def refresh
+      Dir.chdir(dir) do
+        buildpack_refresh = Mixlib::ShellOut.new("git fetch origin && git reset --hard origin/master")
+        buildpack_refresh.run_command
+        buildpack_refresh.error!
+      end
+    end
+
+    def install
+      Dir.chdir(buildpack_cache_dir) do
+        buildpack_install = Mixlib::ShellOut.new("git clone \"#{url}\"")
+        buildpack_install.run_command
+        buildpack_install.error!
+      end
+    end
+  end
+end

data/lib/pkgr/cli.rb

@@ -1,164 +1,40 @@
-require 'open-uri'
-require 'fileutils'
+require "thor"
 require 'pkgr'
-require 'uri'
 
 module Pkgr
-  class CLI
-    include Rake::DSL
-
-    class Error < StandardError; end
-
-    attr_reader :errors
-    attr_reader :uri
-    attr_reader :dir
-    attr_reader :config_files
-    attr_reader :version
-    attr_reader :name
-    attr_reader :app
-    attr_reader :host
-    attr_reader :ref
-
-    def initialize(opts = {})
-      @errors = []
-      @uri, @config_files, @version, @name, @host, @ref = opts.values_at(
-        :uri, :config_files, :version, :name, :host, :ref
-      )
-      @app = nil
-    end
-
-    def run
-      raise Error, "Can't run pkgr: #{errors.join(", ")}" unless valid?
-      clone_repository
-      Dir.chdir(dir) do
-        checkout
-        copy_remote_config_files
-        copy_example_config_files
-        setup
-        bundle
-        configure_app
-        generate
-        bump
-        build
-      end
-    end
-
-    def valid?
-      @errors.clear
-      @errors.push("You must pass a repository URI through --uri") if uri.nil?
-      @errors.push("You must pass a version number through --bump") if version.nil?
-      @errors.empty?
-    end
-
-    def build
-      if host.nil?
-        puts "Can't build the package. You must pass the --host option for this."
-      else
-        @app.build_debian_package(host)
-      end
-    end
-
-    def bump
-      @app.bump!(:custom, version)
-    end
-
-    def bundle
-      sh "bundle install"
-      sh "git add -f Gemfile.lock"
-      sh "if git status --porcelain | grep Gemfile.lock; then git commit -m '[pkgr] Update Gemfile.lock.'; fi"
-    end
-
-    def checkout
-      sh "if git branch | grep '#{pkgr_branch}'; then git checkout #{pkgr_branch}; else git checkout -b #{pkgr_branch} #{ref}; fi"
-    end
-
-    def clone_repository
-      parsed_uri = URI.parse(uri)
-      case parsed_uri.scheme
-      when nil, "file"
-        @dir = parsed_uri.path
-      else
-        @dir = File.basename(uri, ".git")
-        sh "git clone #{uri}"
-      end
-      @dir = File.expand_path @dir
-    end
-
-    def configure_app
-      @app = Pkgr::App.new(dir, "config/pkgr.yml")
-      @app.config['git_ref'] = pkgr_branch
-      @app.config['config_files'].push(*Dir["config/*.yml"].map{|f| File.basename(f)}).uniq!
-      if name.nil?
-        @app.config['name'] = File.basename(dir) if @app.name.nil?
-      else
-        @app.config['name'] = name
-      end
-      raise Error, "The app is not correctly configured: #{@app.errors.join(", ")}" unless @app.valid?
-      @app.write_config
-    end
-
-    # Download the given config files
-    def copy_remote_config_files
-      (config_files || []).each do |file|
-        filename, file_uri = file.split("::")
-        if file_uri.nil?
-          file_uri = filename
-          filename = File.basename(file_uri)
-        end
-
-        file_uri = File.expand_path(file_uri) if URI.parse(file_uri).scheme.nil?
-        target = "config/#{filename}"
-        puts "Copying #{file_uri} into #{target}..."
-        File.open(target, "w+") { |f| f << open(file_uri).read }
-      end
-    end
-
-    def copy_example_config_files
-      [".example", ".dist"].each do |pattern|
-        Dir["config/*.yml#{pattern}"].each do |file|
-          target = File.basename(file, pattern)
-          unless File.exist?("config/#{target}")
-            FileUtils.cp(file, "config/#{target}")
-          end
-        end
-      end
-    end
-
-    def generate
-      @app.generate_required_files
-      sh "git add debian/"
-      sh "if git status --porcelain | grep debian/; then git commit -m '[pkgr] Add debian files.'; fi"
-      sh "git add bin/"
-      sh "if git status --porcelain | grep bin/; then git commit -m '[pkgr] Add executable file.'; fi"
-    end
-
-    def pkgr_branch
-      "pkgr-#{ref}"
-    end
-
-    def setup
-      Pkgr.setup(dir)
-
-      gemfile = File.read("Gemfile")
-      unless gemfile =~ /^gem 'pkgr'/
-        File.open("Gemfile", "a") do |f|
-          f.puts
-          f.puts "gem 'pkgr'"
-        end
-      end
-
-      unless gemfile =~ /^gem 'thin'/
-        File.open("Gemfile", "a") do |f|
-          f.puts
-          f.puts "gem 'thin'"
-        end
-      end
-
-      sh "git add Gemfile"
-      sh" if git status --porcelain | grep Gemfile; then git commit -m '[pkgr] Update Gemfile.'; fi"
-      sh "git add -f config/*.yml"
-      sh" if git status --porcelain | grep config/*.yml; then git commit -m '[pkgr] Update configuration files.'; fi"
+  class CLI < Thor
+    class_option :verbose,    :type => :boolean, :default => false, :desc => "Run verbosely"
+    class_option :debug,      :type => :boolean, :default => false, :desc => "Run very verbosely"
+    class_option :name,       :type => :string, :desc => "Application name (if directory given, it will default to the directory name)"
+
+    desc "package TARBALL", "Package the given tarball or directory"
+
+    method_option :target,              :type => :string, :default => "deb", :desc => "Target package to build (only 'deb' supported for now)"
+    method_option :changelog,           :type => :string, :desc => "Changelog"
+    method_option :architecture,        :type => :string, :default => "x86_64", :desc => "Target architecture for the package"
+    method_option :homepage,            :type => :string, :desc => "Project homepage"
+    method_option :description,         :type => :string, :desc => "Project description"
+    method_option :version,             :type => :string, :desc => "Package version (if git directory given, it will use the latest git tag available)"
+    method_option :iteration,           :type => :string, :default => Time.now.strftime("%Y%m%d%H%M%S"), :desc => "Package iteration (you should keep the default here)"
+    method_option :user,                :type => :string, :desc => "User to run the app under (defaults to your app name)"
+    method_option :group,               :type => :string, :desc => "Group to run the app under (defaults to your app name)"
+    method_option :compile_cache_dir,   :type => :string, :desc => "Where to store the files cached between packaging runs"
+    method_option :dependencies,        :type => :array,  :default => [], :desc => "Specific system dependencies that you want to install with the package"
+    method_option :build_dependencies,  :type => :array,  :default => [], :desc => "Specific system dependencies that must be present before building"
+    method_option :before_precompile,   :type => :string, :desc => "Provide a script to run just before the buildpack compilation"
+    method_option :host,                :type => :string, :desc => "Remote host to build on (default: local machine)"
+    method_option :auto,                :type => :boolean, :default => false, :desc => "Automatically attempt to install missing dependencies"
+
+    def package(tarball)
+      Pkgr.level = Logger::INFO if options[:verbose]
+      Pkgr.level = Logger::DEBUG if options[:debug]
+
+      packager = Dispatcher.new(tarball, options)
+      packager.call
+    rescue Pkgr::Errors::Base => e
+      Pkgr.error "#{e.class.name} : #{e.message}"
+      puts "* ERROR: #{e.message}"
+      exit 1
     end
   end
-
 end