omnibus 1.0.0

data/lib/omnibus/builder.rb

@@ -0,0 +1,323 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'forwardable'
+
+module Omnibus
+  class Builder
+
+    # Proxies method calls to either a Builder object or the Software that the
+    # builder belongs to. Provides compatibility with our DSL where we never
+    # yield objects to blocks and hopefully hides some of the confusion that
+    # can arise from instance_eval.
+    class DSLProxy
+      extend Forwardable
+
+      # @todo def_delegators :@builder, :patch, :command, :ruby, ...
+
+      def_delegator :@builder, :patch
+      def_delegator :@builder, :command
+      def_delegator :@builder, :ruby
+      def_delegator :@builder, :gem
+      def_delegator :@builder, :bundle
+      def_delegator :@builder, :rake
+      def_delegator :@builder, :block
+      def_delegator :@builder, :name
+
+      def initialize(builder, software)
+        @builder, @software = builder, software
+      end
+
+      def eval_block(&block)
+        instance_eval(&block)
+      end
+
+      def respond_to?(method)
+        super || @software.respond_to?(method)
+      end
+
+      def methods
+        super | @software.methods
+      end
+
+      def method_missing(method_name, *args, &block)
+        if @software.respond_to?(method_name)
+          @software.send(method_name, *args, &block)
+        else
+          super
+        end
+      end
+
+    end
+
+
+    # @todo code duplication with {Fetcher::ErrorReporter}
+    class ErrorReporter
+
+      # @todo fetcher isn't even used
+      def initialize(error, fetcher)
+        @error, @fetcher = error, fetcher
+      end
+
+      # @todo this isn't necessary
+      def e
+        @error
+      end
+
+      def explain(why)
+        $stderr.puts "* " * 40
+        $stderr.puts why
+        $stderr.puts "Exception:"
+        $stderr.puts indent("#{e.class}: #{e.message.strip}", 2)
+        Array(e.backtrace).each {|l| $stderr.puts indent(l, 4) }
+        $stderr.puts "* " * 40
+      end
+
+      private
+
+      def indent(string, n)
+        string.split("\n").map {|l| " ".rjust(n) << l }.join("\n")
+      end
+
+    end
+
+    # @todo Look at using Bundler.with_clean_env{ ... } instead
+    BUNDLER_BUSTER = {  "RUBYOPT"         => nil,
+                        "BUNDLE_BIN_PATH" => nil,
+                        "BUNDLE_GEMFILE"  => nil,
+                        "GEM_PATH"        => nil,
+                        "GEM_HOME"        => nil }
+
+    attr_reader :build_commands
+
+    def initialize(software, &block)
+      @software = software
+      @build_commands = []
+      @dsl_proxy = DSLProxy.new(self, software)
+      @dsl_proxy.eval_block(&block) if block_given?
+    end
+
+    def name
+      @software.name
+    end
+
+    def command(*args)
+      @build_commands << args
+    end
+
+    def patch(*args)
+      args = args.dup.pop
+
+      # we'll search for a patch file in the project root AND
+      # the omnibus-software gem
+      candidate_roots = [Omnibus.project_root]
+      candidate_roots << Omnibus.omnibus_software_root if Omnibus.omnibus_software_root
+
+      candidate_paths = candidate_roots.map do |root|
+        File.expand_path("#{root}/config/patches/#{name}/#{args[:source]}")
+      end
+
+      source = candidate_paths.find{|path| File.exists?(path) }
+
+      plevel = args[:plevel] || 1
+      if args[:target] 
+        target = File.expand_path("#{project_dir}/#{args[:target]}")
+        @build_commands << 
+         "cat #{source} | patch -p#{plevel} #{target}"
+      else
+        @build_commands << 
+         "patch -d #{project_dir} -p#{plevel} -i #{source}"
+      end
+    end
+
+    # @todo all these ruby commands (ruby, gem, bundle, rake) could
+    #   all be collapsed into a single underlying implementation, since
+    #   they all just differ on the executable being called
+    def ruby(*args)
+      @build_commands << bundle_bust(*prepend_cmd("#{install_dir}/embedded/bin/ruby", *args))
+    end
+
+    def gem(*args)
+      @build_commands << bundle_bust(*prepend_cmd("#{install_dir}/embedded/bin/gem", *args))
+    end
+
+    def bundle(*args)
+      @build_commands << bundle_bust(*prepend_cmd("#{install_dir}/embedded/bin/bundle", *args))
+    end
+
+    def rake(*args)
+      @build_commands << bundle_bust(*prepend_cmd("#{install_dir}/embedded/bin/rake", *args))
+    end
+
+    def block(&rb_block)
+      @build_commands << rb_block
+    end
+
+    def project_dir
+      @software.project_dir
+    end
+
+    def install_dir
+      @software.install_dir
+    end
+
+    def log(message)
+      puts "[builder:#{name}] #{message}"
+    end
+
+    def build
+      log "building #{name}"
+      time_it("#{name} build") do
+        @build_commands.each do |cmd|
+          execute(cmd)
+        end
+      end
+    end
+
+    def execute(cmd)
+      case cmd
+      when Proc
+        execute_proc(cmd)
+      else
+        execute_sh(cmd)
+      end
+    end
+
+    private
+
+    def execute_proc(cmd)
+      cmd.call
+    rescue Exception => e
+      # In Ruby 1.9, Procs have a #source_location method with file/line info.
+      # Too bad we can't use it :(
+      ErrorReporter.new(e, self).explain("Failed to build #{name} while running ruby block build step")
+      raise
+    end
+
+    def execute_sh(cmd)
+      retries ||= 0
+      shell = nil
+      cmd_args = Array(cmd)
+      options = {
+        :cwd => project_dir,
+        :timeout => 5400
+      }
+      options[:live_stream] = STDOUT if ENV['DEBUG']
+      if cmd_args.last.is_a? Hash
+        cmd_options = cmd_args.last
+        cmd_args[cmd_args.size - 1] = options.merge(cmd_options)
+      else
+        cmd_args << options
+      end
+
+      cmd_string = cmd_args[0..-2].join(' ')
+      cmd_opts_for_display = to_kv_str(cmd_args.last)
+
+      log "Executing: `#{cmd_string}` with #{cmd_opts_for_display}"
+
+      shell = Mixlib::ShellOut.new(*cmd)
+      shell.environment["HOME"] = "/tmp" unless ENV["HOME"]
+
+      cmd_name = cmd_string.split(/\s+/).first
+      time_it("#{cmd_name} command") do
+        shell.run_command
+        shell.error!
+      end
+    rescue Exception => e
+      # Getting lots of errors from github, particularly with erlang/rebar
+      # projects fetching tons of deps via git all the time. This isn't a
+      # particularly elegant way to solve that problem. But it should work.
+      if retries >= 3
+        ErrorReporter.new(e, self).explain("Failed to build #{name} while running `#{cmd_string}` with #{cmd_opts_for_display}")
+        raise
+      else
+        time_to_sleep = 5 * (2 ** retries)
+        retries +=1
+        log "Failed to execute cmd #{cmd} #{retries} time(s). Retrying in #{time_to_sleep}s."
+        sleep(time_to_sleep)
+        retry
+      end
+    end
+
+    def prepend_cmd(str, *cmd_args)
+      if cmd_args.size == 1
+        # command as a string, no opts
+        "#{str} #{cmd_args.first}"
+      elsif cmd_args.size == 2 && cmd_args.last.is_a?(Hash)
+        # command as a string w/ opts
+        ["#{str} #{cmd_args.first}", cmd_args.last]
+      elsif cmd_args.size == 0
+        raise ArgumentError, "I don't even"
+      else
+        # cmd given as argv array
+        cmd_args.dup.unshift(str)
+      end
+    end
+
+    def bundle_bust(*cmd_args)
+      if cmd_args.last.is_a?(Hash)
+        cmd_args = cmd_args.dup
+        cmd_opts = cmd_args.pop.dup
+        cmd_opts[:env] = cmd_opts[:env] ? BUNDLER_BUSTER.merge(cmd_opts[:env]) : BUNDLER_BUSTER
+        cmd_args << cmd_opts
+      else
+        cmd_args << {:env => BUNDLER_BUSTER}
+      end
+    end
+
+
+    def time_it(what)
+      start = Time.now
+      yield
+    rescue Exception
+      elapsed = Time.now - start
+      log "#{what} failed, #{elapsed.to_f}s"
+      raise
+    else
+      elapsed = Time.now - start
+      log "#{what} succeeded, #{elapsed.to_f}s"
+    end
+
+    # Convert a hash to a string in the form `key=value`. It should work with
+    # whatever input is given but is designed to make the options to ShellOut
+    # look nice.
+    def to_kv_str(hash, join_str=",")
+      hash.inject([]) do |kv_pair_strs, (k,v)|
+        val_str = case v
+        when Hash
+          %Q["#{to_kv_str(v, " ")}"]
+        else
+          v.to_s
+        end
+        kv_pair_strs << "#{k}=#{val_str}"
+      end.join(join_str)
+    end
+
+  end
+
+  # @todo What's the point of this class?  Can we not just detect that
+  #   there are no commands in {Omnibus::Builder#build} and output the
+  #   appropriate message?  Seems like a lot of extra ceremony.
+  class NullBuilder < Builder
+
+    def build
+      log "Nothing to build for #{name}"
+    end
+
+  end
+
+end

data/lib/omnibus/clean_tasks.rb

@@ -0,0 +1,30 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Adds source and build artifacts to Rake's CLEAN list, and install
+# and cache artifacts to its CLOBBER list, ensuring that the `clean`
+# and `clobber` Rake tasks do the right thing for Omnibus projects.
+#
+# @see https://github.com/jimweirich/rake/blob/master/lib/rake/clean.rb
+
+require 'rake/clean'
+
+::CLEAN.include("#{Omnibus.config.source_dir}/**/*",
+                "#{Omnibus.config.build_dir}/**/*")
+
+::CLOBBER.include("#{Omnibus.config.cache_dir}/**/*",
+                  "#{Omnibus.config.package_dir}/**/*")

data/lib/omnibus/cli.rb

@@ -0,0 +1,35 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'omnibus/cli/application'
+require 'omnibus/cli/base'
+require 'omnibus/cli/build'
+
+module Omnibus
+  module CLI
+
+    class Error < StandardError
+      attr_reader :original
+
+      def initialize(msg, original=nil)
+        super(msg)
+        @original = original
+      end
+    end
+
+  end
+end

data/lib/omnibus/cli/application.rb

@@ -0,0 +1,136 @@
+#
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'omnibus'
+require 'omnibus/cli/base'
+require 'omnibus/cli/build'
+require 'omnibus/cli/cache'
+
+module Omnibus
+  module CLI
+    class Application < Base
+
+      method_option :purge,
+        :type => :boolean,
+        :default => false,
+        :desc => "Remove ALL files generated during the build (including packages)."
+      method_option :path,
+        :aliases => [:p],
+        :type => :string,
+        :default => Dir.pwd,
+        :desc => "Path to the Omnibus project root."
+      desc "clean PROJECT", "Remove temporary files generated during the build process."
+      def clean(project_name)
+        project = load_project!(project_name)
+
+        deletion_list = []
+        deletion_list << Dir.glob("#{Omnibus.config.source_dir}/**/*")
+        deletion_list << Dir.glob("#{Omnibus.config.build_dir}/**/*")
+
+        if options[:purge]
+          deletion_list << Dir.glob("#{Omnibus.config.package_dir}/**/*")
+          deletion_list << Dir.glob("#{Omnibus.config.cache_dir}/**/*")
+          deletion_list << project.install_path if File.exist?(project.install_path)
+        end
+
+        deletion_list.flatten!
+        deletion_list.each{|f| remove_file(f) }
+      end
+
+      desc "project PROJECT", "Creates a skeletal Omnibus project"
+      def project(name)
+        name = name.chomp("/") # remove trailing slash if present
+        target = File.join(Dir.pwd, "omnibus-#{name}")
+        install_path = File.join("/opt", name)
+        opts = {
+          :name => name,
+          :install_path => install_path
+        }
+
+        # core project files
+        template(File.join("Gemfile.erb"), File.join(target, "Gemfile"), opts)
+        template(File.join("gitignore.erb"), File.join(target, ".gitignore"), opts)
+        template(File.join("README.md.erb"), File.join(target, "README.md"), opts)
+        template(File.join("omnibus.rb.example.erb"), File.join(target, "omnibus.rb.example"), opts)
+
+        # project definition
+        template(File.join("project.rb.erb"), File.join(target, "config", "projects", "#{name}.rb"), opts)
+
+        # example software definitions
+        config_software = File.join(target, "config", "software")
+        template(File.join("software", "c-example.rb.erb"),
+                 File.join(config_software, "c-example.rb"), opts)
+        template(File.join("software", "erlang-example.rb.erb"),
+                 File.join(config_software, "erlang-example.rb"), opts)
+        template(File.join("software", "ruby-example.rb.erb"),
+                 File.join(config_software, "ruby-example.rb"), opts)
+
+        # Vagrant build lab
+        template(File.join("Berksfile.erb"), File.join(target, "Berksfile"), opts)
+        template(File.join("Vagrantfile.erb"), File.join(target, "Vagrantfile"), opts)
+
+        # render out stub packge scripts
+        %w{ makeselfinst postinst postrm }.each do |package_script|
+          script_path = File.join(target, "package-scripts", name, package_script)
+          template_path = File.join("package_scripts", "#{package_script}.erb")
+          # render the package script
+          template(template_path, script_path, opts)
+          # ensure the package script is executable
+          FileUtils.chmod(0755, script_path)
+        end
+      end
+
+      desc "version", "Display version information"
+      def version
+        say("Omnibus: #{Omnibus::VERSION}", :yellow)
+      end
+
+      ###########################################################################
+      # Subcommands
+      ###########################################################################
+
+      desc "build [COMMAND]", "Perform build-related tasks"
+      subcommand "build", Omnibus::CLI::Build
+
+      desc "cache [COMMAND]", "Perform cache management tasks"
+      subcommand "cache", Omnibus::CLI::Cache
+
+      ###########################################################################
+      # Class Methods
+      ###########################################################################
+
+      # Override start so we can catch and process any exceptions bubbling up
+      def self.start(*args)
+        begin
+          super
+        rescue => e
+          error_msg = "Something went wrong...the Omnibus just ran off the road!"
+          error_msg << "\n\nError raised was:\n\n\t#{$!}"
+          error_msg << "\n\nBacktrace:\n\n\t#{e.backtrace.join("\n\t")}"
+          if e.respond_to?(:original) && e.original
+            error_msg << "\n\nOriginal Error:\n\n\t#{e.original}"
+            error_msg << "\n\nOriginal Backtrace:\n\n\t#{e.original.backtrace.join("\n\t")}"
+          end
+          # TODO - we need a proper UI class
+          Thor::Base.shell.new.say(error_msg, :red)
+          exit 1
+        end
+      end
+
+    end
+  end
+end