vanagon 0.3.18

data/lib/vanagon/driver.rb

@@ -0,0 +1,112 @@
+require 'vanagon/project'
+require 'vanagon/platform'
+require 'vanagon/component'
+require 'vanagon/utilities'
+require 'vanagon/common'
+require 'vanagon/errors'
+require 'tmpdir'
+require 'logger'
+
+class Vanagon
+  class Driver
+    include Vanagon::Utilities
+    attr_accessor :platform, :project, :target, :workdir, :verbose, :preserve
+
+    def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil })
+      @verbose = false
+      @preserve = false
+
+      @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
+      components = options[:components] || []
+      target = options[:target]
+      engine = options[:engine] || 'pooler'
+
+      @platform = Vanagon::Platform.load_platform(platform, File.join(@@configdir, "platforms"))
+      @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
+
+      # If a target has been given, we don't want to make any assumptions about how to tear it down.
+      engine = 'base' if target
+      require "vanagon/engine/#{engine}"
+      @engine = Object.const_get("Vanagon::Engine::#{engine.capitalize}").new(@platform, target)
+
+      @@logger = Logger.new('vanagon_hosts.log')
+      @@logger.progname = 'vanagon'
+    rescue LoadError => e
+      raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{@engine_name}'.")
+    end
+
+    def cleanup_workdir
+      FileUtils.rm_rf(@workdir)
+    end
+
+    def self.configdir
+      @@configdir
+    end
+
+    def self.logger
+      @@logger
+    end
+
+    # Returns the set difference between the build_requires and the components to get a list of external dependencies that need to be installed.
+    def list_build_dependencies
+      @project.components.map(&:build_requires).flatten.uniq - @project.components.map(&:name)
+    end
+
+    def install_build_dependencies
+      unless list_build_dependencies.empty?
+        if @platform.build_dependencies && @platform.build_dependencies.command && !@platform.build_dependencies.command.empty?
+          @engine.dispatch("#{@platform.build_dependencies.command} #{list_build_dependencies.join(' ')} #{@platform.build_dependencies.suffix}")
+        elsif @platform.respond_to?(:install_build_dependencies)
+          @engine.dispatch(@platform.install_build_dependencies(list_build_dependencies))
+        else
+          raise Vanagon::Error, "No method defined to install build dependencies for #{@platform.name}"
+        end
+      end
+    end
+
+    def run
+      # Simple sanity check for the project
+      if @project.version.nil? or @project.version.empty?
+        raise Vanagon::Error, "Project requires a version set, all is lost."
+      end
+      @workdir = Dir.mktmpdir
+      @engine.startup(@workdir)
+
+      puts "Target is #{@engine.target}"
+
+      install_build_dependencies
+      @project.fetch_sources(@workdir)
+      @project.make_makefile(@workdir)
+      @project.make_bill_of_materials(@workdir)
+      @project.generate_packaging_artifacts(@workdir)
+      @engine.ship_workdir(@workdir)
+      @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make})")
+      @engine.retrieve_built_artifact
+      @engine.teardown unless @preserve
+      cleanup_workdir unless @preserve
+    rescue => e
+      puts e
+      puts e.backtrace.join("\n")
+      raise e
+    end
+
+    def prepare(workdir = nil)
+      @workdir = workdir ? FileUtils.mkdir_p(workdir).first : Dir.mktmpdir
+      @engine.startup(@workdir)
+
+      puts "Devkit on #{@engine.target}"
+
+      install_build_dependencies
+      @project.fetch_sources(@workdir)
+      @project.make_makefile(@workdir)
+      @project.make_bill_of_materials(@workdir)
+      # Builds only the project, skipping packaging into an artifact.
+      @engine.ship_workdir(@workdir)
+      @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make} #{@project.name}-project)")
+    rescue => e
+      puts e
+      puts e.backtrace.join("\n")
+      raise e
+    end
+  end
+end

data/lib/vanagon/engine/base.rb

@@ -0,0 +1,82 @@
+require 'vanagon/utilities'
+require 'vanagon/errors'
+
+class Vanagon
+  class Engine
+    class Base
+      attr_accessor :target, :remote_workdir
+
+      def initialize(platform, target = nil)
+        @platform = platform
+        @required_attributes = ["ssh_port"]
+        @target = target if target
+        @target_user = "root"
+      end
+
+      # This method is used to obtain a vm to build upon
+      # For the base class we just return the target that was passed in
+      def select_target
+        @target or raise Vanagon::Error, '#select_target has not been implemented for your engine.'
+      end
+
+      # Dispatches the command for execution
+      def dispatch(command, return_output = false)
+        Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", command, @platform.ssh_port, return_command_output: return_output)
+      end
+
+      # Steps needed to tear down or clean up the system after the build is
+      # complete
+      def teardown
+      end
+
+      # Applies the steps needed to extend the system to build packages against
+      # the target system
+      def setup
+        unless @platform.provisioning.empty?
+          script = @platform.provisioning.join(' && ')
+          dispatch(script)
+        end
+      end
+
+      # This method will take care of validation and target selection all at
+      # once as an easy shorthand to call from the driver
+      def startup(workdir)
+        validate_platform
+        select_target
+        setup
+        get_remote_workdir
+      end
+
+      def get_remote_workdir
+        @remote_workdir ||= dispatch("mktemp -d -p /var/tmp 2>/dev/null || mktemp -d -t 'tmp'", true)
+      end
+
+      def ship_workdir(workdir)
+        Vanagon::Utilities.rsync_to("#{workdir}/*", "#{@target_user}@#{@target}", @remote_workdir, @platform.ssh_port)
+      end
+
+      def retrieve_built_artifact
+        FileUtils.mkdir_p("output")
+        Vanagon::Utilities.rsync_from("#{@remote_workdir}/output/*", "#{@target_user}@#{@target}", "output", @platform.ssh_port)
+      end
+
+      # Ensures that the platform defines the attributes that the engine needs to function.
+      #
+      # @raise [Vanagon::Error] an error is raised if a needed attribute is not defined
+      def validate_platform
+        missing_attrs = []
+        @required_attributes.each do |attr|
+          if (!@platform.instance_variables.include?("@#{attr}".to_sym)) or @platform.instance_variable_get("@#{attr}".to_sym).nil?
+            missing_attrs << attr
+          end
+        end
+
+        if missing_attrs.empty?
+          return true
+        else
+          raise Vanagon::Error, "The following required attributes were not set in '#{@platform.name}': #{missing_attrs.join(', ')}."
+        end
+      end
+    end
+  end
+end

data/lib/vanagon/engine/docker.rb

@@ -0,0 +1,40 @@
+require 'vanagon/engine/base'
+
+class Vanagon
+  class Engine
+    class Docker < Base
+      # Both the docker_image and the docker command itself are required for
+      # the docker engine to work
+      def initialize(platform, target = nil)
+        @docker_cmd = Vanagon::Utilities.find_program_on_path('docker')
+        super
+        @required_attributes << "docker_image"
+      end
+
+      # This method is used to obtain a vm to build upon using
+      # a docker container.
+      # @raise [Vanagon::Error] if a target cannot be obtained
+      def select_target
+        Vanagon::Utilities.ex("#{@docker_cmd} run -d --name #{@platform.docker_image}-builder -p #{@platform.ssh_port}:22 #{@platform.docker_image}")
+        @target = 'localhost'
+
+        # Wait for ssh to come up in the container
+        Vanagon::Utilities.retry_with_timeout do
+          Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", 'exit', @platform.ssh_port)
+        end
+      rescue => e
+        raise Vanagon::Error.wrap(e, "Something went wrong getting a target vm to build on using docker. Ssh was not up in the container after 5 seconds.")
+      end
+
+      # This method is used to tell the vmpooler to delete the instance of the
+      # vm that was being used so the pool can be replenished.
+      def teardown
+        Vanagon::Utilities.ex("#{@docker_cmd} stop #{@platform.docker_image}-builder")
+        Vanagon::Utilities.ex("#{@docker_cmd} rm #{@platform.docker_image}-builder")
+      rescue Vanagon::Error => e
+        warn "There was a problem tearing down the docker container #{@platform.docker_image}-builder (#{e.message})."
+      end
+    end
+  end
+end
+

data/lib/vanagon/engine/local.rb

@@ -0,0 +1,40 @@
+require 'vanagon/utilities'
+require 'vanagon/errors'
+require 'benchmark'
+
+class Vanagon
+  class Engine
+    class Local
+      attr_accessor :target
+
+      def initialize(platform, target = nil)
+        @platform = platform
+        @target = "local machine"
+      end
+
+      # Dispatches the command for execution
+      def dispatch(command)
+        puts Benchmark.measure { local_command(command, @workdir) }
+      end
+
+      # Steps needed to tear down or clean up the system after the build is
+      # complete
+      def teardown
+      end
+
+      # This method will take care of validation and target selection all at
+      # once as an easy shorthand to call from the driver
+      def startup(workdir)
+        @workdir = workdir
+        script = @platform.provisioning.join(' && ')
+        dispatch(script)
+      end
+
+      def ship_workdir(workdir)
+      end
+
+      def retrieve_built_artifact
+      end
+    end
+  end
+end

data/lib/vanagon/engine/pooler.rb

@@ -0,0 +1,85 @@
+require 'vanagon/engine/base'
+
+class Vanagon
+  class Engine
+    class Pooler < Base
+      attr_reader :token
+
+      # The vmpooler_template is required to use the pooler engine
+      def initialize(platform, target = nil)
+        @pooler = "http://vmpooler.delivery.puppetlabs.net"
+        @token = load_token
+        super
+        @required_attributes << "vmpooler_template"
+      end
+
+      # This method loads the pooler token from one of two locations
+      # @return [String, nil] token for use with the vmpooler
+      def load_token
+        if ENV['VMPOOLER_TOKEN']
+          token = ENV['VMPOOLER_TOKEN']
+        else
+          token_file = File.expand_path("~/.vanagon-token")
+          if File.exist?(token_file)
+            token = File.open(token_file).read.chomp
+          end
+        end
+        token
+      end
+
+      # This method is used to obtain a vm to build upon using the Puppet Labs'
+      # vmpooler (https://github.com/puppetlabs/vmpooler)
+      # @raise [Vanagon::Error] if a target cannot be obtained
+      def select_target
+        response = Vanagon::Utilities.http_request(
+          "#{@pooler}/vm",
+          'POST',
+          '{"' + @platform.vmpooler_template + '":"1"}',
+          { 'X-AUTH-TOKEN' => @token }
+        )
+        if response and response["ok"]
+          @target = response[@platform.vmpooler_template]['hostname'] + '.' + response['domain']
+          Vanagon::Driver.logger.info "Reserving #{@target} (#{@platform.vmpooler_template}) [#{@token ? 'token used' : 'no token used'}]"
+
+          tags = {
+            'tags' => {
+              'jenkins_build_url' => ENV['BUILD_URL'],
+              'project' => ENV['JOB_NAME'] || 'vanagon',
+              'created_by' => ENV['USER'] || ENV['USERNAME'] || 'unknown'
+            }
+          }
+
+          response_tag = Vanagon::Utilities.http_request(
+            "#{@pooler}/vm/#{response[@platform.vmpooler_template]['hostname']}",
+            'PUT',
+            tags.to_json,
+            { 'X-AUTH-TOKEN' => @token }
+          )
+        else
+          raise Vanagon::Error, "Something went wrong getting a target vm to build on, maybe the pool for #{@platform.vmpooler_template} is empty?"
+        end
+      end
+
+      # This method is used to tell the vmpooler to delete the instance of the
+      # vm that was being used so the pool can be replenished.
+      def teardown
+        response = Vanagon::Utilities.http_request(
+          "#{@pooler}/vm/#{@target}",
+          "DELETE",
+          nil,
+          { 'X-AUTH-TOKEN' => @token }
+        )
+        if response and response["ok"]
+          Vanagon::Driver.logger.info "#{@target} has been destroyed"
+          puts "#{@target} has been destroyed"
+        else
+          Vanagon::Driver.logger.info "#{@target} could not be destroyed"
+          warn "#{@target} could not be destroyed"
+        end
+      rescue Vanagon::Error => e
+        Vanagon::Driver.logger.info "#{@target} could not be destroyed (#{e.message})"
+        warn "#{@target} could not be destroyed (#{e.message})"
+      end
+    end
+  end
+end

data/lib/vanagon/errors.rb

@@ -0,0 +1,28 @@
+class Vanagon
+
+  # An error class that accepts a wrapped error message
+  #
+  class Error < StandardError
+    attr_accessor :original
+
+    # Generate a wrapped exception
+    #
+    # @param original [Exception] The exception to wrap
+    # @param mesg [String]
+    #
+    # @return [Vanagon::Error]
+    def self.wrap(original, mesg)
+      new(mesg).tap do |e|
+        e.set_backtrace(caller(4))
+        e.original = original
+      end
+    end
+
+    # @overload initialize(mesg)
+    #   @param mesg [String] The exception mesg
+    #
+    def initialize(mesg)
+      super(mesg)
+    end
+  end
+end

data/lib/vanagon/extensions/string.rb

@@ -0,0 +1,11 @@
+# String is generally pretty functional, but sometimes you need
+# a couple of small convienence methods to make working with really
+# long or really funky strings easier. This will indent all lines
+# to the margin of the first line, preserving sunsequent indentation
+# while still helping reign in HEREDOCS.
+class String
+  # @return [String]
+  def undent
+    gsub(/^.{#{slice(/^ +/).length}}/, '')
+  end
+end

data/lib/vanagon/optparse.rb

@@ -0,0 +1,62 @@
+require 'optparse'
+
+class Vanagon
+  class OptParse
+    def initialize(banner, options = [])
+      @options = Hash[options.map { |v| [v, nil] }]
+      @optparse = OptionParser.new do |opts|
+        opts.banner = banner
+
+        if @options.include?(:workdir)
+          opts.on('-w DIR', '--workdir DIR', "Working directory where build source should be put (defaults to a tmpdir)") do |dir|
+            @options[:workdir] = dir
+          end
+        end
+
+        if @options.include?(:configdir)
+          opts.on('-c', '--configdir DIR', 'Configs dir (defaults to $pwd/configs)') do |dir|
+            @options[:configdir] = dir
+          end
+        end
+
+        if @options.include?(:target)
+          opts.on('-t HOST', '--target HOST', 'Configure a target machine for build and packaging (defaults to grabbing one from the pooler)') do |name|
+            @options[:target] = name
+          end
+        end
+
+        if @options.include?(:engine)
+          opts.on('-e ENGINE', '--engine ENGINE', "A custom engine to use (defaults to the pooler) [base, local, docker, pooler currently supported]") do |engine|
+            @options[:engine] = engine
+          end
+        end
+
+        if @options.include?(:preserve)
+          opts.on('-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)') do |flag|
+            @options[:preserve] = flag
+          end
+        end
+
+        if @options.include?(:verbose)
+          opts.on('-v', '--verbose', 'Verbose output (does nothing)') do |flag|
+            @options[:verbose] = flag
+          end
+        end
+
+        opts.on('-h', '--help', 'Display help') do
+          puts opts
+          exit 1
+        end
+      end
+    end
+
+    def parse!(args)
+      @optparse.parse!(args)
+      @options
+    end
+
+    def to_s
+      @optparse.to_s
+    end
+  end
+end