vanagon 0.16.0 → 0.19.0

data/lib/vanagon/component/source.rb

@@ -1,4 +1,5 @@
 require 'fustigit'
+require 'vanagon/logger'
 require 'vanagon/component/source/http'
 require 'vanagon/component/source/git'
 require 'vanagon/component/source/local'
@@ -66,7 +67,7 @@ class Vanagon
           timeout = 5
           if Vanagon::Component::Source::Git.valid_remote?(uri, timeout)
             if uri =~ /^http/
-              warn "Passing git URLs as http(s) addresses is deprecated! Please prefix your source URL with `git:`"
+              VanagonLogger.info "Passing git URLs as http(s) addresses is deprecated! Please prefix your source URL with `git:`"
             end
             return :git
           end

data/lib/vanagon/component/source/git.rb

@@ -1,5 +1,6 @@
 require 'vanagon/utilities'
 require 'vanagon/errors'
+require 'vanagon/logger'
 # This stupid library requires a capital 'E' in its name
 # but it provides a wealth of useful constants
 require 'English'
@@ -81,7 +82,7 @@ class Vanagon
         # There is no md5 to manually verify here, so this is a noop.
         def verify
           # nothing to do here, so just tell users that and return
-          warn "Nothing to verify for '#{dirname}' (using Git reference '#{ref}')"
+          VanagonLogger.info "Nothing to verify for '#{dirname}' (using Git reference '#{ref}')"
         end
 
         # The dirname to reference when building from the repo
@@ -131,8 +132,8 @@ class Vanagon
         # Clone a remote repo, make noise about it, and fail entirely
         # if we're unable to retrieve the remote repo
         def clone!
-          warn "Cloning Git repo '#{url}'"
-          warn "Successfully cloned '#{dirname}'" if clone
+          VanagonLogger.info "Cloning Git repo '#{url}'"
+          VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
         rescue ::Git::GitExecuteError
           raise Vanagon::InvalidRepo, "Unable to clone from '#{url}'"
         end
@@ -141,7 +142,7 @@ class Vanagon
         # Checkout desired ref/sha, make noise about it, and fail
         # entirely if we're unable to checkout that given ref/sha
         def checkout!
-          warn "Checking out '#{ref}' from Git repo '#{dirname}'"
+          VanagonLogger.info "Checking out '#{ref}' from Git repo '#{dirname}'"
           clone.checkout(ref)
         rescue ::Git::GitExecuteError
           raise Vanagon::CheckoutFailed, "unable to checkout #{ref} from '#{url}'"
@@ -151,7 +152,7 @@ class Vanagon
         # Attempt to update submodules, and do not panic
         # if there are no submodules to initialize
         def update_submodules
-          warn "Attempting to update submodules for repo '#{dirname}'"
+          VanagonLogger.info "Attempting to update submodules for repo '#{dirname}'"
           clone.update_submodules(init: true)
         end
         private :update_submodules
@@ -163,7 +164,7 @@ class Vanagon
         def describe
           clone.describe(ref, tags: true)
         rescue ::Git::GitExecuteError
-          warn "Directory '#{dirname}' cannot be versioned by Git. Maybe it hasn't been tagged yet?"
+          VanagonLogger.info "Directory '#{dirname}' cannot be versioned by Git. Maybe it hasn't been tagged yet?"
         end
         private :describe
       end

data/lib/vanagon/component/source/http.rb

@@ -1,4 +1,5 @@
 require 'vanagon/utilities'
+require 'vanagon/logger'
 require 'vanagon/component/source/local'
 require 'net/http'
 require 'uri'
@@ -93,7 +94,7 @@ class Vanagon
         #
         # @raise [RuntimeError] an exception is raised if the sum does not match the sum of the file
         def verify # rubocop:disable Metrics/AbcSize
-          warn "Verifying file: #{file} against sum: '#{sum}'"
+          VanagonLogger.info "Verifying file: #{file} against sum: '#{sum}'"
           actual = get_sum(File.join(workdir, file), sum_type)
           return true if sum == actual
 
@@ -107,7 +108,7 @@ class Vanagon
           uri = URI.parse(target_url.to_s)
           target_file ||= File.basename(uri.path)
 
-          warn "Downloading file '#{target_file}' from url '#{target_url}'"
+          VanagonLogger.info "Downloading file '#{target_file}' from url '#{target_url}'"
 
           Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
             http.request(Net::HTTP::Get.new(uri, headers)) do |response|

data/lib/vanagon/component/source/local.rb

@@ -1,4 +1,5 @@
 require 'vanagon/utilities'
+require 'vanagon/logger'
 
 class Vanagon
   class Component
@@ -57,7 +58,7 @@ class Vanagon
         #
         # @raise [RuntimeError, Vanagon::Error] an exception is raised if the URI scheme cannot be handled
         def copy
-          warn "Copying file '#{url.basename}' to workdir"
+          VanagonLogger.info "Copying file '#{url.basename}' to workdir"
 
           FileUtils.cp_r(url, file)
         end

data/lib/vanagon/component/source/rewrite.rb

@@ -1,4 +1,5 @@
 require 'vanagon/component/source'
+require 'vanagon/logger'
 
 class Vanagon
   class Component
@@ -16,7 +17,7 @@ class Vanagon
           # @deprecated Please use the component DSL method #mirror(<URI>)
           #   instead. This method will be removed before Vanagon 1.0.0.
           def register_rewrite_rule(protocol, rule)
-            warn <<-HERE.undent
+            VanagonLogger.info <<-HERE.undent
               rewrite rule support is deprecated and will be removed before Vanagon 1.0.0.
               Rewritten URLs will be automatically converted into mirror URLs for now but
               please use the component DSL method '#mirror url' to define new mirror URL
@@ -76,7 +77,7 @@ class Vanagon
           def parse_and_rewrite(uri)
             return uri if rewrite_rules.empty?
             if !!uri.match(/^git:http/)
-              warn <<-HERE.undent
+              VanagonLogger.info <<-HERE.undent
                 `fustigit` parsing doesn't get along with how we specify the source
                 type by prefixing `git`. As `rewrite_rules` are deprecated, we'll
                 replace `git:http` with `http` in your uri. At some point this will

data/lib/vanagon/driver.rb

@@ -4,6 +4,7 @@ require 'vanagon/component'
 require 'vanagon/utilities'
 require 'vanagon/common'
 require 'vanagon/errors'
+require 'vanagon/logger'
 require 'tmpdir'
 require 'logger'
 
@@ -20,33 +21,29 @@ class Vanagon
       @retry_count ||= @project.retry_count || ENV["VANAGON_RETRY_COUNT"] || 1
     end
 
-    def initialize(platform, project, options = { workdir: nil, configdir: nil, target: nil, engine: nil, components: nil, skipcheck: false, verbose: false, preserve: false, only_build: nil, remote_workdir: nil }) # rubocop:disable Metrics/AbcSize
-      @verbose = options[:verbose]
-      @preserve = options[:preserve]
+    def initialize(platform, project, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+      @options = options
+      @verbose = options[:verbose] || false
+      @preserve = options[:preserve] || false
       @workdir = options[:workdir] || Dir.mktmpdir
 
       @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
       components = options[:components] || []
       only_build = options[:only_build]
-      target = options[:target]
 
       @platform = Vanagon::Platform.load_platform(platform, File.join(@@configdir, "platforms"))
-      @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
+      @project = Vanagon::Project.load_project(
+        project, File.join(@@configdir, "projects"), @platform, components
+      )
       @project.settings[:verbose] = options[:verbose]
-      @project.settings[:skipcheck] = options[:skipcheck]
+      @project.settings[:skipcheck] = options[:skipcheck] || false
       filter_out_components(only_build) if only_build
       loginit('vanagon_hosts.log')
 
       @remote_workdir = options[:"remote-workdir"]
 
-      if options[:engine]
-        # Use the explicitly configured engine.
-        load_engine_object(options[:engine], @platform, target)
-      else
-        # Use 'pooler' as a default, but also apply selection logic that may
-        # choose something different based on platform configuration.
-        load_engine('pooler', @platform, target)
-      end
+      engine = pick_engine(options)
+      load_engine_object(engine, @platform, options[:target])
     end
 
     def filter_out_components(only_build)
@@ -54,29 +51,31 @@ class Vanagon
       # flatten all the results in to one array and set project.components to that.
       @project.components = only_build.flat_map { |comp| @project.filter_component(comp) }.uniq
       if @verbose
-        warn "Only building:"
-        @project.components.each { |comp| warn comp.name }
+        VanagonLogger.info "Only building:"
+        @project.components.each { |comp| VanagonLogger.info comp.name }
       end
     end
 
-    def load_engine(engine_type, platform, target)
-      if engine_type != 'always_be_scheduling'
-        if platform.build_hosts
-          engine_type = 'hardware'
-        elsif platform.aws_ami
-          engine_type = 'ec2'
-        elsif platform.docker_image
-          engine_type = 'docker'
-        elsif target
-          engine_type = 'base'
-        end
+    def pick_engine(options) # rubocop:disable Metrics/PerceivedComplexity
+      if options[:engine] && !options[:target]
+        options[:engine]
+      elsif @platform.build_hosts
+        'hardware'
+      elsif @platform.aws_ami
+        'ec2'
+      elsif @platform.docker_image
+        'docker'
+      elsif options[:target]
+        'base'
+      else
+        'always_be_scheduling'
       end
-      load_engine_object(engine_type, platform, target)
     end
 
     def load_engine_object(engine_type, platform, target)
       require "vanagon/engine/#{engine_type}"
-      @engine = Object::const_get("Vanagon::Engine::#{camelize(engine_type)}").new(platform, target, remote_workdir: remote_workdir)
+      @engine = Object::const_get("Vanagon::Engine::#{camelize(engine_type)}")
+        .new(platform, target, remote_workdir: remote_workdir)
     rescue StandardError, ScriptError => e
       raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine_type}'")
     end
@@ -103,7 +102,9 @@ class Vanagon
       { "name" => @engine.build_host_name, "engine" => @engine.name }
     end
 
-    # Returns the set difference between the build_requires and the components to get a list of external dependencies that need to be installed.
+    # Returns the set difference between the build_requires and the
+    # components to get a list of external dependencies that need to
+    # be installed.
     def list_build_dependencies
       @project.components.map(&:build_requires).flatten.uniq - @project.components.map(&:name)
     end
@@ -134,7 +135,7 @@ class Vanagon
       end
 
       @engine.startup(workdir)
-      warn "Target is #{@engine.target}"
+      VanagonLogger.info "Target is #{@engine.target}"
       Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
         install_build_dependencies
       end
@@ -159,8 +160,8 @@ class Vanagon
         @engine.teardown
         cleanup_workdir
       end
-      warn e
-      warn e.backtrace.join("\n")
+      VanagonLogger.error(e)
+      VanagonLogger.error e.backtrace.join("\n")
       raise e
     ensure
       if ["hardware", "ec2"].include?(@engine.name)
@@ -174,7 +175,7 @@ class Vanagon
         raise Vanagon::Error, "Project requires a version set, all is lost."
       end
 
-      warn "rendering Makefile"
+      VanagonLogger.info "rendering Makefile"
       @project.fetch_sources(workdir, retry_count, timeout)
       @project.make_bill_of_materials(workdir)
       @project.generate_packaging_artifacts(workdir)

data/lib/vanagon/engine/always_be_scheduling.rb

@@ -1,5 +1,7 @@
-require 'vanagon/engine/base'
 require 'json'
+require 'vanagon/engine/base'
+require 'vanagon/logger'
+require 'yaml'
 
 class Vanagon
   class Engine
@@ -82,9 +84,15 @@ class Vanagon
     # {"name":"aix-53-ppc","engine":"always_be_scheduling"}
     # ```
     class AlwaysBeScheduling < Base
+      attr_reader :token
+      attr_reader :token_vmpooler
+
       def initialize(platform, target, **opts)
         super
 
+        @available_abs_endpoint = "https://abs-prod.k8s.infracore.puppet.net/api/v2"
+        @token_vmpooler = ENV['VMPOOLER_TOKEN']
+        @token = load_token
         Vanagon::Driver.logger.debug "AlwaysBeScheduling engine invoked."
       end
 
@@ -94,6 +102,7 @@ class Vanagon
       end
 
       # return the platform name as the "host" name
+      # order of preference: abs_resource_name, vmpooler_template or name
       def build_host_name
         if @platform.abs_resource_name
           @platform.abs_resource_name
@@ -103,6 +112,268 @@ class Vanagon
           @platform.name
         end
       end
+
+      # Retrieve the ABS token from an environment variable
+      # ("ABS_TOKEN") or from a number of potential configuration
+      # files (~/.vanagon-token or ~/.vmfloaty.yml).
+      # @return [String, nil] token for use with the vmpooler
+      def load_token
+        ENV['ABS_TOKEN'] || token_from_file
+      end
+
+      # a wrapper method around retrieving a token,
+      # with an explicitly ordered preference for a Vanagon-specific
+      # token file or a preexisting vmfoaty yaml file.
+      #
+      # @return [String, nil] token for use with the vmpooler
+      def token_from_file
+        read_vanagon_token || read_vmfloaty_token
+      end
+      private :token_from_file
+
+      # Read an ABS/vmpooler token from the plaintext vanagon-token file,
+      # as outlined in the project README.
+      # The first line should be the ABS token
+      # and the second (optional) line should be the vmpooler token
+      # Saves the second line into the @token_vmpooler variable
+      #
+      # @return [String, nil] the vanagon vmpooler token value
+      def read_vanagon_token(path = "~/.vanagon-token")
+        absolute_path = File.expand_path(path)
+        return nil unless File.exist?(absolute_path)
+
+        VanagonLogger.info "Reading ABS token from: #{path}"
+        contents = File.read(absolute_path).chomp
+        lines = contents.each_line.map(&:chomp)
+
+        abs = lines.shift
+        @token_vmpooler = lines.shift
+
+        VanagonLogger.info "Please add a second line with the vmpooler token to be able to modify or see the VM in floaty/bit-bar" if @token_vmpooler.nil?
+        return abs
+      end
+      private :read_vanagon_token
+
+      # Read a vmpooler token from the yaml formatted vmfloaty config,
+      # as outlined by the vmfloaty project:
+      # https://github.com/puppetlabs/vmfloaty
+      #
+      # It returns the top-level token value or the first 'abs' service
+      # token value it finds in the services list
+      # it sets @token_vmpooler with the optional vmpooler token
+      # if the abs services has a vmpooler_fallback, and that service
+      # has a token eg
+      #
+      # TOP-LEVEL DEFINED  => would get only the abs token
+      # user: 'jdoe'
+      # url: 'https://abs.example.net'
+      # token: '456def789'
+      #
+      # MULTIPLE SERVICES DEFINED => would get the abs token in 'abs-prod' and the vmpooler token in 'vmpooler-prod'
+      # user: 'jdoe'
+      # services:
+      #   abs-prod:
+      #     type: 'abs'
+      #     url: 'https://abs.example.net/api/v2'
+      #     token: '123abc456'
+      #     vmpooler_fallback: 'vmpooler-prod'
+      #   vmpooler-dev:
+      #     type: 'vmpooler'
+      #     url: 'https://vmpooler-dev.example.net'
+      #     token: '987dsa654'
+      #   vmpooler-prod:
+      #     type: 'vmpooler'
+      #     url: 'https://vmpooler.example.net'
+      #     token: '456def789'
+      #
+      # @return [String, nil] the vmfloaty vmpooler token value
+      def read_vmfloaty_token(path = "~/.vmfloaty.yml") # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+        absolute_path = File.expand_path(path)
+        return nil unless File.exist?(absolute_path)
+
+        yaml_config = YAML.load_file(absolute_path)
+        abs_token = nil
+        abs_service_name = nil
+        if yaml_config['token'] # top level
+          abs_token = yaml_config['token']
+        elsif yaml_config['services']
+          yaml_config['services'].each do |name, value|
+            if value['type'] == "abs" && value['token']
+              abs_token = value['token']
+              abs_service_name = name
+              vmpooler_fallback = value['vmpooler_fallback']
+              unless vmpooler_fallback.nil? || yaml_config['services'][vmpooler_fallback].nil? || yaml_config['services'][vmpooler_fallback]['token'].nil?
+                @token_vmpooler = yaml_config['services'][vmpooler_fallback]['token']
+              end
+              break
+            end
+          end
+        end
+        message = "Reading ABS token from: #{path}"
+        if abs_service_name
+          message.concat(" for service named #{abs_service_name}")
+          if @token_vmpooler.nil?
+            message.concat(" but there was no vmpooler_fallback value, please add one if you want to be able to modify the VM via bit-bar/floaty")
+          else
+            message.concat(" with a vmpooler_fallback token")
+          end
+        end
+        VanagonLogger.info message
+        return abs_token
+      end
+      private :read_vmfloaty_token
+
+      # This method is used to obtain a vm to build upon using Puppet's internal
+      # ABS (https://github.com/puppetlabs/always-be-scheduling) which is a level of abstraction for other
+      # engines using similar APIs
+      # @raise [Vanagon::Error] if a target cannot be obtained
+      def select_target
+        @pooler = select_target_from(@available_abs_endpoint)
+        raise Vanagon::Error, "Something went wrong getting a target vm to build on" if @pooler.empty?
+      end
+
+      # Attempt to provision a host from a specific pooler.
+      def select_target_from(pooler) # rubocop:disable Metrics/AbcSize
+        request_object = build_request_object
+
+        VanagonLogger.info "Requesting VMs with job_id: #{@saved_job_id}.  Will poll for up to an hour."
+        #the initial request is always replied with "come back again"
+        response = Vanagon::Utilities.http_request_generic(
+          "#{pooler}/request",
+          'POST',
+          request_object.to_json,
+          { 'X-AUTH-TOKEN' => @token }
+        )
+
+        unless response.code == "202"
+          VanagonLogger.info "failed to request ABS with code #{response.code}"
+          if valid_json?(response.body)
+            response_json = JSON.parse(response.body)
+            VanagonLogger.info "reason: #{response_json['reason']}"
+          end
+          return ''
+        end
+        response_body = check_queue(pooler, request_object)
+
+        return '' unless response_body["ok"]
+        @target = response_body[build_host_name]['hostname']
+        Vanagon::Driver.logger.info "Reserving #{@target} (#{build_host_name}) [#{@token ? 'token used' : 'no token used'}]"
+        return pooler
+      end
+
+      # main loop where the status of the request is checked, to see if the request
+      # has been allocated
+      def check_queue(pooler, request_object)
+        retries = 360 # ~ one hour
+        response_body = nil
+        begin
+          (1..retries).each do |i|
+            response = Vanagon::Utilities.http_request_generic(
+              "#{pooler}/request",
+              'POST',
+              request_object.to_json,
+              { 'X-AUTH-TOKEN' => @token }
+            )
+            response_body = validate_queue_status_response(response.code, response.body)
+            break if response_body
+
+            sleep_seconds = 10 if i >= 10
+            sleep_seconds = i if i < 10
+            VanagonLogger.info "Waiting #{sleep_seconds} seconds to check if ABS request has been filled. (x#{i})"
+
+            sleep(sleep_seconds)
+          end
+        rescue SystemExit, Interrupt
+          VanagonLogger.error "\nVanagon interrupted during mains ABS polling. Make sure you delete the requested job_id #{@saved_job_id}"
+          raise
+        end
+        response_body = translated(response_body, @saved_job_id)
+        response_body
+      end
+
+      def validate_queue_status_response(status_code, body)
+        case status_code
+        when "200"
+          return JSON.parse(body) unless body.empty? || !valid_json?(body)
+        when "202"
+          return nil
+        when "401"
+          raise Vanagon::Error, "HTTP #{status_code}: The token provided could not authenticate.\n#{body}"
+        when "503"
+          return nil
+        else
+          raise Vanagon::Error, "HTTP #{status_code}: request to ABS failed!\n#{body}"
+        end
+      end
+
+      # This method is used to tell the ABS to delete the job_id requested
+      # otherwise the resources will eventually get allocated asynchronously
+      # and will keep running until the end of their lifetime.
+      def teardown # rubocop:disable Metrics/AbcSize
+        request_object = {
+            'job_id' => @saved_job_id,
+        }
+
+        response = Vanagon::Utilities.http_request_generic(
+          "#{@available_abs_endpoint}/return",
+          "POST",
+          request_object.to_json,
+          { 'X-AUTH-TOKEN' => @token }
+        )
+        if response && response.body == 'OK'
+          Vanagon::Driver.logger.info "#{@saved_job_id} has been scheduled for removal"
+          VanagonLogger.info "#{@saved_job_id} has been scheduled for removal"
+        else
+          Vanagon::Driver.logger.info "#{@saved_job_id} could not be scheduled for removal: #{response.body}"
+          VanagonLogger.info "#{@saved_job_id} could not be scheduled for removal"
+        end
+      rescue Vanagon::Error => e
+        Vanagon::Driver.logger.info "#{@saved_job_id} could not be scheduled for removal (#{e.message})"
+        VanagonLogger.info "#{@saved_job_id} could not be scheduled for removal (#{e.message})"
+      end
+
+      private
+
+      def translated(response_body, job_id)
+        vmpooler_formatted_body = { 'job_id' => job_id }
+
+        response_body.each do |host| # in this context there should be only one host
+          vmpooler_formatted_body[host['type']] = { 'hostname' => host['hostname'] }
+        end
+        vmpooler_formatted_body['ok'] = true
+
+        vmpooler_formatted_body
+      end
+
+      def build_request_object
+        user = ENV['USER'] || ENV['USERNAME'] || 'unknown'
+
+        @saved_job_id = user + "-" + DateTime.now.strftime('%Q')
+        request_object = {
+            :resources => { build_host_name => 1 },
+            :job       => {
+                :id   => @saved_job_id,
+                :tags => {
+                    :jenkins_build_url => ENV['BUILD_URL'],
+                    :project           => ENV['JOB_NAME'] || 'vanagon',
+                    :created_by        => user,
+                    :user              => user
+                },
+            },
+            :priority => 1, # DO NOT use priority 1 in automated CI runs
+        }
+        unless @token_vmpooler.nil?
+          request_object[:vm_token] = @token_vmpooler
+        end
+        request_object
+      end
+
+      def valid_json?(json)
+        JSON.parse(json)
+        return true
+      rescue TypeError, JSON::ParserError
+        return false
+      end
     end
   end
 end