vanagon 0.13.1 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1fbe8fc537feff6f6fe49b077602c6f2fa1dff5
-  data.tar.gz: 577e9e051c48bc9d2c9da9f7b1b08dabdd62e87c
+  metadata.gz: 072c1ccfd65eb990500787ebd5cfa03d25a4ae09
+  data.tar.gz: 7e3a2687a3e8a8b367fe195d1d853ce782d46005
 SHA512:
-  metadata.gz: cd644943daee7fcaa360bd40a1285c7c341f6736b8750a7157373662de3329b8c200ac713bff758a75c9aea74f54d1a3f2eedb7aa3d7034a529917ed819c08c1
-  data.tar.gz: c98eb69b2ac57756d663ff61d83d2c1c3dbf35d925fd9f4c7cc4219585c4f01208ee895a950871c293077ea080f754b1389f269a0d25d6b32e03f292c14a3034
+  metadata.gz: d015eede00d49017508043fd1f235b46b2ef61b4e608abd75cb8fec863fafe4d5b0f63a4dbe1a26c6a32cae47424f8ec63ae33603cb943a960b8588b9126982f
+  data.tar.gz: 86c8a5b9ea8dce5c3d0b3fb83f0c1280e8dc94ce37f1256f5c0707bc229e570e4570f6a83fae8b0138655e270a3711c80a91ff27f33f07f34a3979aeb3dda3a6

data/README.md

@@ -11,7 +11,6 @@
 - [Configuration and Usage](#configuration-and-usage)
   - [`build` usage](#build-usage)
   - [`inspect` usage](#inspect-usage)
-  - [`devkit` usage](#devkit-usage)
 - [Engines](#engines)
   - [Amazon Ec2](#amazon-ec2)
 - [Contributing](#contributing)
@@ -33,13 +32,6 @@ Vanagon builds up a Makefile and packaging files (specfile for RPM,
 control/rules/etc for DEB) and copies them to a remote host, where make can be
 invoked to build all of the components and make a package of the contents.
 
-Vanagon also provides a devkit command that will prepare a machine as a
-development environment for the entire project, or restricted to individual
-components of the project. The devkit command installs all required build tools,
-creates a master makefile for the project, and configures, builds, and installs
-all components. The result is an environment where you can work on individual
-components, then rebuild the project and test the installed artifacts.
-
 ### How is it pronounced?
 
 Vanagon (/ˈvænəgɪn/) sounds like "van again." It does not sound like "van wagon" or "van and gone."
@@ -255,43 +247,6 @@ build-time configuration, environment variables, and expected artifacts.
 
 ---
 
-### `devkit` usage
-
-The devkit command has positional arguments and position independent flagged
-arguments.
-
-#### Arguments (position dependent)
-
-##### project name
-As in `build` arguments.
-
-##### platform name
-As in `build` arguments.
-
-##### component names <optional>
-Specifies specific components that should be built. If components are not
-specified, then all components in the project will be built. If components
-are specified as arguments, then any in the project that aren't specified
-as arguments will be retrieved from packages rather than built from source.
-
-#### Flagged arguments
-
-**Note:** command flags can be used anywhere in the command.
-
-Supports all flagged arguments from the `build` command.
-
-##### -t HOST, --target HOST
-As in the `build` target host optional argument.
-
-#### Flags
-
-**Note:** command flags can be used anywhere in the command.
-
-##### -h, --help
-Display command-line help.
-
----
-
 Engines
 ---
 

data/bin/ship

@@ -12,4 +12,17 @@ else
   require 'packaging'
   Pkg::Util::RakeUtils.load_packaging_tasks
   Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship', 'artifacts', 'output')
+  begin
+    Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship_to_artifactory', 'output')
+  rescue
+    warn <<-DOC
+  Unable to ship packages to artifactory. Please make sure you are pointing to a
+  recent version of packaging in your Gemfile. Please also make sure you include
+  the artifactory gem in your Gemfile.
+
+  i.e.,
+  gem 'packaging', :github => 'puppetlabs/packaging', branch: '1.0.x'
+  gem 'artifactory'
+    DOC
+  end
 end

data/lib/vanagon/component.rb

@@ -92,6 +92,15 @@ class Vanagon
     # preinstall_actions is a two-dimensional Array, describing scripts that
     # should be executed before a given component is installed.
     attr_accessor :preinstall_actions
+    # install_triggers is a one-dimensional Array of OpenStructs, describing scripts that
+    # should be executed when a package is installed or upgraded
+    attr_accessor :install_triggers
+    # interest_triggers is a one-dimensional Array of OpenStructs, describing scripts that
+    # should be executed when a package identifies an interest trigger
+    attr_accessor :interest_triggers
+    # activate_triggers is a one-dimentional Array of Strings, describing scripts that
+    # should be executed when a package identifies an activate trigger
+    attr_accessor :activate_triggers
     # postinstall_actions is a two-dimensional Array, describing scripts that
     # should be executed after a given component is installed.
     attr_accessor :postinstall_actions
@@ -106,6 +115,12 @@ class Vanagon
     # usually a String, but not required to be.
     attr_accessor :cleanup_source
 
+    # When dealing with compiled artifacts generated by setting
+    # `project.generate_archives true`, you only need to install the component.
+    # By setting install_only to true the component 'build' will exclude the
+    # unpack, patch, configure, build, and check steps.
+    attr_accessor :install_only
+
     # Loads a given component from the configdir
     #
     # @param name [String] the name of the component
@@ -152,9 +167,13 @@ class Vanagon
       @environment = Vanagon::Environment.new
       @sources = []
       @preinstall_actions = []
+      @install_triggers = []
+      @interest_triggers = []
+      @activate_triggers = []
       @postinstall_actions = []
       @preremove_actions = []
       @postremove_actions = []
+      @install_only = false
     end
 
     # Adds the given file to the list of files and returns @files.
@@ -193,7 +212,9 @@ class Vanagon
 
     # @return [Set] a list of unique mirror URIs that should be used to
     #   retrieve the upstream source before attempting to retrieve from
-    #   whatever URI was defined for #uri.
+    #   whatever URI was defined for #uri. If no mirrors are set and the
+    #   deprecated rewrite system has been configured, this will return
+    #   rewritten URIs
     def mirrors
       @mirrors ||= Set.new [deprecated_rewrite_url].compact
     end
@@ -243,10 +264,12 @@ class Vanagon
     end
 
     # This method is deprecated and private. It will return
-    # a rewritten URL if there's any value for #url.
+    # a rewritten URL if there's any value for #url and a
+    # rewrite rule for that URL has been set
     def deprecated_rewrite_url
       return nil unless url
-      Vanagon::Component::Source::Rewrite.parse_and_rewrite(url)
+      rewritten_url = Vanagon::Component::Source::Rewrite.parse_and_rewrite(url)
+      url == rewritten_url ? nil : rewritten_url
     end
     private :deprecated_rewrite_url
 

data/lib/vanagon/component/dsl.rb

@@ -416,21 +416,61 @@ class Vanagon
         eos
       end
 
+      # Checks that the array of pkg_state is valid (install AND/OR upgrade).
+      # Returns vanagon error if invalid
+      #
+      # @param pkg_state [Array] array of pkg_state input to test
+      def check_pkg_state_array(pkg_state)
+        if pkg_state.empty? || (pkg_state - ["install", "upgrade"]).any?
+          raise Vanagon::Error, "#{pkg_state} should be an array containing one or more of ['install', 'upgrade']"
+        end
+      end
+
       # Adds action to run during the preinstall phase of packaging
       #
-      # @param pkg_state [Array] the state in which the scripts should execute. Can be
-      #   one or multiple of 'install' and 'upgrade'.
+      # @param pkg_state [String, Array] the package state during which the scripts should execute.
+      #   Accepts either a single string ("install" or "upgrade"), or an Array of Strings (["install", "upgrade"]).
       # @param scripts [Array] the Bourne shell compatible scriptlet(s) to execute
       def add_preinstall_action(pkg_state, scripts)
         pkg_state = Array(pkg_state)
         scripts = Array(scripts)
-
-        if pkg_state.empty? || !(pkg_state - ["install", "upgrade"]).empty?
-          raise Vanagon::Error, "#{pkg_state} should be an array containing one or more of ['install', 'upgrade']"
-        end
+        check_pkg_state_array(pkg_state)
         @component.preinstall_actions << OpenStruct.new(:pkg_state => pkg_state, :scripts => scripts)
       end
 
+      # Adds trigger for scripts to be run on specified pkg_state.
+      #
+      # @param pkg_state [String, Array] the package state during which the scripts should execute.
+      #   Accepts either a single string ("install" or "upgrade"), or an Array of Strings (["install", "upgrade"]).
+      # @param scripts [Array] the rpm pkg scriptlet(s) to execute
+      # @param pkg [String] the package the trigger will be set in
+      def add_rpm_install_triggers(pkg_state, scripts, pkg)
+        pkg_state = Array(pkg_state)
+        scripts = Array(scripts)
+        check_pkg_state_array(pkg_state)
+        @component.install_triggers << OpenStruct.new(:pkg_state => pkg_state, :scripts => scripts, :pkg => pkg)
+      end
+
+      # Adds interest trigger based on the specified packaging state and interest name.
+      #
+      # @param pkg_state [String, Array] the package state during which the scripts should execute.
+      #   Accepts either a single string ("install" or "upgrade"), or an Array of Strings (["install", "upgrade"]).
+      # @param scripts [Array] the scripts to run for the interest trigger
+      # @param interest_name [String] the name of the interest trigger
+      def add_debian_interest_triggers(pkg_state, scripts, interest_name)
+        pkg_state = Array(pkg_state)
+        scripts = Array(scripts)
+        check_pkg_state_array(pkg_state)
+        @component.interest_triggers << OpenStruct.new(:pkg_state => pkg_state, :scripts => scripts, :interest_name => interest_name)
+      end
+
+      # Adds activate trigger name to be watched
+      #
+      # @param activate_name [String] the activate trigger name
+      def add_debian_activate_triggers(activate_name)
+        @component.activate_triggers << OpenStruct.new(:activate_name => activate_name)
+      end
+
       # Adds action to run during the postinstall phase of packaging
       #
       # @param pkg_state [Array] the state in which the scripts should execute. Can be
@@ -439,10 +479,7 @@ class Vanagon
       def add_postinstall_action(pkg_state, scripts)
         pkg_state = Array(pkg_state)
         scripts = Array(scripts)
-
-        if pkg_state.empty? || !(pkg_state - ["install", "upgrade"]).empty?
-          raise Vanagon::Error, "#{pkg_state} should be an array containing one or more of ['install', 'upgrade']"
-        end
+        check_pkg_state_array(pkg_state)
         @component.postinstall_actions << OpenStruct.new(:pkg_state => pkg_state, :scripts => scripts)
       end
 
@@ -479,6 +516,10 @@ class Vanagon
       def license(license)
         @component.license = license
       end
+
+      def install_only(install_only)
+        @component.install_only = install_only
+      end
     end
   end
 end

data/lib/vanagon/component/rules.rb

@@ -47,8 +47,8 @@ class Vanagon
       # in the returned rules.
       #
       # @return [Array<Makefile::Rule>]
-      def rules
-        list = [
+      def rules # rubocop:disable Metrics/AbcSize
+        list_of_rules = [
           component_rule,
           unpack_rule,
           patch_rule,
@@ -59,11 +59,20 @@ class Vanagon
           clean_rule,
           clobber_rule,
         ]
+
+        if component.install_only
+          list_of_rules = [
+            component_rule,
+            install_rule,
+            clean_rule,
+            clobber_rule,
+          ]
+        end
         if project.cleanup
-          list << cleanup_rule
+          list_of_rules << cleanup_rule
         end
 
-        list
+        list_of_rules
       end
 
       # Generate a top level rule to build this component.
@@ -148,13 +157,20 @@ class Vanagon
 
       # Install this component.
       rule("install") do |r|
-        r.dependencies = ["#{component.name}-check"]
+        r.dependencies = ["#{component.name}-check"] unless component.install_only
         unless component.install.empty?
-          r.recipe << andand_multiline(
-            component.environment_variables,
-            "cd #{component.get_build_dir}",
-            component.install
-          )
+          if component.install_only
+            r.recipe << andand_multiline(
+              component.environment_variables,
+              component.install
+            )
+          else
+            r.recipe << andand_multiline(
+              component.environment_variables,
+              "cd #{component.get_build_dir}",
+              component.install
+            )
+          end
         end
 
         after_install_patches = component.patches.select { |patch| patch.after == "install" }

data/lib/vanagon/component/source/http.rb

@@ -43,11 +43,12 @@ class Vanagon
         # Constructor for the Http source type
         #
         # @param url [String] url of the http source to fetch
-        # @param sum [String] sum to verify the download against
+        # @param sum [String] sum to verify the download against or URL to fetch
+        #                     sum from
         # @param workdir [String] working directory to download into
         # @param sum_type [String] type of sum we are verifying
         # @raise [RuntimeError] an exception is raised is sum is nil
-        def initialize(url, sum:, workdir:, sum_type:, **options)
+        def initialize(url, sum:, workdir:, sum_type:, **options) # rubocop:disable Metrics/AbcSize
           unless sum
             fail "sum is required to validate the http source"
           end
@@ -62,6 +63,20 @@ class Vanagon
           @sum = sum
           @workdir = workdir
           @sum_type = sum_type
+
+          if Vanagon::Component::Source::Http.valid_url?(@sum)
+            sum_file = download(@sum)
+            File.open(File.join(@workdir, sum_file)) do |file|
+              # the sha1 files generated during archive creation  are formatted
+              # "<sha1sum> <filename>". This will also work for sources that
+              # only contain the checksum.
+              remote_sum = file.read.split.first
+              unless remote_sum
+                fail "Downloaded checksum file seems to be empty, make sure you have the correct URL"
+              end
+              @sum = remote_sum
+            end
+          end
         end
 
         # Download the source from the url specified. Sets the full path to the

data/lib/vanagon/driver.rb

@@ -77,7 +77,7 @@ class Vanagon
     end
 
     def cleanup_workdir
-      FileUtils.rm_rf(@workdir)
+      FileUtils.rm_rf(workdir)
     end
 
     def self.configdir
@@ -115,25 +115,29 @@ class Vanagon
         raise Vanagon::Error, "Project requires a version set, all is lost."
       end
 
-      @engine.startup(@workdir)
+      @engine.startup(workdir)
 
       $stderr.puts "Target is #{@engine.target}"
       retry_task { install_build_dependencies }
-      retry_task { @project.fetch_sources(@workdir) }
+      retry_task { @project.fetch_sources(workdir) }
 
-      @project.make_makefile(@workdir)
-      @project.make_bill_of_materials(@workdir)
-      @project.generate_packaging_artifacts(@workdir)
+      @project.make_makefile(workdir)
+      @project.make_bill_of_materials(workdir)
+      @project.generate_packaging_artifacts(workdir)
       @project.save_manifest_json
-      @engine.ship_workdir(@workdir)
+      @engine.ship_workdir(workdir)
       @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make})")
       @engine.retrieve_built_artifact
 
-      unless @preserve
+      if %i[never on-failure].include? @preserve
         @engine.teardown
         cleanup_workdir
       end
     rescue => e
+      if [:never].include? @preserve
+        @engine.teardown
+        cleanup_workdir
+      end
       $stderr.puts e
       $stderr.puts e.backtrace.join("\n")
       raise e
@@ -143,36 +147,17 @@ class Vanagon
       end
     end
 
-    def render
+    def render # rubocop:disable Metrics/AbcSize
       # Simple sanity check for the project
       if @project.version.nil? or @project.version.empty?
         raise Vanagon::Error, "Project requires a version set, all is lost."
       end
 
       $stderr.puts "rendering Makefile"
-      retry_task { @project.fetch_sources(@workdir) }
-      @project.make_bill_of_materials(@workdir)
-      @project.generate_packaging_artifacts(@workdir)
-      @project.make_makefile(@workdir)
-    end
-
-    def prepare(workdir = nil) # rubocop:disable Metrics/AbcSize
-      @workdir = workdir ? FileUtils.mkdir_p(workdir).first : Dir.mktmpdir
-      @engine.startup(@workdir)
-
-      $stderr.puts "Devkit on #{@engine.target}"
-
-      install_build_dependencies
-      @project.fetch_sources(@workdir)
-      @project.make_makefile(@workdir)
-      @project.make_bill_of_materials(@workdir)
-      # Builds only the project, skipping packaging into an artifact.
-      @engine.ship_workdir(@workdir)
-      @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make} #{@project.name}-project)")
-    rescue => e
-      $stderr.puts e
-      $stderr.puts e.backtrace.join("\n")
-      raise e
+      retry_task { @project.fetch_sources(workdir) }
+      @project.make_bill_of_materials(workdir)
+      @project.generate_packaging_artifacts(workdir)
+      @project.make_makefile(workdir)
     end
 
     # Retry the provided block, use the retry count and timeout

data/lib/vanagon/engine/pooler.rb

@@ -10,7 +10,7 @@ class Vanagon
       def initialize(platform, target = nil, **opts)
         super
 
-        @pooler = "http://vmpooler.delivery.puppetlabs.net"
+        @available_poolers = ["http://vmpooler.delivery.puppetlabs.net", "https://nspooler-service-prod-1.delivery.puppetlabs.net"]
         @token = load_token
         @required_attributes << "vmpooler_template"
       end
@@ -76,36 +76,58 @@ class Vanagon
       private :read_vmfloaty_token
 
       # This method is used to obtain a vm to build upon using the Puppet Labs'
-      # vmpooler (https://github.com/puppetlabs/vmpooler)
+      # vmpooler (https://github.com/puppetlabs/vmpooler) or other pooler technologies
+      # leveraging the same API
       # @raise [Vanagon::Error] if a target cannot be obtained
-      def select_target # rubocop:disable Metrics/AbcSize
+      def select_target
+        @available_poolers.each do |current_pooler|
+          @pooler = select_target_from(current_pooler)
+          break unless @pooler.empty?
+        end
+        raise Vanagon::Error, "Something went wrong getting a target vm to build on, maybe the pool for #{build_host_name} is empty?" if @pooler.empty?
+      end
+
+      # Attempt to provision a host from a specific pooler.
+      #
+      def select_target_from(pooler) # rubocop:disable Metrics/AbcSize
         response = Vanagon::Utilities.http_request(
-          "#{@pooler}/vm",
+          "#{pooler}/vm",
           'POST',
           '{"' + build_host_name + '":"1"}',
           { 'X-AUTH-TOKEN' => @token }
         )
         if response["ok"]
-          @target = response[build_host_name]['hostname'] + '.' + response['domain']
+          @target = response[build_host_name]['hostname']
+          # The nspooler does not use 'domain' in the response: 'hostname' just returns the fqdn.
+          # in the future we should make the two APIs the same in this sense, but for now, just check
+          # if 'domain' is a thing and use it if so.
+          if response['domain']
+            @target += '.' + response['domain']
+          end
           Vanagon::Driver.logger.info "Reserving #{@target} (#{build_host_name}) [#{@token ? 'token used' : 'no token used'}]"
-
-          tags = {
-            'tags' => {
-              'jenkins_build_url' => ENV['BUILD_URL'],
-              'project' => ENV['JOB_NAME'] || 'vanagon',
-              'created_by' => ENV['USER'] || ENV['USERNAME'] || 'unknown'
-            }
-          }
-
-          Vanagon::Utilities.http_request(
-            "#{@pooler}/vm/#{response[build_host_name]['hostname']}",
-            'PUT',
-            tags.to_json,
-            { 'X-AUTH-TOKEN' => @token }
-          )
+          add_tags_to_target(pooler, response[build_host_name]['hostname'])
         else
-          raise Vanagon::Error, "Something went wrong getting a target vm to build on, maybe the pool for #{build_host_name} is empty?"
+          pooler = ''
         end
+        pooler
+      end
+
+      # Add tags to a provisioned target using the pooler API
+      #
+      def add_tags_to_target(pooler, hostname)
+        tags = {
+          'tags' => {
+            'jenkins_build_url' => ENV['BUILD_URL'],
+            'project' => ENV['JOB_NAME'] || 'vanagon',
+            'created_by' => ENV['USER'] || ENV['USERNAME'] || 'unknown'
+          }
+        }
+        Vanagon::Utilities.http_request(
+          "#{pooler}/vm/#{hostname}",
+          'PUT',
+          tags.to_json,
+          { 'X-AUTH-TOKEN' => @token }
+        )
       end
 
       # This method is used to tell the vmpooler to delete the instance of the