jenkins_pipeline_builder 0.10.12 → 0.10.13

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTM5YTI4ZDgwN2M1YjZjNzQ3MWY5NzliM2U4MTA4NmQ3MjQ5N2Y4NA==
+    MzcxMzVkZDI1ODAzOWFkZGQxMWIwOTg1MzE1YTdjN2M2OGUwNjUwZA==
   data.tar.gz: !binary |-
-    MWFhY2QyYzc5MWQyOGM5MmE3YTM2MzhhNGY5ZWUxODdmMTQ0YjFiMg==
+    YzRkZjVjNWU0OTg2OGNkMWE3NDMxOTUxYjg2YTBlZjFhMDMxNTdiZQ==
 SHA512:
   metadata.gz: !binary |-
-    NThjODRiYzA3ZDY1NDJmY2NjZWJjOTY3MTY0YmJmMTkyZTllODc4YTc0YmI5
-    MDhhODgyYWM3ZjdlZTQwZmFhY2U4YWJkZTQ4MjYwNzg4YmYyZWQ4ZGZmNjJh
-    NTMzMzJlOTBlOGY4MGViZjFiMWU0Yzk3NjYyNjI2MWQ2YzM0MjI=
+    NDk0MDhhMmYwMGUxYThmZGEwMmU0Y2U1Y2ZhNmIxNDE1ZWI1NTM5MGI5MmRi
+    MDM3NTY0NTcwNzdkZGFkOTMwZDA4ZDkwYzBlMjhmNzY4OTg1ZmY3MDVmY2Nl
+    M2EzMThkMDBiZTRjZWViYzBmNzgyOTA4YTY3MjNlMjU4MTljNzk=
   data.tar.gz: !binary |-
-    NzhmYjZkNGRlMTVmYTBlNDBkNDRkMjE5YzMzZGIwYzZlMjRkMjI2YjkwYzlm
-    ODY2NGUwMDlkODVlNjcyZmNjNzIzMDk3NWJkMTJkMDdlMGFmZmE1NDE1NGNl
-    NzhkMjRmNmM4MDBkOTIzNTFhMDFkYmViNDc0MzQyNzcxYzNhNGY=
+    OTA4NThkMzdmYTVjZTY0YzcyZTk5ZjQ5Y2ZmNzM5MWViM2ExOWYwNzFlZmRk
+    MjBiZTM1YTRjMWUwNGUwODY0ZjQwYjUxNDdjZjJiNGUyMTdhN2Q1MWM1Mjg4
+    ODEwMjg5OWQ3YTAwOTlhMTc1Mzc1YWExMjU3MDdhMDc3ZTlmYzA=

data/.rubocop.yml

@@ -5,9 +5,9 @@ Metrics/CyclomaticComplexity:
 Metrics/PerceivedComplexity:
   Max: 11
 Metrics/MethodLength:
-  Max: 27
+  Max: 30
 Metrics/ClassLength:
-  Max: 350
+  Max: 175
 
 #######
 # Finalized settings

data/README.md

@@ -3,19 +3,13 @@ jenkins-pipeline-builder
 
 [![Build Status](https://travis-ci.org/constantcontact/jenkins_pipeline_builder.svg)](https://travis-ci.org/constantcontact/jenkins_pipeline_builder)  [![Gem Version](https://badge.fury.io/rb/jenkins_pipeline_builder.svg)](http://badge.fury.io/rb/jenkins_pipeline_builder)
 
-YAML driven CI Jenkins Pipeline Builder enabling to version your artifact pipelines alongside with the artifact source
-itself.
-
-This Gem uses this methodolody by itself. Notice the 'pipeline' folder where we have a declaration of the Gem's build
-pipeline.
+YAML/JSON driven jenkins job generator that lets you version your artifact pipelines alongside with the artifact source itself.
 
 # Background
 
-This project is inspired by a great work done by Arangamani with [jenkins_api_client](https://github.com/arangamani/jenkins_api_client) and
-amazing progress done by the Open Stack community with their [jenkins-job-builder](http://ci.openstack.org/jenkins-job-builder/)
+This project is inspired by the great work done by Arangamani with [jenkins_api_client](https://github.com/arangamani/jenkins_api_client) and amazing progress done by the Open Stack community with their [jenkins-job-builder](http://ci.openstack.org/jenkins-job-builder/)
 
-The YAML structure very closely resembles the OpenStack Job Builder, but, in comparison to Python version, is 100%
-pure Ruby and uses Jenkins API Client and has additional functionlity of building different types of Jenkins views.
+The YAML structure began very closely resembling the OpenStack Job Builder, but, has evolved since then. Under the covers it uses the Jenkins API Client and is very extensible. Any plugin that is not currently supported can be added locally.
 
 # JenkinsPipelineBuilder
 
@@ -47,8 +41,8 @@ Or install it yourself as:
 For more info see [jenkins_api_client](https://github.com/arangamani/jenkins_api_client).
 Supplying credentials to the client is optional, as not all Jenkins instances
 require authentication. This project supports two types of password-based
-authentication. You can just you the plain password by using <tt>password</tt>
-parameter. If you don't prefer leaving plain passwords in the credentials file,
+authentication. You can just use the plain password by using <tt>password</tt>
+parameter. If you don't want to leave plain passwords in the credentials file,
 you can encode your password in base64 format and use <tt>password_base64</tt>
 parameter to specify the password either in the arguments or in the credentials
 file. To use the client without credentials, just leave out the
@@ -66,7 +60,7 @@ initializing the client.
 
 ### Basic usage
 
-Create all your Job description files in a folder (Ex.: ./pipeline). Follow the Job/View/Project DSL.
+Create all your Job description files in a folder (e.g.: ./pipeline). Follow the Job/View/Project DSL.
 Try to extract the reusable values out of jobs into the project.
 
 Put the right information about the location of your Jenkins server and the appropriate credentials

data/lib/jenkins_pipeline_builder/compiler.rb

@@ -22,42 +22,21 @@
 
 module JenkinsPipelineBuilder
   class Compiler
-    def self.resolve_value(value, settings, job_collection)
-      # pull@ designates that this is a reference to a job that will be generated
-      # for a pull request, so we want to save the resolution for the second pass
-      pull_job = value.to_s.match(/{{pull@(.*)}}/)
-      return pull_job[1] if pull_job
+    attr_reader :generator, :job_collection
 
-      settings = settings.with_indifferent_access
-      value_s = value.to_s.clone
-      # First we try to do job name correction
-      vars = value_s.scan(/{{job@(.*)}}/).flatten
-      if vars.count > 0
-        vars.select! do |var|
-          var_val = job_collection[var.to_s]
-          value_s.gsub!("{{job@#{var}}}", var_val[:value][:name]) unless var_val.nil?
-          var_val.nil?
-        end
-      end
-      # Then we look for normal values to replace
-      vars = value_s.scan(/{{([^{}@]+)}}/).flatten
-      vars.select! do |var|
-        var_val = settings[var]
-        value_s.gsub!("{{#{var}}}", var_val.to_s) unless var_val.nil?
-        var_val.nil?
-      end
-      return nil if vars.count != 0
-      value_s
+    def initialize(generator)
+      @generator = generator
+      @job_collection = generator.job_collection.collection
     end
 
-    def self.get_settings_bag(item_bag, settings_bag = {})
+    def get_settings_bag(item_bag, settings_bag = {})
       item = item_bag[:value]
       bag = {}
       return unless item.is_a?(Hash)
       item.keys.each do |k|
         val = item[k]
         next unless val.is_a? String
-        new_value = resolve_value(val, settings_bag, {})
+        new_value = resolve_value(val, settings_bag)
         return nil if new_value.nil?
         bag[k] = new_value
       end
@@ -65,30 +44,51 @@ module JenkinsPipelineBuilder
       my_settings_bag.merge(bag)
     end
 
-    def self.compile(item, settings = {}, job_collection = {})
-      success, item = handle_enable(item, settings, job_collection)
+    def compile(item, settings = {})
+      success, item = handle_enable(item, settings)
       return false, item unless success
 
       case item
       when String
-        return compile_string item, settings, job_collection
+        return compile_string item, settings
       when Hash
-        return compile_hash item, settings, job_collection
+        return compile_hash item, settings
       when Array
-        return compile_array item, settings, job_collection
+        return compile_array item, settings
       end
       [true, item]
     end
 
-    def self.compile_string(item, settings, job_collection)
+    def handle_enable(item, settings)
+      return true, item unless item.is_a? Hash
+      if item.key?(:enabled) && item.key?(:parameters) && item.length == 2
+        enabled_switch = resolve_value(item[:enabled], settings)
+        return [true, {}] if enabled_switch == 'false'
+        if enabled_switch != 'true'
+          return [false, { 'value error' => "Invalid value for #{item[:enabled]}: #{enabled_switch}" }]
+        end
+        if item[:parameters].is_a? Hash
+          item = item.merge item[:parameters]
+          item.delete :parameters
+          item.delete :enabled
+        else
+          item = item[:parameters]
+        end
+      end
+      [true, item]
+    end
+
+    private
+
+    def compile_string(item, settings)
       errors = {}
-      new_value = resolve_value(item, settings, job_collection)
+      new_value = resolve_value(item, settings)
       errors[item] =  "Failed to resolve #{item}" if new_value.nil?
       return false, errors unless errors.empty?
       [true, new_value]
     end
 
-    def self.compile_array(item, settings, job_collection)
+    def compile_array(item, settings)
       errors = {}
       result = []
       item.each do |value|
@@ -96,7 +96,7 @@ module JenkinsPipelineBuilder
           errors[item] = "found a nil value when processing following array:\n #{item.inspect}"
           break
         end
-        success, payload = compile(value, settings, job_collection)
+        success, payload = compile(value, settings)
         unless success
           errors.merge!(payload)
           next
@@ -111,50 +111,65 @@ module JenkinsPipelineBuilder
       [true, result]
     end
 
-    def self.handle_enable(item, settings, job_collection)
-      return true, item unless item.is_a? Hash
-      if item.key?(:enabled) && item.key?(:parameters) && item.length == 2
-        enabled_switch = resolve_value(item[:enabled], settings, job_collection)
-        return [true, {}] if enabled_switch == 'false'
-        if enabled_switch != 'true'
-          return [false, { 'value error' => "Invalid value for #{item[:enabled]}: #{enabled_switch}" }]
-        end
-        if item[:parameters].is_a? Hash
-          item = item.merge item[:parameters]
-          item.delete :parameters
-          item.delete :enabled
-        else
-          item = item[:parameters]
-        end
+    def compile_item(key, value, errors, settings)
+      if value.nil?
+        errors[key] = "key: #{key} has a nil value, this is often a yaml syntax error. Skipping children and siblings"
+        return false, errors[key]
       end
-      [true, item]
+      success, payload = compile(value, settings)
+      unless success
+        errors.merge!(payload)
+        return false, payload
+      end
+      if payload.nil?
+        errors[key] = "Failed to resolve:\n===>key: #{key}\n\n===>value: #{value}\n\n===>of: #{item}"
+        return false, errors[key]
+      end
+      [true, payload]
     end
 
-    def self.compile_hash(item, settings, job_collection)
-      success, item = handle_enable(item, settings, job_collection)
+    def compile_hash(item, settings)
+      success, item = handle_enable(item, settings)
       return false, item unless success
 
       errors = {}
       result = {}
 
       item.each do |key, value|
-        if value.nil?
-          errors[key] = "key: #{key} has a nil value, this is often a yaml syntax error. Skipping children and siblings"
-          break
-        end
-        success, payload = compile(value, settings, job_collection)
-        unless success
-          errors.merge!(payload)
-          next
-        end
-        if payload.nil?
-          errors[key] = "Failed to resolve:\n===>key: #{key}\n\n===>value: #{value}\n\n===>of: #{item}"
-          next
-        end
+        success, payload = compile_item(key, value, errors, settings)
+        next unless success
         result[key] = payload unless payload == {}
       end
       return false, errors unless errors.empty?
       [true, result]
     end
+
+    def resolve_value(value, settings)
+      # pull@ designates that this is a reference to a job that will be generated
+      # for a pull request, so we want to save the resolution for the second pass
+      pull_job = value.to_s.match(/{{pull@(.*)}}/)
+      return pull_job[1] if pull_job
+
+      settings = settings.with_indifferent_access
+      value_s = value.to_s.clone
+      # First we try to do job name correction
+      vars = value_s.scan(/{{job@(.*)}}/).flatten
+      if vars.count > 0
+        vars.select! do |var|
+          var_val = job_collection[var.to_s]
+          value_s.gsub!("{{job@#{var}}}", var_val[:value][:name]) unless var_val.nil?
+          var_val.nil?
+        end
+      end
+      # Then we look for normal values to replace
+      vars = value_s.scan(/{{([^{}@]+)}}/).flatten
+      vars.select! do |var|
+        var_val = settings[var]
+        value_s.gsub!("{{#{var}}}", var_val.to_s) unless var_val.nil?
+        var_val.nil?
+      end
+      return nil if vars.count != 0
+      value_s
+    end
   end
 end

data/lib/jenkins_pipeline_builder/extension_set.rb

@@ -29,6 +29,10 @@ module JenkinsPipelineBuilder
       end
     end
 
+    def clear_installed_version
+      @version = nil
+    end
+
     def installed_version=(version)
       version = version.match(/\d+\.\d+/)
       @version = Gem::Version.new version

data/lib/jenkins_pipeline_builder/extensions.rb

@@ -77,6 +77,32 @@ module JenkinsPipelineBuilder
       errors.empty?
     end
 
+    def execute(value, n_xml)
+      errors = []
+      check_parameters value
+      errors.each do |error|
+        logger.error error
+      end
+      return false if errors.any?
+
+      n_builders = n_xml.xpath(path).first
+      n_builders.instance_exec(value, &before) if before
+      Nokogiri::XML::Builder.with(n_builders) do |builder|
+        builder.instance_exec value, &xml
+      end
+      n_builders.instance_exec(value, &after) if after
+      true
+    end
+
+    def check_parameters(value)
+      return if parameters && parameters.empty?
+      return unless value.is_a? Hash
+      value.each_key do |key|
+        next if parameters && parameters.include?(key)
+        errors << "Extension #{extension.name} does not support parameter #{key}"
+      end
+    end
+
     def errors
       errors = {}
       EXT_METHODS.keys.each do |name|

data/lib/jenkins_pipeline_builder/extensions/builders.rb

@@ -43,6 +43,12 @@ builder do
                       properties job[:config][:predefined_build_parameters]
                     end
                   end
+                  if job[:config].key? :properties_file
+                    send('hudson.plugins.parameterizedtrigger.FileBuildParameters') do
+                      propertiesFile job[:config][:properties_file][:file]
+                      failTriggerOnMissing job[:config][:properties_file][:skip_if_missing] || 'false'
+                    end
+                  end
                 end
               end
               killPhaseOnJobResultCondition job[:kill_phase_on] || 'FAILURE'

data/lib/jenkins_pipeline_builder/generator.rb

@@ -25,19 +25,8 @@ require 'json'
 
 module JenkinsPipelineBuilder
   class Generator
-    attr_reader :debug
     attr_accessor :no_files, :job_templates, :logger, :module_registry, :job_collection
 
-    # Initialize a Client object with Jenkins Api Client
-    #
-    # @param args [Hash] Arguments to connect to Jenkins server
-    #
-    # @option args [String] :something some option description
-    #
-    # @return [JenkinsPipelineBuilder::Generator] a client generator
-    #
-    # @raise [ArgumentError] when required options are not provided.
-    #
     def initialize
       @job_templates = {}
       @extensions = {}
@@ -53,10 +42,6 @@ module JenkinsPipelineBuilder
       JenkinsPipelineBuilder.client
     end
 
-    # Creates an instance to the View class by passing a reference to self
-    #
-    # @return [JenkinsApi::Client::System] An object to System subclass
-    #
     def view
       JenkinsPipelineBuilder::View.new(self)
     end
@@ -68,7 +53,7 @@ module JenkinsPipelineBuilder
       if job_collection.projects.any?
         errors = publish_project(project_name)
       else
-        errors = publish_jobs(standalone job_collection.jobs)
+        errors = publish_jobs(job_collection.standalone_jobs)
       end
       errors.each do |k, v|
         logger.error "Encountered errors compiling: #{k}:"
@@ -85,18 +70,13 @@ module JenkinsPipelineBuilder
       job_collection.projects.each do |project|
         next unless project[:name] == project_name || project_name.nil?
         logger.info "Using Project #{project}"
-        pull_job = find_pull_request_generator(project)
-        p_success, p_payload = compile_pull_request_generator(pull_job[:name], project)
-        unless p_success
-          errors[pull_job[:name]] = p_payload
-          next
+
+        pull_request_generator = JenkinsPipelineBuilder::PullRequestGenerator.new project, self
+
+        unless pull_request_generator.valid?
+          errors[pull_request_generator.pull_generator[:name]] = pull_request_generator.errors
         end
-        jobs = filter_pull_request_jobs(pull_job)
-        pull = JenkinsPipelineBuilder::PullRequestGenerator.new(project, jobs, p_payload)
-        @job_collection.collection.merge! pull.jobs
-        pull_errors = create_pull_request_jobs(pull)
-        errors.merge! pull_errors
-        purge_pull_request_jobs(pull)
+
       end
       errors.each do |k, v|
         logger.error "Encountered errors compiling: #{k}:"
@@ -112,84 +92,51 @@ module JenkinsPipelineBuilder
     end
 
     def dump(job_name)
-      @logger.info "Debug #{@debug}"
-      @logger.info "Dumping #{job_name} into #{job_name}.xml"
+      logger.info "Debug #{JenkinsPipelineBuilder.debug}"
+      logger.info "Dumping #{job_name} into #{job_name}.xml"
       xml = client.job.get_config(job_name)
       File.open(job_name + '.xml', 'w') { |f| f.write xml }
     end
 
-    #
-    # BEGIN PRIVATE METHODS
-    #
-
-    private
-
-    # Converts standalone jobs to the format that they have when loaded as part of a project.
-    # This addresses an issue where #pubish_jobs assumes that each job will be wrapped
-    # with in a hash a referenced under a key called :result, which is what happens when
-    # it is loaded as part of a project.
-    #
-    # @return An array of jobs
-    #
-    def standalone(jobs)
-      jobs.map! { |job| { result: job } }
+    def resolve_job_by_name(name, settings = {})
+      job = job_collection.get_item(name)
+      fail "Failed to locate job by name '#{name}'" if job.nil?
+      job_value = job[:value]
+      logger.debug "Compiling job #{name}"
+      compiler = Compiler.new self
+      success, payload = compiler.compile(job_value, settings)
+      [success, payload]
     end
 
-    def purge_pull_request_jobs(pull)
-      pull.purge.each do |purge_job|
-        jobs = client.job.list "#{purge_job}.*"
-        jobs.each do |job|
-          client.job.delete job
-        end
-      end
-    end
+    def resolve_project(project)
+      defaults = job_collection.defaults
+      settings = defaults.nil? ? {} : defaults[:value] || {}
+      compiler = Compiler.new self
+      project[:settings] = compiler.get_settings_bag(project, settings) unless project[:settings]
+      project_body = project[:value]
 
-    def create_pull_request_jobs(pull)
-      errors = {}
-      pull.create.each do |pull_project|
-        success, compiled_project = resolve_project(pull_project)
-        compiled_project[:value][:jobs].each do |i|
-          job = i[:result]
-          job = Job.new job
-          success, payload = job.create_or_update
-          errors[job.name] = payload unless success
+      jobs = prepare_jobs(project_body[:jobs]) if project_body[:jobs]
+      logger.info project
+      process_job_changes(jobs)
+      errors = process_jobs(jobs, project)
+      errors = process_views(project_body[:views], project, errors) if project_body[:views]
+      errors.each do |k, v|
+        puts "Encountered errors processing: #{k}:"
+        v.each do |key, error|
+          puts "  key: #{key} had the following error:"
+          puts "  #{error.inspect}"
         end
       end
-      errors
-    end
+      return false, 'Encountered errors exiting' unless errors.empty?
 
-    def find_pull_request_generator(project)
-      project_jobs = project[:value][:jobs] || []
-      pull_job = nil
-      project_jobs.each do |job|
-        job = job.keys.first if job.is_a? Hash
-        job = @job_collection.collection[job.to_s]
-        pull_job = job if job[:value][:job_type] == 'pull_request_generator'
-      end
-      fail 'No jobs of type pull_request_generator found' unless pull_job
-      pull_job
+      [true, project]
     end
 
-    def filter_pull_request_jobs(pull_job)
-      jobs = {}
-      pull_jobs = pull_job[:value][:jobs] || []
-      pull_jobs.each do |job|
-        if job.is_a? String
-          jobs[job.to_s] = @job_collection.collection[job.to_s]
-        else
-          jobs[job.keys.first.to_s] = @job_collection.collection[job.keys.first.to_s]
-        end
-      end
-      fail 'No jobs found for pull request' if jobs.empty?
-      jobs
-    end
+    #
+    # BEGIN PRIVATE METHODS
+    #
 
-    def compile_pull_request_generator(pull_job, project)
-      defaults = job_collection.defaults
-      settings = defaults.nil? ? {} : defaults[:value] || {}
-      settings = Compiler.get_settings_bag(project, settings)
-      resolve_job_by_name(pull_job, settings)
-    end
+    private
 
     def prepare_jobs(jobs)
       jobs.map! do |job|
@@ -227,29 +174,6 @@ module JenkinsPipelineBuilder
       errors
     end
 
-    def resolve_project(project)
-      defaults = job_collection.defaults
-      settings = defaults.nil? ? {} : defaults[:value] || {}
-      project[:settings] = Compiler.get_settings_bag(project, settings) unless project[:settings]
-      project_body = project[:value]
-
-      jobs = prepare_jobs(project_body[:jobs]) if project_body[:jobs]
-      logger.info project
-      process_job_changes(jobs)
-      errors = process_jobs(jobs, project)
-      errors = process_views(project_body[:views], project, errors) if project_body[:views]
-      errors.each do |k, v|
-        puts "Encountered errors processing: #{k}:"
-        v.each do |key, error|
-          puts "  key: #{key} had the following error:"
-          puts "  #{error.inspect}"
-        end
-      end
-      return false, 'Encountered errors exiting' unless errors.empty?
-
-      [true, project]
-    end
-
     def process_jobs(jobs, project, errors = {})
       jobs.each do |job|
         job_id = job.keys.first
@@ -264,15 +188,6 @@ module JenkinsPipelineBuilder
       errors
     end
 
-    def resolve_job_by_name(name, settings = {})
-      job = job_collection.get_item(name)
-      fail "Failed to locate job by name '#{name}'" if job.nil?
-      job_value = job[:value]
-      logger.debug "Compiling job #{name}"
-      success, payload = Compiler.compile(job_value, settings, @job_collection.collection)
-      [success, payload]
-    end
-
     def publish_project(project_name, errors = {})
       job_collection.projects.each do |project|
         next unless project_name.nil? || project[:name] == project_name