jenkins_pipeline_builder 0.4.2 → 0.5.0

data/lib/jenkins_pipeline_builder/cli/base.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -21,49 +21,41 @@
 #
 
 require 'thor'
-#require "#{File.dirname(__FILE__)}/pipeline.rb"
 
 module JenkinsPipelineBuilder
   module CLI
     class Base < Thor
+      class_option :username, aliases:  '-u', desc:  'Name of Jenkins user'
+      class_option :password, aliases:  '-p', desc:  'Password of Jenkins user'
+      class_option :password_base64, aliases:  '-b', desc:  'Base 64 encoded password of Jenkins user'
+      class_option :server_ip, aliases:  '-s', desc:  'Jenkins server IP address'
+      class_option :server_port, aliases:  '-o', desc:  'Jenkins port'
+      class_option :creds_file, aliases:  '-c', desc:  'Credentials file for communicating with Jenkins server'
+      class_option :debug, type:  :boolean, aliases:  '-d', desc:  'Run in debug mode (no Jenkins changes)', default:  false
 
-      class_option :username, :aliases => "-u", :desc => "Name of Jenkins user"
-      class_option :password, :aliases => "-p",
-                   :desc => "Password of Jenkins user"
-      class_option :password_base64, :aliases => "-b",
-                   :desc => "Base 64 encoded password of Jenkins user"
-      class_option :server_ip, :aliases => "-s",
-                   :desc => "Jenkins server IP address"
-      class_option :server_port, :aliases => "-o", :desc => "Jenkins port"
-      class_option :creds_file, :aliases => "-c",
-                   :desc => "Credentials file for communicating with Jenkins server"
-      class_option :debug, :type => :boolean, :aliases => "-d", :desc => "Run in debug mode (no Jenkins changes)",
-                   :default => false
+      map '-v' => :version
 
-
-      map "-v" => :version
-
-      desc "version", "Shows current version"
+      desc 'version', 'Shows current version'
       # CLI command that returns the version of Jenkins API Client
       def version
         puts JenkinsPipelineBuilder::VERSION
       end
 
-      # Register the CLI::Pipeline class as "pipeline" subcommand to CLI
+      # Register the CLI::Pipeline class as 'pipeline' subcommand to CLI
       register(
-          CLI::Pipeline,
-          'pipeline',
-          'pipeline [subcommand]',
-          'Provides functions to access pipeline functions of the Jenkins CI server'
+        CLI::Pipeline,
+        'pipeline',
+        'pipeline [subcommand]',
+        'Provides functions to access pipeline functions of the Jenkins CI server'
       )
 
-      # Register the CLI::Job class as "view" subcommand to CLI
+      # Register the CLI::Job class as 'view' subcommand to CLI
       register(
-          CLI::View,
-          'view',
-          'view [subcommand]',
-          'Provides functions to access view interface of Jenkins CI server'
+        CLI::View,
+        'view',
+        'view [subcommand]',
+        'Provides functions to access view interface of Jenkins CI server'
       )
     end
   end
-end
+end

data/lib/jenkins_pipeline_builder/cli/helper.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -22,7 +22,11 @@
 
 require 'fileutils'
 require 'yaml'
+require 'json'
 require 'jenkins_api_client'
+require 'open-uri'
+require 'zlib'
+require 'archive/tar/minitar'
 
 module JenkinsPipelineBuilder
   module CLI
@@ -40,29 +44,22 @@ module JenkinsPipelineBuilder
           (options[:password] || options[:password_base64])
           creds = options
         elsif options[:creds_file]
-          creds = YAML.load_file(
-              File.expand_path(options[:creds_file])
-              #File.expand_path(options[:creds_file], __FILE__)
-              #options[:creds_file]
-          )
+          creds = YAML.load_file(File.expand_path(options[:creds_file]))
         elsif File.exist?("#{ENV['HOME']}/.jenkins_api_client/login.yml")
           creds = YAML.load_file(
-              File.expand_path(
-                  "#{ENV['HOME']}/.jenkins_api_client/login.yml", __FILE__
-              )
+            File.expand_path("#{ENV['HOME']}/.jenkins_api_client/login.yml", __FILE__)
           )
         else
-          msg = "Credentials are not set. Please pass them as parameters or"
-          msg << " set them in the default credentials file"
+          msg = 'Credentials are not set. Please pass them as parameters or'
+          msg << ' set them in the default credentials file'
           puts msg
           exit 1
         end
 
-        #creds[:log_level] =  Logger::DEBUG
         client = JenkinsApi::Client.new(creds)
-        generator = JenkinsPipelineBuilder::Generator.new(options, client)
-        generator.debug = options[:debug] #== 'debug'
-        return generator
+        generator = JenkinsPipelineBuilder::Generator.new(client)
+        generator.debug = options[:debug]
+        generator
       end
     end
   end

data/lib/jenkins_pipeline_builder/cli/pipeline.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -35,6 +35,11 @@ module JenkinsPipelineBuilder
       def bootstrap(path, project_name = nil)
         Helper.setup(parent_options).bootstrap(path, project_name)
       end
+
+      desc 'pull_request Path', 'Generates jenkins jobs based on a git pull request.'
+      def pull_request(path, project_name = nil)
+        Helper.setup(parent_options).pull_request(path, project_name)
+      end
     end
   end
 end

data/lib/jenkins_pipeline_builder/cli/view.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -37,4 +37,4 @@ module JenkinsPipelineBuilder
       end
     end
   end
-end
+end

data/lib/jenkins_pipeline_builder/compiler.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -22,17 +22,27 @@
 
 module JenkinsPipelineBuilder
   class Compiler
-    def self.resolve_value(value, settings)
+    def self.resolve_value(value, settings, job_collection)
       settings = settings.with_indifferent_access
       value_s = value.to_s.clone
-      vars = value_s.scan(/{{([^{}]+)}}/).flatten
+      # First we try to do job name correction
+      vars = value_s.scan(/{{job@(.*)}}/).flatten
+      if vars.count > 0
+        vars.select! do |var|
+          var_val = job_collection[var.to_s]
+          value_s.gsub!("{{job@#{var}}}", var_val[:value][:name]) unless var_val.nil?
+          var_val.nil?
+        end
+      end
+      # Then we look for normal values to replace
+      vars = value_s.scan(/{{([^{}@]+)}}/).flatten
       vars.select! do |var|
         var_val = settings[var]
-        value_s.gsub!("{{#{var.to_s}}}", var_val) unless var_val.nil?
+        value_s.gsub!("{{#{var}}}", var_val) unless var_val.nil?
         var_val.nil?
       end
       return nil if vars.count != 0
-      return value_s
+      value_s
     end
 
     def self.get_settings_bag(item_bag, settings_bag = {})
@@ -42,73 +52,65 @@ module JenkinsPipelineBuilder
       item.keys.each do |k|
         val = item[k]
         if val.kind_of? String
-          new_value = resolve_value(val, settings_bag)
+          new_value = resolve_value(val, settings_bag, {})
           return nil if new_value.nil?
           bag[k] = new_value
         end
       end
       my_settings_bag = settings_bag.clone
-      return my_settings_bag.merge(bag)
+      my_settings_bag.merge(bag)
     end
 
-    def self.compile(item, settings = {})
+    def self.compile(item, settings = {}, job_collection = {})
       errors = {}
       case item
-        when String
-          new_value = resolve_value(item, settings)
-          if new_value.nil?
-            errors[item] =  "Failed to resolve #{item}"
+      when String
+        new_value = resolve_value(item, settings, job_collection)
+        errors[item] =  "Failed to resolve #{item}" if new_value.nil?
+        return false, errors unless errors.empty?
+        return true, new_value
+      when Hash
+        result = {}
+        item.each do |key, value|
+          if value.nil?
+            errors[key] = "key: #{key} has a nil value, this is likely a yaml syntax error. Skipping children and siblings"
+            break
           end
-          unless errors.empty?
-            return false, errors
+          success, payload = compile(value, settings, job_collection)
+          unless success
+            errors.merge!(payload)
+            next
           end
-          return true, new_value
-        when Hash
-          result = {}
-          item.each do |key, value|
-            if value.nil?
-              errors[key] = "key: #{key} has a nil value, this is likely a yaml syntax error. Skipping children and siblings"
-              break
-            end
-            success, payload = compile(value, settings)
-            unless success
-              errors.merge!(payload)
-              next
-            end
-            if payload.nil?
-              errors[key] = "Failed to resolve:\n===>key: #{key}\n\n===>value: #{value}\n\n===>of: #{item}"
-              next
-            end
-            result[key] = payload
+          if payload.nil?
+            errors[key] = "Failed to resolve:\n===>key: #{key}\n\n===>value: #{value}\n\n===>of: #{item}"
+            next
           end
-          unless errors.empty?
-            return false, errors
+          result[key] = payload
+        end
+        return false, errors unless errors.empty?
+        return true, result
+      when Array
+        result = []
+        item.each do |value|
+          if value.nil?
+            errors[item] = "found a nil value when processing following array:\n #{item.inspect}"
+            break
           end
-          return true, result
-        when Array
-          result = []
-          item.each do |value|
-            if value.nil?
-              errors[item] = "found a nil value when processing following array:\n #{item.inspect}"
-              break
-            end
-            success, payload = compile(value, settings)
-            unless success
-              errors.merge!(payload)
-              next
-            end
-            if payload.nil?
-              errors[value] = "Failed to resolve:\n===>item #{value}\n\n===>of list: #{item}"
-              next
-            end
-            result << payload
+          success, payload = compile(value, settings, job_collection)
+          unless success
+            errors.merge!(payload)
+            next
           end
-          unless errors.empty?
-            return false, errors
+          if payload.nil?
+            errors[value] = "Failed to resolve:\n===>item #{value}\n\n===>of list: #{item}"
+            next
           end
-          return true, result
+          result << payload
+        end
+        return false, errors unless errors.empty?
+        return true, result
       end
-      return true, item
+      [true, item]
     end
   end
 end

data/lib/jenkins_pipeline_builder/generator.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Igor Moochnick
+# Copyright (c) 2014 Constant Contact
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -35,84 +35,119 @@ module JenkinsPipelineBuilder
     #
     # @raise [ArgumentError] when required options are not provided.
     #
-    def initialize(args, client)
+    def initialize(client)
       @client        = client
       @logger        = @client.logger
-      #@logger.level = (@debug) ? Logger::DEBUG : Logger::INFO;
+      # @logger.level = (@debug) ? Logger::DEBUG : Logger::INFO;
       @job_templates = {}
       @job_collection = {}
-
-      @module_registry = ModuleRegistry.new ({
-          job: {
-              description: JobBuilder.method(:change_description),
-              scm_params: JobBuilder.method(:apply_scm_params),
-              hipchat: JobBuilder.method(:hipchat_notifier),
-              parameters: JobBuilder.method(:build_parameters),
-              priority: JobBuilder.method(:use_specific_priority),
-              discard_old: JobBuilder.method(:discard_old_param),
-              throttle: JobBuilder.method(:throttle_job),
-              prepare_environment: JobBuilder.method(:prepare_environment),
-              builders: {
-                  registry: {
-                      multi_job: Builders.method(:build_multijob),
-                      inject_vars_file: Builders.method(:build_environment_vars_injector),
-                      shell_command: Builders.method(:build_shell_command),
-                      maven3: Builders.method(:build_maven3),
-                      remote_job: Builders.method(:start_remote_job)
-                  },
-                  method:
-                    lambda { |registry, params, n_xml| @module_registry.run_registry_on_path('//builders', registry, params, n_xml) }
-              },
-              publishers: {
-                  registry: {
-                      git: Publishers.method(:push_to_git),
-                      hipchat: Publishers.method(:push_to_hipchat),
-                      description_setter: Publishers.method(:description_setter),
-                      downstream: Publishers.method(:push_to_projects),
-                      junit_result: Publishers.method(:publish_junit),
-                      coverage_result: Publishers.method(:publish_rcov),
-                      post_build_script: Publishers.method(:post_build_script)
-                  },
-                  method:
-                    lambda { |registry, params, n_xml| @module_registry.run_registry_on_path('//publishers', registry, params, n_xml) }
-              },
-              wrappers: {
-                  registry: {
-                      timestamp: Wrappers.method(:console_timestamp),
-                      ansicolor: Wrappers.method(:ansicolor),
-                      artifactory: Wrappers.method(:publish_to_artifactory),
-                      rvm: Wrappers.method(:run_with_rvm),
-                      rvm05: Wrappers.method(:run_with_rvm05),
-                      inject_env_var: Wrappers.method(:inject_env_vars),
-                      inject_passwords: Wrappers.method(:inject_passwords),
-                      maven3artifactory: Wrappers.method(:artifactory_maven3_configurator)
-                  },
-                  method:
-                    lambda { |registry, params, n_xml| @module_registry.run_registry_on_path('//buildWrappers', registry, params, n_xml) }
-              },
-              triggers: {
-                  registry: {
-                      git_push: Triggers.method(:enable_git_push),
-                      scm_polling: Triggers.method(:enable_scm_polling),
-                      periodic_build: Triggers.method(:enable_periodic_build),
-                      upstream: Triggers.method(:enable_upstream_check)
-                  }, 
-                  method:
-                    lambda { |registry, params, n_xml| @module_registry.run_registry_on_path('//triggers', registry, params, n_xml) }
-              }
+      @extensions = {}
+      @remote_depends = {}
+
+      @module_registry = ModuleRegistry.new(
+        job: {
+          description: JobBuilder.method(:change_description),
+          scm_params: JobBuilder.method(:apply_scm_params),
+          hipchat: JobBuilder.method(:hipchat_notifier),
+          parameters: JobBuilder.method(:build_parameters),
+          priority: JobBuilder.method(:use_specific_priority),
+          discard_old: JobBuilder.method(:discard_old_param),
+          throttle: JobBuilder.method(:throttle_job),
+          prepare_environment: JobBuilder.method(:prepare_environment),
+          concurrent_build: JobBuilder.method(:concurrent_build),
+          builders: {
+            registry: {
+              multi_job: Builders.method(:build_multijob),
+              inject_vars_file: Builders.method(:build_environment_vars_injector),
+              shell_command: Builders.method(:build_shell_command),
+              maven3: Builders.method(:build_maven3),
+              blocking_downstream: Builders.method(:blocking_downstream),
+              remote_job: Builders.method(:start_remote_job)
+            },
+            method:
+            ->(registry, params, n_xml) { @module_registry.run_registry_on_path('//builders', registry, params, n_xml) }
+          },
+          publishers: {
+            registry: {
+              git: Publishers.method(:push_to_git),
+              hipchat: Publishers.method(:push_to_hipchat),
+              description_setter: Publishers.method(:description_setter),
+              downstream: Publishers.method(:push_to_projects),
+              junit_result: Publishers.method(:publish_junit),
+              coverage_result: Publishers.method(:publish_rcov),
+              post_build_script: Publishers.method(:post_build_script),
+              groovy_postbuild: Publishers.method(:groovy_postbuild)
+            },
+            method:
+            ->(registry, params, n_xml) { @module_registry.run_registry_on_path('//publishers', registry, params, n_xml) }
+          },
+          wrappers: {
+            registry: {
+              timestamp: Wrappers.method(:console_timestamp),
+              ansicolor: Wrappers.method(:ansicolor),
+              artifactory: Wrappers.method(:publish_to_artifactory),
+              rvm: Wrappers.method(:run_with_rvm),
+              rvm05: Wrappers.method(:run_with_rvm05),
+              inject_env_var: Wrappers.method(:inject_env_vars),
+              inject_passwords: Wrappers.method(:inject_passwords),
+              maven3artifactory: Wrappers.method(:artifactory_maven3_configurator)
+            },
+            method:
+            ->(registry, params, n_xml) { @module_registry.run_registry_on_path('//buildWrappers', registry, params, n_xml) }
+          },
+          triggers: {
+            registry: {
+              git_push: Triggers.method(:enable_git_push),
+              scm_polling: Triggers.method(:enable_scm_polling),
+              periodic_build: Triggers.method(:enable_periodic_build),
+              upstream: Triggers.method(:enable_upstream_check)
+            },
+            method:
+            ->(registry, params, n_xml) { @module_registry.run_registry_on_path('//triggers', registry, params, n_xml) }
           }
-      })
+        }
+      )
+    end
+
+    def load_extensions(path)
+      path = "#{path}/extensions"
+      path = File.expand_path(path, Dir.getwd)
+      if File.directory?(path)
+        @logger.info "Loading extensions from folder #{path}"
+        Dir[File.join(path, '/*.yaml'), File.join(path, '/*.yml')].each do |file|
+          @logger.info "Loading file #{file}"
+          yaml = YAML.load_file(file)
+          yaml.each do |ext|
+            Utils.symbolize_keys_deep!(ext)
+            ext = ext[:extension]
+            name = ext[:name]
+            type = ext[:type]
+            function = ext[:function]
+            fail "Duplicate extension with name '#{name}' was detected." if @extensions.key?(name)
+            @extensions[name.to_s] = { name: name.to_s, type: type, function: function }
+          end
+        end
+      end
+      @extensions.each_value do |ext|
+        name = ext[:name].to_sym
+        registry = @module_registry.registry[:job]
+        function = eval "Proc.new {|params,xml| #{ext[:function]} }"
+        type = ext[:type].downcase.pluralize.to_sym if ext[:type]
+        if type
+          registry[type][:registry][name] = function
+        else
+          registry[name] = function
+        end
+      end
     end
 
-    attr_accessor :client
     def debug=(value)
       @debug = value
-      @logger.level = (value) ? Logger::DEBUG : Logger::INFO;
-    end
-    def debug
-      @debug
+      @logger.level = (value) ? Logger::DEBUG : Logger::INFO
     end
-    # TODO: WTF?
+
+    attr_reader :debug
+    attr_accessor :client
     attr_accessor :no_files
     attr_accessor :job_collection
 
@@ -124,12 +159,12 @@ module JenkinsPipelineBuilder
       JenkinsPipelineBuilder::View.new(self)
     end
 
-    def load_collection_from_path(path, recursively = false)
-      path = File.expand_path(path, relative_to=Dir.getwd)
+    def load_collection_from_path(path, recursively = false, remote = false)
+      path = File.expand_path(path, Dir.getwd)
       if File.directory?(path)
         @logger.info "Generating from folder #{path}"
         Dir[File.join(path, '/*.yaml'), File.join(path, '/*.yml')].each do |file|
-          if File.directory?(file)
+          if File.directory?(file) # TODO: This doesn't work unless the folder contains .yml or .yaml at the end
             if recursively
               load_collection_from_path(File.join(path, file), recursively)
             else
@@ -138,23 +173,35 @@ module JenkinsPipelineBuilder
           end
           @logger.info "Loading file #{file}"
           yaml = YAML.load_file(file)
-          load_job_collection(yaml)
+          load_job_collection(yaml, remote)
         end
       else
         @logger.info "Loading file #{path}"
         yaml = YAML.load_file(path)
-        load_job_collection(yaml)
+        load_job_collection(yaml, remote)
       end
     end
 
-    def load_job_collection(yaml)
+    def load_job_collection(yaml, remote = false)
       yaml.each do |section|
         Utils.symbolize_keys_deep!(section)
         key = section.keys.first
         value = section[key]
+        if key == :dependencies
+          @logger.info 'Resolving Dependencies for remote project'
+          return load_remote_yaml(value)
+        end
+
         name = value[:name]
-        raise "Duplicate item with name '#{name}' was detected." if @job_collection.has_key?(name)
-        @job_collection[name.to_s] = { name: name.to_s, type: key, value: value }
+        if @job_collection.key?(name)
+          if remote
+            @logger.info "Duplicate item with name '#{name}' was detected from the remote folder."
+          else
+            fail "Duplicate item with name '#{name}' was detected."
+          end
+        else
+          @job_collection[name.to_s] = { name: name.to_s, type: key, value: value }
+        end
       end
     end
 
@@ -162,33 +209,116 @@ module JenkinsPipelineBuilder
       @job_collection[name.to_s]
     end
 
-    def resolve_project(project)
-      defaults = get_item('global')
-      settings = defaults.nil? ? {} : defaults[:value] || {}
+    def load_template(path, template)
+      # If we specify what folder the yaml is in, load that
+      if template[:folder]
+        path = File.join(path, template[:folder])
+      else
+        path = File.join(path, template[:name]) unless template[:name] == 'default'
+        # If we are looking for the newest version or no version was set
+        if (template[:version].nil? || template[:version] == 'newest') && File.directory?(path)
+          folders = Dir.entries(path)
+          highest = '0' # Default to v1
+          folders.each do |f|
+            highest = f if f > highest # Note: to_i returns any integers in the folder name
+          end
+          template[:version] = highest unless highest == 0
+        end
+        path = File.join(path, template[:version]) unless template[:version].nil?
+        path = File.join(path, 'pipeline')
+      end
 
-      project[:settings] = Compiler.get_settings_bag(project, settings) unless project[:settings]
-      project_body = project[:value]
+      if File.directory?(path)
+        @logger.info "Loading from #{path}"
+        load_collection_from_path(path, false, true)
+        true
+      else
+        false
+      end
+    end
+
+    def download_yaml(url, file)
+      @remote_depends[url] = file
+      @logger.info "Downloading #{url} to #{file}.tar"
+      open("#{file}.tar", 'w') do |local_file|
+        open(url) do |remote_file|
+          local_file.write(Zlib::GzipReader.new(remote_file).read)
+        end
+      end
 
-      # Process jobs
-      jobs = project_body[:jobs] || []
+      # Extract Tar.gz to 'remote' folder
+      @logger.info "Unpacking #{file}.tar to #{file} folder"
+      Archive::Tar::Minitar.unpack("#{file}.tar", file)
+    end
+
+    def load_remote_yaml(dependencies)
+      ### Load remote YAML
+      # Download Tar.gz
+      dependencies.each do |source|
+        source = source[:source]
+        url = source[:url]
+        file = "remote-#{@remote_depends.length}"
+        if @remote_depends[url]
+          file = @remote_depends[url]
+        else
+          download_yaml(url, file)
+        end
+
+        path = File.expand_path(file, Dir.getwd)
+        # Load templates recursively
+        unless source[:templates]
+          @logger.info 'No specific template specified'
+          # Try to load the folder or the pipeline folder
+          path = File.join(path, 'pipeline') if Dir.entries(path).include? 'pipeline'
+          return load_collection_from_path(path)
+        end
+
+        load_templates(source[:templates])
+      end
+    end
+
+    def load_templates(templates)
+      templates.each do |template|
+        version = template[:version] || 'newest'
+        @logger.info "Loading #{template[:name]} at version #{version}"
+        # Move into the remote folder and look for the template folder
+        remote = Dir.entries(path)
+        if remote.include? template[:name]
+          # We found the template name, load this path
+          @logger.info 'We found the template!'
+          load_template(path, template)
+        else
+          # Many cases we must dig one layer deep
+          remote.each do |file|
+            load_template(File.join(path, file), template)
+          end
+        end
+      end
+    end
+
+    def cleanup_temp_remote
+      @remote_depends.each_value do |file|
+        FileUtils.rm_r file
+        FileUtils.rm_r "#{file}.tar"
+      end
+    end
+
+    def prepare_jobs(jobs)
       jobs.map! do |job|
         job.kind_of?(String) ? { job.to_sym => {} } : job
       end
-      errors = {}
-      @logger.info project
+    end
+
+    def process_job_changes(jobs)
       jobs.each do |job|
         job_id = job.keys.first
-        settings = project[:settings].clone.merge(job[job_id])
-        success, payload = resolve_job_by_name(job_id, settings)
-        if success
-          job[:result] = payload
-        else
-          errors[job_id] = payload
-        end
+        j = get_item(job_id)
+        Utils.hash_merge!(j, job[job_id])
+        j[:value][:name] = j[:job_name] if j[:job_name]
       end
+    end
 
-      # Process views
-      views = project_body[:views] || []
+    def process_views(views, project, errors = {})
       views.map! do |view|
         view.kind_of?(String) ? { view.to_sym => {} } : view
       end
@@ -203,27 +333,55 @@ module JenkinsPipelineBuilder
           errors[view_id] = payload
         end
       end
+      errors
+    end
 
-      errors.each do |k,v|
+    def process_jobs(jobs, project, errors = {})
+      jobs.each do |job|
+        job_id = job.keys.first
+        settings = project[:settings].clone.merge(job[job_id])
+        success, payload = resolve_job_by_name(job_id, settings)
+        if success
+          job[:result] = payload
+        else
+          errors[job_id] = payload
+        end
+      end
+      errors
+    end
+
+    def resolve_project(project)
+      defaults = get_item('global')
+      settings = defaults.nil? ? {} : defaults[:value] || {}
+
+      project[:settings] = Compiler.get_settings_bag(project, settings) unless project[:settings]
+      project_body = project[:value]
+
+      jobs = prepare_jobs(project_body[:jobs]) if project_body[:jobs]
+      @logger.info project
+      process_job_changes(jobs)
+      errors = process_jobs(jobs, project)
+      errors = process_view(project_body[:views], project,  errors) if project_body[:views]
+
+      errors.each do |k, v|
         puts "Encountered errors processing: #{k}:"
         v.each do |key, error|
           puts "  key: #{key} had the following error:"
           puts "  #{error.inspect}"
         end
       end
-      return false, "Encountered errors exiting" unless errors.empty?
+      return false, 'Encountered errors exiting' unless errors.empty?
 
-      return true, project
+      [true, project]
     end
 
     def resolve_job_by_name(name, settings = {})
       job = get_item(name)
-      raise "Failed to locate job by name '#{name}'" if job.nil?
+      fail "Failed to locate job by name '#{name}'" if job.nil?
       job_value = job[:value]
       @logger.debug "Compiling job #{name}"
-
-      success, payload = Compiler.compile(job_value, settings)
-      return success, payload
+      success, payload = Compiler.compile(job_value, settings, @job_collection)
+      [success, payload]
     end
 
     def projects
@@ -231,7 +389,7 @@ module JenkinsPipelineBuilder
       @job_collection.values.each do |item|
         result << item if item[:type] == :project
       end
-      return result
+      result
     end
 
     def jobs
@@ -239,64 +397,96 @@ module JenkinsPipelineBuilder
       @job_collection.values.each do |item|
         result << item if item[:type] == :job
       end
-      return result
+      result
+    end
+
+    def publish_project(project_name, errors = {})
+      projects.each do |project|
+        next if project_name && project[:name] == project_name
+        success, payload = resolve_project(project)
+        if success
+          puts 'successfully resolved project'
+          compiled_project = payload
+        else
+          puts payload
+          return false
+        end
+
+        if compiled_project[:value][:jobs]
+          errors = publish_jobs(compiled_project[:value][:jobs])
+        end
+        if compiled_project[:value][:views]
+          compiled_project[:value][:views].each do |v|
+            compiled_view = v[:result]
+            view.create(compiled_view)
+          end
+        end
+      end
+      errors
+    end
+
+    def publish_jobs(jobs, errors = {})
+      jobs.each do |i|
+        puts "Processing #{i}"
+        job = i[:result]
+        fail "Result is empty for #{i}" if job.nil?
+        success, payload = compile_job_to_xml(job)
+        if success
+          create_or_update(job, payload)
+        else
+          errors[job[:name]] = payload
+        end
+      end
+      errors
     end
 
     def bootstrap(path, project_name)
       @logger.info "Bootstrapping pipeline from path #{path}"
       load_collection_from_path(path)
-
+      cleanup_temp_remote
+      load_extensions(path)
       errors = {}
       # Publish all the jobs if the projects are not found
       if projects.count == 0
-        jobs.each do |i|
-          job = i[:value]
-          success, payload = compile_job_to_xml(job)
-          if success
-            create_or_update(job, payload)
-          else
-            errors[job[:name]] = payload
-          end
-        end
+        errors = publish_jobs(jobs)
       else
-        projects.each do |project|
-          success, payload = resolve_project(project)
-          if payload[:value][:name] == project_name || project_name == nil # If we specify a project name, only use that project
-            if success
-              puts 'successfully resolved project'
-              compiled_project = payload
-            else
-              puts payload
-              return false
-            end
-
-            if compiled_project[:value][:jobs]
-              compiled_project[:value][:jobs].each do |i|
-                puts "Processing #{i}"
-                job = i[:result]
-                fail "Result is empty for #{i}" if job.nil?
-                success, payload = compile_job_to_xml(job)
-                if success
-                  create_or_update(job, payload)
-                else
-                  errors[job[:name]] = payload
-                end
-              end
-            end
+        errors = publish_project(project_name)
+      end
+      errors.each do |k, v|
+        @logger.error "Encountered errors compiling: #{k}:"
+        @logger.error v
+      end
+    end
 
-            if compiled_project[:value][:views]
-              compiled_project[:value][:views].each do |v|
-                _view = v[:result]
-                view.create(_view)
-              end
+    def pull_request(path, project_name)
+      @logger.info "Pull Request Generator Running from path #{path}"
+      load_collection_from_path(path)
+      cleanup_temp_remote
+      load_extensions(path)
+      jobs = {}
+      projects.each do |project|
+        if project[:name] == project_name || project_name.nil?
+          project_body = project[:value]
+          project_jobs = project_body[:jobs] || []
+          @logger.info "Using Project #{project}"
+          pull_job = nil
+          project_jobs.each do |job|
+            job = @job_collection[job.to_s]
+            pull_job = job if job[:value][:job_type] == 'pull_request_generator'
+          end
+          fail 'No Pull Request Found for Project' unless pull_job
+          pull_jobs = pull_job[:value][:jobs] || []
+          pull_jobs.each do |job|
+            if job.is_a? String
+              jobs[job.to_s] = @job_collection[job.to_s]
+            else
+              jobs[job.keys[0].to_s] = @job_collection[job.keys[0].to_s]
             end
           end
+          pull = JenkinsPipelineBuilder::PullRequestGenerator.new(self)
+          pull.run(project, jobs, pull_job)
         end
       end
-      errors.each do |k,v|
-        @logger.error "Encountered errors compiling: #{k}:"
-        @logger.error v
-      end
     end
 
     def dump(job_name)
@@ -323,40 +513,40 @@ module JenkinsPipelineBuilder
     end
 
     def compile_job_to_xml(job)
-      raise 'Job name is not specified' unless job[:name]
+      fail 'Job name is not specified' unless job[:name]
 
       @logger.info "Creating Yaml Job #{job}"
       job[:job_type] = 'free_style' unless job[:job_type]
       case job[:job_type]
-        when 'job_dsl'
-          xml = compile_freestyle_job_to_xml(job)
-          payload = update_job_dsl(job, xml)
-        when 'multi_project'
-          xml = compile_freestyle_job_to_xml(job)
-          payload = adjust_multi_project xml
-        when 'build_flow'
-          xml = compile_freestyle_job_to_xml(job)
-          payload = add_job_dsl(job, xml)
-        when 'free_style'
-          payload = compile_freestyle_job_to_xml job
-        else
-          return false, "Job type: #{job[:job_type]} is not one of job_dsl, multi_project, build_flow or free_style"
+      when 'job_dsl'
+        xml = compile_freestyle_job_to_xml(job)
+        payload = update_job_dsl(job, xml)
+      when 'multi_project'
+        xml = compile_freestyle_job_to_xml(job)
+        payload = adjust_multi_project xml
+      when 'build_flow'
+        xml = compile_freestyle_job_to_xml(job)
+        payload = add_job_dsl(job, xml)
+      when 'free_style', 'pull_request_generator'
+        payload = compile_freestyle_job_to_xml job
+      else
+        return false, "Job type: #{job[:job_type]} is not one of job_dsl, multi_project, build_flow or free_style"
       end
 
-      return true, payload
+      [true, payload]
     end
 
     def adjust_multi_project(xml)
       n_xml = Nokogiri::XML(xml)
-      root = n_xml.root()
+      root = n_xml.root
       root.name = 'com.tikal.jenkins.plugins.multijob.MultiJobProject'
       n_xml.to_xml
     end
 
     def compile_freestyle_job_to_xml(params)
-      if params.has_key?(:template)
+      if params.key?(:template)
         template_name = params[:template]
-        raise "Job template '#{template_name}' can't be resolved." unless @job_templates.has_key?(template_name)
+        fail "Job template '#{template_name}' can't be resolved." unless @job_templates.key?(template_name)
         params.delete(:template)
         template = @job_templates[template_name]
         puts "Template found: #{template}"
@@ -373,7 +563,7 @@ module JenkinsPipelineBuilder
     end
 
     def add_job_dsl(job, xml)
-      n_xml      = Nokogiri::XML(xml)
+      n_xml = Nokogiri::XML(xml)
       n_xml.root.name = 'com.cloudbees.plugins.flow.BuildFlow'
       Nokogiri::XML::Builder.with(n_xml.root) do |xml|
         xml.dsl job[:build_flow]
@@ -392,25 +582,25 @@ module JenkinsPipelineBuilder
     end
 
     def generate_job_dsl_body(params)
-      @logger.info "Generating pipeline"
+      @logger.info 'Generating pipeline'
 
       xml = @client.job.build_freestyle_config(params)
 
       n_xml = Nokogiri::XML(xml)
       if n_xml.xpath('//javaposse.jobdsl.plugin.ExecuteDslScripts').empty?
-        p_xml = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |b_xml|
+        p_xml = Nokogiri::XML::Builder.new(encoding:  'UTF-8') do |b_xml|
           build_job_dsl(params, b_xml)
         end
 
-        n_xml.xpath('//builders').first.add_child("\r\n" + p_xml.doc.root.to_xml(:indent => 4) + "\r\n")
+        n_xml.xpath('//builders').first.add_child("\r\n" + p_xml.doc.root.to_xml(indent:  4) + "\r\n")
         xml = n_xml.to_xml
       end
       xml
     end
 
     def build_job_dsl(job, xml)
-      xml.send('javaposse.jobdsl.plugin.ExecuteDslScripts') {
-        if job.has_key?(:job_dsl)
+      xml.send('javaposse.jobdsl.plugin.ExecuteDslScripts') do
+        if job.key?(:job_dsl)
           xml.scriptText job[:job_dsl]
           xml.usingScriptText true
         else
@@ -419,7 +609,7 @@ module JenkinsPipelineBuilder
         end
         xml.ignoreExisting false
         xml.removedJobAction 'IGNORE'
-      }
+      end
     end
   end
 end