jenkins_pipeline_builder 0.10.11 → 0.10.12

data/lib/jenkins_pipeline_builder/job.rb

@@ -0,0 +1,128 @@
+module JenkinsPipelineBuilder
+  class Job
+    attr_accessor :job
+
+    def initialize(job)
+      @job = job
+    end
+
+    def name
+      job[:name]
+    end
+
+    def logger
+      JenkinsPipelineBuilder.logger
+    end
+
+    def create_or_update
+      success, payload = to_xml
+      return success, payload unless success
+      xml = payload
+      if JenkinsPipelineBuilder.debug || JenkinsPipelineBuilder.file_mode
+        logger.info "Will create job #{job}"
+        logger.info "#{xml}" if @debug
+        FileUtils.mkdir_p(out_dir) unless File.exist?(out_dir)
+        File.open("#{out_dir}/#{name}.xml", 'w') { |f| f.write xml }
+        return [true, nil]
+      end
+
+      if JenkinsPipelineBuilder.client.job.exists?(name)
+        JenkinsPipelineBuilder.client.job.update(name, xml)
+      else
+        JenkinsPipelineBuilder.client.job.create(name, xml)
+      end
+      [true, nil]
+    end
+
+    def to_xml
+      fail 'Job name is not specified' unless name
+
+      logger.info "Creating Yaml Job #{job}"
+      job[:job_type] = 'free_style' unless job[:job_type]
+      case job[:job_type]
+      when 'job_dsl'
+        @xml = setup_freestyle_base(job)
+        payload = update_job_dsl
+      when 'multi_project'
+        @xml = setup_freestyle_base(job)
+        payload = adjust_multi_project
+      when 'build_flow'
+        @xml = setup_freestyle_base(job)
+        payload = add_job_dsl
+      when 'free_style', 'pull_request_generator'
+        payload = setup_freestyle_base job
+      else
+        return false, "Job type: #{job[:job_type]} is not one of job_dsl, multi_project, build_flow or free_style"
+      end
+
+      [true, payload]
+    end
+
+    private
+
+    def out_dir
+      'out/xml'
+    end
+
+    def update_job_dsl
+      n_xml = Nokogiri::XML(@xml)
+      n_builders = n_xml.xpath('//builders').first
+      Nokogiri::XML::Builder.with(n_builders) do |b_xml|
+        build_job_dsl(job, b_xml)
+      end
+      n_xml.to_xml
+    end
+
+    def build_job_dsl(job, xml)
+      xml.send('javaposse.jobdsl.plugin.ExecuteDslScripts') do
+        if job.key?(:job_dsl)
+          xml.scriptText job[:job_dsl]
+          xml.usingScriptText true
+        else
+          xml.targets job[:job_dsl_targets]
+          xml.usingScriptText false
+        end
+        xml.ignoreExisting false
+        xml.removedJobAction 'IGNORE'
+      end
+    end
+
+    def adjust_multi_project
+      n_xml = Nokogiri::XML(@xml)
+      root = n_xml.root
+      root.name = 'com.tikal.jenkins.plugins.multijob.MultiJobProject'
+      n_xml.to_xml
+    end
+
+    def add_job_dsl
+      n_xml = Nokogiri::XML(@xml)
+      n_xml.root.name = 'com.cloudbees.plugins.flow.BuildFlow'
+      Nokogiri::XML::Builder.with(n_xml.root) do |b_xml|
+        b_xml.dsl job[:build_flow]
+      end
+      n_xml.to_xml
+    end
+
+    def setup_freestyle_base(params)
+      # I'm pretty unclear what these templates are...
+      if params.key?(:template)
+        template_name = params[:template]
+        fail "Job template '#{template_name}' can't be resolved." unless @job_templates.key?(template_name)
+        params.delete(:template)
+        template = @job_templates[template_name]
+        puts "Template found: #{template}"
+        params = template.deep_merge(params)
+        puts "Template merged: #{template}"
+      end
+
+      xml = JenkinsPipelineBuilder.client.job.build_freestyle_config(params)
+      n_xml = Nokogiri::XML(xml, &:noblanks)
+
+      logger.debug 'Loading the required modules'
+      JenkinsPipelineBuilder.registry.traverse_registry_path('job', params, n_xml)
+      logger.debug 'Module loading complete'
+
+      n_xml.to_xml
+    end
+  end
+end

data/lib/jenkins_pipeline_builder/job_collection.rb

@@ -0,0 +1,112 @@
+module JenkinsPipelineBuilder
+  class JobCollection
+    attr_accessor :collection, :remote_dependencies
+
+    def initialize
+      @collection = {}
+      @remote_dependencies = RemoteDependencies.new self
+    end
+
+    def clear_remote_dependencies
+      @remote_dependencies = RemoteDependencies.new self
+    end
+
+    def logger
+      JenkinsPipelineBuilder.logger
+    end
+
+    def projects
+      result = []
+      collection.values.each do |item|
+        result << item if item[:type] == :project
+      end
+      result
+    end
+
+    def jobs
+      result = []
+      collection.values.each do |item|
+        result << item if item[:type] == :job
+      end
+      result
+    end
+
+    def defaults
+      collection.each_value do |item|
+        return item if item[:type] == 'defaults' || item[:type] == :defaults
+      end
+      # This is here for historical purposes
+      get_item('global')
+    end
+
+    def get_item(name)
+      collection[name.to_s]
+    end
+
+    def load_from_path(path, remote = false)
+      load_extensions(path)
+      path = File.expand_path(path, Dir.getwd)
+      if File.directory?(path)
+        logger.info "Generating from folder #{path}"
+        Dir[File.join(path, '/*.{yaml,yml}')].each do |file|
+          logger.info "Loading file #{file}"
+          yaml = YAML.load_file(file)
+          load_file(yaml, remote)
+        end
+        Dir[File.join(path, '/*.json')].each do |file|
+          logger.info "Loading file #{file}"
+          json = JSON.parse(IO.read(file))
+          load_file(json, remote)
+        end
+      else
+        logger.info "Loading file #{path}"
+        if path.end_with? 'json'
+          hash = JSON.parse(IO.read(path))
+        else  # elsif path.end_with?("yml") || path.end_with?("yaml")
+          hash = YAML.load_file(path)
+        end
+        load_file(hash, remote)
+      end
+      remote_dependencies.cleanup if remote
+    end
+
+    def load_file(yaml, remote = false)
+      yaml.each do |section|
+        Utils.symbolize_keys_deep!(section)
+        key = section.keys.first
+        value = section[key]
+        if key == :dependencies
+          logger.info 'Resolving Dependencies for remote project'
+          remote_dependencies.load value
+          next
+        end
+        name = value[:name]
+        if collection.key?(name)
+          existing_remote = collection[name.to_s][:remote]
+          # skip if the existing item is local and the new item is remote
+          if remote && !existing_remote
+            next
+          # override if the existing item is remote and the new is local
+          elsif existing_remote && !remote
+            logger.info "Duplicate item with name '#{name}' was detected from the remote folder."
+          else
+            fail "Duplicate item with name '#{name}' was detected."
+          end
+        end
+        collection[name.to_s] = { name: name.to_s, type: key, value: value, remote: remote }
+      end
+    end
+
+    def load_extensions(path)
+      path = "#{path}/extensions"
+      path = File.expand_path(path, Dir.getwd)
+      return unless File.directory?(path)
+      logger.info "Loading extensions from folder #{path}"
+      logger.info Dir.glob("#{path}/*.rb").inspect
+      Dir.glob("#{path}/*.rb").each do |file|
+        logger.info "Loaded #{file}"
+        require file
+      end
+    end
+  end
+end

data/lib/jenkins_pipeline_builder/module_registry.rb

@@ -29,7 +29,7 @@ module JenkinsPipelineBuilder
 
     def versions
       # Return a hash with a default of 1000 so that we'll get the newest in debug
-      return Hash.new { |_| '1000.0' } if JenkinsPipelineBuilder.generator.debug
+      return Hash.new { |_| '1000.0' } if JenkinsPipelineBuilder.debug
       @versions ||= JenkinsPipelineBuilder.client.plugin.list_installed
     end
 

data/lib/jenkins_pipeline_builder/pull_request.rb

@@ -56,7 +56,7 @@ module JenkinsPipelineBuilder
 
     def override_git_2_params(job)
       job[:value][:scm_params] ||= {}
-      job[:value][:scm_params][:changelog_to_branch] = { remote: 'origin', branch: 'pr-{{pull_request_number}}' }
+      job[:value][:scm_params][:changelog_to_branch] = { remote: 'origin', branch: 'pr/{{pull_request_number}}/head' }
     end
 
     # Change the git branch for each job

data/lib/jenkins_pipeline_builder/remote_dependencies.rb

@@ -0,0 +1,112 @@
+module JenkinsPipelineBuilder
+  class RemoteDependencies
+    attr_reader :job_collection, :entries
+
+    def initialize(job_collection)
+      @entries = {}
+      @job_collection = job_collection
+    end
+
+    def logger
+      JenkinsPipelineBuilder.logger
+    end
+
+    def cleanup
+      entries.each_value do |file|
+        FileUtils.rm_r file
+        FileUtils.rm_r "#{file}.tar"
+      end
+    end
+
+    # TODO: Look into remote jobs not working according to sinan
+
+    def load(dependencies)
+      ### Load remote YAML
+      # Download Tar.gz
+      dependencies.each do |source|
+        source = source[:source]
+        url = source[:url]
+
+        file = "remote-#{entries.length}"
+        if entries[url]
+          file = entries[url]
+        else
+          opts = {}
+          opts = { ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE } if source[:verify_ssl] == false
+          download_yaml(url, file, opts)
+        end
+
+        path = File.expand_path(file, Dir.getwd)
+        # Load templates recursively
+        unless source[:templates]
+          logger.info 'No specific template specified'
+          # Try to load the folder or the pipeline folder
+          path = File.join(path, 'pipeline') if Dir.entries(path).include? 'pipeline'
+          return job_collection.load_from_path(path, true)
+        end
+
+        load_templates(path, source[:templates])
+      end
+    end
+
+    private
+
+    def load_template(path, template)
+      # If we specify what folder the yaml is in, load that
+      if template[:folder]
+        path = File.join(path, template[:folder])
+      else
+        path = File.join(path, template[:name]) unless template[:name] == 'default'
+        # If we are looking for the newest version or no version was set
+        if (template[:version].nil? || template[:version] == 'newest') && File.directory?(path)
+          folders = Dir.entries(path)
+          highest = folders.max
+          template[:version] = highest unless highest == 0
+        end
+        path = File.join(path, template[:version]) unless template[:version].nil?
+        path = File.join(path, 'pipeline')
+      end
+
+      if File.directory?(path)
+        logger.info "Loading from #{path}"
+        job_collection.load_from_path(path, true)
+        true
+      else
+        false
+      end
+    end
+
+    def download_yaml(url, file, remote_opts = {})
+      entries[url] = file
+      logger.info "Downloading #{url} to #{file}.tar"
+      open("#{file}.tar", 'w') do |local_file|
+        open(url, remote_opts) do |remote_file|
+          local_file.write(Zlib::GzipReader.new(remote_file).read)
+        end
+      end
+
+      # Extract Tar.gz to 'remote' folder
+      logger.info "Unpacking #{file}.tar to #{file} folder"
+      Archive::Tar::Minitar.unpack("#{file}.tar", file)
+    end
+
+    def load_templates(path, templates)
+      templates.each do |template|
+        version = template[:version] || 'newest'
+        logger.info "Loading #{template[:name]} at version #{version}"
+        # Move into the remote folder and look for the template folder
+        remote = Dir.entries(path)
+        if remote.include? template[:name]
+          # We found the template name, load this path
+          logger.info 'We found the template!'
+          load_template(path, template)
+        else
+          # Many cases we must dig one layer deep
+          remote.each do |file|
+            load_template(File.join(path, file), template)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jenkins_pipeline_builder/version.rb

@@ -21,5 +21,5 @@
 #
 
 module JenkinsPipelineBuilder
-  VERSION = '0.10.11'
+  VERSION = '0.10.12'
 end

data/lib/jenkins_pipeline_builder/view.rb

@@ -67,12 +67,12 @@ module JenkinsPipelineBuilder
     def create(params)
       # Name is a required parameter. Raise an error if not specified
       fail ArgumentError, 'Name is required for creating view' unless params.is_a?(Hash) && params[:name]
-      clean_up_views(params) unless @generator.debug
+      clean_up_views(params) unless JenkinsPipelineBuilder.debug
       params[:type] ||= 'listview'
       create_base_view(params[:name], params[:type], params[:parent_view])
       @logger.debug "Creating a #{params[:type]} view with params: #{params.inspect}"
 
-      if @generator.debug
+      if JenkinsPipelineBuilder.debug
         # pp post_params(params)
         return
       end
@@ -156,7 +156,7 @@ module JenkinsPipelineBuilder
         }.to_json
       }
 
-      if @generator.debug
+      if JenkinsPipelineBuilder.debug
         # pp initial_post_params
         return
       end

data/spec/lib/jenkins_pipeline_builder/cli/helper_spec.rb

@@ -2,17 +2,6 @@ require File.expand_path('../../spec_helper', __FILE__)
 
 describe JenkinsPipelineBuilder::CLI::Helper do
   context '#setup' do
-    let(:generator) do
-      instance_double(
-        JenkinsPipelineBuilder::Generator,
-        :debug= => true
-      )
-    end
-
-    before(:each) do
-      allow(JenkinsPipelineBuilder).to receive(:generator).and_return(generator)
-    end
-
     context 'username and password given' do
       let(:options) do
         {
@@ -92,6 +81,7 @@ describe JenkinsPipelineBuilder::CLI::Helper do
           server_ip: :baz
         }
         expect(JenkinsPipelineBuilder).to receive(:credentials=).with(expected_options)
+        expect(JenkinsPipelineBuilder).to receive(:debug!).and_return true
         described_class.setup(options)
       end
 

data/spec/lib/jenkins_pipeline_builder/generator_spec.rb

@@ -1,11 +1,5 @@
 require File.expand_path('../spec_helper', __FILE__)
 
-def cleanup_compiled_xml(job_name)
-  Dir["#{@generator.out_dir}/#{job_name}*.xml"].each do |file|
-    File.delete(file)
-  end
-end
-
 describe JenkinsPipelineBuilder::Generator do
   after :each do
     JenkinsPipelineBuilder.registry.clear_versions
@@ -23,14 +17,13 @@ describe JenkinsPipelineBuilder::Generator do
   end
 
   after(:each) do
-    @generator.debug = false
-    @generator.job_collection = {}
-    @generator.remote_depends = {}
+    JenkinsPipelineBuilder.no_debug!
+    @generator.job_collection = JenkinsPipelineBuilder::JobCollection.new
   end
 
   describe 'initialized in before(:example)' do
     it 'creates a new generator' do
-      expect(@generator.job_collection).to be_empty
+      expect(@generator.job_collection.collection).to be_empty
       expect(@generator.module_registry).not_to be_nil
     end
 
@@ -38,23 +31,9 @@ describe JenkinsPipelineBuilder::Generator do
       job_name = 'sample_job'
       job_type = 'project'
       job_value = {}
-      @generator.job_collection[job_name] = { name: job_name, type: job_type, value: job_value }
-      expect(@generator.job_collection).not_to be_empty
-      expect(@generator.job_collection[job_name]).not_to be_nil
-    end
-  end
-
-  describe '#debug=' do
-    it 'sets debug mode to false' do
-      @generator.debug = false
-      expect(@generator.debug).to be false
-      expect(@generator.logger.level).to eq(Logger::INFO)
-    end
-
-    it 'sets debug mode to true' do
-      @generator.debug = true
-      expect(@generator.debug).to be true
-      expect(@generator.logger.level).to eq(Logger::DEBUG)
+      @generator.job_collection.collection[job_name] = { name: job_name, type: job_type, value: job_value }
+      expect(@generator.job_collection.collection).not_to be_empty
+      expect(@generator.job_collection.collection[job_name]).not_to be_nil
     end
   end
 
@@ -79,9 +58,8 @@ describe JenkinsPipelineBuilder::Generator do
     end
 
     def bootstrap(fixture_path, job_name)
-      @generator.debug = true
+      JenkinsPipelineBuilder.debug!
       errors = @generator.bootstrap(fixture_path, job_name)
-      cleanup_compiled_xml(job_name)
       errors
     end
 
@@ -114,7 +92,7 @@ describe JenkinsPipelineBuilder::Generator do
     it 'overrides the remote dependencies with local ones' do
       errors = bootstrap(fixture_path('local_override/remote_and_local'), 'TemplatePipeline')
       expect(errors).to be_empty
-      expect(@generator.job_collection['{{name}}-10'][:value][:description]).to eq('Overridden stuff')
+      expect(@generator.job_collection.collection['{{name}}-10'][:value][:description]).to eq('Overridden stuff')
     end
 
     it 'fails to override when there are duplicate local items' do
@@ -133,7 +111,7 @@ describe JenkinsPipelineBuilder::Generator do
     before :each do
       allow(JenkinsPipelineBuilder.client).to receive(:plugin).and_return double(
         list_installed: { 'description' => '20.0', 'git' => '20.0' })
-      @generator.debug = true
+      JenkinsPipelineBuilder.debug!
     end
     let(:jobs) do
       {
@@ -174,7 +152,6 @@ describe JenkinsPipelineBuilder::Generator do
       )
       success = @generator.pull_request(path, job_name)
       expect(success).to be_truthy
-      cleanup_compiled_xml(job_name)
     end
 
     it 'correclty creates jobs when there are multiple pulls open' do
@@ -196,18 +173,21 @@ describe JenkinsPipelineBuilder::Generator do
       expect(JenkinsPipelineBuilder::PullRequestGenerator).to receive(:new).once.and_return(
         double(purge: purge, create: create, jobs: jobs)
       )
-      expect(@generator).to receive(:compile_job_to_xml).once.with(
+      job1 = double name: 'job name'
+      job2 = double name: 'job name'
+      expect(JenkinsPipelineBuilder::Job).to receive(:new).once.with(
         name: 'PullRequest-PR1-10-SampleJob', scm_branch: 'origin/pr/1/head', scm_params: {
           refspec: 'refs/pull/*:refs/remotes/origin/pr/*'
         }
-      )
-      expect(@generator).to receive(:compile_job_to_xml).once.with(
+      ).and_return job1
+      expect(JenkinsPipelineBuilder::Job).to receive(:new).once.with(
         name: 'PullRequest-PR2-10-SampleJob', scm_branch: 'origin/pr/2/head', scm_params: {
           refspec: 'refs/pull/*:refs/remotes/origin/pr/*'
         }
-      )
+      ).and_return job2
+      expect(job1).to receive(:create_or_update).and_return true
+      expect(job2).to receive(:create_or_update).and_return true
       expect(@generator.pull_request(path, job_name)).to be_truthy
-      cleanup_compiled_xml(job_name)
     end
     # Things to check for
     # Fail - no PR job type found
@@ -226,29 +206,28 @@ describe JenkinsPipelineBuilder::Generator do
       }]
     end
 
+    before :each do
+      expect(@generator.job_collection).to receive(:load_file).once.with(view_hash, false).and_return(true)
+      expect(@generator.job_collection).to receive(:load_file).once.with(project_hash, false).and_return(true)
+    end
+
     it 'loads a yaml collection from a path' do
       path = File.expand_path('../fixtures/generator_tests/test_yaml_files', __FILE__)
-      expect(@generator).to receive(:load_job_collection).once.with(view_hash, false).and_return(true)
-      expect(@generator).to receive(:load_job_collection).once.with(project_hash, false).and_return(true)
-      @generator.send(:load_collection_from_path, path)
+      @generator.job_collection.send(:load_from_path, path)
     end
     it 'loads a json collection from a path' do
       path = File.expand_path('../fixtures/generator_tests/test_json_files', __FILE__)
-      expect(@generator).to receive(:load_job_collection).once.with(view_hash, false).and_return(true)
-      expect(@generator).to receive(:load_job_collection).once.with(project_hash, false).and_return(true)
-      @generator.send(:load_collection_from_path, path)
+      @generator.job_collection.send(:load_from_path, path)
     end
     it 'loads both yaml and json files from a path' do
       path = File.expand_path('../fixtures/generator_tests/test_combo_files', __FILE__)
-      expect(@generator).to receive(:load_job_collection).once.with(view_hash, false).and_return(true)
-      expect(@generator).to receive(:load_job_collection).once.with(project_hash, false).and_return(true)
-      @generator.send(:load_collection_from_path, path)
+      @generator.job_collection.send(:load_from_path, path)
     end
   end
 
   describe '#dump' do
     it "writes a job's config XML to a file" do
-      @generator.debug = true
+      JenkinsPipelineBuilder.debug!
       job_name = 'test_job'
       body = ''
       test_path = File.expand_path('../fixtures/generator_tests', __FILE__)