autoproj-jenkins 0.1.0

data/lib/autoproj/jenkins/render_template.rb

@@ -0,0 +1,92 @@
+module Autoproj::Jenkins
+    class UnusedTemplateParameters < ArgumentError; end
+    class UnknownTemplateParameter < ArgumentError; end
+
+    # @api private
+    #
+    # A rendering context for ERB templates
+    #
+    # It ensures that the templates are restricted to use the parameters that
+    # are provided to them
+    class TemplateRenderingContext < BasicObject
+        def initialize(template_path, allow_unused: false, indent: 0, **parameters)
+            @allow_unused = allow_unused
+            @indent = indent
+            @template_path = template_path
+            @parameters = parameters
+            @used_parameters = ::Set.new
+        end
+
+        def __verify
+            return if @allow_unused
+
+            unused_parameters = @parameters.keys.to_set -
+                @used_parameters
+            if !unused_parameters.empty?
+                ::Kernel.raise ::Autoproj::Jenkins::UnusedTemplateParameters, "#{unused_parameters.size} unused parameters: #{unused_parameters.map(&:to_s).sort.join(", ")}"
+            end
+        end
+
+        def render_template(path, escape: false, indent: 0, **parameters)
+            result = ::Autoproj::Jenkins.render_template(path, template_path: @template_path, indent: indent, **parameters)
+            @used_parameters.merge(parameters.keys)
+            if escape
+                result = escape_to_groovy(result)
+            end
+
+            indent_string = " " * indent
+            # We assume that the ERB tag for render_template is indented
+            # properly
+            result.split("\n").join("\n#{indent_string}")
+        end
+
+        def read_and_escape_file(path)
+            file_data = (::Autoproj::Jenkins.template_path + path).read
+            escape_to_groovy(file_data)
+        end
+
+        def escape_to_groovy(content)
+            content.
+                gsub("\n", "\\n").
+                gsub("\"", "\\\"")
+        end
+
+        def method_missing(m)
+            if @parameters.has_key?(m)
+                result = @parameters.fetch(m)
+                @used_parameters << m
+                result
+            else
+                ::Kernel.raise ::Autoproj::Jenkins::UnknownTemplateParameter, "#{m} is not a known template parameter"
+            end
+        end
+    end
+
+    # The path to autoproj-jenkins templates
+    def self.template_path
+        Pathname.new(__dir__) + "templates"
+    end
+
+    # Test if a template with the given basename exists
+    def self.has_template?(template_name)
+        (template_path + "#{template_name}.erb").exist?
+    end
+
+    # Create a template rendering context for the given parameters
+    #
+    # @param [String] template_name the name of the template to be rendered,
+    #   without the .erb extension
+    # @param [Hash<Symbol,Object>] parameters the template parameters
+    # @param [Pathname] template_path where the templates are located on disk
+    # @return [String]
+    def self.render_template(template_name, allow_unused: false, template_path: self.template_path, **parameters)
+        context = TemplateRenderingContext.new(template_path, allow_unused: allow_unused, **parameters)
+        template_path = template_path + "#{template_name}.erb"
+        template = ERB.new(template_path.read)
+        template.filename = template_path.to_s
+        b = context.instance_eval { Kernel.binding }
+        result = template.result(b)
+        context.__verify
+        result
+    end
+end

data/lib/autoproj/jenkins/server.rb

@@ -0,0 +1,79 @@
+module Autoproj::Jenkins
+    # The interface to a Jenkins server
+    class Server
+        # The underlying client
+        #
+        # @return [JenkinsApi::Client]
+        attr_reader :client
+
+        # The job-related API
+        #
+        # @return [JenkinsApi::Client::Job]
+        attr_reader :jobs
+
+        def initialize(**options)
+            @client = JenkinsApi::Client.new(log_location: STDERR, **options)
+            @jobs = JenkinsApi::Client::Job.new(client)
+        end
+
+        # Create a job using a rendered template
+        # 
+        # @param [String] job_name the name of the new job
+        # @param [String] template the name of the template
+        # @param [Hash] parameters parameters for the template rendering
+        # @raise (see Autoproj::Jenkins.render_template)
+        def create_job(job_name, template, **parameters)
+            xml = Autoproj::Jenkins.render_template(template, **parameters)
+            jobs.create(job_name, xml)
+        end
+
+        # Test whether the server already has a job
+        #
+        # @param [String] job_name
+        def has_job?(job_name)
+            jobs.exists?(job_name)
+        end
+
+        # Read a job configuration
+        #
+        # @return [REXML::Document]
+        def read_job_config(job_name)
+            xml_config = jobs.get_config(job_name)
+            REXML::Document.new(xml_config)
+        end
+
+        # Update a job configuration
+        #
+        # @param [String] job_name
+        # @param [REXML::Document] xml the configuration XML document
+        def update_job_config(job_name, xml)
+            jobs.update(job_name, xml.to_s)
+        end
+
+        # Delete a job
+        #
+        # @param [String] job_name
+        def delete_job(job_name)
+            jobs.delete(job_name)
+        end
+
+        # Trigger a job
+        def trigger_job(job_name)
+            jobs.build(job_name)
+        end
+
+        # Update the pipeline of the given job
+        def update_job_pipeline(job_name, template, **parameters)
+            config = read_job_config(job_name)
+            pipeline = Autoproj::Jenkins.render_template(template, **parameters)
+            config.elements['//definition/script'].text = pipeline
+            update_job_config(job_name, config)
+        end
+
+        # Add a job as a downstream job of another
+        def add_downstream_project(upstream_job_name, downstream_job_name)
+            jobs.add_downstream_projects(upstream_job_name, downstream_job_name, 'success', false)
+        end
+    end
+end
+

data/lib/autoproj/jenkins/templates/abort-if-upstream-failed.pipeline.erb

@@ -0,0 +1,18 @@
+def upstream_jobs = [<%= upstream_jobs.each_key.map { |job_name| "'#{job_name}'" }.join(", ") %>]
+
+// Iterate list -- NOTE: we cannot use groovy style or even modern java style iteration
+for (int i = 0; i < upstream_jobs.size(); i++) {
+    def job = Jenkins.instance.getItem(upstream_jobs[i])
+    if (!job) {
+        error("cannot find upstream job ${upstream_jobs[i]}")
+    }
+
+    def result = job.getLastBuild().getResult()
+    if (result != Result.SUCCESS && result != Result.UNSTABLE)
+    {
+        echo "${upstream_jobs[i]} did not finish successfully, aborting this build"
+        currentBuild.result = 'NOT_BUILT';
+        return;
+    }
+}
+

data/lib/autoproj/jenkins/templates/bootstrap.pipeline.erb

@@ -0,0 +1,27 @@
+def fullWorkspaceDir = pwd()
+env.HOME = fullWorkspaceDir
+<% if autoproj_install_path %>
+sh 'cp -f "<%= autoproj_install_path %>" ./autoproj_install'
+<% else %>
+sh 'wget -O autoproj_install https://raw.githubusercontent.com/rock-core/autoproj/master/bin/autoproj_install'
+<% end %>
+def gemfile = "<%= read_and_escape_file gemfile %>"
+writeFile file: 'Gemfile', text: gemfile
+def config = "<%= render_template('buildconf-config.yml', vcs: vcs, escape: true) %>"
+writeFile file: 'seed.yml', text: config
+dir('dev') {
+    sh "ruby ../autoproj_install --skip-stage2 --seed-config=../seed.yml --gems-path=${env.JENKINS_HOME}/cache/gems --gemfile=../Gemfile"
+    <%= render_template("import-#{vcs.type}.pipeline",
+                        patch: false,
+                        allow_unused: true,
+                        package_dir: 'autoproj',
+                        vcs: vcs,
+                        credentials: vcs_credentials.for(vcs),
+                        package_name: 'autoproj/') %>
+
+    <%= render_template('setup-git-credentials.pipeline', credentials: vcs_credentials[:git]) %>
+    sh ".autoproj/bin/aup autoproj/ --force-reset"
+}
+
+def autoproj = "${fullWorkspaceDir}/dev/.autoproj/bin/autoproj"
+env.AUTOPROJ_CURRENT_ROOT = "${fullWorkspaceDir}/dev"

data/lib/autoproj/jenkins/templates/buildconf-Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+gem 'autoproj', '>= 2.0.0.a'
+gem 'autoproj-jenkins'

data/lib/autoproj/jenkins/templates/buildconf-config.yml.erb

@@ -0,0 +1,12 @@
+osdeps_mode: 'all'
+GITORIOUS: http,http,http
+GITHUB: http,http,http
+separate_prefixes: true
+ROCK_SELECTED_FLAVOR: master
+ROCK_FLAVOR: master
+ROCK_BRANCH: master
+USE_OCL: false
+rtt_target: gnulinux
+rtt_corba_implementation: omniorb
+autoproj_test_utility_default: true
+<%= ::YAML.dump('manifest_source' => vcs.to_hash).gsub(/^---\s*/, '') %>

data/lib/autoproj/jenkins/templates/buildconf-vagrant-Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+gem 'autoproj', path: '/opt/autoproj'
+gem 'autobuild', path: '/opt/autobuild'
+gem 'autoproj-jenkins', path: '/opt/autoproj-jenkins'

data/lib/autoproj/jenkins/templates/buildconf.pipeline.erb

@@ -0,0 +1,43 @@
+node(label: 'autoproj-jenkins') {
+    stage 'bootstrap'
+    <%= render_template('bootstrap.pipeline', vcs: vcs, gemfile: gemfile, autoproj_install_path: autoproj_install_path, vcs_credentials: vcs_credentials, indent: 4) %>
+
+    dir('dev') {
+        def arg_packages = "<%= package_names.map { |name| "'#{name}'" }.join(" ") %>"
+
+        stage 'checkout and update packages'
+        sh "${autoproj} test enable"
+        <%= render_template('setup-git-credentials.pipeline', credentials: vcs_credentials[:git], indent: 8) %>
+        sh "${autoproj} update ${arg_packages} --force-reset -k"
+
+        stage 'update package jobs and trigger them'
+        def arg_dev      = "<%= " --dev" if dev %>"
+        def arg_job_prefix = "<%= " '--job-prefix=#{job_prefix}'" if !job_prefix.empty? %>"
+        <% if vcs_credentials.empty?  %>
+        def arg_vcs_credentials = ""
+        <% else %>
+            <% arg_vcs_credentials = vcs_credentials.credentials.flat_map do |vcs_type, vcs_c|
+                  vcs_c.map do |c|
+                      "#{vcs_type}:#{c.protocol}://#{c.host}"
+                  end
+               end.join(" ") %>
+            def arg_vcs_credentials = " '--vcs-credentials=<%= arg_vcs_credentials %>'"
+        <% end %>
+
+        <% if credentials_id %>
+        def jobNames = withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: '<%= credentials_id %>',
+                          usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
+            def credentials = " --username '${env.USERNAME}' --password '${env.PASSWORD}'"
+            sh(script: "${autoproj} jenkins update${arg_dev}${arg_job_prefix} '${env.JENKINS_URL}' ${arg_packages}${credentials}${arg_vcs_credentials}", returnStdout: true).split("\n")
+        }
+        <% else %>
+        def jobNames = sh(
+            script: "${autoproj} jenkins update${arg_dev}${arg_job_prefix} 'http://localhost:8080' ${arg_packages}${arg_vcs_credentials}",
+            returnStdout: true).split("\n")
+        <% end %>
+        for (job_name in jobNames) {
+            build job: job_name, wait: false
+        }
+    }
+}
+

data/lib/autoproj/jenkins/templates/buildconf.xml.erb

@@ -0,0 +1,24 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<flow-definition plugin="workflow-job@2.3">
+    <actions/>
+    <description></description>
+    <keepDependencies>false</keepDependencies>
+    <properties/>
+    <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.9">
+        <script>
+        </script>
+        <sandbox>true</sandbox>
+    </definition>
+    <triggers/>
+    <quietPeriod><%= quiet_period %></quietPeriod>
+    <jenkins.model.BuildDiscarderProperty>
+        <strategy class="hudson.tasks.LogRotator">
+            <daysToKeep>30</daysToKeep>
+            <numToKeep>-1</numToKeep>
+            <artifactDaysToKeep>7</artifactDaysToKeep>
+            <artifactNumToKeep>-1</artifactNumToKeep>
+        </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+</flow-definition>
+
+

data/lib/autoproj/jenkins/templates/handle-downstream.pipeline.erb

@@ -0,0 +1,20 @@
+def downstreamDir = "artifacts/downstream"
+def targetArtifactPath = "${fullWorkspaceDir}/${downstreamDir}/<%= job_name %>-prefix.zip"
+
+dir(downstreamDir) { deleteDir() }
+dir("${downstreamDir}/<%= job_name %>") {
+    sh "rsync '${jobPackagePrefix}/' './' -a --delete"
+    sh "${autoproj} jenkins relativize ./ '${fullWorkspaceDir}' '@WORKSPACE_ROOT@'"
+    zip zipFile: targetArtifactPath, glob: "<%= artifact_glob %>"
+}
+dir(downstreamDir) {
+    archiveArtifacts artifacts: "*.zip"
+    deleteDir()
+}
+
+if (triggeredByUser) { // Defined in handle-upstream.pipeline.erb
+    <% downstream_jobs.each_key do |job_name| %>
+    build job: "<%= job_name %>", wait: false
+    <% end %>
+}
+

data/lib/autoproj/jenkins/templates/import-archive.pipeline.erb

@@ -0,0 +1,2 @@
+sh "${autoproj} update --force-reset '<%= package_name %>'"
+

data/lib/autoproj/jenkins/templates/import-git.pipeline.erb

@@ -0,0 +1,16 @@
+dir('<%= package_dir %>') {
+    <% if patch %>
+    if (fileExists('.git')) {
+        sh "${autoproj} unpatch ."
+    }
+    <% end %>
+
+    <% if credentials
+           arg_credentials = "credentialsId: '#{credentials.jenkins_id}', "
+       end %>
+    <%= vcs.type %> <%= arg_credentials %> <%= Hash[url: vcs.url].merge(vcs.options).map { |k, v| "#{k}: '#{v}'" }.join(", ") %>
+    <% if patch %>
+    sh "${autoproj} patch ."
+    <% end %>
+}
+

data/lib/autoproj/jenkins/templates/import-svn.pipeline.erb

@@ -0,0 +1,13 @@
+dir('<%= package_dir %>') {
+    <% raise "SVN credentials are not supported at the moment" if credentials %>
+    <% if patch %>
+    if (fileExists('.svn')) {
+        sh "${autoproj} unpatch ."
+    }
+    <% end %>
+    svn url: '<%= vcs.url %>'
+    <% if patch %>
+    sh "${autoproj} patch ."
+    <% end %>
+}
+

data/lib/autoproj/jenkins/templates/jenkins_dependency_overrides.rb.erb

@@ -0,0 +1,5 @@
+setup_package("<%= package_name %>") do |pkg|
+    <% upstream_jobs.each_value do |pkg_name| %>
+    pkg.depends_on "<%= pkg_name %>"
+    <% end %>
+end

data/lib/autoproj/jenkins/templates/package-Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gem 'autoproj', '>= 2.0.0.a'

data/lib/autoproj/jenkins/templates/package.pipeline.erb

@@ -0,0 +1,102 @@
+import java.nio.file.Paths
+
+def jobPackageName = "<%= package_name %>"
+def upstreamJobNames = [<%= upstream_jobs.each_key.map { |job_name| "'#{job_name}'" }.join(", ") %>]
+def upstreamPackageNames = [<%= upstream_jobs.each_value.map { |package_name| "'#{package_name}'" }.join(", ") %>]
+
+stage 'waiting for upstream jobs to finish'
+<%= render_template('wait-upstream.pipeline') %>
+
+node(label: 'autoproj-jenkins') {
+    dir('dev/install/log') { deleteDir() }
+
+    stage 'bootstrap'
+    <%= render_template('bootstrap.pipeline', vcs: buildconf_vcs, gemfile: gemfile, autoproj_install_path: autoproj_install_path, vcs_credentials: vcs_credentials, indent: 4) %>
+    stage 'install upstream artifacts'
+
+    def upstreamDir = "artifacts/upstream"
+    def packagePrefixes = sh(script: "${autoproj} locate --no-cache --prefix '${jobPackageName}' ${upstreamPackageNames.join(" ")}", returnStdout: true).
+    split("\n")
+
+    def jobPackagePrefix = packagePrefixes[0]
+    def upstreamPackagePrefixes = packagePrefixes.tail()
+
+    parallel(makeUpstreamArtifactImporters(
+            autoproj, fullWorkspaceDir, upstreamDir,
+            upstreamJobNames, upstreamPackagePrefixes, upstreamBuilds)
+    )
+    // We don't need the upstream artifacts anymore, clear some disk space
+    dir(upstreamDir) { deleteDir() }
+
+    if (fileExists("lastPrefix")) {
+        sh "mv lastPrefix '${jobPackagePrefix}'"
+    }
+
+    def jenkins_dependency_overrides = "<%= render_template 'jenkins_dependency_overrides.rb', escape: true, package_name: package_name, upstream_jobs: upstream_jobs %>"
+    writeFile file: 'dev/autoproj/overrides.d/99_jenkins_dependency_overrides.rb',
+        text: jenkins_dependency_overrides
+
+    dir('dev') {
+        stage 'prepare build'
+        <%= render_template("import-#{vcs.type}.pipeline",
+                            patch: true,
+                            allow_unused: true,
+                            package_dir: package_dir,
+                            vcs: vcs,
+                            credentials: vcs_credentials.for(vcs),
+                            package_name: package_name,
+                            indent: 8) %>
+        sh "${autoproj} test disable '<%= package_name %>'"
+        sh "${autoproj} osdeps '<%= package_name %>' 'pkg-config'"
+
+        stage 'build'
+        try {
+            sh "${autoproj} build --force --deps=f '<%= package_name %>' -p1"
+        }
+        catch(err) {
+            archive includes: 'install/<%= package_name %>/log/<%= package_name %>-*.log'
+            throw(err)
+        }
+        archive includes: 'install/<%= package_name %>/log/<%= package_name %>-*.log'
+    }
+
+    stage 'trigger dependencies'
+    <%= render_template('handle-downstream.pipeline',
+                        job_name: job_name,
+                        artifact_glob: artifact_glob,
+                        downstream_jobs: downstream_jobs,
+                        indent: 4) %>
+
+    stage 'tests'
+    def test_timestamp_path = "${fullWorkspaceDir}/test-timestamp"
+    touch file: test_timestamp_path
+    def test_output_path    = "${fullWorkspaceDir}/test"
+    def autoproj_test_failed = false
+    dir('dev')
+    {
+        try {
+            sh "${autoproj} test enable '<%= package_name %>'"
+            sh "${autoproj} osdeps '<%= package_name %>'"
+            sh "${autoproj} build --deps=f '<%= package_name %>' -p1"
+            sh "${autoproj} test -p=1 '<%= package_name %>'"
+        }
+        catch(err) { autoproj_test_failed = true }
+
+        try { sh "${autoproj} jenkins postprocess-tests --after=${test_timestamp_path} ${test_output_path} '<%= package_name %>'" }
+        catch(err) { autoproj_test_failed = true }
+    }
+    try { junit allowEmptyResults: true, keepLongStdio: true, testResults: "test/*.xml" }
+    catch(err) { autoproj_test_failed = true }
+
+    if (autoproj_test_failed)
+    {
+        currentBuild.result = 'UNSTABLE'
+    }
+
+    // Move the current package prefix to a separate folder, to ensure that
+    // other workspaces don't have access to it. It's not strictly required,
+    // but is a good sanity check
+    dir('lastPrefix') { deleteDir() }
+    sh "mv '${jobPackagePrefix}' lastPrefix"
+}
+

data/lib/autoproj/jenkins/templates/package.xml.erb

@@ -0,0 +1,23 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<flow-definition plugin="workflow-job@2.3">
+    <actions/>
+    <description></description>
+    <keepDependencies>false</keepDependencies>
+    <properties/>
+    <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.9">
+        <script />
+        <sandbox>false</sandbox>
+    </definition>
+    <triggers/>
+    <publishers/>
+    <quietPeriod><%= quiet_period %></quietPeriod>
+    <jenkins.model.BuildDiscarderProperty>
+        <strategy class="hudson.tasks.LogRotator">
+            <daysToKeep>30</daysToKeep>
+            <numToKeep>-1</numToKeep>
+            <artifactDaysToKeep>7</artifactDaysToKeep>
+            <artifactNumToKeep>-1</artifactNumToKeep>
+        </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+</flow-definition>
+

data/lib/autoproj/jenkins/templates/setup-git-credentials.pipeline.erb

@@ -0,0 +1,8 @@
+<% credentials.each do |cred| %>
+sh "git config --global credential.<%= cred.protocol %>://<%= cred.host %>.helper cache"
+withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: '<%= cred.jenkins_id %>',
+                  usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
+    def credentials = "protocol=<%= cred.protocol %>\nhost=<%= cred.host %>\nusername=${env.USERNAME}\npassword=${env.PASSWORD}\n"
+    sh "echo '${credentials}' | git credential approve"
+}
+<% end %>