elastic-apm 3.3.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaf3f0eada3b3bfe036dc55802688666e3df0ee48848b45dd73aa6861e625dc6
-  data.tar.gz: feb50b6f3d2da29af6591f31a62ea95189ab64868fb5403e3d85ceec548868d8
+  metadata.gz: 32d176ac5c22246df60ecc92f1baad37011a872af4e64f842b5a43c699a6ced3
+  data.tar.gz: bf0505e728d8f1bd0bb941574a844a8990d14e13a94a7c9f2a6c2d0f9eb5d35b
 SHA512:
-  metadata.gz: b7bcd197586b7782f25fa814ec0430c33f7b21abaa6b79c81fe93ee07654df1448eea4257f8c29b6f6dce1b7778c0b88f6e825a1338bdba591fd4934b9bcaa09
-  data.tar.gz: 181bf8aa045473ef55cb794199638722e1bd7f142e558077928a94e03ca75812d21633c6781e734d7029b12e9b4d54232498f604a6be26f70daaffcd1e91f03e
+  metadata.gz: d57b6bdbc7e78bea95e197d33f3eef4db7a35b23f9e450c636c3c1e3a5dd75ee8ee195709944ac51d6af7a3ab2451fd19d6139f0dedf9e86a4827732d5097f2a
+  data.tar.gz: 924b9bbf782a0fa4e376137e9a44d10cc01be71455ca7614ab1374c70f0926d174eb1078b985cd60c1693e2e0ec1cdd8446502bd40007359262dcc31241250aa

data/.ci/.jenkins_exclude.yml

@@ -1,5 +1,28 @@
 exclude:
-  - RUBY_VERSION: ruby:2.7-rc
+  # Ruby 2.3
+  # Only includes rails-5.2, sinatra-2.0
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-6.0
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-5.1
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-5.0
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-4.2
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: sinatra-1.4
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: grape-1.3
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-master
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: sinatra-master
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: grape-master
+
+  - RUBY_VERSION: ruby:2.7
     FRAMEWORK: rails-4.2
   - RUBY_VERSION: jruby:9.2
     FRAMEWORK: rails-4.2
@@ -18,22 +41,18 @@ exclude:
 
   - RUBY_VERSION: ruby:2.4
     FRAMEWORK: rails-6.0
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: rails-6.0
   - RUBY_VERSION: jruby:9.1
     FRAMEWORK: rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
     FRAMEWORK: rails-6.0
 
   # Only test master on newest ruby
-  - RUBY_VERSION: ruby:2.7-rc
+  - RUBY_VERSION: ruby:2.6
     FRAMEWORK: rails-master
   - RUBY_VERSION: ruby:2.5
     FRAMEWORK: rails-master
   - RUBY_VERSION: ruby:2.4
     FRAMEWORK: rails-master
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: rails-master
   - RUBY_VERSION: jruby:9.2
     FRAMEWORK: rails-master
   - RUBY_VERSION: jruby:9.1
@@ -53,8 +72,6 @@ exclude:
     FRAMEWORK: sinatra-master
   - RUBY_VERSION: ruby:2.4
     FRAMEWORK: sinatra-master
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: sinatra-master
   - RUBY_VERSION: jruby:9.2
     FRAMEWORK: sinatra-master
   - RUBY_VERSION: jruby:9.1
@@ -70,14 +87,12 @@ exclude:
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
     FRAMEWORK: sinatra-master
 
-  - RUBY_VERSION: ruby:2.7-rc
+  - RUBY_VERSION: ruby:2.6
     FRAMEWORK: grape-master
   - RUBY_VERSION: ruby:2.5
     FRAMEWORK: grape-master
   - RUBY_VERSION: ruby:2.4
     FRAMEWORK: grape-master
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: grape-master
   - RUBY_VERSION: jruby:9.2
     FRAMEWORK: grape-master
   - RUBY_VERSION: jruby:9.1
@@ -93,25 +108,29 @@ exclude:
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
     FRAMEWORK: grape-master
 
-  - RUBY_VERSION: ruby:2.7-rc
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+  # Unsupported
+  - RUBY_VERSION: jruby:9.1
+    FRAMEWORK: grape-1.3
+  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
+    FRAMEWORK: grape-1.3
+
+  - RUBY_VERSION: ruby:2.6
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: ruby:2.5
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: ruby:2.4
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: jruby:9.2
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: jruby:9.1
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-13-jdk
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-12-jdk
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-11-jdk
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-8-jdk
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
-    FRAMEWORK: grape-1.2,sinatra-2.0,rails-6.0
+    FRAMEWORK: grape-1.3,sinatra-2.0,rails-6.0

data/.ci/.jenkins_framework.yml

@@ -8,6 +8,6 @@ FRAMEWORK:
   - sinatra-2.0
   - sinatra-1.4
 
-  - grape-1.2
+  - grape-1.3
 
-  - grape-1.2,sinatra-2.0,rails-6.0
+  - grape-1.3,sinatra-2.0,rails-6.0

data/.ci/.jenkins_ruby.yml

@@ -1,5 +1,5 @@
 RUBY_VERSION:
-  - ruby:2.7-rc
+  - ruby:2.7
   - ruby:2.6
   - ruby:2.5
   - ruby:2.4

data/.ci/Jenkinsfile

@@ -5,16 +5,17 @@ import co.elastic.matrix.*
 import groovy.transform.Field
 
 /**
- This is required to store the results of the tests.
+This is the parallel tasks generator,
+it is need as field to store the results of the tests.
 */
-@Field def rubyTasksFailed = false
+@Field def rubyTasksGen
 
 pipeline {
   agent { label 'linux && immutable' }
   environment {
     REPO = 'apm-agent-ruby'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    PIPELINE_LOG_LEVEL='DEBUG'
+    PIPELINE_LOG_LEVEL = 'INFO'
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     CODECOV_SECRET = 'secret/apm-team/ci/apm-agent-ruby-codecov'
@@ -24,6 +25,7 @@ pipeline {
     ITS_PIPELINE = 'apm-integration-tests-selector-mbp/master'
     RELEASE_SECRET = 'secret/apm-team/ci/apm-agent-ruby-rubygems-release'
     OPBEANS_REPO = 'opbeans-ruby'
+    REFERENCE_REPO = '/var/lib/jenkins/.git-references/apm-agent-ruby.git'
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -36,7 +38,7 @@ pipeline {
     quietPeriod(10)
   }
   triggers {
-    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
+    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
@@ -51,16 +53,25 @@ pipeline {
       steps {
         pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
         deleteDir()
-        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true)
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true, reference: "${env.REFERENCE_REPO}")
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+        script {
+          dir("${BASE_DIR}"){
+            // Skip all the stages except docs for PR's with asciidoc and md changes only
+            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true)
+          }
+        }
       }
     }
     stage('Sanity checks') {
       when {
         beforeAgent true
-        anyOf {
-          not { changeRequest() }
-          expression { return params.Run_As_Master_Branch }
+        allOf {
+          expression { return env.ONLY_DOCS == "false" }
+          anyOf {
+            not { changeRequest() }
+            expression { return params.Run_As_Master_Branch }
+          }
         }
       }
       options { skipDefaultCheckout() }
@@ -83,168 +94,177 @@ pipeline {
     /**
     Execute unit tests.
     */
-    stage('Test') {
-      options { skipDefaultCheckout() }
-      steps {
-        withGithubNotify(context: 'Tests', tab: 'tests') {
-          deleteDir()
-          unstash "source"
-          dir("${BASE_DIR}"){
-            script {
-              def ruby = readYaml(file: '.ci/.jenkins_ruby.yml')
-              def testTasks = [:]
-              ruby['RUBY_VERSION'].each{ rubyVersion ->
-                testTasks[rubyVersion] = { runJob(rubyVersion) }
-              }
-              parallel(testTasks)
-              }
+    stage('Tests') {
+      when {
+        beforeAgent true
+        expression { return env.ONLY_DOCS == "false" }
+      }
+      failFast false
+      parallel {
+        stage('Tests') {
+          options { skipDefaultCheckout() }
+          steps {
+            withGithubNotify(context: 'Tests', tab: 'tests') {
+              runTests('.ci/.jenkins_framework.yml')
             }
           }
-        }
-      }
-      stage('Benchmarks') {
-        options { skipDefaultCheckout() }
-        when {
-          beforeAgent true
-          allOf {
-            anyOf {
-              branch 'master'
-              branch "\\d+\\.\\d+"
-              branch "v\\d?"
-              tag "v\\d+\\.\\d+\\.\\d+*"
-              expression { return params.Run_As_Master_Branch }
+          post {
+            always {
+              convergeCoverage()
             }
-            expression { return params.bench_ci }
           }
         }
-        stages {
-          stage('Clean Workspace') {
-            agent { label 'metal' }
-            steps {
-              echo "Cleaning Workspace"
-            }
-            post {
-              always {
-                cleanWs()
-              }
+        stage('Master Tests frameworks') {
+          options { skipDefaultCheckout() }
+          steps {
+            catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE', message: "The tests for the master framework have failed. Let's warn instead.") {
+              runTests('.ci/.jenkins_master_framework.yml')
             }
           }
-          /**
-            Run the benchmarks and store the results on ES.
-            The result JSON files are also archive into Jenkins.
-          */
-          stage('Run Benchmarks') {
-            steps {
-              withGithubNotify(context: 'Run Benchmarks') {
-                deleteDir()
-                unstash 'source'
-                dir("${BASE_DIR}"){
-                  script {
-                    def versions = readYaml(file: ".ci/.jenkins_ruby.yml")
-                    def benchmarkTask = [:]
-                    // TODO: benchmark for the jruby:9.2 and similar versions got some issues with
-                    //       NoMethodError: undefined method `[]' for nil:NilClass
-                    //      <main> at bench/report.rb:48
-                    versions['RUBY_VERSION'].findAll { !it.contains('9.2') }.each{ v ->
-                      benchmarkTask[v] = runBenchmark(v)
-                    }
-                    parallel(benchmarkTask)
-                  }
-                }
-              }
+        }
+        stage('Integration Tests') {
+          agent none
+          when {
+            beforeAgent true
+            anyOf {
+              changeRequest()
+              expression { return !params.Run_As_Master_Branch }
             }
           }
+          steps {
+            githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+            build(job: env.ITS_PIPELINE, propagate: false, wait: true,
+                  parameters: [ string(name: 'INTEGRATION_TEST', value: 'Ruby'),
+                                string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO} --opbeans-ruby-agent-branch ${env.GIT_BASE_COMMIT}"),
+                                string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
+                                string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+                                string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT) ])
+          }
         }
       }
-      stage('Populate Test failures') {
-        when {
-          expression { return rubyTasksFailed }
-        }
-        steps {
-          error('There were some failures when running the "Test" stage')
-        }
-      }
-      stage('Integration Tests') {
-        agent none
-        when {
-          beforeAgent true
+    }
+    stage('Benchmarks') {
+      options { skipDefaultCheckout() }
+      when {
+        beforeAgent true
+        allOf {
           anyOf {
-            changeRequest()
-            expression { return !params.Run_As_Master_Branch }
+            branch 'master'
+            branch "\\d+\\.\\d+"
+            branch "v\\d?"
+            tag pattern: 'v\\d+.*', comparator: "REGEXP"
+            expression { return params.Run_As_Master_Branch }
+            expression { return env.GITHUB_COMMENT?.contains('benchmark tests') }
           }
-        }
-        steps {
-          log(level: 'INFO', text: 'Launching Async ITs')
-          build(job: env.ITS_PIPELINE, propagate: false, wait: false,
-                parameters: [string(name: 'AGENT_INTEGRATION_TEST', value: 'Ruby'),
-                             string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO}"),
-                             string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
-                             string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-                             string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)]
-          )
-          githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+          expression { return params.bench_ci }
         }
       }
-      stage('Release') {
-        options { skipDefaultCheckout() }
-        environment {
-          RUBY_DOCKER_TAG = 'ruby:2.6'
-          HOME = '/var/lib/jenkins'
-        }
-        when {
-          beforeAgent true
-          tag pattern: 'v\\d+.*', comparator: 'REGEXP'
+      stages {
+        stage('Clean Workspace') {
+          agent { label 'metal' }
+          steps {
+            echo "Cleaning Workspace"
+          }
+          post {
+            always {
+              cleanWs()
+            }
+          }
         }
-        stages {
-          stage('Release') {
-            steps {
-              withGithubNotify(context: 'Release') {
-                deleteDir()
-                unstash 'source'
+        /**
+          Run the benchmarks and store the results on ES.
+          The result JSON files are also archive into Jenkins.
+        */
+        stage('Run Benchmarks') {
+          steps {
+            withGithubNotify(context: 'Run Benchmarks') {
+              deleteDir()
+              unstash 'source'
+              dir("${BASE_DIR}"){
                 script {
-                  dir(BASE_DIR){
-                    docker.image("${env.RUBY_DOCKER_TAG}").inside('-v /etc/passwd:/etc/passwd -v ${HOME}/.ssh:${HOME}/.ssh') {
-                      withEnv(["HOME=${env.WORKSPACE}/${env.BASE_DIR ?: ''}"]) {
-                        rubygemsLogin.withApi(secret: "${env.RELEASE_SECRET}") {
-                          sshagent(['f6c7695a-671e-4f4f-a331-acdce44ff9ba']) {
-                            sh '.ci/prepare-git-context.sh'
-                            sh 'gem install rake yard rspec'
-                            sh 'rake release'
-                          }
-                        }
-                      }
-                    }
+                  def versions = readYaml(file: ".ci/.jenkins_ruby.yml")
+                  def benchmarkTask = [:]
+                  versions['RUBY_VERSION'].each{ v ->
+                    benchmarkTask[v] = runBenchmark(v)
                   }
+                  parallel(benchmarkTask)
                 }
               }
             }
           }
-          stage('Opbeans') {
-            environment {
-              REPO_NAME = "${OPBEANS_REPO}"
+        }
+      }
+    }
+    stage('Release') {
+      options { skipDefaultCheckout() }
+      environment {
+        RUBY_DOCKER_TAG = 'ruby:2.6'
+        HOME = '/var/lib/jenkins'
+      }
+      when {
+        beforeAgent true
+        tag pattern: 'v\\d+.*', comparator: 'REGEXP'
+      }
+      stages {
+        stage('Release') {
+          steps {
+            deleteDir()
+            unstash 'source'
+            prepareRelease(){
+              sh 'rake release'
             }
-            steps {
-              deleteDir()
-              dir("${OPBEANS_REPO}"){
-                git credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
-                    url: "git@github.com:elastic/${OPBEANS_REPO}.git"
-                sh script: ".ci/bump-version.sh ${env.BRANCH_NAME}", label: 'Bump version'
-                // The opbeans pipeline will trigger a release for the master branch
-                gitPush()
-                // The opbeans pipeline will trigger a release for the release tag
-                gitCreateTag(tag: "${env.BRANCH_NAME}")
+          }
+        }
+        stage('Update Branch') {
+          steps {
+            deleteDir()
+            unstash 'source'
+            prepareRelease(){
+              catchError(buildResult: 'SUCCESS', message: 'Update branch task failed', stageResult: 'UNSTABLE') {
+                sh 'rake release:update_branch'
               }
             }
           }
+          post {
+            unsuccessful {
+              emailext subject: "[${env.REPO}] Syncup post-release stage failed.", to: "${NOTIFY_TO}",
+                        body: "Please go to ${env.BUILD_URL} to review the logs. Most likely you need to update the branch manually."
+            }
+          }
+        }
+        stage('Opbeans') {
+          environment {
+            REPO_NAME = "${OPBEANS_REPO}"
+            VERSION = "${env.BRANCH_NAME.replaceAll('^v', '')}"
+          }
+          steps {
+            deleteDir()
+            // Let's wait for the Gem to be available
+            sh label: 'Wait for gem', script: """#!/usr/bin/env bash
+              source /usr/local/bin/bash_standard_lib.sh
+              (retry 10 curl --silent --show-error --fail -I https://rubygems.org/gems/elastic-apm/versions/${env.VERSION})
+            """
+            dir("${OPBEANS_REPO}"){
+              git credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                  url: "git@github.com:elastic/${OPBEANS_REPO}.git"
+              // It's required to transform the tag value to the gem version
+              sh script: ".ci/bump-version.sh ${env.VERSION}", label: 'Bump version'
+              // The opbeans pipeline will trigger a release for the master branch
+              gitPush()
+              // The opbeans pipeline will trigger a release for the release tag
+              gitCreateTag(tag: "${env.BRANCH_NAME}")
+            }
+          }
         }
       }
     }
-    post {
-      cleanup {
-        notifyBuildResult()
-      }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
     }
   }
+}
 
 /**
   Run benchmarks for a Ruby version, then report the results to the Elasticsearch server.
@@ -256,26 +276,64 @@ def runBenchmark(version){
       //  - docker.elastic.co/observability-ci/jruby:9.2-12-jdk to jruby-9.2-12-jdk
       //  - jruby:9.1 to jruby-9.1
       def transformedVersion = version.replaceAll('.*/', '').replaceAll(':', '-')
-      env.HOME = "${env.WORKSPACE}/${transformedVersion}"
-      dir("${transformedVersion}"){
-        deleteDir()
-        unstash 'source'
-        dir("${BASE_DIR}"){
-          retry(2){
-            sleep randomNumber(min:10, max: 30)
-            dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+      withEnv(["HOME=${env.WORKSPACE}/${transformedVersion}"]){
+        dir("${transformedVersion}"){
+          deleteDir()
+          unstash 'source'
+          dir("${BASE_DIR}"){
+            retry(2){
+              sleep randomNumber(min:10, max: 30)
+              dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+            }
+            try{
+              sh """./spec/scripts/benchmarks.sh "${version}" "${REFERENCE_REPO}" """
+            } catch(e){
+              throw e
+            } finally {
+              archiveArtifacts(
+                allowEmptyArchive: true,
+                artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error",
+                onlyIfSuccessful: false)
+              sendBenchmarks(file: "benchmark-${transformedVersion}.bulk",
+                index: "benchmark-ruby", archive: true)
+            }
           }
-          try{
-            sh "./spec/scripts/benchmarks.sh ${version}"
-          } catch(e){
-            throw e
-          } finally {
-            archiveArtifacts(
-              allowEmptyArchive: true,
-              artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error",
-              onlyIfSuccessful: false)
-            sendBenchmarks(file: "benchmark-${transformedVersion}.bulk",
-              index: "benchmark-ruby", archive: true)
+        }
+      }
+    }
+  }
+}
+
+class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
+
+  public RubyParallelTaskGenerator(Map params){
+    super(params)
+  }
+
+  /**
+  build a clousure that launch and agent and execute the corresponding test script,
+  then store the results.
+  */
+  public Closure generateStep(x, y){
+    return {
+      steps.sleep steps.randomNumber(min:10, max: 30)
+      steps.node('linux && immutable'){
+        // Label is transformed to avoid using the internal docker registry in the x coordinate
+        // TODO: def label = "${tag}:${x?.drop(x?.lastIndexOf('/')+1)}#${y}"
+        def label = "${tag}:${x}#${y}"
+        try {
+          steps.runScript(label: label, ruby: x, framework: y)
+          saveResult(x, y, 1)
+        } catch(e){
+          saveResult(x, y, 0)
+          steps.error("${label} tests failed : ${e.toString()}\n")
+        } finally {
+          steps.junit(allowEmptyResults: false,
+            keepLongStdio: true,
+            testResults: "**/spec/junit-reports/**/ruby-agent-junit.xml")
+          steps.dir("${steps.env.BASE_DIR}"){
+            steps.archiveArtifacts(artifacts: 'coverage/matrix_results/', defaultExcludes: false)
+            steps.stash(name: steps.normalise("coverage-${x}-${y}"), includes: 'coverage/matrix_results/', allowEmpty: true)
           }
         }
       }
@@ -283,19 +341,79 @@ def runBenchmark(version){
   }
 }
 
-def runJob(rubyVersion){
-  try {
-    build( job: "apm-agent-ruby/apm-agent-ruby-downstream/${env.BRANCH_NAME}",
-      parameters: [
-        string(name: 'RUBY_VERSION', value: "${rubyVersion}"),
-        string(name: 'BRANCH_SPECIFIER', value: "${env.GIT_BASE_COMMIT}")
-      ],
-      quietPeriod: 15
+/**
+  Run all the tests for the given file with the frameworks to test
+*/
+def runTests(frameworkFile) {
+  deleteDir()
+  unstash "source"
+  dir("${BASE_DIR}"){
+    rubyTasksGen = new RubyParallelTaskGenerator(
+      xKey: 'RUBY_VERSION',
+      yKey: 'FRAMEWORK',
+      xFile: ".ci/.jenkins_ruby.yml",
+      yFile: frameworkFile,
+      exclusionFile: ".ci/.jenkins_exclude.yml",
+      tag: "Ruby",
+      name: "Ruby",
+      steps: this
     )
-  } catch(e) {
-    rubyTasksFailed = true
-    warnError('Test Failures') {
-      error("Downstream job for '${rubyVersion}' failed")
+    def testTasks = rubyTasksGen.generateParallelTests()
+    parallel(testTasks)
+  }
+}
+
+/**
+  Run tests for a Ruby version and framework version.
+*/
+def runScript(Map params = [:]){
+  def label = params.label
+  def ruby = params.ruby
+  def framework = params.framework
+  log(level: 'INFO', text: "${label}")
+  retry(2){
+    withEnv(["HOME=${env.WORKSPACE}", "PATH=${env.PATH}:${env.WORKSPACE}/bin"]) {
+      deleteDir()
+      unstash 'source'
+      dir("${BASE_DIR}"){
+        sleep randomNumber(min:10, max: 30)
+        dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+        sh("./spec/scripts/spec.sh ${ruby} ${framework}")
+      }
     }
   }
 }
+
+def convergeCoverage() {
+  deleteDir()
+  unstash('source')
+  dir("${BASE_DIR}"){
+    rubyTasksGen.dumpMatrix('-')?.each {
+      unstash(normalise("coverage-${it}"))
+    }
+    sh(script: './spec/scripts/coverage_converge.sh')
+    cobertura coberturaReportFile: 'coverage/coverage.xml', onlyStable: false
+  }
+}
+
+def prepareRelease(Closure body){
+  dir("${env.BASE_DIR}"){
+    docker.image("${env.RUBY_DOCKER_TAG}").inside('-v ${REFERENCE_REPO}:${REFERENCE_REPO} -v /etc/passwd:/etc/passwd -v ${HOME}/.ssh:${HOME}/.ssh') {
+      withEnv(["HOME=${env.WORKSPACE}/${env.BASE_DIR ?: ''}"]) {
+        rubygemsLogin.withApi(secret: "${env.RELEASE_SECRET}") {
+          withGitRelease(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken') {
+            sh 'gem install rake yard rspec'
+            body()
+          }
+        }
+      }
+    }
+  }
+}
+
+// Transform the versions like:
+//  - docker.elastic.co/observability-ci/jruby:9.2-12-jdk to jruby-9.2-12-jdk
+//  - jruby:9.1 to jruby-9.1
+def normalise(def what) {
+  return what.replaceAll('.*/', '').replaceAll(':', '-')
+}