elastic-apm 2.9.1 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47f1adcea0273ee666706ddea65216aa0d8ba4d15acb14be64a425b77e1c188c
-  data.tar.gz: 7aa7372590f8c43585681cb84464af8bf984557699c43c09a4614f71e0cfc8b2
+  metadata.gz: eb99c67f61f3f3bc30f7fb7bc64accea3725295db3d7bf90c18f2ce33d183e1f
+  data.tar.gz: 1a45da759179fa4b284a14b580f9f318fe27d4d4ba60fc87d5360acbf1e5ab94
 SHA512:
-  metadata.gz: d4afe772c0f242885882347f6c5ed3b6e319c533ab24f88e52ab3e5c68be16c33243529a642ff3644936218808b7163e7c60536c3632aa11c28d8072efc3ca21
-  data.tar.gz: 49a6baa392426cb6f51b1327b23fc0f0d7257685c26737e222613595fb68be80b44f00b39d309535b918315e7138ba48ae9100837a92160c62265bafa2140764
+  metadata.gz: 731f900b1466cf7f017fbdf747c969e7748cb9fdedf4f9a2dd748d5dbf7da31a3ce96a249ecab002bf62c9961e9c9336b983f570f51775f352b8dba43f5a04cb
+  data.tar.gz: 52518d1fe04e4def36990303cee09643580dee8313ae2809f4e7f68fbc6783ba6028459987c291f153e29a5181ec3a289409d5bcc5fb138fbfae242b4f07747d

data/.ci/.jenkins_codecov.yml

@@ -0,0 +1,5 @@
+---
+# Tuple of ruby version and frameworks which are in charge to send data to codecov.
+# # is the separator between the ruby version and the framework.
+ENABLED:
+  - ruby:2.6#rails-6.0

data/.ci/.jenkins_exclude.yml

@@ -13,6 +13,15 @@ exclude:
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
     FRAMEWORK: rails-4.2
 
+  - RUBY_VERSION: ruby:2.4
+    FRAMEWORK: rails-6.0
+  - RUBY_VERSION: ruby:2.3
+    FRAMEWORK: rails-6.0
+  - RUBY_VERSION: jruby:9.1
+    FRAMEWORK: rails-6.0
+  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
+    FRAMEWORK: rails-6.0
+
   # Only test master on newest ruby
   - RUBY_VERSION: ruby:2.5
     FRAMEWORK: rails-master
@@ -52,22 +61,3 @@ exclude:
   - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
     FRAMEWORK: sinatra-master
 
-  # Rails 6 beta
-  - RUBY_VERSION: ruby:2.5
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: ruby:2.4
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: ruby:2.3
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: jruby:9.2
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: jruby:9.1
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-12-jdk
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-11-jdk
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.2-8-jdk
-    FRAMEWORK: rails-6.0-rc1
-  - RUBY_VERSION: docker.elastic.co/observability-ci/jruby:9.1-7-jdk
-    FRAMEWORK: rails-6.0-rc1

data/.ci/.jenkins_framework.yml

@@ -1,5 +1,5 @@
 FRAMEWORK:
-  - rails-6.0-rc1
+  - rails-6.0
   - rails-5.2
   - rails-5.1
   - rails-5.0
@@ -7,6 +7,3 @@ FRAMEWORK:
 
   - sinatra-2.0
   - sinatra-1.4
-
-  - rails-master
-  - sinatra-master

data/.ci/.jenkins_master_framework.yml

@@ -0,0 +1,3 @@
+FRAMEWORK:
+  - rails-master
+  - sinatra-master

data/.ci/Jenkinsfile

@@ -11,11 +11,11 @@ it is need as field to store the results of the tests.
 @Field def rubyTasksGen
 
 pipeline {
-  agent any
+  agent { label 'linux && immutable' }
   environment {
     REPO = 'apm-agent-ruby'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    PIPELINE_LOG_LEVEL='INFO'
+    PIPELINE_LOG_LEVEL='DEBUG'
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     CODECOV_SECRET = 'secret/apm-team/ci/apm-agent-ruby-codecov'
@@ -39,7 +39,6 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
-    booleanParam(name: 'doc_ci', defaultValue: true, description: 'Enable build docs.')
     booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable run benchmarks.')
   }
   stages {
@@ -47,14 +46,33 @@ pipeline {
     Checkout the code and stash it, to use it on other stages.
     */
     stage('Checkout') {
-      agent { label 'master || immutable' }
+      agent { label 'immutable' }
       options { skipDefaultCheckout() }
       steps {
         deleteDir()
-        gitCheckout(basedir: "${BASE_DIR}")
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true)
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
       }
     }
+    stage('Sanity checks') {
+      agent { label 'linux && immutable && docker' }
+      options { skipDefaultCheckout() }
+      environment {
+        HOME = "${env.WORKSPACE}"
+        PATH = "${env.WORKSPACE}/bin:${env.PATH}"
+      }
+      steps {
+        withGithubNotify(context: 'Sanity checks') {
+          deleteDir()
+          unstash 'source'
+          dir(BASE_DIR) {
+            catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE', message: 'Sanity checks failed but keep running the build') {
+              preCommit(commit: "${GIT_BASE_COMMIT}", junit: true)
+            }
+          }
+        }
+      }
+    }
     /**
     Execute unit tests.
     */
@@ -67,18 +85,12 @@ pipeline {
           unstash "source"
           dir("${BASE_DIR}"){
             script {
-              rubyTasksGen = new RubyParallelTaskGenerator(
-                xKey: 'RUBY_VERSION',
-                yKey: 'FRAMEWORK',
-                xFile: ".ci/.jenkins_ruby.yml",
-                yFile: ".ci/.jenkins_framework.yml",
-                exclusionFile: ".ci/.jenkins_exclude.yml",
-                tag: "Ruby",
-                name: "Ruby",
-                steps: this
-                )
-                def mapPatallelTasks = rubyTasksGen.generateParallelTests()
-                parallel(mapPatallelTasks)
+              def ruby = readYaml(file: '.ci/.jenkins_ruby.yml')
+              def testTasks = [:]
+              ruby['RUBY_VERSION'].each{ rubyVersion ->
+                testTasks[rubyVersion] = { runJob(rubyVersion) }
+              }
+              parallel(testTasks)
               }
             }
           }
@@ -136,31 +148,6 @@ pipeline {
           }
         }
       }
-      /**
-      Build the documentation.
-      */
-      stage('Documentation') {
-        agent { label 'linux && immutable' }
-        options { skipDefaultCheckout() }
-        when {
-          beforeAgent true
-          allOf {
-            anyOf {
-              branch 'master'
-              branch "\\d+\\.\\d+"
-              branch "v\\d?"
-              tag "v\\d+\\.\\d+\\.\\d+*"
-              expression { return params.Run_As_Master_Branch }
-            }
-            expression { return params.doc_ci }
-          }
-        }
-        steps {
-          deleteDir()
-          unstash 'source'
-          buildDocs(docsDir: "${BASE_DIR}/docs", archive: true)
-        }
-      }
       stage('Integration Tests') {
         agent none
         when {
@@ -176,93 +163,21 @@ pipeline {
           log(level: 'INFO', text: 'Launching Async ITs')
           build(job: env.ITS_PIPELINE, propagate: false, wait: false,
                 parameters: [string(name: 'AGENT_INTEGRATION_TEST', value: 'Ruby'),
-                             string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK}/${env.REPO}"),
+                             string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO}"),
                              string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
                              string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-                             string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)])
+                             string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)]
+          )
           githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
         }
       }
     }
     post {
       cleanup {
-        script{
-          if(rubyTasksGen?.results){
-            writeJSON(file: 'results.json', json: toJSON(rubyTasksGen.results), pretty: 2)
-            def mapResults = ["Ruby": rubyTasksGen.results]
-            def processor = new ResultsProcessor()
-            processor.processResults(mapResults)
-            archiveArtifacts allowEmptyArchive: true, artifacts: 'results.json,results.html', defaultExcludes: false
-            catchError(buildResult: 'SUCCESS') {
-              def datafile = readFile(file: "results.json")
-              def json = getVaultSecret(secret: 'secret/apm-team/ci/apm-server-benchmark-cloud')
-              sendDataToElasticsearch(es: json.data.url, data: datafile, restCall: '/jenkins-builds-test-results/_doc/')
-            }
-          }
-        }
-      notifyBuildResult()
-    }
-  }
-}
-
-/**
-Parallel task generator for the integration tests.
-*/
-class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
-
-  public RubyParallelTaskGenerator(Map params){
-    super(params)
-  }
-
-  /**
-  build a clousure that launch and agent and execute the corresponding test script,
-  then store the results.
-  */
-  public Closure generateStep(x, y){
-    return {
-      steps.sleep steps.randomNumber(min:10, max: 30)
-      steps.node('linux && immutable'){
-        // Label is transformed to avoid using the internal docker registry in the x coordinate
-        // TODO: def label = "${tag}:${x?.drop(x?.lastIndexOf('/')+1)}#${y}"
-        def label = "${tag}:${x}#${y}"
-        try {
-          steps.runScript(label: label, ruby: x, framework: y)
-          saveResult(x, y, 1)
-        } catch(e){
-          saveResult(x, y, 0)
-          steps.error("${label} tests failed : ${e.toString()}\n")
-        } finally {
-          steps.junit(allowEmptyResults: false,
-            keepLongStdio: true,
-            testResults: "**/spec/ruby-agent-junit.xml")
-          steps.codecov(repo: "${steps.env.REPO}", basedir: "${steps.env.BASE_DIR}",
-            secret: "${steps.env.CODECOV_SECRET}")
-        }
+        notifyBuildResult()
       }
     }
   }
-}
-
-/**
-  Run tests for a Ruby version and framework version.
-*/
-def runScript(Map params = [:]){
-  def label = params.label
-  def ruby = params.ruby
-  def framework = params.framework
-  log(level: 'INFO', text: "${label}")
-  env.HOME = "${env.WORKSPACE}"
-  env.PATH = "${env.PATH}:${env.WORKSPACE}/bin"
-  deleteDir()
-  unstash 'source'
-  dir("${BASE_DIR}"){
-    retry(2){
-      sleep randomNumber(min:10, max: 30)
-      dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
-      sh("./spec/scripts/spec.sh ${ruby} ${framework}")
-    }
-  }
-}
 
 /**
   Run benchmarks for a Ruby version, then report the results to the Elasticsearch server.
@@ -300,3 +215,13 @@ def runBenchmark(version){
     }
   }
 }
+
+def runJob(rubyVersion){
+  build( job: "apm-agent-ruby/apm-agent-ruby-downstream/${env.BRANCH_NAME}",
+    parameters: [
+      string(name: 'RUBY_VERSION', value: "${rubyVersion}"),
+      string(name: 'BRANCH_SPECIFIER', value: "${env.GIT_BASE_COMMIT}")
+    ],
+    quietPeriod: 15
+  )
+}

data/.ci/downstreamTests.groovy

@@ -11,7 +11,7 @@ it is need as field to store the results of the tests.
 @Field def rubyTasksGen
 
 pipeline {
-  agent any
+  agent { label 'linux && immutable' }
   environment {
     REPO="git@github.com:elastic/apm-agent-ruby.git"
     BASE_DIR="src/github.com/elastic/apm-agent-ruby"
@@ -25,7 +25,7 @@ pipeline {
   }
   options {
     timeout(time: 2, unit: 'HOURS')
-    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    buildDiscarder(logRotator(numToKeepStr: '300', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
     disableResume()
@@ -36,49 +36,36 @@ pipeline {
   parameters {
     string(name: 'RUBY_VERSION', defaultValue: "ruby:2.6", description: "Ruby version to test")
     string(name: 'BRANCH_SPECIFIER', defaultValue: "master", description: "Git branch/tag to use")
-    string(name: 'CHANGE_TARGET', defaultValue: "master", description: "Git branch/tag to merge before building")
   }
   stages {
     /**
     Checkout the code and stash it, to use it on other stages.
     */
     stage('Checkout') {
-      agent { label 'master || immutable' }
+      agent { label 'immutable' }
       options { skipDefaultCheckout() }
       steps {
         deleteDir()
-        gitCheckout(basedir: "${BASE_DIR}", 
+        gitCheckout(basedir: "${BASE_DIR}",
           branch: "${params.BRANCH_SPECIFIER}",
           repo: "${REPO}",
-          credentialsId: "${JOB_GIT_CREDENTIALS}",
-          mergeTarget: "${params.CHANGE_TARGET}",
-          reference: '/var/lib/jenkins/apm-agent-ruby.git')
+          credentialsId: "${JOB_GIT_CREDENTIALS}")
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
       }
     }
-    /**
-    Execute unit tests.
-    */
     stage('Test') {
       agent { label 'linux && immutable' }
       options { skipDefaultCheckout() }
       steps {
-        deleteDir()
-        unstash "source"
-        dir("${BASE_DIR}"){
-          script {
-            rubyTasksGen = new RubyParallelTaskGenerator(
-              xVersions: [ "${params.RUBY_VERSION}" ],
-              yKey: 'FRAMEWORK',
-              yFile: ".ci/.jenkins_framework.yml",
-              exclusionFile: ".ci/.jenkins_exclude.yml",
-              tag: "Ruby",
-              name: "Ruby",
-              steps: this
-            )
-            def mapPatallelTasks = rubyTasksGen.generateParallelTests()
-            parallel(mapPatallelTasks)
-          }
+        runTests('.ci/.jenkins_framework.yml')
+      }
+    }
+    stage('Master Test') {
+      agent { label 'linux && immutable' }
+      options { skipDefaultCheckout() }
+      steps {
+        catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE', message: "The tests for the master framework have failed. Let's warn instead.") {
+          runTests('.ci/.jenkins_master_framework.yml')
         }
       }
     }
@@ -92,6 +79,11 @@ pipeline {
           def processor = new ResultsProcessor()
           processor.processResults(mapResults)
           archiveArtifacts allowEmptyArchive: true, artifacts: 'results.json,results.html', defaultExcludes: false
+          catchError(buildResult: 'SUCCESS') {
+            def datafile = readFile(file: "results.json")
+            def json = getVaultSecret(secret: 'secret/apm-team/ci/jenkins-stats-cloud')
+            sendDataToElasticsearch(es: json.data.url, data: datafile, restCall: '/jenkins-builds-ruby-test-results/_doc/')
+          }
         }
       }
       notifyBuildResult()
@@ -126,11 +118,13 @@ class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
           saveResult(x, y, 0)
           steps.error("${label} tests failed : ${e.toString()}\n")
         } finally {
-          steps.junit(allowEmptyResults: false,
+          steps.junit(allowEmptyResults: true,
             keepLongStdio: true,
             testResults: "**/spec/ruby-agent-junit.xml")
-          steps.codecov(repo: "${steps.env.REPO}", basedir: "${steps.env.BASE_DIR}",
-            secret: "${steps.env.CODECOV_SECRET}")
+          if (steps.isCodecovEnabled(x, y)) {
+            steps.codecov(repo: "${steps.env.REPO}", basedir: "${steps.env.BASE_DIR}",
+                          secret: "${steps.env.CODECOV_SECRET}")
+          }
         }
       }
     }
@@ -157,3 +151,38 @@ def runScript(Map params = [:]){
     }
   }
 }
+
+/**
+* Whether the given ruby version and framework are in charge of sending the
+* codecov results. It does require the workspace.
+*/
+def isCodecovEnabled(ruby, framework){
+  dir(BASE_DIR){
+    def codecovVersions = readYaml(file: '.ci/.jenkins_codecov.yml')
+    return codecovVersions['ENABLED'].any { it.trim() == "${ruby?.trim()}#${framework?.trim()}" }
+  }
+}
+
+/**
+  Run all the tests for the given ruby version and file with the frameworks to test
+*/
+def runTests(frameworkFile) {
+  deleteDir()
+  unstash "source"
+  dir("${BASE_DIR}"){
+    script {
+      rubyTasksGen = new RubyParallelTaskGenerator(
+        xVersions: [ "${params.RUBY_VERSION}" ],
+        xKey: 'RUBY_VERSION',
+        yKey: 'FRAMEWORK',
+        yFile: frameworkFile,
+        exclusionFile: '.ci/.jenkins_exclude.yml',
+        tag: 'Ruby',
+        name: 'Ruby',
+        steps: this
+      )
+      def mapPatallelTasks = rubyTasksGen.generateParallelTests()
+      parallel(mapPatallelTasks)
+    }
+  }
+}

data/.ci/jobs/apm-agent-ruby-downstream.yml

@@ -4,3 +4,34 @@
     display-name: APM Agent Ruby Downstream
     description: APM Agent Ruby Downstream .
     script-path: .ci/downstreamTests.groovy
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: true
+        notification-context: 'apm-ci/downstream'
+        property-strategies:
+          all-branches:
+          - suppress-scm-triggering: true
+        repo: apm-agent-ruby
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'