elastic-apm 3.6.0 → 3.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e226ecd39fc71213084dffaefbd4343c1d5f5d78a4d8fc91786dca9f5e865533
-  data.tar.gz: a0289b4872d55be4132b6982da072844ce71b2a6723877626584c1f5e8d7315f
+  metadata.gz: a6db74de4a0201f5b5dd7e9f96b7ec383ce3276a56d7f9d57e64617dd7cf1a28
+  data.tar.gz: '0319a664e07b2411428134e24ed58f21049acf856cf163c6434fb66934e4cd1f'
 SHA512:
-  metadata.gz: 641923b3afdf3c3fe258053a2f2492092aa1af5a35a50c3783b6e7d0d00e6198f8454c95fb4f2eca0a1ea83f59afd552c600ffaed34efa0aeecf44d40b9948cd
-  data.tar.gz: 427eee529da777c98d4c4c02eb79f5aba46200e92edcac1f31b5bbd38df55f78801d92de063c37a54ffecf2fe3777697495c745f8334d3fdae4509626b1e1350
+  metadata.gz: 7bc1dfb374ea6107f07f690d764205f44c00a588eed767d7621b42640bf00f322d5e8b7c6a3d4958b0753b4755722836a05577fcee5b1a15207a355860a1f71b
+  data.tar.gz: 780227ecdd4592d5f3aaa87b54824b4ed36dc9686052b51314815ebbf3152855dc5ff186c69deb88b86f2eab1d7aa9e4170bfd1454b2fdca3ab82d513eb0280a

data/.ci/Jenkinsfile

@@ -5,21 +5,17 @@ import co.elastic.matrix.*
 import groovy.transform.Field
 
 /**
- This is required to know if there any failures when running the downstream jobs.
+This is the parallel tasks generator,
+it is need as field to store the results of the tests.
 */
-@Field def rubyTasksFailed = false
-
-/**
- This is required to store the build status for the downstream jobs.
-*/
-@Field def rubyDownstreamJobs = [:]
+@Field def rubyTasksGen
 
 pipeline {
   agent { label 'linux && immutable' }
   environment {
     REPO = 'apm-agent-ruby'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    PIPELINE_LOG_LEVEL='DEBUG'
+    PIPELINE_LOG_LEVEL = 'INFO'
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     CODECOV_SECRET = 'secret/apm-team/ci/apm-agent-ruby-codecov'
@@ -42,7 +38,7 @@ pipeline {
     quietPeriod(10)
   }
   triggers {
-    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
+    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
@@ -59,14 +55,23 @@ pipeline {
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true, reference: "${env.REFERENCE_REPO}")
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+        script {
+          dir("${BASE_DIR}"){
+            // Skip all the stages except docs for PR's with asciidoc and md changes only
+            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true)
+          }
+        }
       }
     }
     stage('Sanity checks') {
       when {
         beforeAgent true
-        anyOf {
-          not { changeRequest() }
-          expression { return params.Run_As_Master_Branch }
+        allOf {
+          expression { return env.ONLY_DOCS == "false" }
+          anyOf {
+            not { changeRequest() }
+            expression { return params.Run_As_Master_Branch }
+          }
         }
       }
       options { skipDefaultCheckout() }
@@ -89,43 +94,53 @@ pipeline {
     /**
     Execute unit tests.
     */
-    stage('Test') {
-      options { skipDefaultCheckout() }
-      steps {
-        withGithubNotify(context: 'Tests', tab: 'tests') {
-          deleteDir()
-          unstash "source"
-          dir("${BASE_DIR}"){
-            script {
-              def ruby = readYaml(file: '.ci/.jenkins_ruby.yml')
-              def testTasks = [:]
-              def i = 0
-              ruby['RUBY_VERSION'].each{ rubyVersion ->
-                testTasks[rubyVersion] = { runJob(rubyVersion, i++) }
-              }
-              parallel(testTasks)
+    stage('Tests') {
+      when {
+        beforeAgent true
+        expression { return env.ONLY_DOCS == "false" }
+      }
+      failFast false
+      parallel {
+        stage('Tests') {
+          options { skipDefaultCheckout() }
+          steps {
+            withGithubNotify(context: 'Tests', tab: 'tests') {
+              runTests('.ci/.jenkins_framework.yml')
+            }
+          }
+          post {
+            always {
+              convergeCoverage()
             }
           }
         }
-      }
-    }
-    stage('Integration Tests') {
-      agent none
-      when {
-        beforeAgent true
-        anyOf {
-          changeRequest()
-          expression { return !params.Run_As_Master_Branch }
+        stage('Master Tests frameworks') {
+          options { skipDefaultCheckout() }
+          steps {
+            catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE', message: "The tests for the master framework have failed. Let's warn instead.") {
+              runTests('.ci/.jenkins_master_framework.yml')
+            }
+          }
+        }
+        stage('Integration Tests') {
+          agent none
+          when {
+            beforeAgent true
+            anyOf {
+              changeRequest()
+              expression { return !params.Run_As_Master_Branch }
+            }
+          }
+          steps {
+            githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+            build(job: env.ITS_PIPELINE, propagate: false, wait: true,
+                  parameters: [ string(name: 'INTEGRATION_TEST', value: 'Ruby'),
+                                string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO} --opbeans-ruby-agent-branch ${env.GIT_BASE_COMMIT}"),
+                                string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
+                                string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+                                string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT) ])
+          }
         }
-      }
-      steps {
-        build(job: env.ITS_PIPELINE, propagate: false, wait: false,
-              parameters: [string(name: 'INTEGRATION_TEST', value: 'Ruby'),
-                            string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO} --opbeans-ruby-agent-branch ${env.GIT_BASE_COMMIT}"),
-                            string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
-                            string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-                            string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)])
-        githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
       }
     }
     stage('Benchmarks') {
@@ -137,8 +152,9 @@ pipeline {
             branch 'master'
             branch "\\d+\\.\\d+"
             branch "v\\d?"
-            tag "v\\d+\\.\\d+\\.\\d+*"
+            tag pattern: 'v\\d+.*', comparator: "REGEXP"
             expression { return params.Run_As_Master_Branch }
+            expression { return env.GITHUB_COMMENT?.contains('benchmark tests') }
           }
           expression { return params.bench_ci }
         }
@@ -179,20 +195,6 @@ pipeline {
         }
       }
     }
-    stage('Populate Downstream failures') {
-      when {
-        expression { return rubyTasksFailed }
-      }
-      steps {
-        script {
-          def message = 'Failures when running the "Test" stage'
-          if (notifyBuildResult.isAnyDownstreamJobFailedWithTimeout(rubyDownstreamJobs)) {
-            message += ' related to a timeout issue'
-          }
-          error(message)
-        }
-      }
-    }
     stage('Release') {
       options { skipDefaultCheckout() }
       environment {
@@ -238,7 +240,10 @@ pipeline {
           steps {
             deleteDir()
             // Let's wait for the Gem to be available
-            sh "(retry 10 curl --silent --show-error --fail -I https://rubygems.org/gems/elastic-apm/versions/${env.VERSION})"
+            sh label: 'Wait for gem', script: """#!/usr/bin/env bash
+              source /usr/local/bin/bash_standard_lib.sh
+              (retry 10 curl --silent --show-error --fail -I https://rubygems.org/gems/elastic-apm/versions/${env.VERSION})
+            """
             dir("${OPBEANS_REPO}"){
               git credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
                   url: "git@github.com:elastic/${OPBEANS_REPO}.git"
@@ -256,7 +261,7 @@ pipeline {
   }
   post {
     cleanup {
-      notifyBuildResult(downstreamJobs: rubyDownstreamJobs)
+      notifyBuildResult()
     }
   }
 }
@@ -271,26 +276,64 @@ def runBenchmark(version){
       //  - docker.elastic.co/observability-ci/jruby:9.2-12-jdk to jruby-9.2-12-jdk
       //  - jruby:9.1 to jruby-9.1
       def transformedVersion = version.replaceAll('.*/', '').replaceAll(':', '-')
-      env.HOME = "${env.WORKSPACE}/${transformedVersion}"
-      dir("${transformedVersion}"){
-        deleteDir()
-        unstash 'source'
-        dir("${BASE_DIR}"){
-          retry(2){
-            sleep randomNumber(min:10, max: 30)
-            dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+      withEnv(["HOME=${env.WORKSPACE}/${transformedVersion}"]){
+        dir("${transformedVersion}"){
+          deleteDir()
+          unstash 'source'
+          dir("${BASE_DIR}"){
+            retry(2){
+              sleep randomNumber(min:10, max: 30)
+              dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+            }
+            try{
+              sh """./spec/scripts/benchmarks.sh "${version}" "${REFERENCE_REPO}" """
+            } catch(e){
+              throw e
+            } finally {
+              archiveArtifacts(
+                allowEmptyArchive: true,
+                artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error",
+                onlyIfSuccessful: false)
+              sendBenchmarks(file: "benchmark-${transformedVersion}.bulk",
+                index: "benchmark-ruby", archive: true)
+            }
           }
-          try{
-            sh """./spec/scripts/benchmarks.sh "${version}" "${REFERENCE_REPO}" """
-          } catch(e){
-            throw e
-          } finally {
-            archiveArtifacts(
-              allowEmptyArchive: true,
-              artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error",
-              onlyIfSuccessful: false)
-            sendBenchmarks(file: "benchmark-${transformedVersion}.bulk",
-              index: "benchmark-ruby", archive: true)
+        }
+      }
+    }
+  }
+}
+
+class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
+
+  public RubyParallelTaskGenerator(Map params){
+    super(params)
+  }
+
+  /**
+  build a clousure that launch and agent and execute the corresponding test script,
+  then store the results.
+  */
+  public Closure generateStep(x, y){
+    return {
+      steps.sleep steps.randomNumber(min:10, max: 30)
+      steps.node('linux && immutable'){
+        // Label is transformed to avoid using the internal docker registry in the x coordinate
+        // TODO: def label = "${tag}:${x?.drop(x?.lastIndexOf('/')+1)}#${y}"
+        def label = "${tag}:${x}#${y}"
+        try {
+          steps.runScript(label: label, ruby: x, framework: y)
+          saveResult(x, y, 1)
+        } catch(e){
+          saveResult(x, y, 0)
+          steps.error("${label} tests failed : ${e.toString()}\n")
+        } finally {
+          steps.junit(allowEmptyResults: false,
+            keepLongStdio: true,
+            testResults: "**/spec/junit-reports/**/ruby-agent-junit.xml")
+          steps.dir("${steps.env.BASE_DIR}"){
+            steps.archiveArtifacts(artifacts: 'coverage/matrix_results/', defaultExcludes: false)
+            steps.stash(name: steps.normalise("coverage-${x}-${y}"), includes: 'coverage/matrix_results/', allowEmpty: true)
           }
         }
       }
@@ -298,24 +341,58 @@ def runBenchmark(version){
   }
 }
 
-def runJob(String rubyVersion, int sleep = 1){
-  def buildObject
-  try {
-    def quiet = (sleep * randomNumber(min: 2, max: 6)) + 5
-    buildObject = build(job: "apm-agent-ruby/apm-agent-ruby-downstream/${env.BRANCH_NAME}",
-                        parameters: [
-                          string(name: 'RUBY_VERSION', value: "${rubyVersion}"),
-                          string(name: 'BRANCH_SPECIFIER', value: "${env.GIT_BASE_COMMIT}")
-                        ],
-                        quietPeriod: quiet)
-  } catch(e) {
-    rubyTasksFailed = true
-    buildObject = e
-    warnError('Downstream Failed') {
-      error("Downstream job for '${rubyVersion}' failed")
+/**
+  Run all the tests for the given file with the frameworks to test
+*/
+def runTests(frameworkFile) {
+  deleteDir()
+  unstash "source"
+  dir("${BASE_DIR}"){
+    rubyTasksGen = new RubyParallelTaskGenerator(
+      xKey: 'RUBY_VERSION',
+      yKey: 'FRAMEWORK',
+      xFile: ".ci/.jenkins_ruby.yml",
+      yFile: frameworkFile,
+      exclusionFile: ".ci/.jenkins_exclude.yml",
+      tag: "Ruby",
+      name: "Ruby",
+      steps: this
+    )
+    def testTasks = rubyTasksGen.generateParallelTests()
+    parallel(testTasks)
+  }
+}
+
+/**
+  Run tests for a Ruby version and framework version.
+*/
+def runScript(Map params = [:]){
+  def label = params.label
+  def ruby = params.ruby
+  def framework = params.framework
+  log(level: 'INFO', text: "${label}")
+  retry(2){
+    withEnv(["HOME=${env.WORKSPACE}", "PATH=${env.PATH}:${env.WORKSPACE}/bin"]) {
+      deleteDir()
+      unstash 'source'
+      dir("${BASE_DIR}"){
+        sleep randomNumber(min:10, max: 30)
+        dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+        sh("./spec/scripts/spec.sh ${ruby} ${framework}")
+      }
+    }
+  }
+}
+
+def convergeCoverage() {
+  deleteDir()
+  unstash('source')
+  dir("${BASE_DIR}"){
+    rubyTasksGen.dumpMatrix('-')?.each {
+      unstash(normalise("coverage-${it}"))
     }
-  } finally {
-    rubyDownstreamJobs["${rubyVersion}"] = buildObject
+    sh(script: './spec/scripts/coverage_converge.sh')
+    cobertura coberturaReportFile: 'coverage/coverage.xml', onlyStable: false
   }
 }
 
@@ -324,8 +401,7 @@ def prepareRelease(Closure body){
     docker.image("${env.RUBY_DOCKER_TAG}").inside('-v ${REFERENCE_REPO}:${REFERENCE_REPO} -v /etc/passwd:/etc/passwd -v ${HOME}/.ssh:${HOME}/.ssh') {
       withEnv(["HOME=${env.WORKSPACE}/${env.BASE_DIR ?: ''}"]) {
         rubygemsLogin.withApi(secret: "${env.RELEASE_SECRET}") {
-          sshagent(['f6c7695a-671e-4f4f-a331-acdce44ff9ba']) {
-            sh '.ci/prepare-git-context.sh'
+          withGitRelease(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken') {
             sh 'gem install rake yard rspec'
             body()
           }
@@ -334,3 +410,10 @@ def prepareRelease(Closure body){
     }
   }
 }
+
+// Transform the versions like:
+//  - docker.elastic.co/observability-ci/jruby:9.2-12-jdk to jruby-9.2-12-jdk
+//  - jruby:9.1 to jruby-9.1
+def normalise(def what) {
+  return what.replaceAll('.*/', '').replaceAll(':', '-')
+}

data/.ci/jobs/defaults.yml

@@ -17,7 +17,7 @@
     days-to-keep: '30'
     concurrent: true
     node: linux
-    periodic-folder-trigger: 1d
+    periodic-folder-trigger: 1w
     prune-dead-branches: true
     publishers:
     - email:

data/.ci/packer_cache.sh

@@ -2,13 +2,15 @@
 
 source /usr/local/bin/bash_standard_lib.sh
 
-grep "-" .ci/.jenkins_ruby.yml | grep -v 'observability-ci' | cut -d'-' -f2- | \
-while read -r version;
-do
-    transformedName=${version/:/-}
-    transformedVersion=$(echo "${version}" | cut -d":" -f2)
-    imageName="apm-agent-ruby"
-    registryImageName="docker.elastic.co/observability-ci/${imageName}:${transformedName}"
-    (retry 2 docker pull "${registryImageName}")
-    docker tag "${registryImageName}" "${imageName}:${transformedVersion}"
-done
+if [ -x "$(command -v docker)" ]; then
+  grep "-" .ci/.jenkins_ruby.yml | grep -v 'observability-ci' | cut -d'-' -f2- | \
+  while read -r version;
+  do
+      transformedName=${version/:/-}
+      transformedVersion=$(echo "${version}" | cut -d":" -f2)
+      imageName="apm-agent-ruby"
+      registryImageName="docker.elastic.co/observability-ci/${imageName}:${transformedName}"
+      (retry 2 docker pull "${registryImageName}")
+      docker tag "${registryImageName}" "${imageName}:${transformedVersion}"
+  done
+fi

data/.rspec

@@ -1,3 +1,2 @@
 --color
---require spec_helper
 --exclude-pattern spec/integration/**/*_spec.rb

data/.rubocop.yml

@@ -107,3 +107,6 @@ Style/HashTransformKeys:
 Style/HashTransformValues:
   Enabled: true
 
+Style/FrozenStringLiteralComment:
+  Enabled: false
+

data/CHANGELOG.asciidoc

@@ -35,6 +35,90 @@ endif::[]
 [[release-notes-3.x]]
 === Ruby Agent version 3.x
 
+[[release-notes-3.10.1]]
+==== 3.10.1 (2020-08-26)
+
+[float]
+===== Fixed
+
+- Remove secrets from cookies in errors {pull}863[#863]
+
+[[release-notes-3.10.0]]
+==== 3.10.0 (2020-08-26)
+
+[float]
+===== Added
+
+- Config option `transaction_ignore_urls` to replace `ignore_url_patterns` {pull}844[#844]
+- Prepend `(?-i)` to patterns to make them case-sensitive {pull}846[#846]
+
+[float]
+===== Fixed
+
+- Reverted {pull}839[#839]
+- Improved Kubernetes pod ID detection {pull}859[#859]
+
+[[release-notes-3.9.0]]
+==== 3.9.0 (2020-08-04)
+
+[float]
+===== Fixed
+- Scrub request body of illegal UTF-8 characters {pull}832[#832]
+
+[float]
+===== Added
+
+- Support for DynamoDB {pull}827[#827]
+
+[float]
+===== Fixed
+
+- Fix Rails Engine views' paths being reported as absolute {pull}839[#839]
+- Fix an issue when using Elasticsearch spy without a running agent {pull}830[#830]
+
+[[release-notes-3.8.0]]
+==== 3.8.0 (2020-06-18)
+
+[float]
+===== Added
+
+- Add the option `capture_elasticsearch_queries` {pull}789[#789]
+- Add option to skip patching Kernal#require {pull}812[#812]
+- Add option `service_node_name` {pull}820[#820]
+
+[float]
+===== Fixed
+
+- Allow underscores in hostnames in Net::HTTP spy {pull}804[#804]
+- Don't change log level on logger object via remote config {pull}809[#809]
+- Update and fix the Opentracing bridge {pull}791[#791]
+
+[[release-notes-3.7.0]]
+==== 3.7.0 (2020-05-14)
+
+[float]
+===== Changed
+
+- Make config values dynamic so they can be changed via the remote config {pull}747[#747]
+- Log the updated options from central config in addition to cumulative modifications {pull}758[#758]
+
+[float]
+===== Fixed
+
+- Ensure that the log level is updated for the config's logger when value is changed {pull}755[#755]
+- Set config `false` values to `false`, not `nil` {pull}761[#761]
+- Ensure that the previously running agent's config is used in `ElasticAPM.restart` {pull}763[#763]
+- Handle the Resque spy's payload class value being a String or Class and update docs {pull}768[#768]
+- Add finalizer on ProxyPipe::Write so Zlib::GzipWriter is properly closed {pull}787[#787]
+
+[float]
+===== Added
+
+- Add enabled config option, replacing active {pull}669[#669]
+- Add recording config option {pull}765[#765]
+- Add support for `SuckerPunch` {pull}775[#775]
+- Support forking: Detect forking and restart agent tasks and background threads {pull}783[#783]
+
 [[release-notes-3.6.0]]
 ==== 3.6.0 (2020-03-10)