elastic-apm 3.7.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86463e0604357bc3c88e3507de76b0a2e92ba2a1311aab220b6b092df9523319
-  data.tar.gz: 97b3fcbc28f6b46de4e72f536b314380884f6dd192747258f861bf060aa588ab
+  metadata.gz: 32d176ac5c22246df60ecc92f1baad37011a872af4e64f842b5a43c699a6ced3
+  data.tar.gz: bf0505e728d8f1bd0bb941574a844a8990d14e13a94a7c9f2a6c2d0f9eb5d35b
 SHA512:
-  metadata.gz: 56a3b688bf4646a11888677fad884937b9f83b04b2cb47d5740f9cb0ce3e1c754abbe861e4888418f6ec168695f940d8ac24c4de3461c61a2d4f7360953d3f30
-  data.tar.gz: 9743b2bc369dfedf3ff28893062171bb1af0a2af201f845701966409ab4c0063a4584257f445fabc6c14eba8a170ddbac9e5fe391554be53652f801cf2d482d5
+  metadata.gz: d57b6bdbc7e78bea95e197d33f3eef4db7a35b23f9e450c636c3c1e3a5dd75ee8ee195709944ac51d6af7a3ab2451fd19d6139f0dedf9e86a4827732d5097f2a
+  data.tar.gz: 924b9bbf782a0fa4e376137e9a44d10cc01be71455ca7614ab1374c70f0926d174eb1078b985cd60c1693e2e0ec1cdd8446502bd40007359262dcc31241250aa

data/.ci/Jenkinsfile

@@ -5,14 +5,10 @@ import co.elastic.matrix.*
 import groovy.transform.Field
 
 /**
- This is required to know if there any failures when running the downstream jobs.
+This is the parallel tasks generator,
+it is need as field to store the results of the tests.
 */
-@Field def rubyTasksFailed = false
-
-/**
- This is required to store the build status for the downstream jobs.
-*/
-@Field def rubyDownstreamJobs = [:]
+@Field def rubyTasksGen
 
 pipeline {
   agent { label 'linux && immutable' }
@@ -42,7 +38,7 @@ pipeline {
     quietPeriod(10)
   }
   triggers {
-    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
+    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
@@ -98,50 +94,53 @@ pipeline {
     /**
     Execute unit tests.
     */
-    stage('Test') {
-      options { skipDefaultCheckout() }
+    stage('Tests') {
       when {
         beforeAgent true
         expression { return env.ONLY_DOCS == "false" }
       }
-      steps {
-        withGithubNotify(context: 'Tests', tab: 'tests') {
-          deleteDir()
-          unstash "source"
-          dir("${BASE_DIR}"){
-            script {
-              def ruby = readYaml(file: '.ci/.jenkins_ruby.yml')
-              def testTasks = [:]
-              def i = 0
-              ruby['RUBY_VERSION'].each{ rubyVersion ->
-                testTasks[rubyVersion] = { runJob(rubyVersion, i++) }
-              }
-              parallel(testTasks)
+      failFast false
+      parallel {
+        stage('Tests') {
+          options { skipDefaultCheckout() }
+          steps {
+            withGithubNotify(context: 'Tests', tab: 'tests') {
+              runTests('.ci/.jenkins_framework.yml')
+            }
+          }
+          post {
+            always {
+              convergeCoverage()
             }
           }
         }
-      }
-    }
-    stage('Integration Tests') {
-      agent none
-      when {
-        beforeAgent true
-        allOf {
-          expression { return env.ONLY_DOCS == "false" }
-          anyOf {
-            changeRequest()
-            expression { return !params.Run_As_Master_Branch }
+        stage('Master Tests frameworks') {
+          options { skipDefaultCheckout() }
+          steps {
+            catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE', message: "The tests for the master framework have failed. Let's warn instead.") {
+              runTests('.ci/.jenkins_master_framework.yml')
+            }
+          }
+        }
+        stage('Integration Tests') {
+          agent none
+          when {
+            beforeAgent true
+            anyOf {
+              changeRequest()
+              expression { return !params.Run_As_Master_Branch }
+            }
+          }
+          steps {
+            githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+            build(job: env.ITS_PIPELINE, propagate: false, wait: true,
+                  parameters: [ string(name: 'INTEGRATION_TEST', value: 'Ruby'),
+                                string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO} --opbeans-ruby-agent-branch ${env.GIT_BASE_COMMIT}"),
+                                string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
+                                string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+                                string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT) ])
           }
         }
-      }
-      steps {
-        build(job: env.ITS_PIPELINE, propagate: false, wait: false,
-              parameters: [string(name: 'INTEGRATION_TEST', value: 'Ruby'),
-                            string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK?.trim() ?: 'elastic'}/${env.REPO} --opbeans-ruby-agent-branch ${env.GIT_BASE_COMMIT}"),
-                            string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
-                            string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-                            string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)])
-        githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
       }
     }
     stage('Benchmarks') {
@@ -155,6 +154,7 @@ pipeline {
             branch "v\\d?"
             tag pattern: 'v\\d+.*', comparator: "REGEXP"
             expression { return params.Run_As_Master_Branch }
+            expression { return env.GITHUB_COMMENT?.contains('benchmark tests') }
           }
           expression { return params.bench_ci }
         }
@@ -195,20 +195,6 @@ pipeline {
         }
       }
     }
-    stage('Populate Downstream failures') {
-      when {
-        expression { return rubyTasksFailed }
-      }
-      steps {
-        script {
-          def message = 'Failures when running the "Test" stage'
-          if (notifyBuildResult.isAnyDownstreamJobFailedWithTimeout(rubyDownstreamJobs)) {
-            message += ' related to a timeout issue'
-          }
-          error(message)
-        }
-      }
-    }
     stage('Release') {
       options { skipDefaultCheckout() }
       environment {
@@ -272,33 +258,14 @@ pipeline {
         }
       }
     }
-    stage('Coverage converge') {
-      when {
-        beforeAgent true
-        expression { return env.ONLY_DOCS == "false" }
-      }
-      steps{
-        dir("${BASE_DIR}"){
-          unpack_artifacts(rubyDownstreamJobs)
-          sh(script: "./spec/scripts/coverage_converge.sh")
-          cobertura coberturaReportFile: "coverage/coverage.xml"
-        }
-      }
-    }
   }
   post {
     cleanup {
-      notifyBuildResult(downstreamJobs: rubyDownstreamJobs)
+      notifyBuildResult()
     }
   }
 }
 
-def unpack_artifacts(builds) {
-    builds.values().each() { build ->
-      copyArtifacts(projectName: "apm-agent-ruby/apm-agent-ruby-downstream/${env.BRANCH_NAME}", selector: specific(buildNumber: build.number.toString()))
-    }
-}
-
 /**
   Run benchmarks for a Ruby version, then report the results to the Elasticsearch server.
 */
@@ -337,26 +304,95 @@ def runBenchmark(version){
   }
 }
 
-def runJob(String rubyVersion, int sleep = 1){
-  def buildObject
-  def branch = env.CHANGE_ID?.trim() ? env.CHANGE_TARGET : env.BRANCH_NAME
-  try {
-    def quiet = (sleep * randomNumber(min: 2, max: 6)) + 5
-    buildObject = build(job: "apm-agent-ruby/apm-agent-ruby-downstream/${env.BRANCH_NAME}",
-                        parameters: [
-                          string(name: 'RUBY_VERSION', value: "${rubyVersion}"),
-                          string(name: 'BRANCH_SPECIFIER', value: "${env.GIT_BASE_COMMIT}"),
-                          string(name: 'MERGE_TARGET', value: "${branch}")
-                        ],
-                        quietPeriod: quiet)
-  } catch(e) {
-    rubyTasksFailed = true
-    buildObject = e
-    warnError('Downstream Failed') {
-      error("Downstream job for '${rubyVersion}' failed")
+class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
+
+  public RubyParallelTaskGenerator(Map params){
+    super(params)
+  }
+
+  /**
+  build a clousure that launch and agent and execute the corresponding test script,
+  then store the results.
+  */
+  public Closure generateStep(x, y){
+    return {
+      steps.sleep steps.randomNumber(min:10, max: 30)
+      steps.node('linux && immutable'){
+        // Label is transformed to avoid using the internal docker registry in the x coordinate
+        // TODO: def label = "${tag}:${x?.drop(x?.lastIndexOf('/')+1)}#${y}"
+        def label = "${tag}:${x}#${y}"
+        try {
+          steps.runScript(label: label, ruby: x, framework: y)
+          saveResult(x, y, 1)
+        } catch(e){
+          saveResult(x, y, 0)
+          steps.error("${label} tests failed : ${e.toString()}\n")
+        } finally {
+          steps.junit(allowEmptyResults: false,
+            keepLongStdio: true,
+            testResults: "**/spec/junit-reports/**/ruby-agent-junit.xml")
+          steps.dir("${steps.env.BASE_DIR}"){
+            steps.archiveArtifacts(artifacts: 'coverage/matrix_results/', defaultExcludes: false)
+            steps.stash(name: steps.normalise("coverage-${x}-${y}"), includes: 'coverage/matrix_results/', allowEmpty: true)
+          }
+        }
+      }
     }
-  } finally {
-    rubyDownstreamJobs["${rubyVersion}"] = buildObject
+  }
+}
+
+/**
+  Run all the tests for the given file with the frameworks to test
+*/
+def runTests(frameworkFile) {
+  deleteDir()
+  unstash "source"
+  dir("${BASE_DIR}"){
+    rubyTasksGen = new RubyParallelTaskGenerator(
+      xKey: 'RUBY_VERSION',
+      yKey: 'FRAMEWORK',
+      xFile: ".ci/.jenkins_ruby.yml",
+      yFile: frameworkFile,
+      exclusionFile: ".ci/.jenkins_exclude.yml",
+      tag: "Ruby",
+      name: "Ruby",
+      steps: this
+    )
+    def testTasks = rubyTasksGen.generateParallelTests()
+    parallel(testTasks)
+  }
+}
+
+/**
+  Run tests for a Ruby version and framework version.
+*/
+def runScript(Map params = [:]){
+  def label = params.label
+  def ruby = params.ruby
+  def framework = params.framework
+  log(level: 'INFO', text: "${label}")
+  retry(2){
+    withEnv(["HOME=${env.WORKSPACE}", "PATH=${env.PATH}:${env.WORKSPACE}/bin"]) {
+      deleteDir()
+      unstash 'source'
+      dir("${BASE_DIR}"){
+        sleep randomNumber(min:10, max: 30)
+        dockerLogin(secret: "${DOCKER_SECRET}", registry: "${DOCKER_REGISTRY}")
+        sh("./spec/scripts/spec.sh ${ruby} ${framework}")
+      }
+    }
+  }
+}
+
+def convergeCoverage() {
+  deleteDir()
+  unstash('source')
+  dir("${BASE_DIR}"){
+    rubyTasksGen.dumpMatrix('-')?.each {
+      unstash(normalise("coverage-${it}"))
+    }
+    sh(script: './spec/scripts/coverage_converge.sh')
+    cobertura coberturaReportFile: 'coverage/coverage.xml', onlyStable: false
   }
 }
 
@@ -374,3 +410,10 @@ def prepareRelease(Closure body){
     }
   }
 }
+
+// Transform the versions like:
+//  - docker.elastic.co/observability-ci/jruby:9.2-12-jdk to jruby-9.2-12-jdk
+//  - jruby:9.1 to jruby-9.1
+def normalise(def what) {
+  return what.replaceAll('.*/', '').replaceAll(':', '-')
+}

data/.ci/packer_cache.sh

@@ -2,13 +2,15 @@
 
 source /usr/local/bin/bash_standard_lib.sh
 
-grep "-" .ci/.jenkins_ruby.yml | grep -v 'observability-ci' | cut -d'-' -f2- | \
-while read -r version;
-do
-    transformedName=${version/:/-}
-    transformedVersion=$(echo "${version}" | cut -d":" -f2)
-    imageName="apm-agent-ruby"
-    registryImageName="docker.elastic.co/observability-ci/${imageName}:${transformedName}"
-    (retry 2 docker pull "${registryImageName}")
-    docker tag "${registryImageName}" "${imageName}:${transformedVersion}"
-done
+if [ -x "$(command -v docker)" ]; then
+  grep "-" .ci/.jenkins_ruby.yml | grep -v 'observability-ci' | cut -d'-' -f2- | \
+  while read -r version;
+  do
+      transformedName=${version/:/-}
+      transformedVersion=$(echo "${version}" | cut -d":" -f2)
+      imageName="apm-agent-ruby"
+      registryImageName="docker.elastic.co/observability-ci/${imageName}:${transformedName}"
+      (retry 2 docker pull "${registryImageName}")
+      docker tag "${registryImageName}" "${imageName}:${transformedVersion}"
+  done
+fi

data/.rspec

@@ -1,3 +1,2 @@
 --color
---require spec_helper
 --exclude-pattern spec/integration/**/*_spec.rb

data/CHANGELOG.asciidoc

@@ -35,7 +35,23 @@ endif::[]
 [[release-notes-3.x]]
 === Ruby Agent version 3.x
 
+[[release-notes-3.8.0]]
+==== 3.8.0 (2020-06-18)
+
+[float]
+===== Added
+
+- Add the option `capture_elasticsearch_queries` {pull}789[#789]
+- Add option to skip patching Kernal#require {pull}812[#812]
+- Add option `service_node_name` {pull}820[#820]
+
 [float]
+===== Fixed
+
+- Allow underscores in hostnames in Net::HTTP spy {pull}804[#804]
+- Don't change log level on logger object via remote config {pull}809[#809]
+- Update and fix the Opentracing bridge {pull}791[#791]
+
 [[release-notes-3.7.0]]
 ==== 3.7.0 (2020-05-14)
 

data/Gemfile

@@ -39,7 +39,8 @@ gem 'elasticsearch', require: nil
 gem 'fakeredis', require: nil
 gem 'faraday', require: nil
 gem 'graphql', require: nil
-gem 'grpc' if !defined?(JRUBY_VERSION) && RUBY_VERSION < '2.7'
+gem 'google-protobuf', '< 3.12' if !defined?(JRUBY_VERSION) && RUBY_VERSION < '2.5'
+gem 'grpc' if !defined?(JRUBY_VERSION)
 gem 'json-schema', require: nil
 gem 'mongo', require: nil
 gem 'opentracing', require: nil

data/bin/run-tests

@@ -8,7 +8,10 @@ runRspec(){
   if [ -n "${case}" ]; then
     bn="$(basename ${case} _spec.rb)/"
   fi
-  bundle exec rspec -f progress -f JUnit -o spec/junit-reports/${bn}ruby-agent-junit.xml ${case}
+  bundle exec rspec \
+    -f progress \
+    -r yarjuf -f JUnit -o spec/junit-reports/${bn}ruby-agent-junit.xml \
+    ${case}
 }
 specific_spec=$1
 

data/docker-compose.yml

@@ -26,11 +26,13 @@ services:
       - /tmp:exec,mode=1777
     depends_on:
       - mongodb
+    user: ${USER_ID}
 
   ruby_rspec:
     image: apm-agent-ruby:${RUBY_VERSION}
     environment:
       MONGODB_URL: 'mongodb:27017'
+    user: ${USER_ID}
 
 volumes:
   vendor:

data/docs/configuration.asciidoc

@@ -287,6 +287,16 @@ If your service handles data like this, we advise to only enable this feature wi
 
 Whether or not to attach the request headers to transactions and errors.
 
+[float]
+[[config-capture-elasticsearch-queries]]
+==== `capture_elasticsearch_queries`
+|============
+| Environment                                 | `Config` key                    | Default
+| `ELASTIC_APM_CAPTURE_ELASTICSEARCH_QUERIES` | `capture_elasticsearch_queries` | `false`
+|============
+
+Whether or not to capture the body from requests in Elasticsearch.
+
 [float]
 [[config-capture-env]]
 ==== `capture_env`
@@ -707,6 +717,23 @@ NOTE: The service name must conform to this regular expression: `^[a-zA-Z0-9 _-]
 In layman's terms: Your service name must only contain characters from the ASCII
 alphabet, numbers, dashes, underscores and spaces.
 
+[float]
+[[config-service-node-name]]
+==== `service_node_name`
+
+ [options="header"]
+|============
+| Environment                     | `Config` key        | Default | Example
+| `ELASTIC_APM_SERVICE_NODE_NAME` | `service_node_name` | `nil`   | `"my-app-1"`
+|============
+
+The name of the given service node. This is optional, and if omitted, the APM
+Server will fall back on `system.container.id` if available, and finally
+`host.name` if necessary.
+
+This option allows you to set the node name manually to ensure uniqueness and
+meaningfulness.
+
 [float]
 [[config-service-version]]
 ==== `service_version`
@@ -890,3 +917,13 @@ The unit is provided as suffix directly after the number, without any separation
  * `gb` (gigabytes)
 
 NOTE: we use the power-of-two sizing convention, e.g. `1 kilobyte == 1024 bytes`
+
+[float]
+[[special-configuration]]
+=== Special configuration
+
+Elastic APM patches `Kernel#require` to auto-detect and instrument supported third party libraries. It does so with the utmost care but in rare cases it can conflict with some libraries.
+
+To get around this patch, set the environment variable `ELASTIC_APM_SKIP_REQUIRE_PATCH` to `"1"`.
+
+If you choose to do so, the agent might need some additional tweaking to make sure the third party libraries are picked up and instrumented. Make sure you require the agent _after_ you require your other dependencies.