elastic-apm 2.8.1 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11b66b86db45e066da0c6055bd5cb58894a5ba0eca8fb9d891b5b918fcd8d7c0
-  data.tar.gz: 87bfdbcc9b57f128f032d8e617dfe4540fcf8afeea91f80621d5e1a6b476d47e
+  metadata.gz: 7a73b88b204253aeff8ed4da74f11be5e6a1bdd77cf3848495f823d225569177
+  data.tar.gz: 0e962b3216d0c8dc28666175e2db57bc2fe8826e52abe047f1be4b8903fe2a04
 SHA512:
-  metadata.gz: a514f4463ee2294d1e825fd9ba0b42a764d8380e9ea4ba90254acf92ebd7ffec8d4f2a329a77307732e7206a22009c3cb91218650e499235d8ca2cae3cf1d8ae
-  data.tar.gz: 93e558bab35f323f77116031333ce065fa00b101871ea80e913b082612552857dadc9074c6c4219bf97010eddc6b9abc996fa2dac68454aed1dcb91048c9bd6c
+  metadata.gz: e5302421b067475c581dde24caa1dce14c755017915b0e69420f8c281785aea06a244231a1f9946a6cd14ddc216f3c0fb303c5d3be78dc2ac49233c7f6371d90
+  data.tar.gz: 700cb6b1445390110721447fdf3096e4dc25077c5e46c45b71818888b650af56460e9a94645cb51c45f37912a95e6244f58c9bdc4d89517fb3b7cf61d7bcd353

data/.ci/bin/check_paths_for_matches.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""
+Small wrapper script for jenkins to see if a regex pattern matches any of the
+paths generated by diffing between two commits.
+"""
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+debug = "DEBUG" in os.environ
+
+
+def check_paths_for_matches(pattern, git_commit, git_previous_commit):
+    """Check if any paths between GIT_PREVIOUS_COMMIT and GIT_COMMIT match our pattern.
+
+    For merge commits only actual path changes are included rather than changes
+    from all parents. If GIT_PREVIOUS_COMMIT is not populated, use just the
+    paths that GIT_COMMIT represents. If GIT_PREVIOUS_COMMIT is not populated
+    and GIT_COMMIT is a merge commit, use all path changes from each parent. If
+    GIT_PREVIOUS_COMMIT is the same as GIT_COMMIT, that should generate no path
+    changes.
+    """
+    # Handle case where GIT_PREVIOUS_COMMIT isn't set (e.g. the first build),
+    if not git_previous_commit:
+        command = [
+            "git",
+            "diff-tree",
+            "-m",
+            "--no-commit-id",
+            "--name-only",
+            "-r",
+            git_commit,
+        ]
+    else:
+        command = ["git", "diff", "--name-only", git_previous_commit, git_commit]
+
+    # Run the command and populate paths.
+    completed_process = subprocess.run(
+        command, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    paths = completed_process.stdout.decode().strip().split("\n")
+
+    # Look for any matches of pattern -> path.
+    possible_matches = [(path, pattern.match(path)) for path in paths]
+    if any([match for path, match in possible_matches]):
+        if debug:
+            print("matching change(s) found for {}".format(git_commit))
+            for path, match in possible_matches:
+                if match:
+                    print(path)
+        sys.stdout.write("match")
+        sys.stdout.flush()
+        exit(0)
+    else:
+        if debug:
+            print("no matching change(s) found for {}".format(git_commit))
+        exit(1)
+
+
+if __name__ == "__main__":
+    # Change our working directory so we're in $WORKSPACE.
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+    # Define and parse arguments.
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pattern", help="A regular expression pattern.")
+    parser.add_argument(
+        "--git-commit", help="The contents of the GIT_COMMIT environmental variable."
+    )
+    parser.add_argument(
+        "--git-previous-commit",
+        nargs="?",
+        help="The contents of the GIT_PREVIOUS_COMMIT environmental variable.",
+    )
+    args = parser.parse_args()
+
+    compiled_pattern = re.compile(args.pattern)
+    check_paths_for_matches(compiled_pattern, args.git_commit, args.git_previous_commit)

data/.ci/jobs/apm-agent-ruby-mbp.yml

@@ -0,0 +1,5 @@
+---
+- job:
+    name: apm-agent-ruby/apm-agent-ruby-mbp
+    display-name: APM Agent Ruby
+    description: APM Agent Ruby

data/.ci/jobs/apm-agent-ruby.yml

@@ -0,0 +1,4 @@
+---
+- job:
+    name: apm-agent-ruby
+    project-type: folder

data/.ci/jobs/defaults.yml

@@ -0,0 +1,59 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: apm-ci
+
+##### JOB DEFAULTS
+
+- job:
+    view: APM-CI
+    project-type: multibranch
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 100
+    number-to-keep: '5'
+    days-to-keep: '1'
+    concurrent: true
+    node: linux
+    script-path: Jenkinsfile
+    scm:
+    - github:
+        branch-discovery: all
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: true
+        repo: apm-agent-ruby
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - regular-branches: true
+        - change-request:
+            ignore-target-only-changes: false
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'
+    periodic-folder-trigger: 1d
+    prune-dead-branches: true
+    publishers:
+    - email:
+        recipients: infra-root+build@elastic.co

data/CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 
 This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 2.9.0 (2019-06-25)
+
+### Security
+
+- **NB:** If you are using a custom CA cert via `server_ca_cert`, versions of the agent prior to 2.9.0 may not have validated the certificate of APM Server correctly.
+
+### Added
+
+- Add `transaction.type` to errors ([#434](https://github.com/elastic/apm-agent-ruby/pull/434))
+- Add cookies to `request.cookies` ([#448](https://github.com/elastic/apm-agent-ruby/pull/448))
+
+### Fixed
+
+- Fix support for older versions of Http.rb ([#438](https://github.com/elastic/apm-agent-ruby/pull/434))
+- Strip `Cookie` and `Set-Cookie` from headers ([#448](https://github.com/elastic/apm-agent-ruby/pull/448))
+- Fix disabling SSL verification ([#449](https://github.com/elastic/apm-agent-ruby/pull/449))
+
 ## 2.8.1 (2019-05-29)
 
 ### Fixed
@@ -59,7 +76,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 
 Both APIs are backwards compatible with fallbacks and deprecation warnings, scheduled for removal in next major release.
 
-### Added 
+### Added
 
 - Configuration options to use an HTTP proxy ([#352](https://github.com/elastic/apm-agent-ruby/pull/352))
 
@@ -120,7 +137,7 @@ Both APIs are backwards compatible with fallbacks and deprecation warnings, sche
 ### Added
 
 - Support for [OpenTracing](https://opentracing.io) ([#273](https://github.com/elastic/apm-agent-ruby/pull/273))
-- Add capture_* options ([#279](https://github.com/elastic/apm-agent-ruby/pull/279))
+- Add capture\_\* options ([#279](https://github.com/elastic/apm-agent-ruby/pull/279))
 - Evaluate the config file as ERB ([#288](https://github.com/elastic/apm-agent-ruby/pull/288))
 
 ### Changed

data/Jenkinsfile

@@ -13,13 +13,16 @@ it is need as field to store the results of the tests.
 pipeline {
   agent any
   environment {
-    BASE_DIR="src/github.com/elastic/apm-agent-ruby"
+    REPO = 'apm-agent-ruby'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     PIPELINE_LOG_LEVEL='INFO'
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     CODECOV_SECRET = 'secret/apm-team/ci/apm-agent-ruby-codecov'
     DOCKER_REGISTRY = 'docker.elastic.co'
     DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
+    GITHUB_CHECK_ITS_NAME = 'Integration Tests'
+    ITS_PIPELINE = 'apm-integration-tests-selector-mbp/master'
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -59,22 +62,24 @@ pipeline {
       agent { label 'linux && immutable' }
       options { skipDefaultCheckout() }
       steps {
-        deleteDir()
-        unstash "source"
-        dir("${BASE_DIR}"){
-          script {
-            rubyTasksGen = new RubyParallelTaskGenerator(
-              xKey: 'RUBY_VERSION',
-              yKey: 'FRAMEWORK',
-              xFile: "./spec/.jenkins_ruby.yml",
-              yFile: "./spec/.jenkins_framework.yml",
-              exclusionFile: "./spec/.jenkins_exclude.yml",
-              tag: "Ruby",
-              name: "Ruby",
-              steps: this
-              )
-              def mapPatallelTasks = rubyTasksGen.generateParallelTests()
-              parallel(mapPatallelTasks)
+        withGithubNotify(context: 'Tests', tab: 'tests') {
+          deleteDir()
+          unstash "source"
+          dir("${BASE_DIR}"){
+            script {
+              rubyTasksGen = new RubyParallelTaskGenerator(
+                xKey: 'RUBY_VERSION',
+                yKey: 'FRAMEWORK',
+                xFile: "./spec/.jenkins_ruby.yml",
+                yFile: "./spec/.jenkins_framework.yml",
+                exclusionFile: "./spec/.jenkins_exclude.yml",
+                tag: "Ruby",
+                name: "Ruby",
+                steps: this
+                )
+                def mapPatallelTasks = rubyTasksGen.generateParallelTests()
+                parallel(mapPatallelTasks)
+              }
             }
           }
         }
@@ -113,16 +118,18 @@ pipeline {
           stage('Run Benchmarks') {
             agent { label 'linux && immutable' }
             steps {
-              deleteDir()
-              unstash 'source'
-              dir("${BASE_DIR}"){
-                script {
-                  def versions = readYaml(file: "./spec/.jenkins_ruby.yml")
-                  def benchmarkTask = [:]
-                  versions['RUBY_VERSION'].each{ v ->
-                    benchmarkTask[v] = runBenchmark(v)
+              withGithubNotify(context: 'Run Benchmarks') {
+                deleteDir()
+                unstash 'source'
+                dir("${BASE_DIR}"){
+                  script {
+                    def versions = readYaml(file: "./spec/.jenkins_ruby.yml")
+                    def benchmarkTask = [:]
+                    versions['RUBY_VERSION'].each{ v ->
+                      benchmarkTask[v] = runBenchmark(v)
+                    }
+                    parallel(benchmarkTask)
                   }
-                  parallel(benchmarkTask)
                 }
               }
             }
@@ -154,6 +161,28 @@ pipeline {
           buildDocs(docsDir: "${BASE_DIR}/docs", archive: true)
         }
       }
+      stage('Integration Tests') {
+        agent none
+        when {
+          beforeAgent true
+          allOf {
+            anyOf {
+              environment name: 'GIT_BUILD_CAUSE', value: 'pr'
+              expression { return !params.Run_As_Master_Branch }
+            }
+          }
+        }
+        steps {
+          log(level: 'INFO', text: 'Launching Async ITs')
+          build(job: env.ITS_PIPELINE, propagate: false, wait: false,
+                parameters: [string(name: 'AGENT_INTEGRATION_TEST', value: 'Ruby'),
+                             string(name: 'BUILD_OPTS', value: "--ruby-agent-version ${env.GIT_BASE_COMMIT} --ruby-agent-version-state ${env.GIT_BUILD_CAUSE} --ruby-agent-repo ${env.CHANGE_FORK}/${env.REPO}"),
+                             string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_ITS_NAME),
+                             string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+                             string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT)])
+          githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
+        }
+      }
     }
     post {
       always{
@@ -164,24 +193,17 @@ pipeline {
             def processor = new ResultsProcessor()
             processor.processResults(mapResults)
             archiveArtifacts allowEmptyArchive: true, artifacts: 'results.json,results.html', defaultExcludes: false
+            catchError(buildResult: 'SUCCESS') {
+              def datafile = readFile(file: "results.json")
+              def json = getVaultSecret(secret: 'secret/apm-team/ci/apm-server-benchmark-cloud')
+              sendDataToElasticsearch(es: json.data.url, data: datafile, restCall: '/jenkins-builds-test-results/_doc/')
+            }
           }
         }
-      }
-      success {
-        echoColor(text: '[SUCCESS]', colorfg: 'green', colorbg: 'default')
-      }
-      aborted {
-        echoColor(text: '[ABORTED]', colorfg: 'magenta', colorbg: 'default')
-      }
-      failure {
-        echoColor(text: '[FAILURE]', colorfg: 'red', colorbg: 'default')
-        step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${NOTIFY_TO}", sendToIndividuals: false])
-      }
-      unstable {
-        echoColor(text: '[UNSTABLE]', colorfg: 'yellow', colorbg: 'default')
-      }
+      notifyBuildResult()
     }
   }
+}
 
 /**
 Parallel task generator for the integration tests.
@@ -213,7 +235,7 @@ class RubyParallelTaskGenerator extends DefaultParallelTaskGenerator {
           steps.junit(allowEmptyResults: false,
             keepLongStdio: true,
             testResults: "**/spec/ruby-agent-junit.xml")
-          steps.codecov(repo: 'apm-agent-ruby', basedir: "${steps.env.BASE_DIR}",
+          steps.codecov(repo: "${steps.env.REPO}", basedir: "${steps.env.BASE_DIR}",
             secret: "${steps.env.CODECOV_SECRET}")
         }
       }
@@ -267,8 +289,8 @@ def runBenchmark(version){
             throw e
           } finally {
             archiveArtifacts(
-              allowEmptyArchive: true, 
-              artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error", 
+              allowEmptyArchive: true,
+              artifacts: "**/benchmark-${transformedVersion}.raw,**/benchmark-${transformedVersion}.error",
               onlyIfSuccessful: false)
             sendBenchmarks(file: "benchmark-${transformedVersion}.bulk",
               index: "benchmark-ruby", archive: true)

data/docs/configuration.asciidoc

@@ -292,11 +292,14 @@ Get an array of enabled spies with `ElasticAPM.agent.config.enabled_spies`.
 The name of the environment this service is deployed in,
 e.g. "production" or "staging".
 
+Environments allow you to easily filter data on a global level in the APM UI.
+It's important to be consistent when naming environments across agents.
+See {kibana-ref}/filters.html#environment-selector[environment selector] in the Kibana UI for more information.
+
 Defaults to `ENV['RAILS_ENV'] || ENV['RACK_ENV']`.
 
-NOTE: Kibana will not differentiate between different environments. To avoid
-mixing data from multiple environments, consider appending the environment to
-`service_name`.
+NOTE: This feature is fully supported in the APM UI in Kibana versions >= 7.2.
+You must use the query bar to filter for a specific environment in versions prior to 7.2.
 
 [float]
 [[config-filter-exception-types]]

data/lib/elastic_apm/context_builder.rb

@@ -38,6 +38,8 @@ module ElasticAPM
       request.headers = headers if config.capture_headers?
       request.env = env if config.capture_env?
 
+      request.cookies = req.cookies
+
       context
     end
     # rubocop:enable Metrics/MethodLength, Metrics/AbcSize

data/lib/elastic_apm/error_builder.rb

@@ -52,12 +52,15 @@ module ElasticAPM
       error.culprit = stacktrace.frames.first.function
     end
 
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def add_current_transaction_fields(error, transaction)
       return unless transaction
 
       error.transaction_id = transaction.id
-      error.transaction = { sampled: transaction.sampled? }
+      error.transaction = {
+        sampled: transaction.sampled?,
+        type: transaction.type
+      }
       error.trace_id = transaction.trace_id
       error.parent_id = ElasticAPM.current_span&.id || transaction.id
 
@@ -66,6 +69,6 @@ module ElasticAPM
       Util.reverse_merge!(error.context.tags, transaction.context.tags)
       Util.reverse_merge!(error.context.custom, transaction.context.custom)
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
   end
 end

data/lib/elastic_apm/span.rb

@@ -53,6 +53,7 @@ module ElasticAPM
       stop end_time
 
       build_stacktrace! if should_build_stacktrace?
+      self.original_backtrace = nil # release original
 
       self
     end
@@ -82,7 +83,6 @@ module ElasticAPM
 
     def build_stacktrace!
       @stacktrace = @stacktrace_builder.build(original_backtrace, type: :span)
-      self.original_backtrace = nil # release original
     end
 
     def should_build_stacktrace?

data/lib/elastic_apm/stacktrace/frame.rb

@@ -32,6 +32,8 @@ module ElasticAPM
         to = lineno + padding - 1
         file_lines = read_lines(abs_path, from..to)
 
+        return unless file_lines
+
         self.context_line = file_lines[padding]
         self.pre_context  = file_lines.first(padding)
         self.post_context = file_lines.last(padding)
@@ -43,7 +45,7 @@ module ElasticAPM
       def read_lines(path, range)
         File.readlines(path)[range]
       rescue Errno::ENOENT
-        []
+        nil
       end
     end
   end

data/lib/elastic_apm/transport/connection.rb

@@ -130,11 +130,20 @@ module ElasticAPM
         ].join(' ')
       end
 
-      def build_ssl_context
-        return unless @config.use_ssl? && @config.server_ca_cert
+      def build_ssl_context # rubocop:disable Metrics/MethodLength
+        return unless @config.use_ssl?
 
         OpenSSL::SSL::SSLContext.new.tap do |context|
-          context.ca_file = @config.server_ca_cert
+          if @config.server_ca_cert
+            context.ca_file = @config.server_ca_cert
+          end
+
+          context.verify_mode =
+            if @config.verify_server_cert
+              OpenSSL::SSL::VERIFY_PEER
+            else
+              OpenSSL::SSL::VERIFY_NONE
+            end
         end
       end
     end

data/lib/elastic_apm/transport/connection/proxy_pipe.rb

@@ -9,8 +9,15 @@ module ElasticAPM
       # @api private
       class ProxyPipe
         def initialize(enc = nil, compress: true)
-          @read, wr = IO.pipe(enc)
+          rd, wr = IO.pipe(enc)
+
+          @read = rd
           @write = Write.new(wr, compress: compress)
+
+          # Http.rb<4 calls rewind on the request bodies, but IO::Pipe raises
+          # ~mikker
+          return if HTTP::VERSION.to_i >= 4
+          def rd.rewind; end
         end
 
         attr_reader :read, :write

data/lib/elastic_apm/transport/filters/secrets_filter.rb

@@ -14,7 +14,8 @@ module ElasticAPM
           /secret/i,
           /token/i,
           /api[-._]?key/i,
-          /session[-._]?id/i
+          /session[-._]?id/i,
+          /(set[-_])?cookie/i
         ].freeze
 
         VALUE_FILTERS = [
@@ -30,6 +31,7 @@ module ElasticAPM
         def call(payload)
           strip_from! payload.dig(:transaction, :context, :request, :headers)
           strip_from! payload.dig(:transaction, :context, :request, :env)
+          strip_from! payload.dig(:transaction, :context, :request, :cookies)
           strip_from! payload.dig(:transaction, :context, :response, :headers)
           strip_from! payload.dig(:error, :context, :request, :headers)
           strip_from! payload.dig(:error, :context, :response, :headers)

data/lib/elastic_apm/transport/serializers/error_serializer.rb

@@ -14,7 +14,7 @@ module ElasticAPM
           base = {
             id: error.id,
             transaction_id: error.transaction_id,
-            transaction: error.transaction,
+            transaction: build_transaction(error.transaction),
             trace_id: error.trace_id,
             parent_id: error.parent_id,
 
@@ -61,6 +61,15 @@ module ElasticAPM
             stacktrace: log.stacktrace.to_a
           }
         end
+
+        def build_transaction(transaction)
+          return unless transaction
+
+          {
+            sampled: transaction[:sampled],
+            type: keyword_field(transaction[:type])
+          }
+        end
       end
     end
   end

data/lib/elastic_apm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ElasticAPM
-  VERSION = '2.8.1'
+  VERSION = '2.9.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic-apm
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.9.0
 platform: ruby
 authors:
 - Mikkel Malmberg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-29 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -45,6 +45,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".ci/bin/check_paths_for_matches.py"
+- ".ci/jobs/apm-agent-ruby-mbp.yml"
+- ".ci/jobs/apm-agent-ruby.yml"
+- ".ci/jobs/defaults.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rspec"