dd-trace 4.0.0-pre-071951e → 4.0.0-pre-4e7da80

package/index.d.ts

@@ -77,6 +77,10 @@ export declare interface Tracer extends opentracing.Tracer {
    * span will finish when that callback is called.
    * * The function doesn't accept a callback and doesn't return a promise, in
    * which case the span will finish at the end of the function execution.
+   *
+   * If the `orphanable` option is set to false, the function will not be traced
+   * unless there is already an active span or `childOf` option. Note that this
+   * option is deprecated and has been removed in version 4.0.
    */
   trace<T> (name: string, fn: (span?: Span, fn?: (error?: Error) => any) => T): T;
   trace<T> (name: string, options: TraceOptions & SpanOptions, fn: (span?: Span, done?: (error?: Error) => string) => T): T;
@@ -440,6 +444,11 @@ export declare interface TracerOptions {
        * Whether to enable vulnerability deduplication
        */
       deduplicationEnabled?: boolean
+      /**
+       * Whether to enable vulnerability redaction
+       * @default true
+       */
+      redactionEnabled?: boolean
     }
   };
 
@@ -485,6 +494,13 @@ export declare interface TracerOptions {
    */
   logLevel?: 'error' | 'debug'
 
+  /**
+   * If false, require a parent in order to trace.
+   * @default true
+   * @deprecated since version 4.0
+   */
+  orphanable?: boolean
+
   /**
    * Enables DBM to APM link using tag injection.
    * @default 'disabled'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "4.0.0-pre-071951e",
+  "version": "4.0.0-pre-4e7da80",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -68,9 +68,9 @@
   "dependencies": {
     "@datadog/native-appsec": "^3.1.0",
     "@datadog/native-iast-rewriter": "2.0.1",
-    "@datadog/native-iast-taint-tracking": "^1.4.0",
-    "@datadog/native-metrics": "^1.6.0",
-    "@datadog/pprof": "^2.2.0",
+    "@datadog/native-iast-taint-tracking": "^1.4.1",
+    "@datadog/native-metrics": "^2.0.0",
+    "@datadog/pprof": "^2.2.1",
     "@datadog/sketches-js": "^2.1.0",
     "crypto-randomuuid": "^1.0.0",
     "diagnostics_channel": "^1.1.0",

package/packages/datadog-instrumentations/src/jest.js

@@ -454,6 +454,7 @@ addHook({
 }, jestConfigSyncWrapper)
 
 function jasmineAsyncInstallWraper (jasmineAsyncInstallExport, jestVersion) {
+  log.warn('jest-jasmine2 support is removed from dd-trace@v4. Consider changing to jest-circus as `testRunner`.')
   return function (globalConfig, globalInput) {
     globalInput._ddtrace = global._ddtrace
     shimmer.wrap(globalInput.jasmine.Spec.prototype, 'execute', execute => function (onComplete) {

package/packages/datadog-plugin-http/src/client.js

@@ -45,7 +45,8 @@ class HttpClientPlugin extends Plugin {
           'resource.name': method,
           'span.type': 'http',
           'http.method': method,
-          'http.url': uri
+          'http.url': uri,
+          'out.host': hostname
         }
       })
 

package/packages/datadog-plugin-http2/src/client.js

@@ -52,7 +52,8 @@ class Http2ClientPlugin extends Plugin {
           'resource.name': method,
           'span.type': 'http',
           'http.method': method,
-          'http.url': uri
+          'http.url': uri,
+          'out.host': sessionDetails.host
         }
       })
 

package/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js

@@ -1,9 +1,10 @@
 'use strict'
 const InjectionAnalyzer = require('./injection-analyzer')
+const { COMMAND_INJECTION } = require('../vulnerabilities')
 
 class CommandInjectionAnalyzer extends InjectionAnalyzer {
   constructor () {
-    super('COMMAND_INJECTION')
+    super(COMMAND_INJECTION)
     this.addSub('datadog:child_process:execution:start', ({ command }) => this.analyze(command))
   }
 }

package/packages/dd-trace/src/appsec/iast/analyzers/ldap-injection-analyzer.js

@@ -1,9 +1,10 @@
 'use strict'
 const InjectionAnalyzer = require('./injection-analyzer')
+const { LDAP_INJECTION } = require('../vulnerabilities')
 
 class LdapInjectionAnalyzer extends InjectionAnalyzer {
   constructor () {
-    super('LDAP_INJECTION')
+    super(LDAP_INJECTION)
     this.addSub('datadog:ldapjs:client:search', ({ base, filter }) => this.analyzeAll(base, filter))
   }
 }

package/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js

@@ -4,10 +4,11 @@ const path = require('path')
 const { getIastContext } = require('../iast-context')
 const { storage } = require('../../../../../datadog-core')
 const InjectionAnalyzer = require('./injection-analyzer')
+const { PATH_TRAVERSAL } = require('../vulnerabilities')
 
 class PathTraversalAnalyzer extends InjectionAnalyzer {
   constructor () {
-    super('PATH_TRAVERSAL')
+    super(PATH_TRAVERSAL)
     this.addSub('apm:fs:operation:start', obj => {
       const pathArguments = []
       if (obj.dest) {
@@ -40,13 +41,29 @@ class PathTraversalAnalyzer extends InjectionAnalyzer {
       this.analyze(pathArguments)
     })
 
-    this.exclusionList = [ path.join('node_modules', 'send') + path.sep ]
+    this.exclusionList = [
+      path.join('node_modules', 'send') + path.sep
+    ]
+
+    this.internalExclusionList = [
+      'node:fs',
+      'node:internal/fs',
+      'node:internal\\fs',
+      'fs.js',
+      'internal/fs',
+      'internal\\fs'
+    ]
   }
 
   _isExcluded (location) {
-    let ret = false
+    let ret = true
     if (location && location.path) {
-      ret = this.exclusionList.some(elem => location.path.includes(elem))
+      // Exclude from reporting those vulnerabilities which location is from an internal fs call
+      if (location.isInternal) {
+        ret = this.internalExclusionList.some(elem => location.path.includes(elem))
+      } else {
+        ret = this.exclusionList.some(elem => location.path.includes(elem))
+      }
     }
     return ret
   }
@@ -59,7 +76,7 @@ class PathTraversalAnalyzer extends InjectionAnalyzer {
 
     if (value && value.constructor === Array) {
       for (const val of value) {
-        if (this._isVulnerable(val, iastContext)) {
+        if (this._isVulnerable(val, iastContext) && this._checkOCE(iastContext)) {
           this._report(val, iastContext)
           // no support several evidences in the same vulnerability, just report the 1st one
           break

package/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js

@@ -1,12 +1,48 @@
 'use strict'
+
 const InjectionAnalyzer = require('./injection-analyzer')
+const { SQL_INJECTION } = require('../vulnerabilities')
+const { getRanges } = require('../taint-tracking/operations')
+const { storage } = require('../../../../../datadog-core')
+const { getIastContext } = require('../iast-context')
+const { createVulnerability, addVulnerability } = require('../vulnerability-reporter')
 
 class SqlInjectionAnalyzer extends InjectionAnalyzer {
   constructor () {
-    super('SQL_INJECTION')
-    this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql))
-    this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql))
-    this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text))
+    super(SQL_INJECTION)
+    this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql, 'MYSQL'))
+    this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql, 'MYSQL'))
+    this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text, 'POSTGRES'))
+  }
+
+  _getEvidence (value, iastContext, dialect) {
+    const ranges = getRanges(iastContext, value)
+    return { value, ranges, dialect }
+  }
+
+  analyze (value, dialect) {
+    const store = storage.getStore()
+    const iastContext = getIastContext(store)
+    if (store && !iastContext) return
+    this._reportIfVulnerable(value, iastContext, dialect)
+  }
+
+  _reportIfVulnerable (value, context, dialect) {
+    if (this._isVulnerable(value, context) && this._checkOCE(context)) {
+      this._report(value, context, dialect)
+      return true
+    }
+    return false
+  }
+
+  _report (value, context, dialect) {
+    const evidence = this._getEvidence(value, context, dialect)
+    const location = this._getLocation()
+    if (!this._isExcluded(location)) {
+      const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
+      const vulnerability = createVulnerability(this._type, evidence, spanId, location)
+      addVulnerability(context, vulnerability)
+    }
   }
 }
 

package/packages/dd-trace/src/appsec/iast/analyzers/weak-cipher-analyzer.js

@@ -1,5 +1,6 @@
 'use strict'
 const Analyzer = require('./vulnerability-analyzer')
+const { WEAK_CIPHER } = require('../vulnerabilities')
 
 const INSECURE_CIPHERS = new Set([
   'des', 'des-cbc', 'des-cfb', 'des-cfb1', 'des-cfb8', 'des-ecb', 'des-ede', 'des-ede-cbc', 'des-ede-cfb',
@@ -12,7 +13,7 @@ const INSECURE_CIPHERS = new Set([
 
 class WeakCipherAnalyzer extends Analyzer {
   constructor () {
-    super('WEAK_CIPHER')
+    super(WEAK_CIPHER)
     this.addSub('datadog:crypto:cipher:start', ({ algorithm }) => this.analyze(algorithm))
   }
 

package/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js

@@ -1,5 +1,6 @@
 'use strict'
 const Analyzer = require('./vulnerability-analyzer')
+const { WEAK_HASH } = require('../vulnerabilities')
 
 const INSECURE_HASH_ALGORITHMS = new Set([
   'md4', 'md4WithRSAEncryption', 'RSA-MD4',
@@ -9,7 +10,7 @@ const INSECURE_HASH_ALGORITHMS = new Set([
 
 class WeakHashAnalyzer extends Analyzer {
   constructor () {
-    super('WEAK_HASH')
+    super(WEAK_HASH)
     this.addSub('datadog:crypto:hashing:start', ({ algorithm }) => this.analyze(algorithm))
   }
 

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -45,7 +45,8 @@ function getFirstNonDDPathAndLineFromCallsites (callsites) {
       if (!isExcluded(callsite) && filepath.indexOf(pathLine.ddBasePath) === -1) {
         return {
           path: path.relative(process.cwd(), filepath),
-          line: callsite.getLineNumber()
+          line: callsite.getLineNumber(),
+          isInternal: !path.isAbsolute(filepath)
         }
       }
     }

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/range-utils.js

@@ -0,0 +1,37 @@
+'use strict'
+
+function contains (rangeContainer, rangeContained) {
+  if (rangeContainer.start > rangeContained.start) {
+    return false
+  }
+  return rangeContainer.end >= rangeContained.end
+}
+
+function intersects (rangeA, rangeB) {
+  return rangeB.start < rangeA.end && rangeB.end > rangeA.start
+}
+
+function remove (range, rangeToRemove) {
+  if (!intersects(range, rangeToRemove)) {
+    return [range]
+  } else if (contains(rangeToRemove, range)) {
+    return []
+  } else {
+    const result = []
+    if (rangeToRemove.start > range.start) {
+      const offset = rangeToRemove.start - range.start
+      result.push({ start: range.start, end: range.start + offset })
+    }
+    if (rangeToRemove.end < range.end) {
+      const offset = range.end - rangeToRemove.end
+      result.push({ start: rangeToRemove.end, end: rangeToRemove.end + offset })
+    }
+    return result
+  }
+}
+
+module.exports = {
+  contains,
+  intersects,
+  remove
+}

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/command-sensitive-analyzer.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const iastLog = require('../../../iast-log')
+
+const COMMAND_PATTERN = '^(?:\\s*(?:sudo|doas)\\s+)?\\b\\S+\\b(.*)'
+
+class CommandSensitiveAnalyzer {
+  constructor () {
+    this._pattern = new RegExp(COMMAND_PATTERN, 'gmi')
+  }
+
+  extractSensitiveRanges (evidence) {
+    try {
+      this._pattern.lastIndex = 0
+
+      const regexResult = this._pattern.exec(evidence.value)
+      if (regexResult && regexResult.length > 1) {
+        const start = regexResult.index + (regexResult[0].length - regexResult[1].length)
+        const end = start + regexResult[1].length
+        return [{ start, end }]
+      }
+    } catch (e) {
+      iastLog.debug(e)
+    }
+    return []
+  }
+}
+
+module.exports = CommandSensitiveAnalyzer

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/ldap-sensitive-analyzer.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const iastLog = require('../../../iast-log')
+
+const LDAP_PATTERN = '\\(.*?(?:~=|=|<=|>=)(?<LITERAL>[^)]+)\\)'
+
+class LdapSensitiveAnalyzer {
+  constructor () {
+    this._pattern = new RegExp(LDAP_PATTERN, 'gmi')
+  }
+
+  extractSensitiveRanges (evidence) {
+    try {
+      this._pattern.lastIndex = 0
+      const tokens = []
+
+      let regexResult = this._pattern.exec(evidence.value)
+      while (regexResult != null) {
+        if (!regexResult.groups.LITERAL) continue
+        // Computing indices manually since NodeJs 12 does not support d flag on regular expressions
+        // TODO Get indices from group by adding d flag in regular expression
+        const start = regexResult.index + (regexResult[0].length - regexResult.groups.LITERAL.length - 1)
+        const end = start + regexResult.groups.LITERAL.length
+        tokens.push({ start, end })
+        regexResult = this._pattern.exec(evidence.value)
+      }
+      return tokens
+    } catch (e) {
+      iastLog.debug(e)
+    }
+    return []
+  }
+}
+
+module.exports = LdapSensitiveAnalyzer

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js

@@ -0,0 +1,95 @@
+'use strict'
+
+const iastLog = require('../../../iast-log')
+
+const STRING_LITERAL = '\'(?:\'\'|[^\'])*\''
+const POSTGRESQL_ESCAPED_LITERAL = '\\$([^$]*)\\$.*?\\$\\1\\$'
+const MYSQL_STRING_LITERAL = '"(?:\\\\"|[^"])*"|\'(?:\\\\\'|[^\'])*\''
+const LINE_COMMENT = '--.*$'
+const BLOCK_COMMENT = '/\\*[\\s\\S]*\\*/'
+const EXPONENT = '(?:E[-+]?\\d+[fd]?)?'
+const INTEGER_NUMBER = '(?<!\\w)\\d+'
+const DECIMAL_NUMBER = '\\d*\\.\\d+'
+const HEX_NUMBER = 'x\'[0-9a-f]+\'|0x[0-9a-f]+'
+const BIN_NUMBER = 'b\'[0-9a-f]+\'|0b[0-9a-f]+'
+const NUMERIC_LITERAL =
+  `[-+]?(?:${
+    [
+      HEX_NUMBER,
+      BIN_NUMBER,
+      DECIMAL_NUMBER + EXPONENT,
+      INTEGER_NUMBER + EXPONENT
+    ].join('|')
+  })`
+
+class SqlSensitiveAnalyzer {
+  constructor () {
+    this._patterns = {
+      MYSQL: new RegExp(
+        [
+          NUMERIC_LITERAL,
+          MYSQL_STRING_LITERAL,
+          LINE_COMMENT,
+          BLOCK_COMMENT
+        ].join('|'),
+        'gmi'
+      ),
+      POSTGRES: new RegExp(
+        [
+          NUMERIC_LITERAL,
+          POSTGRESQL_ESCAPED_LITERAL,
+          STRING_LITERAL,
+          LINE_COMMENT,
+          BLOCK_COMMENT
+        ].join('|'),
+        'gmi'
+      )
+    }
+  }
+
+  extractSensitiveRanges (evidence) {
+    try {
+      const pattern = this._patterns[evidence.dialect]
+      pattern.lastIndex = 0
+      const tokens = []
+
+      let regexResult = pattern.exec(evidence.value)
+      while (regexResult != null) {
+        let start = regexResult.index
+        let end = regexResult.index + regexResult[0].length
+        const startChar = evidence.value.charAt(start)
+        if (startChar === '\'' || startChar === '"') {
+          start++
+          end--
+        } else if (end > start + 1) {
+          const nextChar = evidence.value.charAt(start + 1)
+          if (startChar === '/' && nextChar === '*') {
+            start += 2
+            end -= 2
+          } else if (startChar === '-' && startChar === nextChar) {
+            start += 2
+          } else if (startChar.toLowerCase() === 'q' && nextChar === '\'') {
+            start += 3
+            end -= 2
+          } else if (startChar === '$') {
+            const match = regexResult[0]
+            const size = match.indexOf('$', 1) + 1
+            if (size > 1) {
+              start += size
+              end -= size
+            }
+          }
+        }
+
+        tokens.push({ start, end })
+        regexResult = pattern.exec(evidence.value)
+      }
+      return tokens
+    } catch (e) {
+      iastLog.debug(e)
+    }
+    return []
+  }
+}
+
+module.exports = SqlSensitiveAnalyzer

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js

@@ -0,0 +1,144 @@
+'use strict'
+
+const vulnerabilities = require('../../vulnerabilities')
+
+const { contains, intersects, remove } = require('./range-utils')
+
+const CommandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer')
+const LdapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer')
+const SqlSensitiveAnalyzer = require('./sensitive-analyzers/sql-sensitive-analyzer')
+
+// eslint-disable-next-line max-len
+const DEFAULT_IAST_REDACTION_NAME_PATTERN = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)'
+// eslint-disable-next-line max-len
+const DEFAULT_IAST_REDACTION_VALUE_PATTERN = '(?:bearer\\s+[a-z0-9\\._\\-]+|glpat-[\\w\\-]{20}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=\\-]+\\.ey[I-L][\\w=\\-]+(?:\\.[\\w.+/=\\-]+)?|(?:[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY[\\-]{5}|ssh-rsa\\s*[a-z0-9/\\.+]{100,}))'
+
+class SensitiveHandler {
+  constructor () {
+    this._namePattern = new RegExp(DEFAULT_IAST_REDACTION_NAME_PATTERN, 'gmi')
+    this._valuePattern = new RegExp(DEFAULT_IAST_REDACTION_VALUE_PATTERN, 'gmi')
+
+    this._sensitiveAnalyzers = new Map()
+    this._sensitiveAnalyzers.set(vulnerabilities.COMMAND_INJECTION, new CommandSensitiveAnalyzer())
+    this._sensitiveAnalyzers.set(vulnerabilities.LDAP_INJECTION, new LdapSensitiveAnalyzer())
+    this._sensitiveAnalyzers.set(vulnerabilities.SQL_INJECTION, new SqlSensitiveAnalyzer())
+  }
+
+  isSensibleName (name) {
+    this._namePattern.lastIndex = 0
+    return this._namePattern.test(name)
+  }
+
+  isSensibleValue (value) {
+    this._valuePattern.lastIndex = 0
+    return this._valuePattern.test(value)
+  }
+
+  isSensibleSource (source) {
+    return source != null && (this.isSensibleName(source.name) || this.isSensibleValue(source.value))
+  }
+
+  scrubEvidence (vulnerabilityType, evidence, sourcesIndexes, sources) {
+    const sensitiveAnalyzer = this._sensitiveAnalyzers.get(vulnerabilityType)
+    if (sensitiveAnalyzer) {
+      const sensitiveRanges = sensitiveAnalyzer.extractSensitiveRanges(evidence)
+      return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources)
+    }
+    return null
+  }
+
+  toRedactedJson (evidence, sensitive, sourcesIndexes, sources) {
+    const valueParts = []
+    const redactedSources = []
+
+    const { value, ranges } = evidence
+
+    let start = 0
+    let nextTaintedIndex = 0
+    let sourceIndex
+
+    let nextTainted = ranges.shift()
+    let nextSensitive = sensitive.shift()
+
+    for (let i = 0; i < value.length; i++) {
+      if (nextTainted != null && nextTainted.start === i) {
+        this.writeValuePart(valueParts, value.substring(start, i), sourceIndex)
+
+        sourceIndex = sourcesIndexes[nextTaintedIndex]
+
+        while (nextSensitive != null && contains(nextTainted, nextSensitive)) {
+          sourceIndex != null && redactedSources.push(sourceIndex)
+          nextSensitive = sensitive.shift()
+        }
+
+        if (nextSensitive != null && intersects(nextSensitive, nextTainted)) {
+          sourceIndex != null && redactedSources.push(sourceIndex)
+
+          const entries = remove(nextSensitive, nextTainted)
+          nextSensitive = entries.length > 0 ? entries[0] : null
+        }
+
+        this.isSensibleSource(sources[sourceIndex]) && redactedSources.push(sourceIndex)
+
+        if (redactedSources.indexOf(sourceIndex) > -1) {
+          this.writeRedactedValuePart(valueParts, sourceIndex)
+        } else {
+          const substringEnd = Math.min(nextTainted.end, value.length)
+          this.writeValuePart(valueParts, value.substring(nextTainted.start, substringEnd), sourceIndex)
+        }
+
+        start = i + (nextTainted.end - nextTainted.start)
+        i = start - 1
+        nextTainted = ranges.shift()
+        nextTaintedIndex++
+        sourceIndex = null
+      } else if (nextSensitive != null && nextSensitive.start === i) {
+        this.writeValuePart(valueParts, value.substring(start, i), sourceIndex)
+        if (nextTainted != null && intersects(nextSensitive, nextTainted)) {
+          sourceIndex = sourcesIndexes[nextTaintedIndex]
+          sourceIndex != null && redactedSources.push(sourceIndex)
+
+          for (const entry of remove(nextSensitive, nextTainted)) {
+            if (entry.start === i) {
+              nextSensitive = entry
+            } else {
+              sensitive.push(entry)
+            }
+          }
+        }
+
+        this.writeRedactedValuePart(valueParts)
+
+        start = i + (nextSensitive.end - nextSensitive.start)
+        i = start - 1
+        nextSensitive = sensitive.shift()
+      }
+    }
+
+    if (start < value.length) {
+      this.writeValuePart(valueParts, value.substring(start))
+    }
+
+    return { redactedValueParts: valueParts, redactedSources }
+  }
+
+  writeValuePart (valueParts, value, source) {
+    if (value.length > 0) {
+      if (source != null) {
+        valueParts.push({ value, source })
+      } else {
+        valueParts.push({ value })
+      }
+    }
+  }
+
+  writeRedactedValuePart (valueParts, source) {
+    if (source != null) {
+      valueParts.push({ redacted: true, source })
+    } else {
+      valueParts.push({ redacted: true })
+    }
+  }
+}
+
+module.exports = new SensitiveHandler()

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -0,0 +1,113 @@
+const sensitiveHandler = require('./evidence-redaction/sensitive-handler')
+
+class VulnerabilityFormatter {
+  constructor () {
+    this._redactVulnearbilities = true
+  }
+
+  setRedactVulnerabilities (shouldRedactVulnerabilities) {
+    this._redactVulnearbilities = shouldRedactVulnerabilities
+  }
+
+  extractSourcesFromVulnerability (vulnerability) {
+    if (!vulnerability.evidence.ranges) {
+      return []
+    }
+    return vulnerability.evidence.ranges.map(range => (
+      {
+        origin: range.iinfo.type,
+        name: range.iinfo.parameterName,
+        value: range.iinfo.parameterValue
+      }
+    ))
+  }
+
+  getRedactedValueParts (type, evidence, sourcesIndexes, sources) {
+    const scrubbingResult = sensitiveHandler.scrubEvidence(type, evidence, sourcesIndexes, sources)
+    if (scrubbingResult) {
+      const { redactedValueParts, redactedSources } = scrubbingResult
+      redactedSources.forEach(i => {
+        delete sources[i].value
+        sources[i].redacted = true
+      })
+      return { valueParts: redactedValueParts }
+    }
+
+    return this.getUnredactedValueParts(evidence, sourcesIndexes)
+  }
+
+  getUnredactedValueParts (evidence, sourcesIndexes) {
+    const valueParts = []
+    let fromIndex = 0
+    evidence.ranges.forEach((range, rangeIndex) => {
+      if (fromIndex < range.start) {
+        valueParts.push({ value: evidence.value.substring(fromIndex, range.start) })
+      }
+      valueParts.push({ value: evidence.value.substring(range.start, range.end), source: sourcesIndexes[rangeIndex] })
+      fromIndex = range.end
+    })
+    if (fromIndex < evidence.value.length) {
+      valueParts.push({ value: evidence.value.substring(fromIndex) })
+    }
+    return { valueParts }
+  }
+
+  formatEvidence (type, evidence, sourcesIndexes, sources) {
+    if (!evidence.ranges) {
+      return { value: evidence.value }
+    }
+
+    return this._redactVulnearbilities
+      ? this.getRedactedValueParts(type, evidence, sourcesIndexes, sources)
+      : this.getUnredactedValueParts(evidence, sourcesIndexes)
+  }
+
+  formatVulnerability (vulnerability, sourcesIndexes, sources) {
+    const formattedVulnerability = {
+      type: vulnerability.type,
+      hash: vulnerability.hash,
+      evidence: this.formatEvidence(vulnerability.type, vulnerability.evidence, sourcesIndexes, sources),
+      location: {
+        spanId: vulnerability.location.spanId
+      }
+    }
+    if (vulnerability.location.path) {
+      formattedVulnerability.location.path = vulnerability.location.path
+    }
+    if (vulnerability.location.line) {
+      formattedVulnerability.location.line = vulnerability.location.line
+    }
+    return formattedVulnerability
+  }
+
+  toJson (vulnerabilitiesToFormat) {
+    const sources = []
+
+    const vulnerabilities = vulnerabilitiesToFormat.map(vulnerability => {
+      const vulnerabilitySources = this.extractSourcesFromVulnerability(vulnerability)
+      const sourcesIndexes = []
+      vulnerabilitySources.forEach((source) => {
+        let sourceIndex = sources.findIndex(
+          existingSource =>
+            existingSource.origin === source.origin &&
+            existingSource.name === source.name &&
+            existingSource.value === source.value
+        )
+        if (sourceIndex === -1) {
+          sourceIndex = sources.length
+          sources.push(source)
+        }
+        sourcesIndexes.push(sourceIndex)
+      })
+
+      return this.formatVulnerability(vulnerability, sourcesIndexes, sources)
+    })
+
+    return {
+      sources,
+      vulnerabilities
+    }
+  }
+}
+
+module.exports = new VulnerabilityFormatter()

package/packages/dd-trace/src/appsec/iast/vulnerabilities.js

@@ -0,0 +1,8 @@
+module.exports = {
+  WEAK_HASH: 'WEAK_HASH',
+  WEAK_CIPHER: 'WEAK_CIPHER',
+  SQL_INJECTION: 'SQL_INJECTION',
+  PATH_TRAVERSAL: 'PATH_TRAVERSAL',
+  COMMAND_INJECTION: 'COMMAND_INJECTION',
+  LDAP_INJECTION: 'LDAP_INJECTION'
+}

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -1,5 +1,6 @@
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
 const LRU = require('lru-cache')
+const vulnerabilitiesFormatter = require('./vulnerabilities-formatter')
 const VULNERABILITIES_KEY = 'vulnerabilities'
 const IAST_JSON_TAG_KEY = '_dd.iast.json'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
@@ -60,57 +61,6 @@ function isValidVulnerability (vulnerability) {
     vulnerability.location && vulnerability.location.spanId
 }
 
-function formatEvidence (evidence, sourcesIndexes) {
-  if (!evidence.ranges) {
-    return { value: evidence.value }
-  }
-
-  const valueParts = []
-  let fromIndex = 0
-  evidence.ranges.forEach((range, rangeIndex) => {
-    if (fromIndex < range.start) {
-      valueParts.push({ value: evidence.value.substring(fromIndex, range.start) })
-    }
-    valueParts.push({ value: evidence.value.substring(range.start, range.end), source: sourcesIndexes[rangeIndex] })
-    fromIndex = range.end
-  })
-  if (fromIndex < evidence.value.length) {
-    valueParts.push({ value: evidence.value.substring(fromIndex) })
-  }
-  return { valueParts }
-}
-
-function extractSourcesFromVulnerability (vulnerability) {
-  if (!vulnerability.evidence.ranges) {
-    return []
-  }
-  return vulnerability.evidence.ranges.map(range => (
-    {
-      origin: range.iinfo.type,
-      name: range.iinfo.parameterName,
-      value: range.iinfo.parameterValue
-    }
-  ))
-}
-
-function jsonVulnerabilityFromVulnerability (vulnerability, sourcesIndexes) {
-  const jsonVulnerability = {
-    type: vulnerability.type,
-    hash: vulnerability.hash,
-    evidence: formatEvidence(vulnerability.evidence, sourcesIndexes),
-    location: {
-      spanId: vulnerability.location.spanId
-    }
-  }
-  if (vulnerability.location.path) {
-    jsonVulnerability.location.path = vulnerability.location.path
-  }
-  if (vulnerability.location.line) {
-    jsonVulnerability.location.line = vulnerability.location.line
-  }
-  return jsonVulnerability
-}
-
 function sendVulnerabilities (vulnerabilities, rootSpan) {
   if (vulnerabilities && vulnerabilities.length) {
     let span = rootSpan
@@ -124,31 +74,8 @@ function sendVulnerabilities (vulnerabilities, rootSpan) {
     }
 
     if (span && span.addTags) {
-      const jsonToSend = {
-        sources: [],
-        vulnerabilities: []
-      }
-
-      deduplicateVulnerabilities(vulnerabilities).forEach((vulnerability) => {
-        if (isValidVulnerability(vulnerability)) {
-          const sourcesIndexes = []
-          const vulnerabilitySources = extractSourcesFromVulnerability(vulnerability)
-          vulnerabilitySources.forEach((source) => {
-            let sourceIndex = jsonToSend.sources.findIndex(
-              existingSource =>
-                existingSource.origin === source.origin &&
-                existingSource.name === source.name &&
-                existingSource.value === source.value
-            )
-            if (sourceIndex === -1) {
-              sourceIndex = jsonToSend.sources.length
-              jsonToSend.sources.push(source)
-            }
-            sourcesIndexes.push(sourceIndex)
-          })
-          jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability, sourcesIndexes))
-        }
-      })
+      const validAndDedupVulnerabilities = deduplicateVulnerabilities(vulnerabilities).filter(isValidVulnerability)
+      const jsonToSend = vulnerabilitiesFormatter.toJson(validAndDedupVulnerabilities)
 
       if (jsonToSend.vulnerabilities.length > 0) {
         const tags = {}
@@ -194,6 +121,7 @@ function deduplicateVulnerabilities (vulnerabilities) {
 
 function start (config, _tracer) {
   deduplicationEnabled = config.iast.deduplicationEnabled
+  vulnerabilitiesFormatter.setRedactVulnerabilities(config.iast.redactionEnabled)
   if (deduplicationEnabled) {
     startClearCacheTimer()
   }

package/packages/dd-trace/src/config.js

@@ -9,6 +9,7 @@ const coalesce = require('koalas')
 const tagger = require('./tagger')
 const { isTrue, isFalse } = require('./util')
 const uuid = require('crypto-randomuuid')
+const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags')
 
 const fromEntries = Object.fromEntries || (entries =>
   entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
@@ -397,11 +398,22 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       true
     )
 
+    const DD_IAST_REDACTION_ENABLED = coalesce(
+      iastOptions && iastOptions.redactionEnabled,
+      !isFalse(process.env.DD_IAST_REDACTION_ENABLED),
+      true
+    )
+
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
       true
     )
 
+    const DD_TRACE_GIT_METADATA_ENABLED = coalesce(
+      process.env.DD_TRACE_GIT_METADATA_ENABLED,
+      true
+    )
+
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
     const sampler = {
@@ -506,7 +518,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       requestSampling: DD_IAST_REQUEST_SAMPLING,
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
       maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS,
-      deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED
+      deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED,
+      redactionEnabled: DD_IAST_REDACTION_ENABLED
     }
 
     this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY)
@@ -515,6 +528,19 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.isGitUploadEnabled = this.isCiVisibility &&
       (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
 
+    this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED)
+
+    if (this.gitMetadataEnabled) {
+      this.repositoryUrl = coalesce(
+        process.env.DD_GIT_REPOSITORY_URL,
+        this.tags[GIT_REPOSITORY_URL]
+      )
+      this.commitSHA = coalesce(
+        process.env.DD_GIT_COMMIT_SHA,
+        this.tags[GIT_COMMIT_SHA]
+      )
+    }
+
     this.stats = {
       enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
     }

package/packages/dd-trace/src/constants.js

@@ -25,5 +25,7 @@ module.exports = {
   ERROR_MESSAGE: 'error.message',
   ERROR_STACK: 'error.stack',
   COMPONENT: 'component',
-  CLIENT_PORT_KEY: 'network.destination.port'
+  CLIENT_PORT_KEY: 'network.destination.port',
+  SCI_REPOSITORY_URL: '_dd.git.repository_url',
+  SCI_COMMIT_SHA: '_dd.git.commit.sha'
 }

package/packages/dd-trace/src/git_metadata_tagger.js

@@ -0,0 +1,17 @@
+const { SCI_COMMIT_SHA, SCI_REPOSITORY_URL } = require('./constants')
+
+class GitMetadataTagger {
+  constructor (config) {
+    this._config = config
+  }
+
+  tagGitMetadata (spanContext) {
+    if (this._config.gitMetadataEnabled) {
+      // These tags are added only to the local root span
+      spanContext._trace.tags[SCI_COMMIT_SHA] = this._config.commitSHA
+      spanContext._trace.tags[SCI_REPOSITORY_URL] = this._config.repositoryUrl
+    }
+  }
+}
+
+module.exports = GitMetadataTagger

package/packages/dd-trace/src/plugins/util/ci.js

@@ -20,7 +20,9 @@ const {
   CI_STAGE_NAME,
   CI_ENV_VARS,
   GIT_COMMIT_COMMITTER_NAME,
-  GIT_COMMIT_COMMITTER_EMAIL
+  GIT_COMMIT_COMMITTER_EMAIL,
+  CI_NODE_LABELS,
+  CI_NODE_NAME
 } = require('./tags')
 
 // Receives a string with the form 'John Doe <john.doe@gmail.com>'
@@ -108,7 +110,9 @@ module.exports = {
         GIT_COMMIT: JENKINS_GIT_COMMIT,
         GIT_URL: JENKINS_GIT_REPOSITORY_URL,
         GIT_URL_1: JENKINS_GIT_REPOSITORY_URL_1,
-        DD_CUSTOM_TRACE_ID
+        DD_CUSTOM_TRACE_ID,
+        NODE_NAME,
+        NODE_LABELS
       } = env
 
       tags = {
@@ -119,7 +123,18 @@ module.exports = {
         [GIT_COMMIT_SHA]: JENKINS_GIT_COMMIT,
         [GIT_REPOSITORY_URL]: JENKINS_GIT_REPOSITORY_URL || JENKINS_GIT_REPOSITORY_URL_1,
         [CI_WORKSPACE_PATH]: WORKSPACE,
-        [CI_ENV_VARS]: JSON.stringify({ DD_CUSTOM_TRACE_ID })
+        [CI_ENV_VARS]: JSON.stringify({ DD_CUSTOM_TRACE_ID }),
+        [CI_NODE_NAME]: NODE_NAME
+      }
+
+      if (NODE_LABELS) {
+        let nodeLabels
+        try {
+          nodeLabels = JSON.stringify(NODE_LABELS.split(' '))
+          tags[CI_NODE_LABELS] = nodeLabels
+        } catch (e) {
+          // ignore errors
+        }
       }
 
       const isTag = JENKINS_GIT_BRANCH && JENKINS_GIT_BRANCH.includes('tags/')
@@ -159,7 +174,9 @@ module.exports = {
         CI_COMMIT_TIMESTAMP,
         CI_COMMIT_AUTHOR,
         CI_PROJECT_URL: GITLAB_PROJECT_URL,
-        CI_JOB_ID: GITLAB_CI_JOB_ID
+        CI_JOB_ID: GITLAB_CI_JOB_ID,
+        CI_RUNNER_ID,
+        CI_RUNNER_TAGS
       } = env
 
       const { name, email } = parseEmailAndName(CI_COMMIT_AUTHOR)
@@ -186,7 +203,9 @@ module.exports = {
           CI_PROJECT_URL: GITLAB_PROJECT_URL,
           CI_PIPELINE_ID: GITLAB_PIPELINE_ID,
           CI_JOB_ID: GITLAB_CI_JOB_ID
-        })
+        }),
+        [CI_NODE_LABELS]: CI_RUNNER_TAGS,
+        [CI_NODE_NAME]: CI_RUNNER_ID
       }
     }
 
@@ -448,9 +467,17 @@ module.exports = {
         BUILDKITE_BUILD_CHECKOUT_PATH,
         BUILDKITE_BUILD_AUTHOR,
         BUILDKITE_BUILD_AUTHOR_EMAIL,
-        BUILDKITE_MESSAGE
+        BUILDKITE_MESSAGE,
+        BUILDKITE_AGENT_ID
       } = env
 
+      const extraTags = Object.keys(env).filter(envVar =>
+        envVar.startsWith('BUILDKITE_AGENT_META_DATA_')
+      ).map((metadataKey) => {
+        const key = metadataKey.replace('BUILDKITE_AGENT_META_DATA_', '').toLowerCase()
+        return `${key}:${env[metadataKey]}`
+      })
+
       tags = {
         [CI_PROVIDER_NAME]: 'buildkite',
         [CI_PIPELINE_ID]: BUILDKITE_BUILD_ID,
@@ -469,7 +496,9 @@ module.exports = {
         [CI_ENV_VARS]: JSON.stringify({
           BUILDKITE_BUILD_ID,
           BUILDKITE_JOB_ID
-        })
+        }),
+        [CI_NODE_NAME]: BUILDKITE_AGENT_ID,
+        [CI_NODE_LABELS]: JSON.stringify(extraTags)
       }
     }
 
@@ -546,6 +575,32 @@ module.exports = {
       }
     }
 
+    if (env.CF_BUILD_ID) {
+      const {
+        CF_BUILD_ID,
+        CF_PIPELINE_NAME,
+        CF_BUILD_URL,
+        CF_STEP_NAME,
+        CF_BRANCH
+      } = env
+      tags = {
+        [CI_PROVIDER_NAME]: 'codefresh',
+        [CI_PIPELINE_ID]: CF_BUILD_ID,
+        [CI_PIPELINE_NAME]: CF_PIPELINE_NAME,
+        [CI_PIPELINE_URL]: CF_BUILD_URL,
+        [CI_JOB_NAME]: CF_STEP_NAME,
+        [CI_ENV_VARS]: JSON.stringify({
+          CF_BUILD_ID
+        })
+      }
+
+      const isTag = CF_BRANCH && CF_BRANCH.includes('tags/')
+      const refKey = isTag ? GIT_TAG : GIT_BRANCH
+      const ref = normalizeRef(CF_BRANCH)
+
+      tags[refKey] = ref
+    }
+
     normalizeTag(tags, CI_WORKSPACE_PATH, resolveTilde)
     normalizeTag(tags, GIT_REPOSITORY_URL, filterSensitiveInfoFromRepository)
     normalizeTag(tags, GIT_BRANCH, normalizeRef)

package/packages/dd-trace/src/plugins/util/tags.js

@@ -19,6 +19,8 @@ const CI_WORKSPACE_PATH = 'ci.workspace_path'
 const CI_JOB_URL = 'ci.job.url'
 const CI_JOB_NAME = 'ci.job.name'
 const CI_STAGE_NAME = 'ci.stage.name'
+const CI_NODE_NAME = 'ci.node.name'
+const CI_NODE_LABELS = 'ci.node.labels'
 
 const CI_ENV_VARS = '_dd.ci.env_vars'
 
@@ -43,5 +45,7 @@ module.exports = {
   CI_JOB_URL,
   CI_JOB_NAME,
   CI_STAGE_NAME,
-  CI_ENV_VARS
+  CI_ENV_VARS,
+  CI_NODE_NAME,
+  CI_NODE_LABELS
 }

package/packages/dd-trace/src/span_processor.js

@@ -3,6 +3,7 @@
 const log = require('./log')
 const format = require('./format')
 const SpanSampler = require('./span_sampler')
+const GitMetadataTagger = require('./git_metadata_tagger')
 
 const { SpanStatsProcessor } = require('./span_stats')
 
@@ -18,6 +19,7 @@ class SpanProcessor {
 
     this._stats = new SpanStatsProcessor(config)
     this._spanSampler = new SpanSampler(config.sampler)
+    this._gitMetadataTagger = new GitMetadataTagger(config)
   }
 
   process (span) {
@@ -32,6 +34,7 @@ class SpanProcessor {
     if (started.length === finished.length || finished.length >= flushMinSpans) {
       this._prioritySampler.sample(spanContext)
       this._spanSampler.sample(spanContext)
+      this._gitMetadataTagger.tagGitMetadata(spanContext)
 
       for (const span of started) {
         if (span._duration !== undefined) {

package/packages/dd-trace/src/tracer.js

@@ -7,6 +7,7 @@ const { storage } = require('../../datadog-core')
 const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
 const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants')
+const { MAJOR } = require('../../../version')
 
 const SPAN_TYPE = tags.SPAN_TYPE
 const RESOURCE_NAME = tags.RESOURCE_NAME
@@ -26,6 +27,10 @@ class DatadogTracer extends Tracer {
       childOf: this.scope().active()
     }, options)
 
+    if (!options.childOf && options.orphanable === false && MAJOR < 4) {
+      return fn(null, () => {})
+    }
+
     const span = this.startSpan(name, options)
 
     addTags(span, options)
@@ -77,6 +82,10 @@ class DatadogTracer extends Tracer {
         optionsObj = optionsObj.apply(this, arguments)
       }
 
+      if (optionsObj && optionsObj.orphanable === false && !tracer.scope().active() && MAJOR < 4) {
+        return fn.apply(this, arguments)
+      }
+
       const lastArgId = arguments.length - 1
       const cb = arguments[lastArgId]