@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/remote_config/index.js

@@ -0,0 +1,574 @@
+'use strict'
+
+const uuid = require('../../../../vendor/dist/crypto-randomuuid')
+const tracerVersion = require('../../../../package.json').version
+const request = require('../exporters/common/request')
+const log = require('../log')
+const { getExtraServices } = require('../service-naming/extra-services')
+const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../plugins/util/tags')
+const tagger = require('../tagger')
+const { getAgentUrl } = require('../agent/url')
+const processTags = require('../process-tags')
+const Scheduler = require('./scheduler')
+const { UNACKNOWLEDGED, ACKNOWLEDGED, ERROR } = require('./apply_states')
+
+const clientId = uuid()
+
+const DEFAULT_CAPABILITY = Buffer.alloc(1).toString('base64') // 0x00
+
+const kSupportsAckCallback = Symbol('kSupportsAckCallback')
+
+// There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest
+// with their own separated Client.ClientState.
+class RemoteConfig {
+  #handlers = new Map()
+  #products = new Set()
+  #batchHandlers = new Map()
+
+  constructor (config) {
+    const pollInterval = Math.floor(config.remoteConfig.pollInterval * 1000)
+
+    this.url = getAgentUrl(config)
+
+    tagger.add(config.tags, {
+      '_dd.rc.client_id': clientId,
+    })
+
+    const tags = config.repositoryUrl
+      ? {
+          ...config.tags,
+          [GIT_REPOSITORY_URL]: config.repositoryUrl,
+          [GIT_COMMIT_SHA]: config.commitSHA,
+        }
+      : config.tags
+
+    const appliedConfigs = this.appliedConfigs = new Map()
+
+    this.scheduler = new Scheduler((cb) => this.poll(cb), pollInterval)
+
+    this.state = {
+      client: {
+        state: { // updated by `parseConfig()` and `poll()`
+          root_version: 1,
+          targets_version: 0,
+          // Use getter so `apply_*` can be updated async and still affect the content of `config_states`
+          get config_states () {
+            const configs = []
+            for (const conf of appliedConfigs.values()) {
+              configs.push({
+                id: conf.id,
+                version: conf.version,
+                product: conf.product,
+                apply_state: conf.apply_state,
+                apply_error: conf.apply_error,
+              })
+            }
+            return configs
+          },
+          has_error: false,
+          error: '',
+          backend_client_state: '',
+        },
+        id: clientId,
+        products: /** @type {string[]} */ ([]), // updated by `updateProducts()`
+        is_tracer: true,
+        client_tracer: {
+          runtime_id: config.tags['runtime-id'],
+          language: 'node',
+          tracer_version: tracerVersion,
+          service: config.service,
+          env: config.env,
+          app_version: config.version,
+          extra_services: /** @type {string[]} */ ([]),
+          tags: Object.entries(tags).map((pair) => pair.join(':')),
+          [processTags.REMOTE_CONFIG_FIELD_NAME]: processTags.tagsArray,
+        },
+        capabilities: DEFAULT_CAPABILITY, // updated by `updateCapabilities()`
+      },
+      cached_target_files: /** @type {RcCachedTargetFile[]} */ ([]), // updated by `parseConfig()`
+    }
+  }
+
+  /**
+   * @param {bigint} mask
+   * @param {boolean} value
+   */
+  updateCapabilities (mask, value) {
+    const hex = Buffer.from(this.state.client.capabilities, 'base64').toString('hex')
+
+    let num = BigInt(`0x${hex}`)
+
+    if (value) {
+      num |= mask
+    } else {
+      num &= ~mask
+    }
+
+    let str = num.toString(16)
+
+    if (str.length % 2) str = `0${str}`
+
+    this.state.client.capabilities = Buffer.from(str, 'hex').toString('base64')
+  }
+
+  /**
+   * Subscribe to a product and register a per-config handler.
+   *
+   * This is the common API for products that can be handled one config at a time.
+   * It **implies subscription** (equivalent to calling `subscribeProducts(product)`).
+   *
+   * @param {string} product
+   * @param {Function} handler
+   */
+  setProductHandler (product, handler) {
+    this.#handlers.set(product, handler)
+    this.subscribeProducts(product)
+  }
+
+  /**
+   * Remove the per-config handler for a product and unsubscribe from it.
+   *
+   * If you only want to stop receiving configs (but keep the handler attached for later),
+   * call `unsubscribeProducts(product)` instead.
+   *
+   * @param {string} product
+   */
+  removeProductHandler (product) {
+    this.#handlers.delete(product)
+    this.unsubscribeProducts(product)
+  }
+
+  /**
+   * Subscribe to one or more products with Remote Config (receive configs for them).
+   *
+   * This only affects subscription/polling and does **not** register any handler.
+   *
+   * @param {...string} products
+   */
+  subscribeProducts (...products) {
+    const hadProducts = this.#products.size > 0
+    for (const product of products) {
+      this.#products.add(product)
+    }
+    this.updateProducts()
+    if (!hadProducts && this.#products.size > 0) {
+      this.scheduler.start()
+    }
+  }
+
+  /**
+   * Unsubscribe from one or more products (stop receiving configs for them).
+   *
+   * This does **not** remove registered handlers; use `removeProductHandler(product)`
+   * if you want to detach a handler as well.
+   *
+   * @param {...string} products
+   */
+  unsubscribeProducts (...products) {
+    const hadProducts = this.#products.size > 0
+    for (const product of products) {
+      this.#products.delete(product)
+    }
+    this.updateProducts()
+    if (hadProducts && this.#products.size === 0) {
+      this.scheduler.stop()
+    }
+  }
+
+  updateProducts () {
+    this.state.client.products = [...this.#products]
+  }
+
+  /**
+   * Register a handler that will be invoked once per RC update, with the update batch filtered
+   * down to the specified products. This is useful for consumers that need to process multiple
+   * configs at once (e.g. WAF updates spanning ASM/ASM_DD/ASM_DATA) and then do one-time reconciliation.
+   *
+   * This does **not** implicitly subscribe to the products; call `subscribeProducts()` separately.
+   *
+   * @param {string[]} products
+   * @param {(transaction: RcBatchUpdateTransaction) => void} handler
+   */
+  setBatchHandler (products, handler) {
+    this.#batchHandlers.set(handler, new Set(products))
+  }
+
+  /**
+   * Remove a previously-registered batch handler.
+   *
+   * @param {Function} handler
+   */
+  removeBatchHandler (handler) {
+    this.#batchHandlers.delete(handler)
+  }
+
+  // TODO: Cache the return value of this method, to avoid recomputing it on every poll
+  getPayload () {
+    this.state.client.client_tracer.extra_services = getExtraServices()
+
+    return JSON.stringify(this.state)
+  }
+
+  poll (cb) {
+    const options = {
+      url: this.url,
+      method: 'POST',
+      path: '/v0.7/config',
+      headers: {
+        'Content-Type': 'application/json; charset=utf-8',
+      },
+    }
+
+    request(this.getPayload(), options, (err, data, statusCode) => {
+      // 404 means RC is disabled, ignore it
+      if (statusCode === 404) return cb()
+
+      if (err) {
+        log.errorWithoutTelemetry('[RC] Error in request', err)
+        return cb()
+      }
+
+      // if error was just sent, reset the state
+      if (this.state.client.state.has_error) {
+        this.state.client.state.has_error = false
+        this.state.client.state.error = ''
+      }
+
+      if (data && data !== '{}') { // '{}' means the tracer is up to date
+        try {
+          this.parseConfig(JSON.parse(data))
+        } catch (err) {
+          log.error('[RC] Could not parse remote config response', err)
+
+          this.state.client.state.has_error = true
+          this.state.client.state.error = err.toString()
+        }
+      }
+
+      cb()
+    })
+  }
+
+  // `client_configs` is the list of config paths to have applied
+  // `targets` is the signed index with metadata for config files
+  // `target_files` is the list of config files containing the actual config data
+  parseConfig ({
+    client_configs: clientConfigs = [],
+    targets,
+    target_files: targetFiles = [],
+  }) {
+    const toUnapply = /** @type {RcConfigState[]} */ ([])
+    const toApply = /** @type {RcConfigState[]} */ ([])
+    const toModify = /** @type {RcConfigState[]} */ ([])
+    const transactionByPath = new Map()
+    const transactionHandledPaths = new Set()
+    const transactionOutcomes = new Map()
+
+    for (const appliedConfig of this.appliedConfigs.values()) {
+      if (!clientConfigs.includes(appliedConfig.path)) {
+        toUnapply.push(appliedConfig)
+        transactionByPath.set(appliedConfig.path, appliedConfig)
+      }
+    }
+
+    targets = fromBase64JSON(targets)
+
+    if (targets) {
+      for (const path of clientConfigs) {
+        const meta = targets.signed.targets[path]
+        if (!meta) throw new Error(`Unable to find target for path ${path}`)
+
+        const current = this.appliedConfigs.get(path)
+
+        const newConf = /** @type {RcConfigState} */ ({})
+
+        if (current) {
+          if (current.hashes.sha256 === meta.hashes.sha256) continue
+
+          toModify.push(newConf)
+        } else {
+          toApply.push(newConf)
+        }
+
+        const file = targetFiles.find(file => file.path === path)
+        if (!file) throw new Error(`Unable to find file for path ${path}`)
+
+        // TODO: verify signatures
+        //       verify length
+        //       verify hash
+        //       verify _type
+        // TODO: new Date(meta.signed.expires) ignore the Targets data if it has expired ?
+
+        const { product, id } = parseConfigPath(path)
+
+        Object.assign(newConf, {
+          path,
+          product,
+          id,
+          version: meta.custom.v,
+          apply_state: UNACKNOWLEDGED,
+          apply_error: '',
+          length: meta.length,
+          hashes: meta.hashes,
+          file: fromBase64JSON(file.raw),
+        })
+        transactionByPath.set(path, newConf)
+      }
+
+      this.state.client.state.targets_version = targets.signed.version
+      this.state.client.state.backend_client_state = targets.signed.custom.opaque_backend_state
+    }
+
+    if (toUnapply.length || toApply.length || toModify.length) {
+      const transaction = createUpdateTransaction(
+        { toUnapply, toApply, toModify },
+        transactionHandledPaths,
+        transactionOutcomes
+      )
+
+      if (this.#batchHandlers.size) {
+        for (const [handler, products] of this.#batchHandlers) {
+          const transactionView = filterTransactionByProducts(transaction, products)
+          if (transactionView.toUnapply.length || transactionView.toApply.length || transactionView.toModify.length) {
+            handler(transactionView)
+          }
+        }
+      }
+
+      applyOutcomes(transactionByPath, transactionOutcomes)
+
+      this.dispatch(toUnapply, 'unapply', transactionHandledPaths)
+      this.dispatch(toApply, 'apply', transactionHandledPaths)
+      this.dispatch(toModify, 'modify', transactionHandledPaths)
+
+      this.state.cached_target_files = /** @type {RcCachedTargetFile[]} */ ([])
+
+      for (const conf of this.appliedConfigs.values()) {
+        const hashes = []
+        for (const hash of Object.entries(conf.hashes)) {
+          hashes.push({ algorithm: hash[0], hash: hash[1] })
+        }
+        this.state.cached_target_files.push({
+          path: conf.path,
+          length: conf.length,
+          hashes,
+        })
+      }
+    }
+  }
+
+  /**
+   * Dispatch a list of config changes to per-product handlers, skipping any paths
+   * marked as handled by a batch handler.
+   *
+   * @param {RcConfigState[]} list
+   * @param {'apply' | 'modify' | 'unapply'} action
+   * @param {Set<string>} handledPaths
+   */
+  dispatch (list, action, handledPaths) {
+    for (const item of list) {
+      if (!handledPaths.has(item.path)) {
+        this.#callHandlerFor(action, item)
+      }
+
+      if (action === 'unapply') {
+        this.appliedConfigs.delete(item.path)
+      } else {
+        this.appliedConfigs.set(item.path, item)
+      }
+    }
+  }
+
+  /**
+   * @param {'apply' | 'modify' | 'unapply'} action
+   * @param {RcConfigState} item
+   */
+  #callHandlerFor (action, item) {
+    // in case the item was already handled by a batch hook
+    if (item.apply_state !== UNACKNOWLEDGED && action !== 'unapply') return
+
+    const handler = this.#handlers.get(item.product)
+
+    if (!handler) return
+
+    try {
+      if (supportsAckCallback(handler)) {
+        // If the handler accepts an `ack` callback, expect that to be called and set `apply_state` accordingly
+        // TODO: do we want to pass old and new config ?
+        handler(action, item.file, item.id, (err) => {
+          if (err) {
+            item.apply_state = ERROR
+            item.apply_error = err.toString()
+          } else if (item.apply_state !== ERROR) {
+            item.apply_state = ACKNOWLEDGED
+          }
+        })
+      } else {
+        // If the handler doesn't accept an `ack` callback, assume `apply_state` is `ACKNOWLEDGED`,
+        // unless it returns a promise, in which case we wait for the promise to be resolved or rejected.
+        // TODO: do we want to pass old and new config ?
+        const result = handler(action, item.file, item.id)
+        if (result instanceof Promise) {
+          result.then(
+            () => { item.apply_state = ACKNOWLEDGED },
+            (err) => {
+              item.apply_state = ERROR
+              item.apply_error = err.toString()
+            }
+          )
+        } else {
+          item.apply_state = ACKNOWLEDGED
+        }
+      }
+    } catch (err) {
+      item.apply_state = ERROR
+      item.apply_error = err.toString()
+    }
+  }
+}
+
+/**
+ * Remote Config “applied config” state tracked by the RC manager.
+ * This is the mutable shape stored in `this.appliedConfigs` and passed to per-product handlers.
+ *
+ * @typedef {object} RcConfigState
+ * @property {string} path
+ * @property {string} product
+ * @property {string} id
+ * @property {number} version
+ * @property {unknown} file
+ * @property {number} apply_state
+ * @property {string} apply_error
+ * @property {number} length
+ * @property {Record<string, string>} hashes
+ */
+
+/**
+ * Target file metadata cached in `state.cached_target_files` and sent back to the agent.
+ *
+ * @typedef {object} RcCachedTargetFile
+ * @property {string} path
+ * @property {number} length
+ * @property {Array<{algorithm: string, hash: string}>} hashes
+ */
+
+/**
+ * Remote Config batch update transaction passed to batch handlers registered via
+ * `RemoteConfig.setBatchHandler()`.
+ *
+ * @typedef {object} RcBatchUpdateTransaction
+ * @property {RcConfigState[]} toUnapply
+ * @property {RcConfigState[]} toApply
+ * @property {RcConfigState[]} toModify
+ * @property {(path: string) => void} ack
+ * @property {(path: string, err: unknown) => void} error
+ */
+
+/**
+ * Create an immutable "view" of the batch changes and attach explicit outcome reporting.
+ *
+ * @param {{toUnapply: RcConfigState[], toApply: RcConfigState[], toModify: RcConfigState[]}} changes
+ * @param {Set<string>} handledPaths
+ * @param {Map<string, {state: number, error: string}>} outcomes
+ * @returns {RcBatchUpdateTransaction}
+ */
+function createUpdateTransaction ({ toUnapply, toApply, toModify }, handledPaths, outcomes) {
+  return {
+    toUnapply,
+    toApply,
+    toModify,
+    ack (path) {
+      outcomes.set(path, { state: ACKNOWLEDGED, error: '' })
+      handledPaths.add(path)
+    },
+    error (path, err) {
+      outcomes.set(path, { state: ERROR, error: err ? err.toString() : 'Error' })
+      handledPaths.add(path)
+    },
+  }
+}
+
+/**
+ * Create a filtered "view" of the transaction for a given product set, while preserving
+ * the outcome methods (ack/error).
+ *
+ * @param {RcBatchUpdateTransaction} transaction
+ * @param {Set<string>} products
+ * @returns {RcBatchUpdateTransaction}
+ */
+function filterTransactionByProducts (transaction, products) {
+  const toUnapply = []
+  const toApply = []
+  const toModify = []
+
+  for (const item of transaction.toUnapply) {
+    if (products.has(item.product)) toUnapply.push(item)
+  }
+
+  for (const item of transaction.toApply) {
+    if (products.has(item.product)) toApply.push(item)
+  }
+
+  for (const item of transaction.toModify) {
+    if (products.has(item.product)) toModify.push(item)
+  }
+
+  return {
+    toUnapply,
+    toApply,
+    toModify,
+    ack: transaction.ack,
+    error: transaction.error,
+  }
+}
+
+function applyOutcomes (byPath, outcomes) {
+  for (const [path, outcome] of outcomes) {
+    const item = byPath.get(path)
+    if (item) {
+      item.apply_state = outcome.state
+      item.apply_error = outcome.error
+    }
+  }
+}
+
+function fromBase64JSON (str) {
+  if (!str) return null
+
+  return JSON.parse(Buffer.from(str, 'base64').toString())
+}
+
+const configPathRegex = /^(?:datadog\/\d+|employee)\/([^/]+)\/([^/]+)\/[^/]+$/
+
+function parseConfigPath (configPath) {
+  const match = configPathRegex.exec(configPath)
+
+  if (!match || !match[1] || !match[2]) {
+    throw new Error(`Unable to parse path ${configPath}`)
+  }
+
+  return {
+    product: match[1],
+    id: match[2],
+  }
+}
+
+function supportsAckCallback (handler) {
+  if (kSupportsAckCallback in handler) return handler[kSupportsAckCallback]
+
+  const numOfArgs = handler.length
+  let result = false
+
+  if (numOfArgs >= 4) {
+    result = true
+  } else if (numOfArgs !== 0) {
+    const source = handler.toString()
+    result = source.slice(0, source.indexOf(')')).includes('...')
+  }
+
+  handler[kSupportsAckCallback] = result
+
+  return result
+}
+
+module.exports = RemoteConfig

package/packages/dd-trace/src/remote_config/scheduler.js

@@ -0,0 +1,30 @@
+'use strict'
+
+class Scheduler {
+  _timer = null
+
+  constructor (callback, interval) {
+    this._callback = callback
+    this._interval = interval
+  }
+
+  start () {
+    if (this._timer) return
+
+    this.runAfterDelay(0)
+  }
+
+  runAfterDelay (interval = this._interval) {
+    this._timer = setTimeout(this._callback, interval, () => this.runAfterDelay())
+
+    this._timer.unref()
+  }
+
+  stop () {
+    clearTimeout(this._timer)
+
+    this._timer = null
+  }
+}
+
+module.exports = Scheduler

package/packages/dd-trace/src/require-package-json.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const path = require('path')
+const fs = require('fs')
+
+/**
+ * Given a package name and a module to start from, find a package's
+ * package.json file, parses it, and returns the result.
+ *
+ * Equivalent to require(`${name}/package.json`) prior to Node 12.
+ *
+ * @typedef { import('module').Module } Module
+ * @param {string} name
+ * @param {Module} module
+ * @returns {object} The parsed package.json
+ */
+function requirePackageJson (name, module) {
+  if (path.isAbsolute(name)) {
+    const candidate = path.join(name, 'package.json')
+    return JSON.parse(fs.readFileSync(candidate, 'utf8'))
+  }
+  for (const modulePath of module.paths) {
+    const candidate = path.join(modulePath, name, 'package.json')
+    try {
+      return JSON.parse(fs.readFileSync(candidate, 'utf8'))
+    } catch {
+      continue
+    }
+  }
+  throw new Error(`could not find ${name}/package.json`)
+}
+
+module.exports = requirePackageJson