dd-trace 4.38.1 → 4.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE-3rdparty.csv +0 -3
  2. package/README.md +8 -18
  3. package/ci/init.js +7 -0
  4. package/ext/exporters.d.ts +1 -0
  5. package/ext/exporters.js +2 -1
  6. package/ext/tags.d.ts +1 -0
  7. package/ext/tags.js +1 -0
  8. package/index.d.ts +18 -3
  9. package/initialize.mjs +52 -0
  10. package/package.json +9 -12
  11. package/packages/datadog-instrumentations/src/amqplib.js +5 -2
  12. package/packages/datadog-instrumentations/src/apollo-server-core.js +0 -1
  13. package/packages/datadog-instrumentations/src/apollo-server.js +0 -1
  14. package/packages/datadog-instrumentations/src/body-parser.js +0 -1
  15. package/packages/datadog-instrumentations/src/check_require_cache.js +67 -5
  16. package/packages/datadog-instrumentations/src/cookie-parser.js +0 -1
  17. package/packages/datadog-instrumentations/src/express.js +0 -1
  18. package/packages/datadog-instrumentations/src/graphql.js +0 -2
  19. package/packages/datadog-instrumentations/src/helpers/hooks.js +1 -0
  20. package/packages/datadog-instrumentations/src/helpers/register.js +5 -2
  21. package/packages/datadog-instrumentations/src/http/server.js +0 -1
  22. package/packages/datadog-instrumentations/src/jest.js +6 -3
  23. package/packages/datadog-instrumentations/src/mocha/common.js +48 -0
  24. package/packages/datadog-instrumentations/src/mocha/main.js +487 -0
  25. package/packages/datadog-instrumentations/src/mocha/utils.js +306 -0
  26. package/packages/datadog-instrumentations/src/mocha/worker.js +51 -0
  27. package/packages/datadog-instrumentations/src/mocha.js +4 -673
  28. package/packages/datadog-instrumentations/src/openai.js +188 -17
  29. package/packages/datadog-instrumentations/src/playwright.js +4 -3
  30. package/packages/datadog-instrumentations/src/router.js +1 -1
  31. package/packages/datadog-instrumentations/src/selenium.js +13 -6
  32. package/packages/datadog-plugin-graphql/src/resolve.js +4 -0
  33. package/packages/datadog-plugin-mocha/src/index.js +82 -8
  34. package/packages/datadog-plugin-next/src/index.js +1 -2
  35. package/packages/datadog-plugin-openai/src/index.js +219 -73
  36. package/packages/dd-trace/src/appsec/addresses.js +4 -2
  37. package/packages/dd-trace/src/appsec/blocking.js +19 -25
  38. package/packages/dd-trace/src/appsec/channels.js +2 -1
  39. package/packages/dd-trace/src/appsec/graphql.js +10 -3
  40. package/packages/dd-trace/src/appsec/index.js +11 -4
  41. package/packages/dd-trace/src/appsec/rasp.js +35 -0
  42. package/packages/dd-trace/src/appsec/remote_config/capabilities.js +2 -1
  43. package/packages/dd-trace/src/appsec/remote_config/index.js +1 -0
  44. package/packages/dd-trace/src/appsec/rule_manager.js +15 -25
  45. package/packages/dd-trace/src/appsec/sdk/user_blocking.js +2 -5
  46. package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +3 -1
  47. package/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js +5 -1
  48. package/packages/dd-trace/src/config.js +97 -22
  49. package/packages/dd-trace/src/constants.js +2 -0
  50. package/packages/dd-trace/src/encode/0.4.js +47 -8
  51. package/packages/dd-trace/src/exporter.js +1 -0
  52. package/packages/dd-trace/src/flare/file.js +44 -0
  53. package/packages/dd-trace/src/flare/index.js +98 -0
  54. package/packages/dd-trace/src/log/channels.js +54 -29
  55. package/packages/dd-trace/src/log/writer.js +7 -49
  56. package/packages/dd-trace/src/opentelemetry/span.js +8 -0
  57. package/packages/dd-trace/src/opentracing/propagation/text_map.js +57 -12
  58. package/packages/dd-trace/src/plugins/index.js +1 -0
  59. package/packages/dd-trace/src/plugins/util/ip_extractor.js +1 -1
  60. package/packages/dd-trace/src/plugins/util/test.js +6 -0
  61. package/packages/dd-trace/src/priority_sampler.js +8 -4
  62. package/packages/dd-trace/src/profiler.js +2 -1
  63. package/packages/dd-trace/src/profiling/config.js +1 -0
  64. package/packages/dd-trace/src/profiling/profiler.js +1 -1
  65. package/packages/dd-trace/src/profiling/{ssi-telemetry.js → ssi-heuristics.js} +64 -36
  66. package/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js +4 -9
  67. package/packages/dd-trace/src/proxy.js +49 -15
  68. package/packages/dd-trace/src/ritm.js +13 -1
  69. package/packages/dd-trace/src/sampling_rule.js +2 -1
  70. package/packages/dd-trace/src/startup-log.js +19 -15
  71. package/packages/dd-trace/src/telemetry/index.js +6 -2
  72. package/packages/dd-trace/src/tracer.js +3 -0
  73. package/packages/dd-trace/src/plugins/util/ip_blocklist.js +0 -51
@@ -15,6 +15,14 @@ const RE_TAB = /\t/g
15
15
  // TODO: In the future we should refactor config.js to make it requirable
16
16
  let MAX_TEXT_LEN = 128
17
17
 
18
+ let encodingForModel
19
+ try {
20
+ // eslint-disable-next-line import/no-extraneous-dependencies
21
+ encodingForModel = require('tiktoken').encoding_for_model
22
+ } catch {
23
+ // we will use token count estimations in this case
24
+ }
25
+
18
26
  class OpenApiPlugin extends TracingPlugin {
19
27
  static get id () { return 'openai' }
20
28
  static get operation () { return 'request' }
@@ -112,6 +120,10 @@ class OpenApiPlugin extends TracingPlugin {
112
120
  }
113
121
  }
114
122
 
123
+ if (payload.stream) {
124
+ tags['openai.request.stream'] = payload.stream
125
+ }
126
+
115
127
  switch (methodName) {
116
128
  case 'createFineTune':
117
129
  case 'fine_tuning.jobs.create':
@@ -175,12 +187,21 @@ class OpenApiPlugin extends TracingPlugin {
175
187
  span.addTags(tags)
176
188
  }
177
189
 
178
- finish ({ headers, body, method, path }) {
179
- if (headers.constructor.name === 'Headers') {
180
- headers = Object.fromEntries(headers)
190
+ finish (response) {
191
+ const span = this.activeSpan
192
+ const error = !!span.context()._tags.error
193
+
194
+ let headers, body, method, path
195
+ if (!error) {
196
+ headers = response.headers
197
+ body = response.body
198
+ method = response.method
199
+ path = response.path
181
200
  }
182
201
 
183
- const span = this.activeSpan
202
+ if (!error && headers?.constructor.name === 'Headers') {
203
+ headers = Object.fromEntries(headers)
204
+ }
184
205
  const methodName = span._spanContext._tags['resource.name']
185
206
 
186
207
  body = coerceResponseBody(body, methodName)
@@ -188,88 +209,98 @@ class OpenApiPlugin extends TracingPlugin {
188
209
  const fullStore = storage.getStore()
189
210
  const store = fullStore.openai
190
211
 
191
- if (path.startsWith('https://') || path.startsWith('http://')) {
212
+ if (!error && (path.startsWith('https://') || path.startsWith('http://'))) {
192
213
  // basic checking for if the path was set as a full URL
193
214
  // not using a full regex as it will likely be "https://api.openai.com/..."
194
215
  path = new URL(path).pathname
195
216
  }
196
217
  const endpoint = lookupOperationEndpoint(methodName, path)
197
218
 
198
- const tags = {
199
- 'openai.request.endpoint': endpoint,
200
- 'openai.request.method': method.toUpperCase(),
219
+ const tags = error
220
+ ? {}
221
+ : {
222
+ 'openai.request.endpoint': endpoint,
223
+ 'openai.request.method': method.toUpperCase(),
201
224
 
202
- 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints
203
- 'openai.organization.name': headers['openai-organization'],
225
+ 'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints
226
+ 'openai.organization.name': headers['openai-organization'],
204
227
 
205
- 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined
206
- 'openai.response.id': body.id, // common creation value, numeric epoch
207
- 'openai.response.deleted': body.deleted, // common boolean field in delete responses
228
+ 'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined
229
+ 'openai.response.id': body.id, // common creation value, numeric epoch
230
+ 'openai.response.deleted': body.deleted, // common boolean field in delete responses
208
231
 
209
- // The OpenAI API appears to use both created and created_at in different places
210
- // Here we're conciously choosing to surface this inconsistency instead of normalizing
211
- 'openai.response.created': body.created,
212
- 'openai.response.created_at': body.created_at
213
- }
232
+ // The OpenAI API appears to use both created and created_at in different places
233
+ // Here we're conciously choosing to surface this inconsistency instead of normalizing
234
+ 'openai.response.created': body.created,
235
+ 'openai.response.created_at': body.created_at
236
+ }
214
237
 
215
238
  responseDataExtractionByMethod(methodName, tags, body, store)
216
239
  span.addTags(tags)
217
240
 
218
241
  super.finish()
219
- this.sendLog(methodName, span, tags, store, false)
220
- this.sendMetrics(headers, body, endpoint, span._duration)
242
+ this.sendLog(methodName, span, tags, store, error)
243
+ this.sendMetrics(headers, body, endpoint, span._duration, error, tags)
221
244
  }
222
245
 
223
- error (...args) {
224
- super.error(...args)
225
-
226
- const span = this.activeSpan
227
- const methodName = span._spanContext._tags['resource.name']
228
-
229
- const fullStore = storage.getStore()
230
- const store = fullStore.openai
231
-
232
- // We don't know most information about the request when it fails
246
+ sendMetrics (headers, body, endpoint, duration, error, spanTags) {
247
+ const tags = [`error:${Number(!!error)}`]
248
+ if (error) {
249
+ this.metrics.increment('openai.request.error', 1, tags)
250
+ } else {
251
+ tags.push(`org:${headers['openai-organization']}`)
252
+ tags.push(`endpoint:${endpoint}`) // just "/v1/models", no method
253
+ tags.push(`model:${headers['openai-model'] || body.model}`)
254
+ }
233
255
 
234
- const tags = ['error:1']
235
- this.metrics.distribution('openai.request.duration', span._duration * 1000, tags)
236
- this.metrics.increment('openai.request.error', 1, tags)
256
+ this.metrics.distribution('openai.request.duration', duration * 1000, tags)
237
257
 
238
- this.sendLog(methodName, span, {}, store, true)
239
- }
258
+ const promptTokens = spanTags['openai.response.usage.prompt_tokens']
259
+ const promptTokensEstimated = spanTags['openai.response.usage.prompt_tokens_estimated']
240
260
 
241
- sendMetrics (headers, body, endpoint, duration) {
242
- const tags = [
243
- `org:${headers['openai-organization']}`,
244
- `endpoint:${endpoint}`, // just "/v1/models", no method
245
- `model:${headers['openai-model']}`,
246
- 'error:0'
247
- ]
261
+ const completionTokens = spanTags['openai.response.usage.completion_tokens']
262
+ const completionTokensEstimated = spanTags['openai.response.usage.completion_tokens_estimated']
248
263
 
249
- this.metrics.distribution('openai.request.duration', duration * 1000, tags)
264
+ if (!error) {
265
+ if (promptTokensEstimated) {
266
+ this.metrics.distribution(
267
+ 'openai.tokens.prompt', promptTokens, [...tags, 'openai.estimated:true'])
268
+ } else {
269
+ this.metrics.distribution('openai.tokens.prompt', promptTokens, tags)
270
+ }
271
+ if (completionTokensEstimated) {
272
+ this.metrics.distribution(
273
+ 'openai.tokens.completion', completionTokens, [...tags, 'openai.estimated:true'])
274
+ } else {
275
+ this.metrics.distribution('openai.tokens.completion', completionTokens, tags)
276
+ }
250
277
 
251
- if (body && body.usage) {
252
- const promptTokens = body.usage.prompt_tokens
253
- const completionTokens = body.usage.completion_tokens
254
- this.metrics.distribution('openai.tokens.prompt', promptTokens, tags)
255
- this.metrics.distribution('openai.tokens.completion', completionTokens, tags)
256
- this.metrics.distribution('openai.tokens.total', promptTokens + completionTokens, tags)
278
+ if (promptTokensEstimated || completionTokensEstimated) {
279
+ this.metrics.distribution(
280
+ 'openai.tokens.total', promptTokens + completionTokens, [...tags, 'openai.estimated:true'])
281
+ } else {
282
+ this.metrics.distribution('openai.tokens.total', promptTokens + completionTokens, tags)
283
+ }
257
284
  }
258
285
 
259
- if (headers['x-ratelimit-limit-requests']) {
260
- this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags)
261
- }
286
+ if (headers) {
287
+ if (headers['x-ratelimit-limit-requests']) {
288
+ this.metrics.gauge('openai.ratelimit.requests', Number(headers['x-ratelimit-limit-requests']), tags)
289
+ }
262
290
 
263
- if (headers['x-ratelimit-remaining-requests']) {
264
- this.metrics.gauge('openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags)
265
- }
291
+ if (headers['x-ratelimit-remaining-requests']) {
292
+ this.metrics.gauge(
293
+ 'openai.ratelimit.remaining.requests', Number(headers['x-ratelimit-remaining-requests']), tags
294
+ )
295
+ }
266
296
 
267
- if (headers['x-ratelimit-limit-tokens']) {
268
- this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags)
269
- }
297
+ if (headers['x-ratelimit-limit-tokens']) {
298
+ this.metrics.gauge('openai.ratelimit.tokens', Number(headers['x-ratelimit-limit-tokens']), tags)
299
+ }
270
300
 
271
- if (headers['x-ratelimit-remaining-tokens']) {
272
- this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags)
301
+ if (headers['x-ratelimit-remaining-tokens']) {
302
+ this.metrics.gauge('openai.ratelimit.remaining.tokens', Number(headers['x-ratelimit-remaining-tokens']), tags)
303
+ }
273
304
  }
274
305
  }
275
306
 
@@ -287,6 +318,89 @@ class OpenApiPlugin extends TracingPlugin {
287
318
  }
288
319
  }
289
320
 
321
+ function countPromptTokens (methodName, payload, model) {
322
+ let promptTokens = 0
323
+ let promptEstimated = false
324
+ if (methodName === 'chat.completions.create') {
325
+ const messages = payload.messages
326
+ for (const message of messages) {
327
+ const content = message.content
328
+ const { tokens, estimated } = countTokens(content, model)
329
+ promptTokens += tokens
330
+ promptEstimated = estimated
331
+ }
332
+ } else if (methodName === 'completions.create') {
333
+ let prompt = payload.prompt
334
+ if (!Array.isArray(prompt)) prompt = [prompt]
335
+
336
+ for (const p of prompt) {
337
+ const { tokens, estimated } = countTokens(p, model)
338
+ promptTokens += tokens
339
+ promptEstimated = estimated
340
+ }
341
+ }
342
+
343
+ return { promptTokens, promptEstimated }
344
+ }
345
+
346
+ function countCompletionTokens (body, model) {
347
+ let completionTokens = 0
348
+ let completionEstimated = false
349
+ if (body?.choices) {
350
+ for (const choice of body.choices) {
351
+ const message = choice.message || choice.delta // delta for streamed responses
352
+ const text = choice.text
353
+ const content = text || message?.content
354
+
355
+ const { tokens, estimated } = countTokens(content, model)
356
+ completionTokens += tokens
357
+ completionEstimated = estimated
358
+ }
359
+ }
360
+
361
+ return { completionTokens, completionEstimated }
362
+ }
363
+
364
+ function countTokens (content, model) {
365
+ if (encodingForModel) {
366
+ try {
367
+ // try using tiktoken if it was available
368
+ const encoder = encodingForModel(model)
369
+ const tokens = encoder.encode(content).length
370
+ encoder.free()
371
+ return { tokens, estimated: false }
372
+ } catch {
373
+ // possible errors from tiktoken:
374
+ // * model not available for token counts
375
+ // * issue encoding content
376
+ }
377
+ }
378
+
379
+ return {
380
+ tokens: estimateTokens(content),
381
+ estimated: true
382
+ }
383
+ }
384
+
385
+ // If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens
386
+ // Approximate using the following assumptions:
387
+ // * English text
388
+ // * 1 token ~= 4 chars
389
+ // * 1 token ~= ¾ words
390
+ function estimateTokens (content) {
391
+ let estimatedTokens = 0
392
+ if (typeof content === 'string') {
393
+ const estimation1 = content.length / 4
394
+
395
+ const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g)
396
+ const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string
397
+ estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2)
398
+ } else if (Array.isArray(content) && typeof content[0] === 'number') {
399
+ estimatedTokens = content.length
400
+ }
401
+ return estimatedTokens
402
+ }
403
+
290
404
  function createEditRequestExtraction (tags, payload, store) {
291
405
  const instruction = payload.instruction
292
406
  tags['openai.request.instruction'] = instruction
@@ -298,7 +412,8 @@ function retrieveModelRequestExtraction (tags, payload) {
298
412
  }
299
413
 
300
414
  function createChatCompletionRequestExtraction (tags, payload, store) {
301
- if (!defensiveArrayLength(payload.messages)) return
415
+ const messages = payload.messages
416
+ if (!defensiveArrayLength(messages)) return
302
417
 
303
418
  store.messages = payload.messages
304
419
  for (let i = 0; i < payload.messages.length; i++) {
@@ -344,7 +459,7 @@ function responseDataExtractionByMethod (methodName, tags, body, store) {
344
459
  case 'chat.completions.create':
345
460
  case 'createEdit':
346
461
  case 'edits.create':
347
- commonCreateResponseExtraction(tags, body, store)
462
+ commonCreateResponseExtraction(tags, body, store, methodName)
348
463
  break
349
464
 
350
465
  case 'listFiles':
@@ -580,8 +695,8 @@ function createModerationResponseExtraction (tags, body) {
580
695
  }
581
696
 
582
697
  // createCompletion, createChatCompletion, createEdit
583
- function commonCreateResponseExtraction (tags, body, store) {
584
- usageExtraction(tags, body)
698
+ function commonCreateResponseExtraction (tags, body, store, methodName) {
699
+ usageExtraction(tags, body, methodName)
585
700
 
586
701
  if (!body.choices) return
587
702
 
@@ -600,18 +715,20 @@ function commonCreateResponseExtraction (tags, body, store) {
600
715
  tags[`openai.response.choices.${choiceIdx}.text`] = truncateText(choice.text)
601
716
 
602
717
  // createChatCompletion only
603
- if (choice.message) {
604
- const message = choice.message
718
+ const message = choice.message || choice.delta // delta for streamed responses
719
+ if (message) {
605
720
  tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role
606
721
  tags[`openai.response.choices.${choiceIdx}.message.content`] = truncateText(message.content)
607
722
  tags[`openai.response.choices.${choiceIdx}.message.name`] = truncateText(message.name)
608
723
  if (message.tool_calls) {
609
724
  const toolCalls = message.tool_calls
610
725
  for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) {
611
- tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.name`] =
726
+ tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.name`] =
612
727
  toolCalls[toolIdx].function.name
613
- tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.arguments`] =
728
+ tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.arguments`] =
614
729
  toolCalls[toolIdx].function.arguments
730
+ tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.id`] =
731
+ toolCalls[toolIdx].id
615
732
  }
616
733
  }
617
734
  }
@@ -619,11 +736,40 @@ function commonCreateResponseExtraction (tags, body, store) {
619
736
  }
620
737
 
621
738
  // createCompletion, createChatCompletion, createEdit, createEmbedding
622
- function usageExtraction (tags, body) {
623
- if (typeof body.usage !== 'object' || !body.usage) return
624
- tags['openai.response.usage.prompt_tokens'] = body.usage.prompt_tokens
625
- tags['openai.response.usage.completion_tokens'] = body.usage.completion_tokens
626
- tags['openai.response.usage.total_tokens'] = body.usage.total_tokens
739
+ function usageExtraction (tags, body, methodName) {
740
+ let promptTokens = 0
741
+ let completionTokens = 0
742
+ let totalTokens = 0
743
+ if (body && body.usage) {
744
+ promptTokens = body.usage.prompt_tokens
745
+ completionTokens = body.usage.completion_tokens
746
+ totalTokens = body.usage.total_tokens
747
+ } else if (['chat.completions.create', 'completions.create'].includes(methodName)) {
748
+ // estimate tokens based on method name for completions and chat completions
749
+ const { model } = body
750
+ let promptEstimated = false
751
+ let completionEstimated = false
752
+
753
+ // prompt tokens
754
+ const payload = storage.getStore().openai
755
+ const promptTokensCount = countPromptTokens(methodName, payload, model)
756
+ promptTokens = promptTokensCount.promptTokens
757
+ promptEstimated = promptTokensCount.promptEstimated
758
+
759
+ // completion tokens
760
+ const completionTokensCount = countCompletionTokens(body, model)
761
+ completionTokens = completionTokensCount.completionTokens
762
+ completionEstimated = completionTokensCount.completionEstimated
763
+
764
+ // total tokens
765
+ totalTokens = promptTokens + completionTokens
766
+ if (promptEstimated) tags['openai.response.usage.prompt_tokens_estimated'] = true
767
+ if (completionEstimated) tags['openai.response.usage.completion_tokens_estimated'] = true
768
+ }
769
+
770
+ if (promptTokens) tags['openai.response.usage.prompt_tokens'] = promptTokens
771
+ if (completionTokens) tags['openai.response.usage.completion_tokens'] = completionTokens
772
+ if (totalTokens) tags['openai.response.usage.total_tokens'] = totalTokens
627
773
  }
628
774
 
629
775
  function truncateApiKey (apiKey) {
@@ -15,10 +15,12 @@ module.exports = {
15
15
  HTTP_INCOMING_GRAPHQL_RESOLVERS: 'graphql.server.all_resolvers',
16
16
  HTTP_INCOMING_GRAPHQL_RESOLVER: 'graphql.server.resolver',
17
17
 
18
- HTTP_OUTGOING_BODY: 'server.response.body',
18
+ HTTP_INCOMING_RESPONSE_BODY: 'server.response.body',
19
19
 
20
20
  HTTP_CLIENT_IP: 'http.client_ip',
21
21
 
22
22
  USER_ID: 'usr.id',
23
- WAF_CONTEXT_PROCESSOR: 'waf.context.processor'
23
+ WAF_CONTEXT_PROCESSOR: 'waf.context.processor',
24
+
25
+ HTTP_OUTGOING_URL: 'server.io.net.url'
24
26
  }
@@ -8,7 +8,6 @@ const detectedSpecificEndpoints = {}
8
8
  let templateHtml = blockedTemplates.html
9
9
  let templateJson = blockedTemplates.json
10
10
  let templateGraphqlJson = blockedTemplates.graphqlJson
11
- let blockingConfiguration
12
11
 
13
12
  const specificBlockingTypes = {
14
13
  GRAPHQL: 'graphql'
@@ -22,13 +21,13 @@ function addSpecificEndpoint (method, url, type) {
22
21
  detectedSpecificEndpoints[getSpecificKey(method, url)] = type
23
22
  }
24
23
 
25
- function getBlockWithRedirectData (rootSpan) {
26
- let statusCode = blockingConfiguration.parameters.status_code
24
+ function getBlockWithRedirectData (rootSpan, actionParameters) {
25
+ let statusCode = actionParameters.status_code
27
26
  if (!statusCode || statusCode < 300 || statusCode >= 400) {
28
27
  statusCode = 303
29
28
  }
30
29
  const headers = {
31
- Location: blockingConfiguration.parameters.location
30
+ Location: actionParameters.location
32
31
  }
33
32
 
34
33
  rootSpan.addTags({
@@ -48,10 +47,9 @@ function getSpecificBlockingData (type) {
48
47
  }
49
48
  }
50
49
 
51
- function getBlockWithContentData (req, specificType, rootSpan) {
50
+ function getBlockWithContentData (req, specificType, rootSpan, actionParameters) {
52
51
  let type
53
52
  let body
54
- let statusCode
55
53
 
56
54
  const specificBlockingType = specificType || detectedSpecificEndpoints[getSpecificKey(req.method, req.url)]
57
55
  if (specificBlockingType) {
@@ -64,7 +62,7 @@ function getBlockWithContentData (req, specificType, rootSpan) {
64
62
  // parse the Accept header, ex: Accept: text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8
65
63
  const accept = req.headers.accept?.split(',').map((str) => str.split(';', 1)[0].trim())
66
64
 
67
- if (!blockingConfiguration || blockingConfiguration.parameters.type === 'auto') {
65
+ if (!actionParameters || actionParameters.type === 'auto') {
68
66
  if (accept?.includes('text/html') && !accept.includes('application/json')) {
69
67
  type = 'text/html; charset=utf-8'
70
68
  body = templateHtml
@@ -73,7 +71,7 @@ function getBlockWithContentData (req, specificType, rootSpan) {
73
71
  body = templateJson
74
72
  }
75
73
  } else {
76
- if (blockingConfiguration.parameters.type === 'html') {
74
+ if (actionParameters.type === 'html') {
77
75
  type = 'text/html; charset=utf-8'
78
76
  body = templateHtml
79
77
  } else {
@@ -83,11 +81,7 @@ function getBlockWithContentData (req, specificType, rootSpan) {
83
81
  }
84
82
  }
85
83
 
86
- if (blockingConfiguration?.type === 'block_request' && blockingConfiguration.parameters.status_code) {
87
- statusCode = blockingConfiguration.parameters.status_code
88
- } else {
89
- statusCode = 403
90
- }
84
+ const statusCode = actionParameters?.status_code || 403
91
85
 
92
86
  const headers = {
93
87
  'Content-Type': type,
@@ -101,27 +95,31 @@ function getBlockWithContentData (req, specificType, rootSpan) {
101
95
  return { body, statusCode, headers }
102
96
  }
103
97
 
104
- function getBlockingData (req, specificType, rootSpan) {
105
- if (blockingConfiguration?.type === 'redirect_request' && blockingConfiguration.parameters.location) {
106
- return getBlockWithRedirectData(rootSpan)
98
+ function getBlockingData (req, specificType, rootSpan, actionParameters) {
99
+ if (actionParameters?.location) {
100
+ return getBlockWithRedirectData(rootSpan, actionParameters)
107
101
  } else {
108
- return getBlockWithContentData(req, specificType, rootSpan)
102
+ return getBlockWithContentData(req, specificType, rootSpan, actionParameters)
109
103
  }
110
104
  }
111
105
 
112
- function block (req, res, rootSpan, abortController, type) {
106
+ function block (req, res, rootSpan, abortController, actionParameters) {
113
107
  if (res.headersSent) {
114
108
  log.warn('Cannot send blocking response when headers have already been sent')
115
109
  return
116
110
  }
117
111
 
118
- const { body, headers, statusCode } = getBlockingData(req, type, rootSpan)
112
+ const { body, headers, statusCode } = getBlockingData(req, null, rootSpan, actionParameters)
119
113
 
120
114
  res.writeHead(statusCode, headers).end(body)
121
115
 
122
116
  abortController?.abort()
123
117
  }
124
118
 
119
+ function getBlockingAction (actions) {
120
+ return actions?.block_request || actions?.redirect_request
121
+ }
122
+
125
123
  function setTemplates (config) {
126
124
  if (config.appsec.blockedTemplateHtml) {
127
125
  templateHtml = config.appsec.blockedTemplateHtml
@@ -142,15 +140,11 @@ function setTemplates (config) {
142
140
  }
143
141
  }
144
142
 
145
- function updateBlockingConfiguration (newBlockingConfiguration) {
146
- blockingConfiguration = newBlockingConfiguration
147
- }
148
-
149
143
  module.exports = {
150
144
  addSpecificEndpoint,
151
145
  block,
152
146
  specificBlockingTypes,
153
147
  getBlockingData,
154
- setTemplates,
155
- updateBlockingConfiguration
148
+ getBlockingAction,
149
+ setTemplates
156
150
  }
@@ -17,5 +17,6 @@ module.exports = {
17
17
  setCookieChannel: dc.channel('datadog:iast:set-cookie'),
18
18
  nextBodyParsed: dc.channel('apm:next:body-parsed'),
19
19
  nextQueryParsed: dc.channel('apm:next:query-parsed'),
20
- responseBody: dc.channel('datadog:express:response:json:start')
20
+ responseBody: dc.channel('datadog:express:response:json:start'),
21
+ httpClientRequestStart: dc.channel('apm:http:client:request:start')
21
22
  }
@@ -1,7 +1,12 @@
1
1
  'use strict'
2
2
 
3
3
  const { storage } = require('../../../datadog-core')
4
- const { addSpecificEndpoint, specificBlockingTypes, getBlockingData } = require('./blocking')
4
+ const {
5
+ addSpecificEndpoint,
6
+ specificBlockingTypes,
7
+ getBlockingData,
8
+ getBlockingAction
9
+ } = require('./blocking')
5
10
  const waf = require('./waf')
6
11
  const addresses = require('./addresses')
7
12
  const web = require('../plugins/util/web')
@@ -32,10 +37,12 @@ function onGraphqlStartResolve ({ context, resolverInfo }) {
32
37
  if (!resolverInfo || typeof resolverInfo !== 'object') return
33
38
 
34
39
  const actions = waf.run({ ephemeral: { [addresses.HTTP_INCOMING_GRAPHQL_RESOLVER]: resolverInfo } }, req)
35
- if (actions?.includes('block')) {
40
+ const blockingAction = getBlockingAction(actions)
41
+ if (blockingAction) {
36
42
  const requestData = graphqlRequestData.get(req)
37
43
  if (requestData?.isInGraphqlRequest) {
38
44
  requestData.blocked = true
45
+ requestData.wafAction = blockingAction
39
46
  context?.abortController?.abort()
40
47
  }
41
48
  }
@@ -87,7 +94,7 @@ function beforeWriteApolloGraphqlResponse ({ abortController, abortData }) {
87
94
  const rootSpan = web.root(req)
88
95
  if (!rootSpan) return
89
96
 
90
- const blockingData = getBlockingData(req, specificBlockingTypes.GRAPHQL, rootSpan)
97
+ const blockingData = getBlockingData(req, specificBlockingTypes.GRAPHQL, rootSpan, requestData.wafAction)
91
98
  abortData.statusCode = blockingData.statusCode
92
99
  abortData.headers = blockingData.headers
93
100
  abortData.message = blockingData.body
@@ -22,10 +22,11 @@ const apiSecuritySampler = require('./api_security_sampler')
22
22
  const web = require('../plugins/util/web')
23
23
  const { extractIp } = require('../plugins/util/ip_extractor')
24
24
  const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
25
- const { block, setTemplates } = require('./blocking')
25
+ const { block, setTemplates, getBlockingAction } = require('./blocking')
26
26
  const { passportTrackEvent } = require('./passport')
27
27
  const { storage } = require('../../../datadog-core')
28
28
  const graphql = require('./graphql')
29
+ const rasp = require('./rasp')
29
30
 
30
31
  let isEnabled = false
31
32
  let config
@@ -37,6 +38,10 @@ function enable (_config) {
37
38
  appsecTelemetry.enable(_config.telemetry)
38
39
  graphql.enable()
39
40
 
41
+ if (_config.appsec.rasp.enabled) {
42
+ rasp.enable()
43
+ }
44
+
40
45
  setTemplates(_config)
41
46
 
42
47
  RuleManager.loadRules(_config.appsec)
@@ -203,7 +208,7 @@ function onResponseBody ({ req, body }) {
203
208
  // we don't support blocking at this point, so no results needed
204
209
  waf.run({
205
210
  persistent: {
206
- [addresses.HTTP_OUTGOING_BODY]: body
211
+ [addresses.HTTP_INCOMING_RESPONSE_BODY]: body
207
212
  }
208
213
  }, req)
209
214
  }
@@ -223,8 +228,9 @@ function onPassportVerify ({ credentials, user }) {
223
228
  function handleResults (actions, req, res, rootSpan, abortController) {
224
229
  if (!actions || !req || !res || !rootSpan || !abortController) return
225
230
 
226
- if (actions.includes('block')) {
227
- block(req, res, rootSpan, abortController)
231
+ const blockingAction = getBlockingAction(actions)
232
+ if (blockingAction) {
233
+ block(req, res, rootSpan, abortController, blockingAction)
228
234
  }
229
235
  }
230
236
 
@@ -236,6 +242,7 @@ function disable () {
236
242
 
237
243
  appsecTelemetry.disable()
238
244
  graphql.disable()
245
+ rasp.disable()
239
246
 
240
247
  remoteConfig.disableWafUpdate()
241
248
 
@@ -0,0 +1,35 @@
1
+ 'use strict'
2
+
3
+ const { storage } = require('../../../datadog-core')
4
+ const addresses = require('./addresses')
5
+ const { httpClientRequestStart } = require('./channels')
6
+ const waf = require('./waf')
7
+
8
+ function enable () {
9
+ httpClientRequestStart.subscribe(analyzeSsrf)
10
+ }
11
+
12
+ function disable () {
13
+ if (httpClientRequestStart.hasSubscribers) httpClientRequestStart.unsubscribe(analyzeSsrf)
14
+ }
15
+
16
+ function analyzeSsrf (ctx) {
17
+ const store = storage.getStore()
18
+ const req = store?.req
19
+ const url = ctx.args.uri
20
+
21
+ if (!req || !url) return
22
+
23
+ const persistent = {
24
+ [addresses.HTTP_OUTGOING_URL]: url
25
+ }
26
+ // TODO: Currently this is only monitoring, we should
27
+ // block the request if SSRF attempt and
28
+ // generate stack traces
29
+ waf.run({ persistent }, req)
30
+ }
31
+
32
+ module.exports = {
33
+ enable,
34
+ disable
35
+ }