dd-trace 6.19.0 → 6.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +163 -22
- package/package.json +2 -2
- package/packages/datadog-esbuild/index.js +9 -6
- package/packages/datadog-instrumentations/src/ai.js +26 -0
- package/packages/datadog-instrumentations/src/aws-sdk.js +2 -2
- package/packages/datadog-instrumentations/src/helpers/hooks.js +0 -7
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentation-registry.js +2 -2
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/ai.js +25 -0
- package/packages/datadog-plugin-openai/src/stream-helpers.js +50 -1
- package/packages/dd-trace/src/config/generated-config-types.d.ts +4 -0
- package/packages/dd-trace/src/config/remote_config.js +12 -0
- package/packages/dd-trace/src/config/supported-configurations.json +21 -0
- package/packages/dd-trace/src/debugger/constants.js +13 -4
- package/packages/dd-trace/src/debugger/devtools_client/breakpoints.js +14 -1
- package/packages/dd-trace/src/debugger/devtools_client/condition.js +109 -6
- package/packages/dd-trace/src/debugger/devtools_client/config.js +2 -0
- package/packages/dd-trace/src/debugger/devtools_client/index.js +65 -6
- package/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js +22 -10
- package/packages/dd-trace/src/debugger/devtools_client/remote_config.js +12 -8
- package/packages/dd-trace/src/debugger/devtools_client/send.js +14 -0
- package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +43 -5
- package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js +3 -8
- package/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js +2 -122
- package/packages/dd-trace/src/debugger/guardrail-metrics.js +0 -2
- package/packages/dd-trace/src/debugger/index.js +60 -28
- package/packages/dd-trace/src/debugger/inspect-segment.js +92 -23
- package/packages/dd-trace/src/debugger/pause-duration-histogram.js +74 -0
- package/packages/dd-trace/src/debugger/probe_sampler.js +192 -52
- package/packages/dd-trace/src/debugger/redaction.js +139 -0
- package/packages/dd-trace/src/knuth-hash.js +17 -0
- package/packages/dd-trace/src/llmobs/constants/tags.js +2 -0
- package/packages/dd-trace/src/llmobs/experiments/client.js +66 -2
- package/packages/dd-trace/src/llmobs/experiments/evaluator.js +138 -0
- package/packages/dd-trace/src/llmobs/experiments/experiment.js +213 -90
- package/packages/dd-trace/src/llmobs/experiments/index.js +40 -1
- package/packages/dd-trace/src/llmobs/experiments/noop.js +53 -2
- package/packages/dd-trace/src/llmobs/experiments/remote-evaluator.js +110 -0
- package/packages/dd-trace/src/llmobs/experiments/util.js +34 -6
- package/packages/dd-trace/src/llmobs/plugins/openai/index.js +2 -2
- package/packages/dd-trace/src/llmobs/sdk.js +19 -3
- package/packages/dd-trace/src/llmobs/span_processor.js +42 -26
- package/packages/dd-trace/src/llmobs/tagger.js +35 -0
- package/packages/dd-trace/src/llmobs/writers/base.js +1 -6
- package/packages/dd-trace/src/openfeature/constants/constants.js +10 -0
- package/packages/dd-trace/src/openfeature/flagging_provider.js +14 -1
- package/packages/dd-trace/src/openfeature/writers/base.js +39 -18
- package/packages/dd-trace/src/openfeature/writers/flag-eval-evp-hook.js +105 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-aggregation.js +190 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-consumer.js +233 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-context.js +262 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-payload.js +181 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-pii.js +108 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-telemetry.js +154 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluation-worker.js +77 -0
- package/packages/dd-trace/src/openfeature/writers/flag-evaluations.js +398 -0
- package/packages/dd-trace/src/openfeature/writers/util.js +5 -4
- package/packages/dd-trace/src/opentelemetry/context_manager.js +3 -0
- package/packages/dd-trace/src/opentelemetry/span_context.js +6 -2
- package/packages/dd-trace/src/opentelemetry/trace/otlp_transformer.js +6 -0
- package/packages/dd-trace/src/opentelemetry/tracer.js +16 -18
- package/packages/dd-trace/src/opentracing/propagation/text_map.js +116 -25
- package/packages/dd-trace/src/opentracing/propagation/tracestate.js +80 -16
- package/packages/dd-trace/src/opentracing/span.js +14 -9
- package/packages/dd-trace/src/opentracing/tracer.js +10 -3
- package/packages/dd-trace/src/otel-sampling.js +171 -0
- package/packages/dd-trace/src/plugins/index.js +0 -1
- package/packages/dd-trace/src/priority_sampler.js +155 -33
- package/packages/dd-trace/src/remote_config/capabilities.js +9 -0
- package/packages/dd-trace/src/sampler.js +2 -5
- package/packages/dd-trace/src/sampling_rule.js +4 -8
- package/packages/dd-trace/src/span_format.js +6 -0
- package/packages/dd-trace/src/span_processor.js +26 -2
- package/packages/dd-trace/src/standalone/index.js +5 -4
- package/packages/dd-trace/src/standalone/tracesource_priority_sampler.js +25 -5
- package/packages/dd-trace/src/telemetry/metrics.js +4 -3
- package/packages/datadog-instrumentations/src/postgres.js +0 -7
- package/packages/datadog-instrumentations/src/supabase.js +0 -15
|
@@ -3,9 +3,14 @@
|
|
|
3
3
|
module.exports = {
|
|
4
4
|
compile,
|
|
5
5
|
compileSegments,
|
|
6
|
+
getRedactionError,
|
|
7
|
+
getSegmentRedactionErrors,
|
|
6
8
|
templateRequiresEvaluation,
|
|
7
9
|
}
|
|
8
10
|
|
|
11
|
+
const { REDACTED_PLACEHOLDER } = require('../redaction')
|
|
12
|
+
const { isRedactedIdentifier } = require('./snapshot/redaction')
|
|
13
|
+
|
|
9
14
|
const identifierRegex = /^(@[\w$]+|[a-zA-Z_$][\w$]*)$/
|
|
10
15
|
|
|
11
16
|
// The following identifiers have purposefully not been included in this list:
|
|
@@ -39,6 +44,16 @@ const reservedWords = new Set([
|
|
|
39
44
|
|
|
40
45
|
const PRIMITIVE_TYPES = new Set(['string', 'number', 'bigint', 'boolean', 'undefined', 'symbol', 'null'])
|
|
41
46
|
|
|
47
|
+
// What a dynamic template segment renders as when it references a redacted identifier and is therefore not evaluated.
|
|
48
|
+
const REDACTED_SEGMENT = JSON.stringify(REDACTED_PLACEHOLDER)
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* @typedef {{ str: string, dsl?: undefined, json?: undefined }} StaticTemplateSegment
|
|
52
|
+
* @typedef {{ str?: undefined, dsl: string, json: object }} DynamicTemplateSegment
|
|
53
|
+
* @typedef {StaticTemplateSegment|DynamicTemplateSegment} TemplateSegment
|
|
54
|
+
* @typedef {{ expr: string, message: string }} EvaluationError
|
|
55
|
+
*/
|
|
56
|
+
|
|
42
57
|
function templateRequiresEvaluation (segments) {
|
|
43
58
|
if (segments === undefined) return false // There should always be segments, but just in case
|
|
44
59
|
for (const { str } of segments) {
|
|
@@ -47,12 +62,29 @@ function templateRequiresEvaluation (segments) {
|
|
|
47
62
|
return false
|
|
48
63
|
}
|
|
49
64
|
|
|
65
|
+
/**
|
|
66
|
+
* @param {TemplateSegment[]} segments - Template segments to compile into a JavaScript array expression.
|
|
67
|
+
*/
|
|
50
68
|
function compileSegments (segments) {
|
|
51
69
|
let result = '['
|
|
52
70
|
for (let i = 0; i < segments.length; i++) {
|
|
53
71
|
const { str, dsl, json } = segments[i]
|
|
54
|
-
result += str === undefined
|
|
55
|
-
|
|
72
|
+
result += str === undefined ? compileSegment(dsl, json) : JSON.stringify(str)
|
|
73
|
+
if (i !== segments.length - 1) {
|
|
74
|
+
result += ','
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return `${result}]`
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {string} dsl - The original DSL expression.
|
|
82
|
+
* @param {object} json - The JSON expression AST.
|
|
83
|
+
*/
|
|
84
|
+
function compileSegment (dsl, json) {
|
|
85
|
+
if (findRedactedIdentifier(json) !== undefined) return REDACTED_SEGMENT
|
|
86
|
+
|
|
87
|
+
return `(() => {
|
|
56
88
|
try {
|
|
57
89
|
const result = ${compile(json)}
|
|
58
90
|
return typeof result === 'string' ? result : $dd_inspectSegment(result)
|
|
@@ -60,12 +92,83 @@ function compileSegments (segments) {
|
|
|
60
92
|
return { expr: ${JSON.stringify(dsl)}, message: \`\${e.name}: \${e.message}\` }
|
|
61
93
|
}
|
|
62
94
|
})()`
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Get the evaluation errors for the dynamic template segments that are not evaluated because they reference a redacted
|
|
99
|
+
* identifier. The errors are the same every time the template is evaluated, so they are only computed once.
|
|
100
|
+
*
|
|
101
|
+
* @param {TemplateSegment[]} segments - Template segments to inspect.
|
|
102
|
+
* @returns {EvaluationError[]|undefined} The evaluation errors, if any segment references a redacted identifier.
|
|
103
|
+
*/
|
|
104
|
+
function getSegmentRedactionErrors (segments) {
|
|
105
|
+
let errors
|
|
106
|
+
for (const { str, dsl, json } of segments) {
|
|
107
|
+
if (str !== undefined) continue
|
|
108
|
+
const error = getRedactionError(dsl, json)
|
|
109
|
+
if (error !== undefined) {
|
|
110
|
+
errors ??= []
|
|
111
|
+
errors.push(error)
|
|
66
112
|
}
|
|
67
113
|
}
|
|
68
|
-
return
|
|
114
|
+
return errors
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Get the evaluation error to report instead of evaluating an expression, because it reads a redacted identifier.
|
|
119
|
+
*
|
|
120
|
+
* @param {string} expr - The expression to report the error for.
|
|
121
|
+
* @param {object} json - The JSON expression AST.
|
|
122
|
+
* @returns {EvaluationError|undefined} The evaluation error, if the expression reads a redacted identifier.
|
|
123
|
+
*/
|
|
124
|
+
function getRedactionError (expr, json) {
|
|
125
|
+
const identifier = findRedactedIdentifier(json)
|
|
126
|
+
if (identifier === undefined) return
|
|
127
|
+
return { expr, message: `Could not evaluate the expression because '${identifier}' was redacted` }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Find the first redacted identifier referenced by an expression AST.
|
|
132
|
+
*
|
|
133
|
+
* @param {unknown} node - The JSON expression AST node to inspect.
|
|
134
|
+
* @returns {string|undefined} The redacted identifier, if one is referenced.
|
|
135
|
+
*/
|
|
136
|
+
function findRedactedIdentifier (node) {
|
|
137
|
+
if (node === null || typeof node !== 'object') return
|
|
138
|
+
|
|
139
|
+
// Expression nodes are single-key objects: the key is the operator, and the value is that operator's payload.
|
|
140
|
+
const [type, value] = Object.entries(node)[0]
|
|
141
|
+
|
|
142
|
+
// Root references and member/index keys are the only AST positions that name values being read.
|
|
143
|
+
if (type === 'ref') {
|
|
144
|
+
// A non-string `ref` is malformed. Leave it for `compile` to reject with a useful error.
|
|
145
|
+
return typeof value === 'string' && isRedactedIdentifier(value) ? value : undefined
|
|
146
|
+
} else if (type === 'getmember' || type === 'index') {
|
|
147
|
+
const key = findRedactedMemberKey(value[1])
|
|
148
|
+
if (key !== undefined) return key
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Other expression types can wrap redacted reads, e.g. substring(user.password, 0, 4), so walk their children.
|
|
152
|
+
if (!Array.isArray(value)) return findRedactedIdentifier(value)
|
|
153
|
+
|
|
154
|
+
let key
|
|
155
|
+
for (const child of value) {
|
|
156
|
+
key = findRedactedIdentifier(child)
|
|
157
|
+
if (key !== undefined) break
|
|
158
|
+
}
|
|
159
|
+
return key
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @param {unknown} key - A member/index key expression.
|
|
164
|
+
* @returns {string|undefined} The redacted identifier, if the key references one.
|
|
165
|
+
*/
|
|
166
|
+
function findRedactedMemberKey (key) {
|
|
167
|
+
if (typeof key === 'string') {
|
|
168
|
+
return isRedactedIdentifier(key) ? key : undefined
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return findRedactedIdentifier(key)
|
|
69
172
|
}
|
|
70
173
|
|
|
71
174
|
// TODO: Consider storing some of these functions that doesn't require closure access to the current scope on `process`
|
|
@@ -27,4 +27,6 @@ function updateConfig (updates) {
|
|
|
27
27
|
config.url = new URL(updates.url)
|
|
28
28
|
config.dynamicInstrumentation.captureTimeoutNs =
|
|
29
29
|
BigInt(updates.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS) * 1_000_000n
|
|
30
|
+
config.dynamicInstrumentation.evaluationTimeoutNs =
|
|
31
|
+
BigInt(updates.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS) * 1_000_000n
|
|
30
32
|
}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { randomUUID } = require('crypto')
|
|
4
|
-
const {
|
|
4
|
+
const { workerData: { probeSamplerBuffer, pauseDurationBuffer } } = require('worker_threads')
|
|
5
5
|
const { version } = require('../../../../../package.json')
|
|
6
6
|
const processTags = require('../../process-tags')
|
|
7
|
-
const { INSPECT_SEGMENT_GLOBAL_PROPERTY } = require('../constants')
|
|
7
|
+
const { INSPECT_SEGMENT_GLOBAL_PROPERTY, WORKER_ERROR_REASON } = require('../constants')
|
|
8
8
|
const { EVENT_TYPE, INCOMPLETE_REASON } = require('../guardrail-metrics')
|
|
9
|
+
const { PauseDurationHistogram } = require('../pause-duration-histogram')
|
|
9
10
|
const {
|
|
10
11
|
CONDITION_ERROR_FLAG,
|
|
11
12
|
MAX_SAMPLED_PROBES_PER_PAUSE,
|
|
@@ -13,7 +14,7 @@ const {
|
|
|
13
14
|
SAMPLED_PROBE_INDEXES_START,
|
|
14
15
|
SAMPLED_PROBE_OVERFLOW_INDEX,
|
|
15
16
|
} = require('../probe_sampler_constants')
|
|
16
|
-
const { getTakeConditionErrorExpression } = require('./probe_sampler')
|
|
17
|
+
const { getEvaluationTimedOutExpression, getTakeConditionErrorExpression } = require('./probe_sampler')
|
|
17
18
|
const { breakpointToProbes, samplingIndexToProbe } = require('./state')
|
|
18
19
|
const { refreshBreakpoints } = require('./breakpoints')
|
|
19
20
|
const session = require('./session')
|
|
@@ -42,6 +43,7 @@ const getDDTagsExpression = `(() => {
|
|
|
42
43
|
const threadId = config.parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${config.parentThreadId}`
|
|
43
44
|
const threadName = config.parentThreadId === 0 ? 'MainThread' : `WorkerThread:${config.parentThreadId}`
|
|
44
45
|
const sampledProbeIndexes = new Int32Array(probeSamplerBuffer)
|
|
46
|
+
const pauseDurations = new PauseDurationHistogram(pauseDurationBuffer)
|
|
45
47
|
|
|
46
48
|
// WARNING: The code above the line `await session.post('Debugger.resume')` is highly optimized. Please edit with care!
|
|
47
49
|
session.on('Debugger.paused', async ({ params }) => {
|
|
@@ -49,7 +51,9 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
49
51
|
|
|
50
52
|
if (params.reason !== 'other') {
|
|
51
53
|
// This error should not be caught, and should exit the worker thread, effectively stopping the debugging session
|
|
52
|
-
throw new Error(`Unexpected Debugger.paused reason: ${params.reason}`)
|
|
54
|
+
throw Object.assign(new Error(`Unexpected Debugger.paused reason: ${params.reason}`), {
|
|
55
|
+
reason: WORKER_ERROR_REASON.UNEXPECTED_PAUSE_REASON,
|
|
56
|
+
})
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
let maxReferenceDepth = 0
|
|
@@ -58,6 +62,7 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
58
62
|
let maxLength = 0
|
|
59
63
|
let numberOfProbesWithSnapshots = 0
|
|
60
64
|
let probesWithCaptureExpressions = false
|
|
65
|
+
let probesWithTemplates = false
|
|
61
66
|
const probes = []
|
|
62
67
|
// Expressions evaluated on the paused frame in one round trip, in the order of `probes`: the evaluated template for
|
|
63
68
|
// probes whose template requires evaluation, and the recorded error for probes paused to report a condition error
|
|
@@ -125,6 +130,7 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
125
130
|
}
|
|
126
131
|
|
|
127
132
|
if (probe.templateRequiresEvaluation) {
|
|
133
|
+
probesWithTemplates = true
|
|
128
134
|
frameExpressions += `,${probe.template}`
|
|
129
135
|
}
|
|
130
136
|
|
|
@@ -142,6 +148,7 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
142
148
|
const timestamp = Date.now()
|
|
143
149
|
|
|
144
150
|
let evalResults
|
|
151
|
+
const evaluationStart = process.hrtime.bigint()
|
|
145
152
|
const { result } = /** @type {EvaluateOnCallFrameResult} */ (
|
|
146
153
|
await session.post('Debugger.evaluateOnCallFrame', {
|
|
147
154
|
callFrameId: params.callFrames[0].callFrameId,
|
|
@@ -152,6 +159,10 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
152
159
|
includeCommandLineAPI: true,
|
|
153
160
|
})
|
|
154
161
|
)
|
|
162
|
+
// The templates of all probes at the location are evaluated together, so an exceeded budget is attributed to all of
|
|
163
|
+
// them. Evaluation can't be interrupted, so the budget is enforced by reporting and throttling after the fact.
|
|
164
|
+
const templatesTimedOut = probesWithTemplates &&
|
|
165
|
+
process.hrtime.bigint() - evaluationStart > config.dynamicInstrumentation.evaluationTimeoutNs
|
|
155
166
|
if (result?.subtype === 'error') {
|
|
156
167
|
log.error('[debugger:devtools_client] Error evaluating code on call frame: %s', result?.description)
|
|
157
168
|
evalResults = []
|
|
@@ -180,7 +191,8 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
180
191
|
captureExpressionResults.set(probe.id, await evaluateCaptureExpressions(
|
|
181
192
|
params.callFrames[0],
|
|
182
193
|
probe.compiledCaptureExpressions,
|
|
183
|
-
start + config.dynamicInstrumentation.captureTimeoutNs
|
|
194
|
+
start + config.dynamicInstrumentation.captureTimeoutNs,
|
|
195
|
+
config.dynamicInstrumentation.evaluationTimeoutNs
|
|
184
196
|
))
|
|
185
197
|
}
|
|
186
198
|
}
|
|
@@ -188,6 +200,21 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
188
200
|
await session.post('Debugger.resume')
|
|
189
201
|
reportPauseDuration(start)
|
|
190
202
|
|
|
203
|
+
// Ideally these throttles would be installed before resuming so a tight loop could not sample the probe again before
|
|
204
|
+
// the request is handled. That would keep the application paused for another inspector request even when the probe
|
|
205
|
+
// does not immediately run again, so we deliberately resume first. If the race occurs, the probe can perform one
|
|
206
|
+
// additional over-budget evaluation before the throttle takes effect, which is preferable to lengthening every
|
|
207
|
+
// timeout pause.
|
|
208
|
+
for (const probe of probes) {
|
|
209
|
+
// Condition errors are already throttled synchronously by the sampler; preserve that state and its classification.
|
|
210
|
+
if (conditionErrorProbes?.has(probe)) continue
|
|
211
|
+
if ((probe.templateRequiresEvaluation && templatesTimedOut) ||
|
|
212
|
+
captureExpressionResults?.get(probe.id)?.timedOut === true) {
|
|
213
|
+
// TODO: Batch all timed-out probes into a single Runtime.evaluate request.
|
|
214
|
+
throttleProbe(probe)
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
191
218
|
const logger = {
|
|
192
219
|
// We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints`
|
|
193
220
|
// must exist in the same file since the debugger can only pause the main thread in one location.
|
|
@@ -322,6 +349,25 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
322
349
|
message += `{${result.message}}`
|
|
323
350
|
}
|
|
324
351
|
}
|
|
352
|
+
if (probe.templateRedactionErrors !== undefined) {
|
|
353
|
+
if (snapshot.evaluationErrors === undefined) {
|
|
354
|
+
snapshot.evaluationErrors = [...probe.templateRedactionErrors]
|
|
355
|
+
} else {
|
|
356
|
+
snapshot.evaluationErrors.push(...probe.templateRedactionErrors)
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (templatesTimedOut) {
|
|
361
|
+
const error = {
|
|
362
|
+
expr: '',
|
|
363
|
+
message: 'Template evaluation exceeded its time budget of ' +
|
|
364
|
+
`${config.dynamicInstrumentation.evaluationTimeoutNs / 1_000_000n}ms`,
|
|
365
|
+
}
|
|
366
|
+
if (snapshot.evaluationErrors === undefined) {
|
|
367
|
+
snapshot.evaluationErrors = [error]
|
|
368
|
+
} else {
|
|
369
|
+
snapshot.evaluationErrors.push(error)
|
|
370
|
+
}
|
|
325
371
|
}
|
|
326
372
|
} else {
|
|
327
373
|
message = probe.template
|
|
@@ -358,11 +404,24 @@ session.on('Debugger.paused', async ({ params }) => {
|
|
|
358
404
|
*/
|
|
359
405
|
function reportPauseDuration (start) {
|
|
360
406
|
const durationMs = Number(process.hrtime.bigint() - start) / 1_000_000
|
|
361
|
-
|
|
407
|
+
pauseDurations.record(durationMs)
|
|
362
408
|
log.debug('[debugger:devtools_client] Finished processing breakpoints - instrumented thread paused for: ~%d ms',
|
|
363
409
|
durationMs)
|
|
364
410
|
}
|
|
365
411
|
|
|
412
|
+
/**
|
|
413
|
+
* Throttle a probe in the runtime sampler because its template or capture expressions exceeded the evaluation time
|
|
414
|
+
* budget. The result has already been reported, so this only stops the probe from being evaluated again for a while.
|
|
415
|
+
*
|
|
416
|
+
* @param {{ id: string }} probe - The probe to throttle.
|
|
417
|
+
*/
|
|
418
|
+
function throttleProbe (probe) {
|
|
419
|
+
log.debug('[debugger:devtools_client] Evaluation of probe %s exceeded its time budget; throttling probe', probe.id)
|
|
420
|
+
session.post('Runtime.evaluate', { expression: getEvaluationTimedOutExpression(probe.id) }).catch((err) => {
|
|
421
|
+
log.error('[debugger:devtools_client] Error throttling probe %s', probe.id, err)
|
|
422
|
+
})
|
|
423
|
+
}
|
|
424
|
+
|
|
366
425
|
function processDD (result) {
|
|
367
426
|
return result?.trace_id === undefined ? undefined : result
|
|
368
427
|
}
|
|
@@ -7,6 +7,7 @@ const SAMPLER_EXPRESSION = `globalThis[Symbol.for(${JSON.stringify(DD_TRACE_SYMB
|
|
|
7
7
|
|
|
8
8
|
module.exports = {
|
|
9
9
|
compileBreakpointCondition,
|
|
10
|
+
getEvaluationTimedOutExpression,
|
|
10
11
|
getRemoveProbeExpression,
|
|
11
12
|
getTakeConditionErrorExpression,
|
|
12
13
|
isSnapshotProducingProbe,
|
|
@@ -23,6 +24,16 @@ function isSnapshotProducingProbe (probe) {
|
|
|
23
24
|
return probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Build the expression that throttles a probe whose evaluation exceeded its time budget in the worker. Called by the
|
|
29
|
+
* devtools worker and evaluated on the debuggee.
|
|
30
|
+
*
|
|
31
|
+
* @param {string} id - The probe id.
|
|
32
|
+
*/
|
|
33
|
+
function getEvaluationTimedOutExpression (id) {
|
|
34
|
+
return `${SAMPLER_EXPRESSION}?.evaluationTimedOut(${JSON.stringify(id)})`
|
|
35
|
+
}
|
|
36
|
+
|
|
26
37
|
/**
|
|
27
38
|
* Build the expression that removes a probe from runtime sampler state. Called by the devtools worker and evaluated on
|
|
28
39
|
* the debuggee.
|
|
@@ -85,22 +96,23 @@ function compileBreakpointCondition (probes) {
|
|
|
85
96
|
* }} probe - The probe to sample.
|
|
86
97
|
*/
|
|
87
98
|
function compileProbeCondition (probe) {
|
|
88
|
-
const
|
|
89
|
-
|
|
99
|
+
const id = JSON.stringify(probe.id)
|
|
100
|
+
const producesSnapshot = isSnapshotProducingProbe(probe)
|
|
101
|
+
const samplingArgs = `${probe.nsBetweenSampling}n, ${producesSnapshot}`
|
|
90
102
|
|
|
91
103
|
if (probe.condition === undefined) {
|
|
92
|
-
return `$dd_sampled = ${
|
|
104
|
+
return `$dd_sampled = $dd_sampler.makeSampleDecision(${probe.samplingIndex}, ${id}, ${samplingArgs}) || $dd_sampled`
|
|
93
105
|
}
|
|
94
106
|
|
|
95
|
-
//
|
|
96
|
-
|
|
107
|
+
// The sampler times the condition against the evaluation budget, from `shouldEvaluateCondition` until it either
|
|
108
|
+
// returns or throws. A condition that throws or exceeds its budget is reported once per throttle window and skipped
|
|
109
|
+
// at probe entry in between.
|
|
110
|
+
return `if ($dd_sampler.shouldEvaluateCondition(${id}, ${producesSnapshot})) {
|
|
97
111
|
try {
|
|
98
|
-
|
|
99
|
-
$
|
|
100
|
-
}
|
|
112
|
+
$dd_sampled = $dd_sampler.conditionEvaluated(${probe.samplingIndex}, ${id}, (${probe.condition}) === true,
|
|
113
|
+
${samplingArgs}) || $dd_sampled
|
|
101
114
|
} catch ($dd_error) {
|
|
102
|
-
$dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${
|
|
103
|
-
$dd_sampled
|
|
115
|
+
$dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${id}, $dd_error) || $dd_sampled
|
|
104
116
|
}
|
|
105
117
|
}`
|
|
106
118
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { workerData: { probePort } } = require('node:worker_threads')
|
|
4
|
+
const { WORKER_ERROR_REASON } = require('../constants')
|
|
4
5
|
const { addBreakpoint, removeBreakpoint, modifyBreakpoint } = require('./breakpoints')
|
|
5
6
|
const { ackReceived, ackInstalled, ackError } = require('./status')
|
|
6
7
|
const log = require('./log')
|
|
@@ -54,7 +55,8 @@ probePort.on('message', async ({ action, probe, ackId }) => {
|
|
|
54
55
|
await processMsg(action, probe)
|
|
55
56
|
probePort.postMessage({ ackId })
|
|
56
57
|
} catch (err) {
|
|
57
|
-
|
|
58
|
+
// Structured cloning an Error drops its custom properties, so send the reason separately.
|
|
59
|
+
probePort.postMessage({ ackId, error: err, reason: err.reason })
|
|
58
60
|
ackError(err, probe)
|
|
59
61
|
}
|
|
60
62
|
})
|
|
@@ -72,18 +74,20 @@ async function processMsg (action, probe) {
|
|
|
72
74
|
if (action !== 'unapply') ackReceived(probe)
|
|
73
75
|
|
|
74
76
|
if (probe.type !== 'LOG_PROBE') {
|
|
75
|
-
throw new Error(
|
|
77
|
+
throw Object.assign(new Error(
|
|
78
|
+
`Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`
|
|
79
|
+
), { reason: WORKER_ERROR_REASON.UNSUPPORTED_PROBE_TYPE })
|
|
76
80
|
}
|
|
77
81
|
if (!probe.where.sourceFile && !probe.where.lines) {
|
|
78
|
-
throw new Error(
|
|
82
|
+
throw Object.assign(new Error(
|
|
79
83
|
// eslint-disable-next-line @stylistic/max-len
|
|
80
84
|
`Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})`
|
|
81
|
-
)
|
|
85
|
+
), { reason: WORKER_ERROR_REASON.UNSUPPORTED_INSERTION_POINT })
|
|
82
86
|
}
|
|
83
87
|
if (probe.captureSnapshot && probe.captureExpressions?.length > 0) {
|
|
84
|
-
throw new Error(
|
|
88
|
+
throw Object.assign(new Error(
|
|
85
89
|
`Cannot set both captureSnapshot and captureExpressions (probe: ${probe.id}, version: ${probe.version})`
|
|
86
|
-
)
|
|
90
|
+
), { reason: WORKER_ERROR_REASON.CONFLICTING_CAPTURE_OPTIONS })
|
|
87
91
|
}
|
|
88
92
|
|
|
89
93
|
switch (action) {
|
|
@@ -99,8 +103,8 @@ async function processMsg (action, probe) {
|
|
|
99
103
|
ackInstalled(probe)
|
|
100
104
|
break
|
|
101
105
|
default:
|
|
102
|
-
throw new Error(
|
|
106
|
+
throw Object.assign(new Error(
|
|
103
107
|
`Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}`
|
|
104
|
-
)
|
|
108
|
+
), { reason: WORKER_ERROR_REASON.UNKNOWN_REMOTE_CONFIG_ACTION })
|
|
105
109
|
}
|
|
106
110
|
}
|
|
@@ -85,6 +85,20 @@ function send (message, logger, dd, snapshot, processTags, eventType, incomplete
|
|
|
85
85
|
json = JSON.stringify(payload)
|
|
86
86
|
}
|
|
87
87
|
size = Buffer.byteLength(json)
|
|
88
|
+
|
|
89
|
+
if (size > MAX_LOG_PAYLOAD_SIZE_BYTES) {
|
|
90
|
+
// Trimming is best-effort: the pruner can only replace leaf nodes deep inside the snapshot and the fallback only
|
|
91
|
+
// drops captures, so a payload whose bytes sit elsewhere - a very deep stack, say - can stay over the cap. There
|
|
92
|
+
// is nothing to gain from sending it: the intake truncates the log message, then throws the snapshot away, and
|
|
93
|
+
// drops the event anyway if that still doesn't fit.
|
|
94
|
+
log.debug(
|
|
95
|
+
'[debugger:devtools_client] Dropping probe result for probe %s: payload is too large (%d bytes)',
|
|
96
|
+
snapshot.probe.id,
|
|
97
|
+
size
|
|
98
|
+
)
|
|
99
|
+
guardrailMetrics.eventDropped(DROPPED_REASON.PAYLOAD_TOO_LARGE, eventType)
|
|
100
|
+
return
|
|
101
|
+
}
|
|
88
102
|
}
|
|
89
103
|
|
|
90
104
|
if (jsonBuffer.write(json, size)) {
|
|
@@ -106,6 +106,13 @@ function isCollectable (scope) {
|
|
|
106
106
|
* @property {CaptureLimits} limits - Fully resolved capture limits (precomputed at probe setup)
|
|
107
107
|
*/
|
|
108
108
|
|
|
109
|
+
/**
|
|
110
|
+
* @typedef {object} RedactedCaptureExpression
|
|
111
|
+
* @property {string} name - The name of the expression
|
|
112
|
+
* @property {{ expr: string, message: string }} redactionError - Reported instead of evaluating the expression, as it
|
|
113
|
+
* reads a redacted identifier
|
|
114
|
+
*/
|
|
115
|
+
|
|
109
116
|
/**
|
|
110
117
|
* @typedef {object} CaptureExpressionResult
|
|
111
118
|
* @property {() => Record<string, ReturnType<typeof processRemoteObject>>} processCaptureExpressions - Callback to
|
|
@@ -116,6 +123,7 @@ function isCollectable (scope) {
|
|
|
116
123
|
* @property {import('./processor').IncompleteCapture} incomplete - The capture limits enforced on the expressions,
|
|
117
124
|
* including a runtime error for every expression that threw or could not be evaluated. Only fully populated once
|
|
118
125
|
* `processCaptureExpressions` has run.
|
|
126
|
+
* @property {boolean} timedOut - Whether evaluating an expression exceeded the evaluation time budget
|
|
119
127
|
*/
|
|
120
128
|
|
|
121
129
|
/**
|
|
@@ -130,12 +138,20 @@ function isCollectable (scope) {
|
|
|
130
138
|
* Collects raw data while paused, returns a callback to process after resume.
|
|
131
139
|
*
|
|
132
140
|
* @param {import('inspector').Debugger.CallFrame} callFrame - The call frame to evaluate expressions on
|
|
133
|
-
* @param {CompiledCaptureExpression[]} expressions - The compiled expressions with
|
|
141
|
+
* @param {(CompiledCaptureExpression|RedactedCaptureExpression)[]} expressions - The compiled expressions with
|
|
142
|
+
* precomputed capture limits
|
|
134
143
|
* @param {bigint} [deadlineNs] - The deadline in nanoseconds. Defaults to {@link BIGINT_MAX}. If the deadline is
|
|
135
144
|
* reached, the snapshot will be truncated.
|
|
145
|
+
* @param {bigint} [evaluationTimeoutNs] - The time budget in nanoseconds for evaluating a single expression. Defaults
|
|
146
|
+
* to {@link BIGINT_MAX}. Evaluation can't be interrupted, so an exceeded budget is reported as an evaluation error.
|
|
136
147
|
* @returns {Promise<CaptureExpressionResult>} Raw results with deferred processing callback
|
|
137
148
|
*/
|
|
138
|
-
async function evaluateCaptureExpressions (
|
|
149
|
+
async function evaluateCaptureExpressions (
|
|
150
|
+
callFrame,
|
|
151
|
+
expressions,
|
|
152
|
+
deadlineNs = BIGINT_MAX,
|
|
153
|
+
evaluationTimeoutNs = BIGINT_MAX
|
|
154
|
+
) {
|
|
139
155
|
/** @type {{ name: string, remoteObject: object, maxLength: number }[]} */
|
|
140
156
|
const rawResults = []
|
|
141
157
|
/** @type {{ expr: string, message: string }[]} */
|
|
@@ -146,12 +162,20 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
|
|
|
146
162
|
const incomplete = { reasons: 0 }
|
|
147
163
|
/** @type {Record<string, ReturnType<typeof processRemoteObject>> | null} */
|
|
148
164
|
let processedResult = null
|
|
165
|
+
let timedOut = false
|
|
149
166
|
|
|
150
167
|
for (let i = 0; i < expressions.length; i++) {
|
|
151
|
-
const
|
|
168
|
+
const captureExpression = expressions[i]
|
|
169
|
+
if ('redactionError' in captureExpression) {
|
|
170
|
+
evaluationErrors.push(captureExpression.redactionError)
|
|
171
|
+
continue
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const { name, expression, limits } = captureExpression
|
|
152
175
|
const { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } = limits
|
|
153
176
|
|
|
154
177
|
try {
|
|
178
|
+
const evaluationStart = process.hrtime.bigint()
|
|
155
179
|
const { result, exceptionDetails } = /** @type {EvaluateOnCallFrameResult} */ (
|
|
156
180
|
// eslint-disable-next-line no-await-in-loop
|
|
157
181
|
await session.post('Debugger.evaluateOnCallFrame', {
|
|
@@ -159,6 +183,13 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
|
|
|
159
183
|
expression,
|
|
160
184
|
})
|
|
161
185
|
)
|
|
186
|
+
if (process.hrtime.bigint() - evaluationStart > evaluationTimeoutNs) {
|
|
187
|
+
timedOut = true
|
|
188
|
+
evaluationErrors.push({
|
|
189
|
+
expr: name,
|
|
190
|
+
message: `Expression evaluation exceeded its time budget of ${evaluationTimeoutNs / 1_000_000n}ms`,
|
|
191
|
+
})
|
|
192
|
+
}
|
|
162
193
|
|
|
163
194
|
// Handle evaluation exceptions (maybe transient - bad expression, undefined var, etc.)
|
|
164
195
|
if (exceptionDetails) {
|
|
@@ -197,10 +228,16 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
|
|
|
197
228
|
// and are recorded when processed, unless they end up redacted.
|
|
198
229
|
rawResults.push({ name, remoteObject: result, maxLength })
|
|
199
230
|
// Add stub entries for remaining uncaptured expressions. The stubs are used as-is, so record them here.
|
|
200
|
-
|
|
231
|
+
// Redacted expressions are never evaluated, so they are reported the same way as without a timeout.
|
|
201
232
|
for (let j = i + 1; j < expressions.length; j++) {
|
|
233
|
+
const remaining = expressions[j]
|
|
234
|
+
if ('redactionError' in remaining) {
|
|
235
|
+
evaluationErrors.push(remaining.redactionError)
|
|
236
|
+
continue
|
|
237
|
+
}
|
|
238
|
+
incomplete.reasons |= INCOMPLETE_REASON.TIMEOUT
|
|
202
239
|
rawResults.push({
|
|
203
|
-
name:
|
|
240
|
+
name: remaining.name,
|
|
204
241
|
remoteObject: { notCapturedReason: 'timeout' },
|
|
205
242
|
maxLength: 0,
|
|
206
243
|
})
|
|
@@ -238,6 +275,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
|
|
|
238
275
|
evaluationErrors,
|
|
239
276
|
fatalErrors,
|
|
240
277
|
incomplete,
|
|
278
|
+
timedOut,
|
|
241
279
|
}
|
|
242
280
|
}
|
|
243
281
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const { INCOMPLETE_REASON } = require('../../guardrail-metrics')
|
|
4
4
|
const { LARGE_OBJECT_SKIP_THRESHOLD } = require('./constants')
|
|
5
5
|
const { collectionSizeSym, largeCollectionSkipThresholdSym, fieldCountSym, timeBudgetSym } = require('./symbols')
|
|
6
|
-
const {
|
|
6
|
+
const { isRedactedIdentifier } = require('./redaction')
|
|
7
7
|
|
|
8
8
|
module.exports = {
|
|
9
9
|
processRawState: processProperties,
|
|
@@ -347,21 +347,16 @@ function arrayBufferToString (bytes, size) {
|
|
|
347
347
|
}
|
|
348
348
|
|
|
349
349
|
function shouldRedactProperty (prop) {
|
|
350
|
-
return
|
|
350
|
+
return isRedactedIdentifier(prop.name, prop.symbol !== undefined)
|
|
351
351
|
}
|
|
352
352
|
|
|
353
353
|
function shouldRedactMapValue (key) {
|
|
354
354
|
const isSymbol = key.value.type === 'symbol'
|
|
355
355
|
if (!isSymbol && key.value.type !== 'string') return false // WeakMaps uses objects as keys
|
|
356
|
-
|
|
356
|
+
return isRedactedIdentifier(
|
|
357
357
|
isSymbol ? key.value.description : key.value.value,
|
|
358
358
|
isSymbol
|
|
359
359
|
)
|
|
360
|
-
return REDACTED_IDENTIFIERS.has(name)
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
function getNormalizedNameFromProp (prop) {
|
|
364
|
-
return normalizeName(prop.name, prop.symbol !== undefined)
|
|
365
360
|
}
|
|
366
361
|
|
|
367
362
|
function setNotCaptureReasonOnCollection (result, collection, incomplete) {
|