@mastra/observability 0.0.0-vnext-20251104230439 → 0.0.0-vnext-20251119160359
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -3
- package/README.md +18 -20
- package/dist/config.d.ts +343 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/default.d.ts +29 -0
- package/dist/default.d.ts.map +1 -0
- package/dist/exporters/base.d.ts +10 -10
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/cloud.d.ts +2 -2
- package/dist/exporters/cloud.d.ts.map +1 -1
- package/dist/exporters/console.d.ts +2 -2
- package/dist/exporters/console.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +10 -14
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +1 -1
- package/dist/index.cjs +1786 -1649
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1774 -1632
- package/dist/index.js.map +1 -1
- package/dist/{tracers → instances}/base.d.ts +27 -26
- package/dist/instances/base.d.ts.map +1 -0
- package/dist/instances/default.d.ts +8 -0
- package/dist/instances/default.d.ts.map +1 -0
- package/dist/instances/index.d.ts +6 -0
- package/dist/instances/index.d.ts.map +1 -0
- package/dist/model-tracing.d.ts +6 -6
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/registry.d.ts +46 -48
- package/dist/registry.d.ts.map +1 -1
- package/dist/span_processors/index.d.ts +1 -1
- package/dist/span_processors/sensitive-data-filter.d.ts +4 -4
- package/dist/span_processors/sensitive-data-filter.d.ts.map +1 -1
- package/dist/spans/base.d.ts +11 -11
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts +4 -4
- package/dist/spans/default.d.ts.map +1 -1
- package/dist/spans/index.d.ts +1 -1
- package/dist/spans/no-op.d.ts +4 -4
- package/dist/spans/no-op.d.ts.map +1 -1
- package/package.json +16 -11
- package/dist/default-entrypoint.d.ts +0 -16
- package/dist/default-entrypoint.d.ts.map +0 -1
- package/dist/init.d.ts +0 -2
- package/dist/init.d.ts.map +0 -1
- package/dist/tracers/base.d.ts.map +0 -1
- package/dist/tracers/default.d.ts +0 -7
- package/dist/tracers/default.d.ts.map +0 -1
- package/dist/tracers/index.d.ts +0 -6
- package/dist/tracers/index.d.ts.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1,1765 +1,1921 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var base = require('@mastra/core/base');
|
|
4
|
+
var error = require('@mastra/core/error');
|
|
4
5
|
var logger = require('@mastra/core/logger');
|
|
6
|
+
var zod = require('zod');
|
|
5
7
|
var observability = require('@mastra/core/observability');
|
|
6
8
|
var utils = require('@mastra/core/utils');
|
|
7
9
|
var web = require('stream/web');
|
|
8
|
-
var error = require('@mastra/core/error');
|
|
9
10
|
|
|
10
|
-
// src/
|
|
11
|
-
var
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
11
|
+
// src/default.ts
|
|
12
|
+
var SamplingStrategyType = /* @__PURE__ */ ((SamplingStrategyType2) => {
|
|
13
|
+
SamplingStrategyType2["ALWAYS"] = "always";
|
|
14
|
+
SamplingStrategyType2["NEVER"] = "never";
|
|
15
|
+
SamplingStrategyType2["RATIO"] = "ratio";
|
|
16
|
+
SamplingStrategyType2["CUSTOM"] = "custom";
|
|
17
|
+
return SamplingStrategyType2;
|
|
18
|
+
})(SamplingStrategyType || {});
|
|
19
|
+
var samplingStrategySchema = zod.z.discriminatedUnion("type", [
|
|
20
|
+
zod.z.object({
|
|
21
|
+
type: zod.z.literal("always" /* ALWAYS */)
|
|
22
|
+
}),
|
|
23
|
+
zod.z.object({
|
|
24
|
+
type: zod.z.literal("never" /* NEVER */)
|
|
25
|
+
}),
|
|
26
|
+
zod.z.object({
|
|
27
|
+
type: zod.z.literal("ratio" /* RATIO */),
|
|
28
|
+
probability: zod.z.number().min(0, "Probability must be between 0 and 1").max(1, "Probability must be between 0 and 1")
|
|
29
|
+
}),
|
|
30
|
+
zod.z.object({
|
|
31
|
+
type: zod.z.literal("custom" /* CUSTOM */),
|
|
32
|
+
sampler: zod.z.function().args(zod.z.any().optional()).returns(zod.z.boolean())
|
|
33
|
+
})
|
|
34
|
+
]);
|
|
35
|
+
var observabilityInstanceConfigSchema = zod.z.object({
|
|
36
|
+
name: zod.z.string().min(1, "Name is required"),
|
|
37
|
+
serviceName: zod.z.string().min(1, "Service name is required"),
|
|
38
|
+
sampling: samplingStrategySchema.optional(),
|
|
39
|
+
exporters: zod.z.array(zod.z.any()).optional(),
|
|
40
|
+
spanOutputProcessors: zod.z.array(zod.z.any()).optional(),
|
|
41
|
+
includeInternalSpans: zod.z.boolean().optional(),
|
|
42
|
+
requestContextKeys: zod.z.array(zod.z.string()).optional()
|
|
43
|
+
});
|
|
44
|
+
var observabilityConfigValueSchema = zod.z.object({
|
|
45
|
+
serviceName: zod.z.string().min(1, "Service name is required"),
|
|
46
|
+
sampling: samplingStrategySchema.optional(),
|
|
47
|
+
exporters: zod.z.array(zod.z.any()).optional(),
|
|
48
|
+
spanOutputProcessors: zod.z.array(zod.z.any()).optional(),
|
|
49
|
+
includeInternalSpans: zod.z.boolean().optional(),
|
|
50
|
+
requestContextKeys: zod.z.array(zod.z.string()).optional()
|
|
51
|
+
});
|
|
52
|
+
var observabilityRegistryConfigSchema = zod.z.object({
|
|
53
|
+
default: zod.z.object({
|
|
54
|
+
enabled: zod.z.boolean().optional()
|
|
55
|
+
}).optional().nullable(),
|
|
56
|
+
configs: zod.z.union([zod.z.record(zod.z.string(), zod.z.any()), zod.z.array(zod.z.any()), zod.z.null()]).optional(),
|
|
57
|
+
configSelector: zod.z.function().optional()
|
|
58
|
+
}).passthrough().refine(
|
|
59
|
+
(data) => {
|
|
60
|
+
const isDefaultEnabled = data.default?.enabled === true;
|
|
61
|
+
const hasConfigs = data.configs && typeof data.configs === "object" && !Array.isArray(data.configs) ? Object.keys(data.configs).length > 0 : false;
|
|
62
|
+
return !(isDefaultEnabled && hasConfigs);
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
message: 'Cannot specify both "default" (when enabled) and "configs". Use either default observability or custom configs, but not both.'
|
|
66
|
+
}
|
|
67
|
+
).refine(
|
|
68
|
+
(data) => {
|
|
69
|
+
const configCount = data.configs && typeof data.configs === "object" && !Array.isArray(data.configs) ? Object.keys(data.configs).length : 0;
|
|
70
|
+
if (configCount > 1 && !data.configSelector) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
return true;
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
message: 'A "configSelector" function is required when multiple configs are specified to determine which config to use.'
|
|
29
77
|
}
|
|
78
|
+
);
|
|
79
|
+
var BaseExporter = class {
|
|
80
|
+
/** Mastra logger instance */
|
|
81
|
+
logger;
|
|
82
|
+
/** Whether this exporter is disabled */
|
|
83
|
+
isDisabled = false;
|
|
30
84
|
/**
|
|
31
|
-
*
|
|
85
|
+
* Initialize the base exporter with logger
|
|
32
86
|
*/
|
|
33
|
-
|
|
34
|
-
this
|
|
87
|
+
constructor(config = {}) {
|
|
88
|
+
const logLevel = this.resolveLogLevel(config.logLevel);
|
|
89
|
+
this.logger = config.logger ?? new logger.ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
35
90
|
}
|
|
36
91
|
/**
|
|
37
|
-
*
|
|
92
|
+
* Set the logger for the exporter (called by Mastra/ObservabilityInstance during initialization)
|
|
38
93
|
*/
|
|
39
|
-
|
|
40
|
-
this
|
|
94
|
+
__setLogger(logger) {
|
|
95
|
+
this.logger = logger;
|
|
96
|
+
this.logger.debug(`Logger updated for exporter [name=${this.name}]`);
|
|
41
97
|
}
|
|
42
98
|
/**
|
|
43
|
-
*
|
|
99
|
+
* Convert string log level to LogLevel enum
|
|
44
100
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
101
|
+
resolveLogLevel(logLevel) {
|
|
102
|
+
if (!logLevel) {
|
|
103
|
+
return logger.LogLevel.INFO;
|
|
104
|
+
}
|
|
105
|
+
if (typeof logLevel === "number") {
|
|
106
|
+
return logLevel;
|
|
107
|
+
}
|
|
108
|
+
const logLevelMap = {
|
|
109
|
+
debug: logger.LogLevel.DEBUG,
|
|
110
|
+
info: logger.LogLevel.INFO,
|
|
111
|
+
warn: logger.LogLevel.WARN,
|
|
112
|
+
error: logger.LogLevel.ERROR
|
|
113
|
+
};
|
|
114
|
+
return logLevelMap[logLevel] ?? logger.LogLevel.INFO;
|
|
47
115
|
}
|
|
48
116
|
/**
|
|
49
|
-
*
|
|
117
|
+
* Mark the exporter as disabled and log a message
|
|
118
|
+
*
|
|
119
|
+
* @param reason - Reason why the exporter is disabled
|
|
50
120
|
*/
|
|
51
|
-
|
|
52
|
-
this
|
|
53
|
-
|
|
54
|
-
type: observability.AISpanType.MODEL_STEP,
|
|
55
|
-
attributes: {
|
|
56
|
-
stepIndex: this.#stepIndex,
|
|
57
|
-
...payload?.messageId ? { messageId: payload.messageId } : {},
|
|
58
|
-
...payload?.warnings?.length ? { warnings: payload.warnings } : {}
|
|
59
|
-
},
|
|
60
|
-
input: payload?.request
|
|
61
|
-
});
|
|
62
|
-
this.#chunkSequence = 0;
|
|
121
|
+
setDisabled(reason) {
|
|
122
|
+
this.isDisabled = true;
|
|
123
|
+
this.logger.warn(`${this.name} disabled: ${reason}`);
|
|
63
124
|
}
|
|
64
125
|
/**
|
|
65
|
-
*
|
|
126
|
+
* Export a tracing event
|
|
127
|
+
*
|
|
128
|
+
* This method checks if the exporter is disabled before calling _exportEvent.
|
|
129
|
+
* Subclasses should implement _exportEvent instead of overriding this method.
|
|
66
130
|
*/
|
|
67
|
-
|
|
68
|
-
if (
|
|
69
|
-
|
|
70
|
-
const { usage, ...otherOutput } = output;
|
|
71
|
-
const stepResult = payload.stepResult;
|
|
72
|
-
const metadata = payload.metadata;
|
|
73
|
-
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
74
|
-
if (cleanMetadata?.request) {
|
|
75
|
-
delete cleanMetadata.request;
|
|
131
|
+
async exportTracingEvent(event) {
|
|
132
|
+
if (this.isDisabled) {
|
|
133
|
+
return;
|
|
76
134
|
}
|
|
77
|
-
this
|
|
78
|
-
output: otherOutput,
|
|
79
|
-
attributes: {
|
|
80
|
-
usage,
|
|
81
|
-
isContinued: stepResult.isContinued,
|
|
82
|
-
finishReason: stepResult.reason,
|
|
83
|
-
warnings: stepResult.warnings
|
|
84
|
-
},
|
|
85
|
-
metadata: {
|
|
86
|
-
...cleanMetadata
|
|
87
|
-
}
|
|
88
|
-
});
|
|
89
|
-
this.#currentStepSpan = void 0;
|
|
90
|
-
this.#stepIndex++;
|
|
135
|
+
await this._exportTracingEvent(event);
|
|
91
136
|
}
|
|
92
137
|
/**
|
|
93
|
-
*
|
|
138
|
+
* Shutdown the exporter and clean up resources
|
|
139
|
+
*
|
|
140
|
+
* Default implementation just logs. Override to add custom cleanup.
|
|
94
141
|
*/
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
this.#startStepSpan();
|
|
98
|
-
}
|
|
99
|
-
this.#currentChunkSpan = this.#currentStepSpan?.createChildSpan({
|
|
100
|
-
name: `chunk: '${chunkType}'`,
|
|
101
|
-
type: observability.AISpanType.MODEL_CHUNK,
|
|
102
|
-
attributes: {
|
|
103
|
-
chunkType,
|
|
104
|
-
sequenceNumber: this.#chunkSequence
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
this.#accumulator = initialData || {};
|
|
142
|
+
async shutdown() {
|
|
143
|
+
this.logger.info(`${this.name} shutdown complete`);
|
|
108
144
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
145
|
+
};
|
|
146
|
+
var CloudExporter = class extends BaseExporter {
|
|
147
|
+
name = "mastra-cloud-observability-exporter";
|
|
148
|
+
config;
|
|
149
|
+
buffer;
|
|
150
|
+
flushTimer = null;
|
|
151
|
+
constructor(config = {}) {
|
|
152
|
+
super(config);
|
|
153
|
+
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
154
|
+
if (!accessToken) {
|
|
155
|
+
this.setDisabled(
|
|
156
|
+
"MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.\n\u{1F680} Sign up at https://cloud.mastra.ai to see your AI traces online and obtain your access token."
|
|
157
|
+
);
|
|
117
158
|
}
|
|
159
|
+
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
160
|
+
this.config = {
|
|
161
|
+
logger: this.logger,
|
|
162
|
+
logLevel: config.logLevel ?? logger.LogLevel.INFO,
|
|
163
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
164
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
165
|
+
maxRetries: config.maxRetries ?? 3,
|
|
166
|
+
accessToken: accessToken || "",
|
|
167
|
+
endpoint
|
|
168
|
+
};
|
|
169
|
+
this.buffer = {
|
|
170
|
+
spans: [],
|
|
171
|
+
totalSize: 0
|
|
172
|
+
};
|
|
118
173
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
*/
|
|
123
|
-
#endChunkSpan(output) {
|
|
124
|
-
if (!this.#currentChunkSpan) return;
|
|
125
|
-
this.#currentChunkSpan.end({
|
|
126
|
-
output: output !== void 0 ? output : this.#accumulator
|
|
127
|
-
});
|
|
128
|
-
this.#currentChunkSpan = void 0;
|
|
129
|
-
this.#accumulator = {};
|
|
130
|
-
this.#chunkSequence++;
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* Create an event span (for single chunks like tool-call)
|
|
134
|
-
*/
|
|
135
|
-
#createEventSpan(chunkType, output) {
|
|
136
|
-
if (!this.#currentStepSpan) {
|
|
137
|
-
this.#startStepSpan();
|
|
174
|
+
async _exportTracingEvent(event) {
|
|
175
|
+
if (event.type !== observability.TracingEventType.SPAN_ENDED) {
|
|
176
|
+
return;
|
|
138
177
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
if (span) {
|
|
149
|
-
this.#chunkSequence++;
|
|
178
|
+
this.addToBuffer(event);
|
|
179
|
+
if (this.shouldFlush()) {
|
|
180
|
+
this.flush().catch((error) => {
|
|
181
|
+
this.logger.error("Batch flush failed", {
|
|
182
|
+
error: error instanceof Error ? error.message : String(error)
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
} else if (this.buffer.totalSize === 1) {
|
|
186
|
+
this.scheduleFlush();
|
|
150
187
|
}
|
|
151
188
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
189
|
+
addToBuffer(event) {
|
|
190
|
+
if (this.buffer.totalSize === 0) {
|
|
191
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
192
|
+
}
|
|
193
|
+
const spanRecord = this.formatSpan(event.exportedSpan);
|
|
194
|
+
this.buffer.spans.push(spanRecord);
|
|
195
|
+
this.buffer.totalSize++;
|
|
157
196
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
197
|
+
formatSpan(span) {
|
|
198
|
+
const spanRecord = {
|
|
199
|
+
traceId: span.traceId,
|
|
200
|
+
spanId: span.id,
|
|
201
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
202
|
+
name: span.name,
|
|
203
|
+
spanType: span.type,
|
|
204
|
+
attributes: span.attributes ?? null,
|
|
205
|
+
metadata: span.metadata ?? null,
|
|
206
|
+
startedAt: span.startTime,
|
|
207
|
+
endedAt: span.endTime ?? null,
|
|
208
|
+
input: span.input ?? null,
|
|
209
|
+
output: span.output ?? null,
|
|
210
|
+
error: span.errorInfo,
|
|
211
|
+
isEvent: span.isEvent,
|
|
212
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
213
|
+
updatedAt: null
|
|
214
|
+
};
|
|
215
|
+
return spanRecord;
|
|
163
216
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
case "text-delta":
|
|
173
|
-
this.#appendToAccumulator("text", chunk.payload.text);
|
|
174
|
-
break;
|
|
175
|
-
case "text-end": {
|
|
176
|
-
this.#endChunkSpan();
|
|
177
|
-
break;
|
|
217
|
+
shouldFlush() {
|
|
218
|
+
if (this.buffer.totalSize >= this.config.maxBatchSize) {
|
|
219
|
+
return true;
|
|
220
|
+
}
|
|
221
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
222
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
223
|
+
if (elapsed >= this.config.maxBatchWaitMs) {
|
|
224
|
+
return true;
|
|
178
225
|
}
|
|
179
226
|
}
|
|
227
|
+
return false;
|
|
180
228
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
#handleReasoningChunk(chunk) {
|
|
185
|
-
switch (chunk.type) {
|
|
186
|
-
case "reasoning-start":
|
|
187
|
-
this.#startChunkSpan("reasoning");
|
|
188
|
-
break;
|
|
189
|
-
case "reasoning-delta":
|
|
190
|
-
this.#appendToAccumulator("text", chunk.payload.text);
|
|
191
|
-
break;
|
|
192
|
-
case "reasoning-end": {
|
|
193
|
-
this.#endChunkSpan();
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
229
|
+
scheduleFlush() {
|
|
230
|
+
if (this.flushTimer) {
|
|
231
|
+
clearTimeout(this.flushTimer);
|
|
196
232
|
}
|
|
233
|
+
this.flushTimer = setTimeout(() => {
|
|
234
|
+
this.flush().catch((error$1) => {
|
|
235
|
+
const mastraError = new error.MastraError(
|
|
236
|
+
{
|
|
237
|
+
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
238
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
239
|
+
category: error.ErrorCategory.USER
|
|
240
|
+
},
|
|
241
|
+
error$1
|
|
242
|
+
);
|
|
243
|
+
this.logger.trackException(mastraError);
|
|
244
|
+
this.logger.error("Scheduled flush failed", mastraError);
|
|
245
|
+
});
|
|
246
|
+
}, this.config.maxBatchWaitMs);
|
|
197
247
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
switch (chunk.type) {
|
|
203
|
-
case "tool-call-input-streaming-start":
|
|
204
|
-
this.#startChunkSpan("tool-call", {
|
|
205
|
-
toolName: chunk.payload.toolName,
|
|
206
|
-
toolCallId: chunk.payload.toolCallId
|
|
207
|
-
});
|
|
208
|
-
break;
|
|
209
|
-
case "tool-call-delta":
|
|
210
|
-
this.#appendToAccumulator("toolInput", chunk.payload.argsTextDelta);
|
|
211
|
-
break;
|
|
212
|
-
case "tool-call-input-streaming-end":
|
|
213
|
-
case "tool-call": {
|
|
214
|
-
const acc = this.#getAccumulator();
|
|
215
|
-
let toolInput;
|
|
216
|
-
try {
|
|
217
|
-
toolInput = acc.toolInput ? JSON.parse(acc.toolInput) : {};
|
|
218
|
-
} catch {
|
|
219
|
-
toolInput = acc.toolInput;
|
|
220
|
-
}
|
|
221
|
-
this.#endChunkSpan({
|
|
222
|
-
toolName: acc.toolName,
|
|
223
|
-
toolCallId: acc.toolCallId,
|
|
224
|
-
toolInput
|
|
225
|
-
});
|
|
226
|
-
break;
|
|
227
|
-
}
|
|
248
|
+
async flush() {
|
|
249
|
+
if (this.flushTimer) {
|
|
250
|
+
clearTimeout(this.flushTimer);
|
|
251
|
+
this.flushTimer = null;
|
|
228
252
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
* Handle object chunk spans (object, object-result)
|
|
232
|
-
*/
|
|
233
|
-
#handleObjectChunk(chunk) {
|
|
234
|
-
switch (chunk.type) {
|
|
235
|
-
case "object":
|
|
236
|
-
if (!this.#hasActiveChunkSpan()) {
|
|
237
|
-
this.#startChunkSpan("object");
|
|
238
|
-
}
|
|
239
|
-
break;
|
|
240
|
-
case "object-result":
|
|
241
|
-
this.#endChunkSpan(chunk.object);
|
|
242
|
-
break;
|
|
253
|
+
if (this.buffer.totalSize === 0) {
|
|
254
|
+
return;
|
|
243
255
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
case "tool-call-input-streaming-end":
|
|
265
|
-
case "tool-call":
|
|
266
|
-
this.#handleToolCallChunk(chunk);
|
|
267
|
-
break;
|
|
268
|
-
case "reasoning-start":
|
|
269
|
-
case "reasoning-delta":
|
|
270
|
-
case "reasoning-end":
|
|
271
|
-
this.#handleReasoningChunk(chunk);
|
|
272
|
-
break;
|
|
273
|
-
case "object":
|
|
274
|
-
case "object-result":
|
|
275
|
-
this.#handleObjectChunk(chunk);
|
|
276
|
-
break;
|
|
277
|
-
case "step-start":
|
|
278
|
-
this.#startStepSpan(chunk.payload);
|
|
279
|
-
break;
|
|
280
|
-
case "step-finish":
|
|
281
|
-
this.#endStepSpan(chunk.payload);
|
|
282
|
-
break;
|
|
283
|
-
case "raw":
|
|
284
|
-
// Skip raw chunks as they're redundant
|
|
285
|
-
case "start":
|
|
286
|
-
case "finish":
|
|
287
|
-
break;
|
|
288
|
-
// Default: auto-create event span for all other chunk types
|
|
289
|
-
default: {
|
|
290
|
-
let outputPayload = chunk.payload;
|
|
291
|
-
if (outputPayload && typeof outputPayload === "object" && "data" in outputPayload) {
|
|
292
|
-
const typedPayload = outputPayload;
|
|
293
|
-
outputPayload = { ...typedPayload };
|
|
294
|
-
if (typedPayload.data) {
|
|
295
|
-
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
296
|
-
delete outputPayload.data;
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
this.#createEventSpan(chunk.type, outputPayload);
|
|
300
|
-
break;
|
|
301
|
-
}
|
|
256
|
+
const startTime = Date.now();
|
|
257
|
+
const spansCopy = [...this.buffer.spans];
|
|
258
|
+
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
259
|
+
this.resetBuffer();
|
|
260
|
+
try {
|
|
261
|
+
await this.batchUpload(spansCopy);
|
|
262
|
+
const elapsed = Date.now() - startTime;
|
|
263
|
+
this.logger.debug("Batch flushed successfully", {
|
|
264
|
+
batchSize: spansCopy.length,
|
|
265
|
+
flushReason,
|
|
266
|
+
durationMs: elapsed
|
|
267
|
+
});
|
|
268
|
+
} catch (error$1) {
|
|
269
|
+
const mastraError = new error.MastraError(
|
|
270
|
+
{
|
|
271
|
+
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
272
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
273
|
+
category: error.ErrorCategory.USER,
|
|
274
|
+
details: {
|
|
275
|
+
droppedBatchSize: spansCopy.length
|
|
302
276
|
}
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
// src/spans/base.ts
|
|
310
|
-
function isSpanInternal(spanType, flags) {
|
|
311
|
-
if (flags === void 0 || flags === observability.InternalSpans.NONE) {
|
|
312
|
-
return false;
|
|
313
|
-
}
|
|
314
|
-
switch (spanType) {
|
|
315
|
-
// Workflow-related spans
|
|
316
|
-
case observability.AISpanType.WORKFLOW_RUN:
|
|
317
|
-
case observability.AISpanType.WORKFLOW_STEP:
|
|
318
|
-
case observability.AISpanType.WORKFLOW_CONDITIONAL:
|
|
319
|
-
case observability.AISpanType.WORKFLOW_CONDITIONAL_EVAL:
|
|
320
|
-
case observability.AISpanType.WORKFLOW_PARALLEL:
|
|
321
|
-
case observability.AISpanType.WORKFLOW_LOOP:
|
|
322
|
-
case observability.AISpanType.WORKFLOW_SLEEP:
|
|
323
|
-
case observability.AISpanType.WORKFLOW_WAIT_EVENT:
|
|
324
|
-
return (flags & observability.InternalSpans.WORKFLOW) !== 0;
|
|
325
|
-
// Agent-related spans
|
|
326
|
-
case observability.AISpanType.AGENT_RUN:
|
|
327
|
-
return (flags & observability.InternalSpans.AGENT) !== 0;
|
|
328
|
-
// Tool-related spans
|
|
329
|
-
case observability.AISpanType.TOOL_CALL:
|
|
330
|
-
case observability.AISpanType.MCP_TOOL_CALL:
|
|
331
|
-
return (flags & observability.InternalSpans.TOOL) !== 0;
|
|
332
|
-
// Model-related spans
|
|
333
|
-
case observability.AISpanType.MODEL_GENERATION:
|
|
334
|
-
case observability.AISpanType.MODEL_STEP:
|
|
335
|
-
case observability.AISpanType.MODEL_CHUNK:
|
|
336
|
-
return (flags & observability.InternalSpans.MODEL) !== 0;
|
|
337
|
-
// Default: never internal
|
|
338
|
-
default:
|
|
339
|
-
return false;
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
var BaseAISpan = class {
|
|
343
|
-
name;
|
|
344
|
-
type;
|
|
345
|
-
attributes;
|
|
346
|
-
parent;
|
|
347
|
-
startTime;
|
|
348
|
-
endTime;
|
|
349
|
-
isEvent;
|
|
350
|
-
isInternal;
|
|
351
|
-
aiTracing;
|
|
352
|
-
input;
|
|
353
|
-
output;
|
|
354
|
-
errorInfo;
|
|
355
|
-
metadata;
|
|
356
|
-
traceState;
|
|
357
|
-
/** Parent span ID (for root spans that are children of external spans) */
|
|
358
|
-
parentSpanId;
|
|
359
|
-
constructor(options, aiTracing) {
|
|
360
|
-
this.name = options.name;
|
|
361
|
-
this.type = options.type;
|
|
362
|
-
this.attributes = deepClean(options.attributes) || {};
|
|
363
|
-
this.metadata = deepClean(options.metadata);
|
|
364
|
-
this.parent = options.parent;
|
|
365
|
-
this.startTime = /* @__PURE__ */ new Date();
|
|
366
|
-
this.aiTracing = aiTracing;
|
|
367
|
-
this.isEvent = options.isEvent ?? false;
|
|
368
|
-
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
369
|
-
this.traceState = options.traceState;
|
|
370
|
-
if (this.isEvent) {
|
|
371
|
-
this.output = deepClean(options.output);
|
|
372
|
-
} else {
|
|
373
|
-
this.input = deepClean(options.input);
|
|
277
|
+
},
|
|
278
|
+
error$1
|
|
279
|
+
);
|
|
280
|
+
this.logger.trackException(mastraError);
|
|
281
|
+
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
374
282
|
}
|
|
375
283
|
}
|
|
376
|
-
createChildSpan(options) {
|
|
377
|
-
return this.aiTracing.startSpan({ ...options, parent: this, isEvent: false });
|
|
378
|
-
}
|
|
379
|
-
createEventSpan(options) {
|
|
380
|
-
return this.aiTracing.startSpan({ ...options, parent: this, isEvent: true });
|
|
381
|
-
}
|
|
382
284
|
/**
|
|
383
|
-
*
|
|
384
|
-
* Returns undefined for non-MODEL_GENERATION spans
|
|
285
|
+
* Uploads spans to cloud API using fetchWithRetry for all retry logic
|
|
385
286
|
*/
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
return new ModelSpanTracker(this);
|
|
391
|
-
}
|
|
392
|
-
/** Returns `TRUE` if the span is the root span of a trace */
|
|
393
|
-
get isRootSpan() {
|
|
394
|
-
return !this.parent;
|
|
395
|
-
}
|
|
396
|
-
/** Get the closest parent spanId that isn't an internal span */
|
|
397
|
-
getParentSpanId(includeInternalSpans) {
|
|
398
|
-
if (!this.parent) {
|
|
399
|
-
return this.parentSpanId;
|
|
400
|
-
}
|
|
401
|
-
if (includeInternalSpans) return this.parent.id;
|
|
402
|
-
if (this.parent.isInternal) return this.parent.getParentSpanId(includeInternalSpans);
|
|
403
|
-
return this.parent.id;
|
|
404
|
-
}
|
|
405
|
-
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
406
|
-
findParent(spanType) {
|
|
407
|
-
let current = this.parent;
|
|
408
|
-
while (current) {
|
|
409
|
-
if (current.type === spanType) {
|
|
410
|
-
return current;
|
|
411
|
-
}
|
|
412
|
-
current = current.parent;
|
|
413
|
-
}
|
|
414
|
-
return void 0;
|
|
415
|
-
}
|
|
416
|
-
/** Returns a lightweight span ready for export */
|
|
417
|
-
exportSpan(includeInternalSpans) {
|
|
418
|
-
return {
|
|
419
|
-
id: this.id,
|
|
420
|
-
traceId: this.traceId,
|
|
421
|
-
name: this.name,
|
|
422
|
-
type: this.type,
|
|
423
|
-
attributes: this.attributes,
|
|
424
|
-
metadata: this.metadata,
|
|
425
|
-
startTime: this.startTime,
|
|
426
|
-
endTime: this.endTime,
|
|
427
|
-
input: this.input,
|
|
428
|
-
output: this.output,
|
|
429
|
-
errorInfo: this.errorInfo,
|
|
430
|
-
isEvent: this.isEvent,
|
|
431
|
-
isRootSpan: this.isRootSpan,
|
|
432
|
-
parentSpanId: this.getParentSpanId(includeInternalSpans)
|
|
287
|
+
async batchUpload(spans) {
|
|
288
|
+
const headers = {
|
|
289
|
+
Authorization: `Bearer ${this.config.accessToken}`,
|
|
290
|
+
"Content-Type": "application/json"
|
|
433
291
|
};
|
|
292
|
+
const options = {
|
|
293
|
+
method: "POST",
|
|
294
|
+
headers,
|
|
295
|
+
body: JSON.stringify({ spans })
|
|
296
|
+
};
|
|
297
|
+
await utils.fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
434
298
|
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
440
|
-
"logger",
|
|
441
|
-
"experimental_providerMetadata",
|
|
442
|
-
"providerMetadata",
|
|
443
|
-
"steps",
|
|
444
|
-
"tracingContext"
|
|
445
|
-
]);
|
|
446
|
-
function deepClean(value, options = {}, _seen = /* @__PURE__ */ new WeakSet(), _depth = 0) {
|
|
447
|
-
const { keysToStrip = DEFAULT_KEYS_TO_STRIP, maxDepth = 10 } = options;
|
|
448
|
-
if (_depth > maxDepth) {
|
|
449
|
-
return "[MaxDepth]";
|
|
450
|
-
}
|
|
451
|
-
if (value === null || typeof value !== "object") {
|
|
452
|
-
try {
|
|
453
|
-
JSON.stringify(value);
|
|
454
|
-
return value;
|
|
455
|
-
} catch (error) {
|
|
456
|
-
return `[${error instanceof Error ? error.message : String(error)}]`;
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
if (_seen.has(value)) {
|
|
460
|
-
return "[Circular]";
|
|
461
|
-
}
|
|
462
|
-
_seen.add(value);
|
|
463
|
-
if (Array.isArray(value)) {
|
|
464
|
-
return value.map((item) => deepClean(item, options, _seen, _depth + 1));
|
|
299
|
+
resetBuffer() {
|
|
300
|
+
this.buffer.spans = [];
|
|
301
|
+
this.buffer.firstEventTime = void 0;
|
|
302
|
+
this.buffer.totalSize = 0;
|
|
465
303
|
}
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
continue;
|
|
470
|
-
}
|
|
471
|
-
try {
|
|
472
|
-
cleaned[key] = deepClean(val, options, _seen, _depth + 1);
|
|
473
|
-
} catch (error) {
|
|
474
|
-
cleaned[key] = `[${error instanceof Error ? error.message : String(error)}]`;
|
|
304
|
+
async shutdown() {
|
|
305
|
+
if (this.isDisabled) {
|
|
306
|
+
return;
|
|
475
307
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
var DefaultAISpan = class extends BaseAISpan {
|
|
480
|
-
id;
|
|
481
|
-
traceId;
|
|
482
|
-
constructor(options, aiTracing) {
|
|
483
|
-
super(options, aiTracing);
|
|
484
|
-
this.id = generateSpanId();
|
|
485
|
-
if (options.parent) {
|
|
486
|
-
this.traceId = options.parent.traceId;
|
|
487
|
-
} else if (options.traceId) {
|
|
488
|
-
if (isValidTraceId(options.traceId)) {
|
|
489
|
-
this.traceId = options.traceId;
|
|
490
|
-
} else {
|
|
491
|
-
console.error(
|
|
492
|
-
`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`
|
|
493
|
-
);
|
|
494
|
-
this.traceId = generateTraceId();
|
|
495
|
-
}
|
|
496
|
-
} else {
|
|
497
|
-
this.traceId = generateTraceId();
|
|
308
|
+
if (this.flushTimer) {
|
|
309
|
+
clearTimeout(this.flushTimer);
|
|
310
|
+
this.flushTimer = null;
|
|
498
311
|
}
|
|
499
|
-
if (
|
|
500
|
-
|
|
501
|
-
this.
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
|
|
312
|
+
if (this.buffer.totalSize > 0) {
|
|
313
|
+
this.logger.info("Flushing remaining events on shutdown", {
|
|
314
|
+
remainingEvents: this.buffer.totalSize
|
|
315
|
+
});
|
|
316
|
+
try {
|
|
317
|
+
await this.flush();
|
|
318
|
+
} catch (error$1) {
|
|
319
|
+
const mastraError = new error.MastraError(
|
|
320
|
+
{
|
|
321
|
+
id: `CLOUD_EXPORTER_FAILED_TO_FLUSH_REMAINING_EVENTS_DURING_SHUTDOWN`,
|
|
322
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
323
|
+
category: error.ErrorCategory.USER,
|
|
324
|
+
details: {
|
|
325
|
+
remainingEvents: this.buffer.totalSize
|
|
326
|
+
}
|
|
327
|
+
},
|
|
328
|
+
error$1
|
|
505
329
|
);
|
|
330
|
+
this.logger.trackException(mastraError);
|
|
331
|
+
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
506
332
|
}
|
|
507
333
|
}
|
|
334
|
+
this.logger.info("CloudExporter shutdown complete");
|
|
508
335
|
}
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
if (options?.output !== void 0) {
|
|
515
|
-
this.output = deepClean(options.output);
|
|
516
|
-
}
|
|
517
|
-
if (options?.attributes) {
|
|
518
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
519
|
-
}
|
|
520
|
-
if (options?.metadata) {
|
|
521
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
522
|
-
}
|
|
336
|
+
};
|
|
337
|
+
var ConsoleExporter = class extends BaseExporter {
|
|
338
|
+
name = "tracing-console-exporter";
|
|
339
|
+
constructor(config = {}) {
|
|
340
|
+
super(config);
|
|
523
341
|
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
domain: error$1.domain,
|
|
534
|
-
message: error$1.message
|
|
535
|
-
} : {
|
|
536
|
-
message: error$1.message
|
|
342
|
+
async _exportTracingEvent(event) {
|
|
343
|
+
const span = event.exportedSpan;
|
|
344
|
+
const formatAttributes = (attributes) => {
|
|
345
|
+
try {
|
|
346
|
+
return JSON.stringify(attributes, null, 2);
|
|
347
|
+
} catch (error) {
|
|
348
|
+
const errMsg = error instanceof Error ? error.message : "Unknown formatting error";
|
|
349
|
+
return `[Unable to serialize attributes: ${errMsg}]`;
|
|
350
|
+
}
|
|
537
351
|
};
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
352
|
+
const formatDuration = (startTime, endTime) => {
|
|
353
|
+
if (!endTime) return "N/A";
|
|
354
|
+
const duration = endTime.getTime() - startTime.getTime();
|
|
355
|
+
return `${duration}ms`;
|
|
356
|
+
};
|
|
357
|
+
switch (event.type) {
|
|
358
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
359
|
+
this.logger.info(`\u{1F680} SPAN_STARTED`);
|
|
360
|
+
this.logger.info(` Type: ${span.type}`);
|
|
361
|
+
this.logger.info(` Name: ${span.name}`);
|
|
362
|
+
this.logger.info(` ID: ${span.id}`);
|
|
363
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
364
|
+
if (span.input !== void 0) {
|
|
365
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
366
|
+
}
|
|
367
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
368
|
+
this.logger.info("\u2500".repeat(80));
|
|
369
|
+
break;
|
|
370
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
371
|
+
const duration = formatDuration(span.startTime, span.endTime);
|
|
372
|
+
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
373
|
+
this.logger.info(` Type: ${span.type}`);
|
|
374
|
+
this.logger.info(` Name: ${span.name}`);
|
|
375
|
+
this.logger.info(` ID: ${span.id}`);
|
|
376
|
+
this.logger.info(` Duration: ${duration}`);
|
|
377
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
378
|
+
if (span.input !== void 0) {
|
|
379
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
380
|
+
}
|
|
381
|
+
if (span.output !== void 0) {
|
|
382
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
383
|
+
}
|
|
384
|
+
if (span.errorInfo) {
|
|
385
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
386
|
+
}
|
|
387
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
388
|
+
this.logger.info("\u2500".repeat(80));
|
|
389
|
+
break;
|
|
390
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
391
|
+
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
392
|
+
this.logger.info(` Type: ${span.type}`);
|
|
393
|
+
this.logger.info(` Name: ${span.name}`);
|
|
394
|
+
this.logger.info(` ID: ${span.id}`);
|
|
395
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
396
|
+
if (span.input !== void 0) {
|
|
397
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
398
|
+
}
|
|
399
|
+
if (span.output !== void 0) {
|
|
400
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
401
|
+
}
|
|
402
|
+
if (span.errorInfo) {
|
|
403
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
404
|
+
}
|
|
405
|
+
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
406
|
+
this.logger.info("\u2500".repeat(80));
|
|
407
|
+
break;
|
|
408
|
+
default:
|
|
409
|
+
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
565
410
|
}
|
|
566
411
|
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
}
|
|
570
|
-
async export() {
|
|
571
|
-
return JSON.stringify({
|
|
572
|
-
spanId: this.id,
|
|
573
|
-
traceId: this.traceId,
|
|
574
|
-
startTime: this.startTime,
|
|
575
|
-
endTime: this.endTime,
|
|
576
|
-
attributes: this.attributes,
|
|
577
|
-
metadata: this.metadata
|
|
578
|
-
});
|
|
412
|
+
async shutdown() {
|
|
413
|
+
this.logger.info("ConsoleExporter shutdown");
|
|
579
414
|
}
|
|
580
415
|
};
|
|
581
|
-
function
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
416
|
+
function resolveTracingStorageStrategy(config, storage, logger) {
|
|
417
|
+
if (config.strategy && config.strategy !== "auto") {
|
|
418
|
+
const hints = storage.tracingStrategy;
|
|
419
|
+
if (hints.supported.includes(config.strategy)) {
|
|
420
|
+
return config.strategy;
|
|
421
|
+
}
|
|
422
|
+
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
423
|
+
userStrategy: config.strategy,
|
|
424
|
+
storageAdapter: storage.constructor.name,
|
|
425
|
+
supportedStrategies: hints.supported,
|
|
426
|
+
fallbackStrategy: hints.preferred
|
|
427
|
+
});
|
|
589
428
|
}
|
|
590
|
-
return
|
|
429
|
+
return storage.tracingStrategy.preferred;
|
|
591
430
|
}
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
431
|
+
var DefaultExporter = class extends BaseExporter {
|
|
432
|
+
name = "mastra-default-observability-exporter";
|
|
433
|
+
#storage;
|
|
434
|
+
#config;
|
|
435
|
+
#resolvedStrategy;
|
|
436
|
+
buffer;
|
|
437
|
+
#flushTimer = null;
|
|
438
|
+
// Track all spans that have been created, persists across flushes
|
|
439
|
+
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
440
|
+
constructor(config = {}) {
|
|
441
|
+
super(config);
|
|
442
|
+
if (config === void 0) {
|
|
443
|
+
config = {};
|
|
599
444
|
}
|
|
445
|
+
this.#config = {
|
|
446
|
+
...config,
|
|
447
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
448
|
+
maxBufferSize: config.maxBufferSize ?? 1e4,
|
|
449
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
450
|
+
maxRetries: config.maxRetries ?? 4,
|
|
451
|
+
retryDelayMs: config.retryDelayMs ?? 500,
|
|
452
|
+
strategy: config.strategy ?? "auto"
|
|
453
|
+
};
|
|
454
|
+
this.buffer = {
|
|
455
|
+
creates: [],
|
|
456
|
+
updates: [],
|
|
457
|
+
insertOnly: [],
|
|
458
|
+
seenSpans: /* @__PURE__ */ new Set(),
|
|
459
|
+
spanSequences: /* @__PURE__ */ new Map(),
|
|
460
|
+
completedSpans: /* @__PURE__ */ new Set(),
|
|
461
|
+
outOfOrderCount: 0,
|
|
462
|
+
totalSize: 0
|
|
463
|
+
};
|
|
464
|
+
this.#resolvedStrategy = "batch-with-updates";
|
|
600
465
|
}
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
id;
|
|
613
|
-
traceId;
|
|
614
|
-
constructor(options, aiTracing) {
|
|
615
|
-
super(options, aiTracing);
|
|
616
|
-
this.id = "no-op";
|
|
617
|
-
this.traceId = "no-op-trace";
|
|
466
|
+
#strategyInitialized = false;
|
|
467
|
+
/**
|
|
468
|
+
* Initialize the exporter (called after all dependencies are ready)
|
|
469
|
+
*/
|
|
470
|
+
init(options) {
|
|
471
|
+
this.#storage = options.mastra?.getStorage();
|
|
472
|
+
if (!this.#storage) {
|
|
473
|
+
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
this.initializeStrategy(this.#storage);
|
|
618
477
|
}
|
|
619
|
-
|
|
478
|
+
/**
|
|
479
|
+
* Initialize the resolved strategy once storage is available
|
|
480
|
+
*/
|
|
481
|
+
initializeStrategy(storage) {
|
|
482
|
+
if (this.#strategyInitialized) return;
|
|
483
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, storage, this.logger);
|
|
484
|
+
this.#strategyInitialized = true;
|
|
485
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
486
|
+
strategy: this.#resolvedStrategy,
|
|
487
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
488
|
+
storageAdapter: storage.constructor.name,
|
|
489
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
490
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
491
|
+
});
|
|
620
492
|
}
|
|
621
|
-
|
|
493
|
+
/**
|
|
494
|
+
* Builds a unique span key for tracking
|
|
495
|
+
*/
|
|
496
|
+
buildSpanKey(traceId, spanId) {
|
|
497
|
+
return `${traceId}:${spanId}`;
|
|
622
498
|
}
|
|
623
|
-
|
|
499
|
+
/**
|
|
500
|
+
* Gets the next sequence number for a span
|
|
501
|
+
*/
|
|
502
|
+
getNextSequence(spanKey) {
|
|
503
|
+
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
504
|
+
const next = current + 1;
|
|
505
|
+
this.buffer.spanSequences.set(spanKey, next);
|
|
506
|
+
return next;
|
|
624
507
|
}
|
|
625
|
-
|
|
626
|
-
|
|
508
|
+
/**
|
|
509
|
+
* Handles out-of-order span updates by logging and skipping
|
|
510
|
+
*/
|
|
511
|
+
handleOutOfOrderUpdate(event) {
|
|
512
|
+
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
513
|
+
spanId: event.exportedSpan.id,
|
|
514
|
+
traceId: event.exportedSpan.traceId,
|
|
515
|
+
spanName: event.exportedSpan.name,
|
|
516
|
+
eventType: event.type
|
|
517
|
+
});
|
|
627
518
|
}
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
519
|
+
/**
|
|
520
|
+
* Adds an event to the appropriate buffer based on strategy
|
|
521
|
+
*/
|
|
522
|
+
addToBuffer(event) {
|
|
523
|
+
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
524
|
+
if (this.buffer.totalSize === 0) {
|
|
525
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
526
|
+
}
|
|
527
|
+
switch (event.type) {
|
|
528
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
529
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
530
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
531
|
+
this.buffer.creates.push(createRecord);
|
|
532
|
+
this.buffer.seenSpans.add(spanKey);
|
|
533
|
+
this.allCreatedSpans.add(spanKey);
|
|
534
|
+
}
|
|
535
|
+
break;
|
|
536
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
537
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
538
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
539
|
+
this.buffer.updates.push({
|
|
540
|
+
traceId: event.exportedSpan.traceId,
|
|
541
|
+
spanId: event.exportedSpan.id,
|
|
542
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
543
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
544
|
+
});
|
|
545
|
+
} else {
|
|
546
|
+
this.handleOutOfOrderUpdate(event);
|
|
547
|
+
this.buffer.outOfOrderCount++;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
break;
|
|
551
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
552
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
553
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
554
|
+
this.buffer.updates.push({
|
|
555
|
+
traceId: event.exportedSpan.traceId,
|
|
556
|
+
spanId: event.exportedSpan.id,
|
|
557
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
558
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
559
|
+
});
|
|
560
|
+
this.buffer.completedSpans.add(spanKey);
|
|
561
|
+
} else if (event.exportedSpan.isEvent) {
|
|
562
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
563
|
+
this.buffer.creates.push(createRecord);
|
|
564
|
+
this.buffer.seenSpans.add(spanKey);
|
|
565
|
+
this.allCreatedSpans.add(spanKey);
|
|
566
|
+
this.buffer.completedSpans.add(spanKey);
|
|
567
|
+
} else {
|
|
568
|
+
this.handleOutOfOrderUpdate(event);
|
|
569
|
+
this.buffer.outOfOrderCount++;
|
|
570
|
+
}
|
|
571
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
572
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
573
|
+
this.buffer.insertOnly.push(createRecord);
|
|
574
|
+
this.buffer.completedSpans.add(spanKey);
|
|
575
|
+
this.allCreatedSpans.add(spanKey);
|
|
576
|
+
}
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
644
580
|
}
|
|
645
581
|
/**
|
|
646
|
-
*
|
|
647
|
-
* and propagate logger to exporters
|
|
582
|
+
* Checks if buffer should be flushed based on size or time triggers
|
|
648
583
|
*/
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
584
|
+
shouldFlush() {
|
|
585
|
+
if (this.buffer.totalSize >= this.#config.maxBufferSize) {
|
|
586
|
+
return true;
|
|
587
|
+
}
|
|
588
|
+
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
589
|
+
return true;
|
|
590
|
+
}
|
|
591
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
592
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
593
|
+
if (elapsed >= this.#config.maxBatchWaitMs) {
|
|
594
|
+
return true;
|
|
654
595
|
}
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
`[AI Tracing] Initialized [service=${this.config.serviceName}] [instance=${this.config.name}] [sampling=${this.config.sampling.type}]`
|
|
658
|
-
);
|
|
596
|
+
}
|
|
597
|
+
return false;
|
|
659
598
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
599
|
+
/**
|
|
600
|
+
* Resets the buffer after successful flush
|
|
601
|
+
*/
|
|
602
|
+
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
603
|
+
this.buffer.creates = [];
|
|
604
|
+
this.buffer.updates = [];
|
|
605
|
+
this.buffer.insertOnly = [];
|
|
606
|
+
this.buffer.seenSpans.clear();
|
|
607
|
+
this.buffer.spanSequences.clear();
|
|
608
|
+
this.buffer.completedSpans.clear();
|
|
609
|
+
this.buffer.outOfOrderCount = 0;
|
|
610
|
+
this.buffer.firstEventTime = void 0;
|
|
611
|
+
this.buffer.totalSize = 0;
|
|
612
|
+
for (const spanKey of completedSpansToCleanup) {
|
|
613
|
+
this.allCreatedSpans.delete(spanKey);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Schedules a flush using setTimeout
|
|
618
|
+
*/
|
|
619
|
+
scheduleFlush() {
|
|
620
|
+
if (this.#flushTimer) {
|
|
621
|
+
clearTimeout(this.#flushTimer);
|
|
622
|
+
}
|
|
623
|
+
this.#flushTimer = setTimeout(() => {
|
|
624
|
+
this.flush().catch((error) => {
|
|
625
|
+
this.logger.error("Scheduled flush failed", {
|
|
626
|
+
error: error instanceof Error ? error.message : String(error)
|
|
627
|
+
});
|
|
628
|
+
});
|
|
629
|
+
}, this.#config.maxBatchWaitMs);
|
|
630
|
+
}
|
|
631
|
+
/**
|
|
632
|
+
* Serializes span attributes to storage record format
|
|
633
|
+
* Handles all Span types and their specific attributes
|
|
634
|
+
*/
|
|
635
|
+
serializeAttributes(span) {
|
|
636
|
+
if (!span.attributes) {
|
|
637
|
+
return null;
|
|
638
|
+
}
|
|
639
|
+
try {
|
|
640
|
+
return JSON.parse(
|
|
641
|
+
JSON.stringify(span.attributes, (_key, value) => {
|
|
642
|
+
if (value instanceof Date) {
|
|
643
|
+
return value.toISOString();
|
|
644
|
+
}
|
|
645
|
+
if (typeof value === "object" && value !== null) {
|
|
646
|
+
return value;
|
|
647
|
+
}
|
|
648
|
+
return value;
|
|
649
|
+
})
|
|
650
|
+
);
|
|
651
|
+
} catch (error) {
|
|
652
|
+
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
653
|
+
spanId: span.id,
|
|
654
|
+
spanType: span.type,
|
|
655
|
+
error: error instanceof Error ? error.message : String(error)
|
|
656
|
+
});
|
|
657
|
+
return null;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
buildCreateRecord(span) {
|
|
661
|
+
return {
|
|
662
|
+
traceId: span.traceId,
|
|
663
|
+
spanId: span.id,
|
|
664
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
665
|
+
name: span.name,
|
|
666
|
+
scope: null,
|
|
667
|
+
spanType: span.type,
|
|
668
|
+
attributes: this.serializeAttributes(span),
|
|
669
|
+
metadata: span.metadata ?? null,
|
|
670
|
+
links: null,
|
|
671
|
+
startedAt: span.startTime,
|
|
672
|
+
endedAt: span.endTime ?? null,
|
|
673
|
+
input: span.input,
|
|
674
|
+
output: span.output,
|
|
675
|
+
error: span.errorInfo,
|
|
676
|
+
isEvent: span.isEvent
|
|
677
|
+
};
|
|
665
678
|
}
|
|
666
|
-
|
|
667
|
-
return
|
|
679
|
+
buildUpdateRecord(span) {
|
|
680
|
+
return {
|
|
681
|
+
name: span.name,
|
|
682
|
+
scope: null,
|
|
683
|
+
attributes: this.serializeAttributes(span),
|
|
684
|
+
metadata: span.metadata ?? null,
|
|
685
|
+
links: null,
|
|
686
|
+
endedAt: span.endTime ?? null,
|
|
687
|
+
input: span.input,
|
|
688
|
+
output: span.output,
|
|
689
|
+
error: span.errorInfo
|
|
690
|
+
};
|
|
668
691
|
}
|
|
669
|
-
// ============================================================================
|
|
670
|
-
// Public API - Single type-safe span creation method
|
|
671
|
-
// ============================================================================
|
|
672
692
|
/**
|
|
673
|
-
*
|
|
693
|
+
* Handles realtime strategy - processes each event immediately
|
|
674
694
|
*/
|
|
675
|
-
|
|
676
|
-
const
|
|
677
|
-
|
|
678
|
-
return new NoOpAISpan({ ...rest, metadata }, this);
|
|
679
|
-
}
|
|
680
|
-
let traceState;
|
|
681
|
-
if (options.parent) {
|
|
682
|
-
traceState = options.parent.traceState;
|
|
683
|
-
} else {
|
|
684
|
-
traceState = this.computeTraceState(tracingOptions);
|
|
685
|
-
}
|
|
686
|
-
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, metadata, traceState);
|
|
687
|
-
const span = this.createSpan({
|
|
688
|
-
...rest,
|
|
689
|
-
metadata: enrichedMetadata,
|
|
690
|
-
traceState
|
|
691
|
-
});
|
|
695
|
+
async handleRealtimeEvent(event, storage) {
|
|
696
|
+
const span = event.exportedSpan;
|
|
697
|
+
const spanKey = this.buildSpanKey(span.traceId, span.id);
|
|
692
698
|
if (span.isEvent) {
|
|
693
|
-
|
|
699
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
700
|
+
await storage.createSpan(this.buildCreateRecord(event.exportedSpan));
|
|
701
|
+
} else {
|
|
702
|
+
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
703
|
+
}
|
|
694
704
|
} else {
|
|
695
|
-
|
|
696
|
-
|
|
705
|
+
switch (event.type) {
|
|
706
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
707
|
+
await storage.createSpan(this.buildCreateRecord(event.exportedSpan));
|
|
708
|
+
this.allCreatedSpans.add(spanKey);
|
|
709
|
+
break;
|
|
710
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
711
|
+
await storage.updateSpan({
|
|
712
|
+
traceId: span.traceId,
|
|
713
|
+
spanId: span.id,
|
|
714
|
+
updates: this.buildUpdateRecord(span)
|
|
715
|
+
});
|
|
716
|
+
break;
|
|
717
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
718
|
+
await storage.updateSpan({
|
|
719
|
+
traceId: span.traceId,
|
|
720
|
+
spanId: span.id,
|
|
721
|
+
updates: this.buildUpdateRecord(span)
|
|
722
|
+
});
|
|
723
|
+
this.allCreatedSpans.delete(spanKey);
|
|
724
|
+
break;
|
|
725
|
+
default:
|
|
726
|
+
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
727
|
+
}
|
|
697
728
|
}
|
|
698
|
-
return span;
|
|
699
729
|
}
|
|
700
|
-
// ============================================================================
|
|
701
|
-
// Configuration Management
|
|
702
|
-
// ============================================================================
|
|
703
|
-
/**
|
|
704
|
-
* Get current configuration
|
|
705
|
-
*/
|
|
706
|
-
getConfig() {
|
|
707
|
-
return { ...this.config };
|
|
708
|
-
}
|
|
709
|
-
// ============================================================================
|
|
710
|
-
// Plugin Access
|
|
711
|
-
// ============================================================================
|
|
712
730
|
/**
|
|
713
|
-
*
|
|
731
|
+
* Handles batch-with-updates strategy - buffers events and processes in batches
|
|
714
732
|
*/
|
|
715
|
-
|
|
716
|
-
|
|
733
|
+
handleBatchWithUpdatesEvent(event) {
|
|
734
|
+
this.addToBuffer(event);
|
|
735
|
+
if (this.shouldFlush()) {
|
|
736
|
+
this.flush().catch((error) => {
|
|
737
|
+
this.logger.error("Batch flush failed", {
|
|
738
|
+
error: error instanceof Error ? error.message : String(error)
|
|
739
|
+
});
|
|
740
|
+
});
|
|
741
|
+
} else if (this.buffer.totalSize === 1) {
|
|
742
|
+
this.scheduleFlush();
|
|
743
|
+
}
|
|
717
744
|
}
|
|
718
745
|
/**
|
|
719
|
-
*
|
|
746
|
+
* Handles insert-only strategy - only processes SPAN_ENDED events in batches
|
|
720
747
|
*/
|
|
721
|
-
|
|
722
|
-
|
|
748
|
+
handleInsertOnlyEvent(event) {
|
|
749
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
750
|
+
this.addToBuffer(event);
|
|
751
|
+
if (this.shouldFlush()) {
|
|
752
|
+
this.flush().catch((error) => {
|
|
753
|
+
this.logger.error("Batch flush failed", {
|
|
754
|
+
error: error instanceof Error ? error.message : String(error)
|
|
755
|
+
});
|
|
756
|
+
});
|
|
757
|
+
} else if (this.buffer.totalSize === 1) {
|
|
758
|
+
this.scheduleFlush();
|
|
759
|
+
}
|
|
760
|
+
}
|
|
723
761
|
}
|
|
724
762
|
/**
|
|
725
|
-
*
|
|
763
|
+
* Calculates retry delay using exponential backoff
|
|
726
764
|
*/
|
|
727
|
-
|
|
728
|
-
return this.
|
|
765
|
+
calculateRetryDelay(attempt) {
|
|
766
|
+
return this.#config.retryDelayMs * Math.pow(2, attempt);
|
|
729
767
|
}
|
|
730
|
-
// ============================================================================
|
|
731
|
-
// Span Lifecycle Management
|
|
732
|
-
// ============================================================================
|
|
733
768
|
/**
|
|
734
|
-
*
|
|
735
|
-
* This ensures all spans emit events regardless of implementation
|
|
769
|
+
* Flushes the current buffer to storage with retry logic
|
|
736
770
|
*/
|
|
737
|
-
|
|
738
|
-
if (!this
|
|
771
|
+
async flush() {
|
|
772
|
+
if (!this.#storage) {
|
|
773
|
+
this.logger.debug("Cannot flush traces. Mastra storage is not initialized");
|
|
739
774
|
return;
|
|
740
775
|
}
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
this.
|
|
776
|
+
if (this.#flushTimer) {
|
|
777
|
+
clearTimeout(this.#flushTimer);
|
|
778
|
+
this.#flushTimer = null;
|
|
779
|
+
}
|
|
780
|
+
if (this.buffer.totalSize === 0) {
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
const startTime = Date.now();
|
|
784
|
+
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
785
|
+
const bufferCopy = {
|
|
786
|
+
creates: [...this.buffer.creates],
|
|
787
|
+
updates: [...this.buffer.updates],
|
|
788
|
+
insertOnly: [...this.buffer.insertOnly],
|
|
789
|
+
seenSpans: new Set(this.buffer.seenSpans),
|
|
790
|
+
spanSequences: new Map(this.buffer.spanSequences),
|
|
791
|
+
completedSpans: new Set(this.buffer.completedSpans),
|
|
792
|
+
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
793
|
+
firstEventTime: this.buffer.firstEventTime,
|
|
794
|
+
totalSize: this.buffer.totalSize
|
|
758
795
|
};
|
|
796
|
+
this.resetBuffer();
|
|
797
|
+
await this.flushWithRetries(this.#storage, bufferCopy, 0);
|
|
798
|
+
const elapsed = Date.now() - startTime;
|
|
799
|
+
this.logger.debug("Batch flushed", {
|
|
800
|
+
strategy: this.#resolvedStrategy,
|
|
801
|
+
batchSize: bufferCopy.totalSize,
|
|
802
|
+
flushReason,
|
|
803
|
+
durationMs: elapsed,
|
|
804
|
+
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
805
|
+
});
|
|
759
806
|
}
|
|
760
|
-
// ============================================================================
|
|
761
|
-
// Utility Methods
|
|
762
|
-
// ============================================================================
|
|
763
807
|
/**
|
|
764
|
-
*
|
|
808
|
+
* Attempts to flush with exponential backoff retry logic
|
|
765
809
|
*/
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
case observability.SamplingStrategyType.NEVER:
|
|
772
|
-
return false;
|
|
773
|
-
case observability.SamplingStrategyType.RATIO:
|
|
774
|
-
if (sampling.probability === void 0 || sampling.probability < 0 || sampling.probability > 1) {
|
|
775
|
-
this.logger.warn(
|
|
776
|
-
`Invalid sampling probability: ${sampling.probability}. Expected value between 0 and 1. Defaulting to no sampling.`
|
|
777
|
-
);
|
|
778
|
-
return false;
|
|
810
|
+
async flushWithRetries(storage, buffer, attempt) {
|
|
811
|
+
try {
|
|
812
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
813
|
+
if (buffer.creates.length > 0) {
|
|
814
|
+
await storage.batchCreateSpans({ records: buffer.creates });
|
|
779
815
|
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
816
|
+
if (buffer.updates.length > 0) {
|
|
817
|
+
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
818
|
+
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
819
|
+
this.buildSpanKey(b.traceId, b.spanId)
|
|
820
|
+
);
|
|
821
|
+
if (spanCompare !== 0) return spanCompare;
|
|
822
|
+
return a.sequenceNumber - b.sequenceNumber;
|
|
823
|
+
});
|
|
824
|
+
await storage.batchUpdateSpans({ records: sortedUpdates });
|
|
825
|
+
}
|
|
826
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
827
|
+
if (buffer.insertOnly.length > 0) {
|
|
828
|
+
await storage.batchCreateSpans({ records: buffer.insertOnly });
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
for (const spanKey of buffer.completedSpans) {
|
|
832
|
+
this.allCreatedSpans.delete(spanKey);
|
|
833
|
+
}
|
|
834
|
+
} catch (error) {
|
|
835
|
+
if (attempt < this.#config.maxRetries) {
|
|
836
|
+
const retryDelay = this.calculateRetryDelay(attempt);
|
|
837
|
+
this.logger.warn("Batch flush failed, retrying", {
|
|
838
|
+
attempt: attempt + 1,
|
|
839
|
+
maxRetries: this.#config.maxRetries,
|
|
840
|
+
nextRetryInMs: retryDelay,
|
|
841
|
+
error: error instanceof Error ? error.message : String(error)
|
|
842
|
+
});
|
|
843
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
844
|
+
return this.flushWithRetries(storage, buffer, attempt + 1);
|
|
845
|
+
} else {
|
|
846
|
+
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
847
|
+
finalAttempt: attempt + 1,
|
|
848
|
+
maxRetries: this.#config.maxRetries,
|
|
849
|
+
droppedBatchSize: buffer.totalSize,
|
|
850
|
+
error: error instanceof Error ? error.message : String(error)
|
|
851
|
+
});
|
|
852
|
+
for (const spanKey of buffer.completedSpans) {
|
|
853
|
+
this.allCreatedSpans.delete(spanKey);
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
async _exportTracingEvent(event) {
|
|
859
|
+
if (!this.#storage) {
|
|
860
|
+
this.logger.debug("Cannot store traces. Mastra storage is not initialized");
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
if (!this.#strategyInitialized) {
|
|
864
|
+
this.initializeStrategy(this.#storage);
|
|
865
|
+
}
|
|
866
|
+
switch (this.#resolvedStrategy) {
|
|
867
|
+
case "realtime":
|
|
868
|
+
await this.handleRealtimeEvent(event, this.#storage);
|
|
869
|
+
break;
|
|
870
|
+
case "batch-with-updates":
|
|
871
|
+
this.handleBatchWithUpdatesEvent(event);
|
|
872
|
+
break;
|
|
873
|
+
case "insert-only":
|
|
874
|
+
this.handleInsertOnlyEvent(event);
|
|
875
|
+
break;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
async shutdown() {
|
|
879
|
+
if (this.#flushTimer) {
|
|
880
|
+
clearTimeout(this.#flushTimer);
|
|
881
|
+
this.#flushTimer = null;
|
|
882
|
+
}
|
|
883
|
+
if (this.buffer.totalSize > 0) {
|
|
884
|
+
this.logger.info("Flushing remaining events on shutdown", {
|
|
885
|
+
remainingEvents: this.buffer.totalSize
|
|
886
|
+
});
|
|
887
|
+
try {
|
|
888
|
+
await this.flush();
|
|
889
|
+
} catch (error) {
|
|
890
|
+
this.logger.error("Failed to flush remaining events during shutdown", {
|
|
891
|
+
error: error instanceof Error ? error.message : String(error)
|
|
892
|
+
});
|
|
893
|
+
}
|
|
785
894
|
}
|
|
895
|
+
this.logger.info("DefaultExporter shutdown complete");
|
|
896
|
+
}
|
|
897
|
+
};
|
|
898
|
+
var ModelSpanTracker = class {
|
|
899
|
+
#modelSpan;
|
|
900
|
+
#currentStepSpan;
|
|
901
|
+
#currentChunkSpan;
|
|
902
|
+
#accumulator = {};
|
|
903
|
+
#stepIndex = 0;
|
|
904
|
+
#chunkSequence = 0;
|
|
905
|
+
constructor(modelSpan) {
|
|
906
|
+
this.#modelSpan = modelSpan;
|
|
786
907
|
}
|
|
787
908
|
/**
|
|
788
|
-
*
|
|
909
|
+
* Get the tracing context for creating child spans.
|
|
910
|
+
* Returns the current step span if active, otherwise the model span.
|
|
789
911
|
*/
|
|
790
|
-
|
|
791
|
-
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
792
|
-
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
793
|
-
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
794
|
-
if (allKeys.length === 0) {
|
|
795
|
-
return void 0;
|
|
796
|
-
}
|
|
912
|
+
getTracingContext() {
|
|
797
913
|
return {
|
|
798
|
-
|
|
914
|
+
currentSpan: this.#currentStepSpan ?? this.#modelSpan
|
|
799
915
|
};
|
|
800
916
|
}
|
|
801
917
|
/**
|
|
802
|
-
*
|
|
918
|
+
* Report an error on the generation span
|
|
803
919
|
*/
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
return explicitMetadata;
|
|
807
|
-
}
|
|
808
|
-
const extracted = this.extractKeys(requestContext, traceState.requestContextKeys);
|
|
809
|
-
if (Object.keys(extracted).length === 0 && !explicitMetadata) {
|
|
810
|
-
return void 0;
|
|
811
|
-
}
|
|
812
|
-
return {
|
|
813
|
-
...extracted,
|
|
814
|
-
...explicitMetadata
|
|
815
|
-
// Explicit metadata always wins
|
|
816
|
-
};
|
|
920
|
+
reportGenerationError(options) {
|
|
921
|
+
this.#modelSpan?.error(options);
|
|
817
922
|
}
|
|
818
923
|
/**
|
|
819
|
-
*
|
|
924
|
+
* End the generation span
|
|
820
925
|
*/
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
for (const key of keys) {
|
|
824
|
-
const parts = key.split(".");
|
|
825
|
-
const rootKey = parts[0];
|
|
826
|
-
const value = requestContext.get(rootKey);
|
|
827
|
-
if (value !== void 0) {
|
|
828
|
-
if (parts.length > 1) {
|
|
829
|
-
const nestedPath = parts.slice(1).join(".");
|
|
830
|
-
const nestedValue = utils.getNestedValue(value, nestedPath);
|
|
831
|
-
if (nestedValue !== void 0) {
|
|
832
|
-
utils.setNestedValue(result, key, nestedValue);
|
|
833
|
-
}
|
|
834
|
-
} else {
|
|
835
|
-
utils.setNestedValue(result, key, value);
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
return result;
|
|
926
|
+
endGeneration(options) {
|
|
927
|
+
this.#modelSpan?.end(options);
|
|
840
928
|
}
|
|
841
929
|
/**
|
|
842
|
-
*
|
|
930
|
+
* Update the generation span
|
|
843
931
|
*/
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
if (!span) {
|
|
847
|
-
break;
|
|
848
|
-
}
|
|
849
|
-
try {
|
|
850
|
-
span = processor.process(span);
|
|
851
|
-
} catch (error) {
|
|
852
|
-
this.logger.error(`[AI Tracing] Processor error [name=${processor.name}]`, error);
|
|
853
|
-
}
|
|
854
|
-
}
|
|
855
|
-
return span;
|
|
932
|
+
updateGeneration(options) {
|
|
933
|
+
this.#modelSpan?.update(options);
|
|
856
934
|
}
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
935
|
+
/**
|
|
936
|
+
* Start a new Model execution step
|
|
937
|
+
*/
|
|
938
|
+
#startStepSpan(payload) {
|
|
939
|
+
this.#currentStepSpan = this.#modelSpan?.createChildSpan({
|
|
940
|
+
name: `step: ${this.#stepIndex}`,
|
|
941
|
+
type: observability.SpanType.MODEL_STEP,
|
|
942
|
+
attributes: {
|
|
943
|
+
stepIndex: this.#stepIndex,
|
|
944
|
+
...payload?.messageId ? { messageId: payload.messageId } : {},
|
|
945
|
+
...payload?.warnings?.length ? { warnings: payload.warnings } : {}
|
|
946
|
+
},
|
|
947
|
+
input: payload?.request
|
|
948
|
+
});
|
|
949
|
+
this.#chunkSequence = 0;
|
|
865
950
|
}
|
|
866
951
|
/**
|
|
867
|
-
*
|
|
952
|
+
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
868
953
|
*/
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
954
|
+
#endStepSpan(payload) {
|
|
955
|
+
if (!this.#currentStepSpan) return;
|
|
956
|
+
const output = payload.output;
|
|
957
|
+
const { usage, ...otherOutput } = output;
|
|
958
|
+
const stepResult = payload.stepResult;
|
|
959
|
+
const metadata = payload.metadata;
|
|
960
|
+
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
961
|
+
if (cleanMetadata?.request) {
|
|
962
|
+
delete cleanMetadata.request;
|
|
875
963
|
}
|
|
964
|
+
this.#currentStepSpan.end({
|
|
965
|
+
output: otherOutput,
|
|
966
|
+
attributes: {
|
|
967
|
+
usage,
|
|
968
|
+
isContinued: stepResult.isContinued,
|
|
969
|
+
finishReason: stepResult.reason,
|
|
970
|
+
warnings: stepResult.warnings
|
|
971
|
+
},
|
|
972
|
+
metadata: {
|
|
973
|
+
...cleanMetadata
|
|
974
|
+
}
|
|
975
|
+
});
|
|
976
|
+
this.#currentStepSpan = void 0;
|
|
977
|
+
this.#stepIndex++;
|
|
876
978
|
}
|
|
877
979
|
/**
|
|
878
|
-
*
|
|
980
|
+
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
879
981
|
*/
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
this.exportEvent({ type: observability.AITracingEventType.SPAN_ENDED, exportedSpan }).catch((error) => {
|
|
884
|
-
this.logger.error("[AI Tracing] Failed to export span_ended event", error);
|
|
885
|
-
});
|
|
982
|
+
#startChunkSpan(chunkType, initialData) {
|
|
983
|
+
if (!this.#currentStepSpan) {
|
|
984
|
+
this.#startStepSpan();
|
|
886
985
|
}
|
|
986
|
+
this.#currentChunkSpan = this.#currentStepSpan?.createChildSpan({
|
|
987
|
+
name: `chunk: '${chunkType}'`,
|
|
988
|
+
type: observability.SpanType.MODEL_CHUNK,
|
|
989
|
+
attributes: {
|
|
990
|
+
chunkType,
|
|
991
|
+
sequenceNumber: this.#chunkSequence
|
|
992
|
+
}
|
|
993
|
+
});
|
|
994
|
+
this.#accumulator = initialData || {};
|
|
887
995
|
}
|
|
888
996
|
/**
|
|
889
|
-
*
|
|
997
|
+
* Append string content to a specific field in the accumulator
|
|
890
998
|
*/
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
});
|
|
999
|
+
#appendToAccumulator(field, text) {
|
|
1000
|
+
if (this.#accumulator[field] === void 0) {
|
|
1001
|
+
this.#accumulator[field] = text;
|
|
1002
|
+
} else {
|
|
1003
|
+
this.#accumulator[field] += text;
|
|
897
1004
|
}
|
|
898
1005
|
}
|
|
899
1006
|
/**
|
|
900
|
-
*
|
|
1007
|
+
* End the current chunk span.
|
|
1008
|
+
* Safe to call multiple times - will no-op if span already ended.
|
|
901
1009
|
*/
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
await exporter.exportEvent(event);
|
|
907
|
-
this.logger.debug(`[AI Tracing] Event exported [exporter=${exporter.name}] [type=${event.type}]`);
|
|
908
|
-
}
|
|
909
|
-
} catch (error) {
|
|
910
|
-
this.logger.error(`[AI Tracing] Export error [exporter=${exporter.name}]`, error);
|
|
911
|
-
}
|
|
1010
|
+
#endChunkSpan(output) {
|
|
1011
|
+
if (!this.#currentChunkSpan) return;
|
|
1012
|
+
this.#currentChunkSpan.end({
|
|
1013
|
+
output: output !== void 0 ? output : this.#accumulator
|
|
912
1014
|
});
|
|
913
|
-
|
|
1015
|
+
this.#currentChunkSpan = void 0;
|
|
1016
|
+
this.#accumulator = {};
|
|
1017
|
+
this.#chunkSequence++;
|
|
914
1018
|
}
|
|
915
|
-
// ============================================================================
|
|
916
|
-
// Lifecycle Management
|
|
917
|
-
// ============================================================================
|
|
918
1019
|
/**
|
|
919
|
-
*
|
|
1020
|
+
* Create an event span (for single chunks like tool-call)
|
|
920
1021
|
*/
|
|
921
|
-
|
|
922
|
-
this
|
|
923
|
-
|
|
1022
|
+
#createEventSpan(chunkType, output) {
|
|
1023
|
+
if (!this.#currentStepSpan) {
|
|
1024
|
+
this.#startStepSpan();
|
|
1025
|
+
}
|
|
1026
|
+
const span = this.#currentStepSpan?.createEventSpan({
|
|
1027
|
+
name: `chunk: '${chunkType}'`,
|
|
1028
|
+
type: observability.SpanType.MODEL_CHUNK,
|
|
1029
|
+
attributes: {
|
|
1030
|
+
chunkType,
|
|
1031
|
+
sequenceNumber: this.#chunkSequence
|
|
1032
|
+
},
|
|
1033
|
+
output
|
|
1034
|
+
});
|
|
1035
|
+
if (span) {
|
|
1036
|
+
this.#chunkSequence++;
|
|
1037
|
+
}
|
|
924
1038
|
}
|
|
925
1039
|
/**
|
|
926
|
-
*
|
|
1040
|
+
* Check if there is currently an active chunk span
|
|
927
1041
|
*/
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
const shutdownPromises = [...this.exporters.map((e) => e.shutdown()), ...this.processors.map((p) => p.shutdown())];
|
|
931
|
-
await Promise.allSettled(shutdownPromises);
|
|
932
|
-
this.logger.info(`[AI Tracing] Shutdown completed [name=${this.name}]`);
|
|
1042
|
+
#hasActiveChunkSpan() {
|
|
1043
|
+
return !!this.#currentChunkSpan;
|
|
933
1044
|
}
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
super(config);
|
|
1045
|
+
/**
|
|
1046
|
+
* Get the current accumulator value
|
|
1047
|
+
*/
|
|
1048
|
+
#getAccumulator() {
|
|
1049
|
+
return this.#accumulator;
|
|
940
1050
|
}
|
|
941
|
-
|
|
942
|
-
|
|
1051
|
+
/**
|
|
1052
|
+
* Handle text chunk spans (text-start/delta/end)
|
|
1053
|
+
*/
|
|
1054
|
+
#handleTextChunk(chunk) {
|
|
1055
|
+
switch (chunk.type) {
|
|
1056
|
+
case "text-start":
|
|
1057
|
+
this.#startChunkSpan("text");
|
|
1058
|
+
break;
|
|
1059
|
+
case "text-delta":
|
|
1060
|
+
this.#appendToAccumulator("text", chunk.payload.text);
|
|
1061
|
+
break;
|
|
1062
|
+
case "text-end": {
|
|
1063
|
+
this.#endChunkSpan();
|
|
1064
|
+
break;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
943
1067
|
}
|
|
944
|
-
};
|
|
945
|
-
var BaseExporter = class {
|
|
946
|
-
/** Mastra logger instance */
|
|
947
|
-
logger;
|
|
948
|
-
/** Whether this exporter is disabled */
|
|
949
|
-
isDisabled = false;
|
|
950
1068
|
/**
|
|
951
|
-
*
|
|
1069
|
+
* Handle reasoning chunk spans (reasoning-start/delta/end)
|
|
952
1070
|
*/
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
1071
|
+
#handleReasoningChunk(chunk) {
|
|
1072
|
+
switch (chunk.type) {
|
|
1073
|
+
case "reasoning-start":
|
|
1074
|
+
this.#startChunkSpan("reasoning");
|
|
1075
|
+
break;
|
|
1076
|
+
case "reasoning-delta":
|
|
1077
|
+
this.#appendToAccumulator("text", chunk.payload.text);
|
|
1078
|
+
break;
|
|
1079
|
+
case "reasoning-end": {
|
|
1080
|
+
this.#endChunkSpan();
|
|
1081
|
+
break;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
956
1084
|
}
|
|
957
1085
|
/**
|
|
958
|
-
*
|
|
1086
|
+
* Handle tool call chunk spans (tool-call-input-streaming-start/delta/end, tool-call)
|
|
959
1087
|
*/
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
1088
|
+
#handleToolCallChunk(chunk) {
|
|
1089
|
+
switch (chunk.type) {
|
|
1090
|
+
case "tool-call-input-streaming-start":
|
|
1091
|
+
this.#startChunkSpan("tool-call", {
|
|
1092
|
+
toolName: chunk.payload.toolName,
|
|
1093
|
+
toolCallId: chunk.payload.toolCallId
|
|
1094
|
+
});
|
|
1095
|
+
break;
|
|
1096
|
+
case "tool-call-delta":
|
|
1097
|
+
this.#appendToAccumulator("toolInput", chunk.payload.argsTextDelta);
|
|
1098
|
+
break;
|
|
1099
|
+
case "tool-call-input-streaming-end":
|
|
1100
|
+
case "tool-call": {
|
|
1101
|
+
const acc = this.#getAccumulator();
|
|
1102
|
+
let toolInput;
|
|
1103
|
+
try {
|
|
1104
|
+
toolInput = acc.toolInput ? JSON.parse(acc.toolInput) : {};
|
|
1105
|
+
} catch {
|
|
1106
|
+
toolInput = acc.toolInput;
|
|
1107
|
+
}
|
|
1108
|
+
this.#endChunkSpan({
|
|
1109
|
+
toolName: acc.toolName,
|
|
1110
|
+
toolCallId: acc.toolCallId,
|
|
1111
|
+
toolInput
|
|
1112
|
+
});
|
|
1113
|
+
break;
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
963
1116
|
}
|
|
964
1117
|
/**
|
|
965
|
-
*
|
|
1118
|
+
* Handle object chunk spans (object, object-result)
|
|
966
1119
|
*/
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1120
|
+
#handleObjectChunk(chunk) {
|
|
1121
|
+
switch (chunk.type) {
|
|
1122
|
+
case "object":
|
|
1123
|
+
if (!this.#hasActiveChunkSpan()) {
|
|
1124
|
+
this.#startChunkSpan("object");
|
|
1125
|
+
}
|
|
1126
|
+
break;
|
|
1127
|
+
case "object-result":
|
|
1128
|
+
this.#endChunkSpan(chunk.object);
|
|
1129
|
+
break;
|
|
973
1130
|
}
|
|
974
|
-
const logLevelMap = {
|
|
975
|
-
debug: logger.LogLevel.DEBUG,
|
|
976
|
-
info: logger.LogLevel.INFO,
|
|
977
|
-
warn: logger.LogLevel.WARN,
|
|
978
|
-
error: logger.LogLevel.ERROR
|
|
979
|
-
};
|
|
980
|
-
return logLevelMap[logLevel] ?? logger.LogLevel.INFO;
|
|
981
1131
|
}
|
|
982
1132
|
/**
|
|
983
|
-
*
|
|
1133
|
+
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
984
1134
|
*
|
|
985
|
-
*
|
|
1135
|
+
* This should be added to the stream pipeline to automatically
|
|
1136
|
+
* create MODEL_STEP and MODEL_CHUNK spans for each semantic unit in the stream.
|
|
986
1137
|
*/
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1138
|
+
wrapStream(stream) {
|
|
1139
|
+
return stream.pipeThrough(
|
|
1140
|
+
new web.TransformStream({
|
|
1141
|
+
transform: (chunk, controller) => {
|
|
1142
|
+
controller.enqueue(chunk);
|
|
1143
|
+
switch (chunk.type) {
|
|
1144
|
+
case "text-start":
|
|
1145
|
+
case "text-delta":
|
|
1146
|
+
case "text-end":
|
|
1147
|
+
this.#handleTextChunk(chunk);
|
|
1148
|
+
break;
|
|
1149
|
+
case "tool-call-input-streaming-start":
|
|
1150
|
+
case "tool-call-delta":
|
|
1151
|
+
case "tool-call-input-streaming-end":
|
|
1152
|
+
case "tool-call":
|
|
1153
|
+
this.#handleToolCallChunk(chunk);
|
|
1154
|
+
break;
|
|
1155
|
+
case "reasoning-start":
|
|
1156
|
+
case "reasoning-delta":
|
|
1157
|
+
case "reasoning-end":
|
|
1158
|
+
this.#handleReasoningChunk(chunk);
|
|
1159
|
+
break;
|
|
1160
|
+
case "object":
|
|
1161
|
+
case "object-result":
|
|
1162
|
+
this.#handleObjectChunk(chunk);
|
|
1163
|
+
break;
|
|
1164
|
+
case "step-start":
|
|
1165
|
+
this.#startStepSpan(chunk.payload);
|
|
1166
|
+
break;
|
|
1167
|
+
case "step-finish":
|
|
1168
|
+
this.#endStepSpan(chunk.payload);
|
|
1169
|
+
break;
|
|
1170
|
+
case "raw":
|
|
1171
|
+
// Skip raw chunks as they're redundant
|
|
1172
|
+
case "start":
|
|
1173
|
+
case "finish":
|
|
1174
|
+
break;
|
|
1175
|
+
// Default: auto-create event span for all other chunk types
|
|
1176
|
+
default: {
|
|
1177
|
+
let outputPayload = chunk.payload;
|
|
1178
|
+
if (outputPayload && typeof outputPayload === "object" && "data" in outputPayload) {
|
|
1179
|
+
const typedPayload = outputPayload;
|
|
1180
|
+
outputPayload = { ...typedPayload };
|
|
1181
|
+
if (typedPayload.data) {
|
|
1182
|
+
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
1183
|
+
delete outputPayload.data;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
this.#createEventSpan(chunk.type, outputPayload);
|
|
1187
|
+
break;
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
})
|
|
1192
|
+
);
|
|
990
1193
|
}
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1194
|
+
};
|
|
1195
|
+
|
|
1196
|
+
// src/spans/base.ts
|
|
1197
|
+
function isSpanInternal(spanType, flags) {
|
|
1198
|
+
if (flags === void 0 || flags === observability.InternalSpans.NONE) {
|
|
1199
|
+
return false;
|
|
1200
|
+
}
|
|
1201
|
+
switch (spanType) {
|
|
1202
|
+
// Workflow-related spans
|
|
1203
|
+
case observability.SpanType.WORKFLOW_RUN:
|
|
1204
|
+
case observability.SpanType.WORKFLOW_STEP:
|
|
1205
|
+
case observability.SpanType.WORKFLOW_CONDITIONAL:
|
|
1206
|
+
case observability.SpanType.WORKFLOW_CONDITIONAL_EVAL:
|
|
1207
|
+
case observability.SpanType.WORKFLOW_PARALLEL:
|
|
1208
|
+
case observability.SpanType.WORKFLOW_LOOP:
|
|
1209
|
+
case observability.SpanType.WORKFLOW_SLEEP:
|
|
1210
|
+
case observability.SpanType.WORKFLOW_WAIT_EVENT:
|
|
1211
|
+
return (flags & observability.InternalSpans.WORKFLOW) !== 0;
|
|
1212
|
+
// Agent-related spans
|
|
1213
|
+
case observability.SpanType.AGENT_RUN:
|
|
1214
|
+
return (flags & observability.InternalSpans.AGENT) !== 0;
|
|
1215
|
+
// Tool-related spans
|
|
1216
|
+
case observability.SpanType.TOOL_CALL:
|
|
1217
|
+
case observability.SpanType.MCP_TOOL_CALL:
|
|
1218
|
+
return (flags & observability.InternalSpans.TOOL) !== 0;
|
|
1219
|
+
// Model-related spans
|
|
1220
|
+
case observability.SpanType.MODEL_GENERATION:
|
|
1221
|
+
case observability.SpanType.MODEL_STEP:
|
|
1222
|
+
case observability.SpanType.MODEL_CHUNK:
|
|
1223
|
+
return (flags & observability.InternalSpans.MODEL) !== 0;
|
|
1224
|
+
// Default: never internal
|
|
1225
|
+
default:
|
|
1226
|
+
return false;
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
var BaseSpan = class {
|
|
1230
|
+
name;
|
|
1231
|
+
type;
|
|
1232
|
+
attributes;
|
|
1233
|
+
parent;
|
|
1234
|
+
startTime;
|
|
1235
|
+
endTime;
|
|
1236
|
+
isEvent;
|
|
1237
|
+
isInternal;
|
|
1238
|
+
observabilityInstance;
|
|
1239
|
+
input;
|
|
1240
|
+
output;
|
|
1241
|
+
errorInfo;
|
|
1242
|
+
metadata;
|
|
1243
|
+
traceState;
|
|
1244
|
+
/** Parent span ID (for root spans that are children of external spans) */
|
|
1245
|
+
parentSpanId;
|
|
1246
|
+
constructor(options, observabilityInstance) {
|
|
1247
|
+
this.name = options.name;
|
|
1248
|
+
this.type = options.type;
|
|
1249
|
+
this.attributes = deepClean(options.attributes) || {};
|
|
1250
|
+
this.metadata = deepClean(options.metadata);
|
|
1251
|
+
this.parent = options.parent;
|
|
1252
|
+
this.startTime = /* @__PURE__ */ new Date();
|
|
1253
|
+
this.observabilityInstance = observabilityInstance;
|
|
1254
|
+
this.isEvent = options.isEvent ?? false;
|
|
1255
|
+
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
1256
|
+
this.traceState = options.traceState;
|
|
1257
|
+
if (this.isEvent) {
|
|
1258
|
+
this.output = deepClean(options.output);
|
|
1259
|
+
} else {
|
|
1260
|
+
this.input = deepClean(options.input);
|
|
1000
1261
|
}
|
|
1001
|
-
|
|
1262
|
+
}
|
|
1263
|
+
createChildSpan(options) {
|
|
1264
|
+
return this.observabilityInstance.startSpan({ ...options, parent: this, isEvent: false });
|
|
1265
|
+
}
|
|
1266
|
+
createEventSpan(options) {
|
|
1267
|
+
return this.observabilityInstance.startSpan({ ...options, parent: this, isEvent: true });
|
|
1002
1268
|
}
|
|
1003
1269
|
/**
|
|
1004
|
-
*
|
|
1005
|
-
*
|
|
1006
|
-
* Default implementation just logs. Override to add custom cleanup.
|
|
1270
|
+
* Create a ModelSpanTracker for this span (only works if this is a MODEL_GENERATION span)
|
|
1271
|
+
* Returns undefined for non-MODEL_GENERATION spans
|
|
1007
1272
|
*/
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
};
|
|
1012
|
-
var CloudExporter = class extends BaseExporter {
|
|
1013
|
-
name = "mastra-cloud-ai-tracing-exporter";
|
|
1014
|
-
config;
|
|
1015
|
-
buffer;
|
|
1016
|
-
flushTimer = null;
|
|
1017
|
-
constructor(config = {}) {
|
|
1018
|
-
super(config);
|
|
1019
|
-
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
1020
|
-
if (!accessToken) {
|
|
1021
|
-
this.setDisabled(
|
|
1022
|
-
"MASTRA_CLOUD_ACCESS_TOKEN environment variable not set. \u{1F680} Sign up for Mastra Cloud at https://cloud.mastra.ai to see your AI traces online and obtain your access token."
|
|
1023
|
-
);
|
|
1273
|
+
createTracker() {
|
|
1274
|
+
if (this.type !== observability.SpanType.MODEL_GENERATION) {
|
|
1275
|
+
return void 0;
|
|
1024
1276
|
}
|
|
1025
|
-
|
|
1026
|
-
this.config = {
|
|
1027
|
-
logger: this.logger,
|
|
1028
|
-
logLevel: config.logLevel ?? logger.LogLevel.INFO,
|
|
1029
|
-
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
1030
|
-
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
1031
|
-
maxRetries: config.maxRetries ?? 3,
|
|
1032
|
-
accessToken: accessToken || "",
|
|
1033
|
-
endpoint
|
|
1034
|
-
};
|
|
1035
|
-
this.buffer = {
|
|
1036
|
-
spans: [],
|
|
1037
|
-
totalSize: 0
|
|
1038
|
-
};
|
|
1277
|
+
return new ModelSpanTracker(this);
|
|
1039
1278
|
}
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1049
|
-
});
|
|
1050
|
-
});
|
|
1051
|
-
} else if (this.buffer.totalSize === 1) {
|
|
1052
|
-
this.scheduleFlush();
|
|
1279
|
+
/** Returns `TRUE` if the span is the root span of a trace */
|
|
1280
|
+
get isRootSpan() {
|
|
1281
|
+
return !this.parent;
|
|
1282
|
+
}
|
|
1283
|
+
/** Get the closest parent spanId that isn't an internal span */
|
|
1284
|
+
getParentSpanId(includeInternalSpans) {
|
|
1285
|
+
if (!this.parent) {
|
|
1286
|
+
return this.parentSpanId;
|
|
1053
1287
|
}
|
|
1288
|
+
if (includeInternalSpans) return this.parent.id;
|
|
1289
|
+
if (this.parent.isInternal) return this.parent.getParentSpanId(includeInternalSpans);
|
|
1290
|
+
return this.parent.id;
|
|
1054
1291
|
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1292
|
+
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
1293
|
+
findParent(spanType) {
|
|
1294
|
+
let current = this.parent;
|
|
1295
|
+
while (current) {
|
|
1296
|
+
if (current.type === spanType) {
|
|
1297
|
+
return current;
|
|
1298
|
+
}
|
|
1299
|
+
current = current.parent;
|
|
1058
1300
|
}
|
|
1059
|
-
|
|
1060
|
-
this.buffer.spans.push(spanRecord);
|
|
1061
|
-
this.buffer.totalSize++;
|
|
1301
|
+
return void 0;
|
|
1062
1302
|
}
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
name:
|
|
1069
|
-
|
|
1070
|
-
attributes:
|
|
1071
|
-
metadata:
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
input:
|
|
1075
|
-
output:
|
|
1076
|
-
|
|
1077
|
-
isEvent:
|
|
1078
|
-
|
|
1079
|
-
|
|
1303
|
+
/** Returns a lightweight span ready for export */
|
|
1304
|
+
exportSpan(includeInternalSpans) {
|
|
1305
|
+
return {
|
|
1306
|
+
id: this.id,
|
|
1307
|
+
traceId: this.traceId,
|
|
1308
|
+
name: this.name,
|
|
1309
|
+
type: this.type,
|
|
1310
|
+
attributes: this.attributes,
|
|
1311
|
+
metadata: this.metadata,
|
|
1312
|
+
startTime: this.startTime,
|
|
1313
|
+
endTime: this.endTime,
|
|
1314
|
+
input: this.input,
|
|
1315
|
+
output: this.output,
|
|
1316
|
+
errorInfo: this.errorInfo,
|
|
1317
|
+
isEvent: this.isEvent,
|
|
1318
|
+
isRootSpan: this.isRootSpan,
|
|
1319
|
+
parentSpanId: this.getParentSpanId(includeInternalSpans)
|
|
1080
1320
|
};
|
|
1081
|
-
return spanRecord;
|
|
1082
1321
|
}
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
return true;
|
|
1086
|
-
}
|
|
1087
|
-
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
1088
|
-
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
1089
|
-
if (elapsed >= this.config.maxBatchWaitMs) {
|
|
1090
|
-
return true;
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
return false;
|
|
1322
|
+
get externalTraceId() {
|
|
1323
|
+
return this.isValid ? this.traceId : void 0;
|
|
1094
1324
|
}
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
error$1
|
|
1108
|
-
);
|
|
1109
|
-
this.logger.trackException(mastraError);
|
|
1110
|
-
this.logger.error("Scheduled flush failed", mastraError);
|
|
1111
|
-
});
|
|
1112
|
-
}, this.config.maxBatchWaitMs);
|
|
1325
|
+
};
|
|
1326
|
+
var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
1327
|
+
"logger",
|
|
1328
|
+
"experimental_providerMetadata",
|
|
1329
|
+
"providerMetadata",
|
|
1330
|
+
"steps",
|
|
1331
|
+
"tracingContext"
|
|
1332
|
+
]);
|
|
1333
|
+
function deepClean(value, options = {}, _seen = /* @__PURE__ */ new WeakSet(), _depth = 0) {
|
|
1334
|
+
const { keysToStrip = DEFAULT_KEYS_TO_STRIP, maxDepth = 10 } = options;
|
|
1335
|
+
if (_depth > maxDepth) {
|
|
1336
|
+
return "[MaxDepth]";
|
|
1113
1337
|
}
|
|
1114
|
-
|
|
1115
|
-
if (this.flushTimer) {
|
|
1116
|
-
clearTimeout(this.flushTimer);
|
|
1117
|
-
this.flushTimer = null;
|
|
1118
|
-
}
|
|
1119
|
-
if (this.buffer.totalSize === 0) {
|
|
1120
|
-
return;
|
|
1121
|
-
}
|
|
1122
|
-
const startTime = Date.now();
|
|
1123
|
-
const spansCopy = [...this.buffer.spans];
|
|
1124
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
1125
|
-
this.resetBuffer();
|
|
1338
|
+
if (value === null || typeof value !== "object") {
|
|
1126
1339
|
try {
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
flushReason,
|
|
1132
|
-
durationMs: elapsed
|
|
1133
|
-
});
|
|
1134
|
-
} catch (error$1) {
|
|
1135
|
-
const mastraError = new error.MastraError(
|
|
1136
|
-
{
|
|
1137
|
-
id: `CLOUD_AI_TRACING_FAILED_TO_BATCH_UPLOAD`,
|
|
1138
|
-
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
1139
|
-
category: error.ErrorCategory.USER,
|
|
1140
|
-
details: {
|
|
1141
|
-
droppedBatchSize: spansCopy.length
|
|
1142
|
-
}
|
|
1143
|
-
},
|
|
1144
|
-
error$1
|
|
1145
|
-
);
|
|
1146
|
-
this.logger.trackException(mastraError);
|
|
1147
|
-
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
1340
|
+
JSON.stringify(value);
|
|
1341
|
+
return value;
|
|
1342
|
+
} catch (error) {
|
|
1343
|
+
return `[${error instanceof Error ? error.message : String(error)}]`;
|
|
1148
1344
|
}
|
|
1149
1345
|
}
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
*/
|
|
1153
|
-
async batchUpload(spans) {
|
|
1154
|
-
const headers = {
|
|
1155
|
-
Authorization: `Bearer ${this.config.accessToken}`,
|
|
1156
|
-
"Content-Type": "application/json"
|
|
1157
|
-
};
|
|
1158
|
-
const options = {
|
|
1159
|
-
method: "POST",
|
|
1160
|
-
headers,
|
|
1161
|
-
body: JSON.stringify({ spans })
|
|
1162
|
-
};
|
|
1163
|
-
await utils.fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
1346
|
+
if (_seen.has(value)) {
|
|
1347
|
+
return "[Circular]";
|
|
1164
1348
|
}
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
this.buffer.totalSize = 0;
|
|
1349
|
+
_seen.add(value);
|
|
1350
|
+
if (Array.isArray(value)) {
|
|
1351
|
+
return value.map((item) => deepClean(item, options, _seen, _depth + 1));
|
|
1169
1352
|
}
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1353
|
+
const cleaned = {};
|
|
1354
|
+
for (const [key, val] of Object.entries(value)) {
|
|
1355
|
+
if (keysToStrip.has(key)) {
|
|
1356
|
+
continue;
|
|
1173
1357
|
}
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1358
|
+
try {
|
|
1359
|
+
cleaned[key] = deepClean(val, options, _seen, _depth + 1);
|
|
1360
|
+
} catch (error) {
|
|
1361
|
+
cleaned[key] = `[${error instanceof Error ? error.message : String(error)}]`;
|
|
1177
1362
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1363
|
+
}
|
|
1364
|
+
return cleaned;
|
|
1365
|
+
}
|
|
1366
|
+
var DefaultSpan = class extends BaseSpan {
|
|
1367
|
+
id;
|
|
1368
|
+
traceId;
|
|
1369
|
+
constructor(options, observabilityInstance) {
|
|
1370
|
+
super(options, observabilityInstance);
|
|
1371
|
+
this.id = generateSpanId();
|
|
1372
|
+
if (options.parent) {
|
|
1373
|
+
this.traceId = options.parent.traceId;
|
|
1374
|
+
} else if (options.traceId) {
|
|
1375
|
+
if (isValidTraceId(options.traceId)) {
|
|
1376
|
+
this.traceId = options.traceId;
|
|
1377
|
+
} else {
|
|
1378
|
+
console.error(
|
|
1379
|
+
`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`
|
|
1380
|
+
);
|
|
1381
|
+
this.traceId = generateTraceId();
|
|
1382
|
+
}
|
|
1383
|
+
} else {
|
|
1384
|
+
this.traceId = generateTraceId();
|
|
1385
|
+
}
|
|
1386
|
+
if (!options.parent && options.parentSpanId) {
|
|
1387
|
+
if (isValidSpanId(options.parentSpanId)) {
|
|
1388
|
+
this.parentSpanId = options.parentSpanId;
|
|
1389
|
+
} else {
|
|
1390
|
+
console.error(
|
|
1391
|
+
`[Mastra Tracing] Invalid parentSpanId: must be 1-16 hexadecimal characters, got "${options.parentSpanId}". Ignoring parent span ID.`
|
|
1195
1392
|
);
|
|
1196
|
-
this.logger.trackException(mastraError);
|
|
1197
|
-
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
1198
1393
|
}
|
|
1199
1394
|
}
|
|
1200
|
-
this.logger.info("CloudExporter shutdown complete");
|
|
1201
1395
|
}
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1396
|
+
end(options) {
|
|
1397
|
+
if (this.isEvent) {
|
|
1398
|
+
return;
|
|
1399
|
+
}
|
|
1400
|
+
this.endTime = /* @__PURE__ */ new Date();
|
|
1401
|
+
if (options?.output !== void 0) {
|
|
1402
|
+
this.output = deepClean(options.output);
|
|
1403
|
+
}
|
|
1404
|
+
if (options?.attributes) {
|
|
1405
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
1406
|
+
}
|
|
1407
|
+
if (options?.metadata) {
|
|
1408
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
1409
|
+
}
|
|
1207
1410
|
}
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
}
|
|
1233
|
-
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
1234
|
-
this.logger.info("\u2500".repeat(80));
|
|
1235
|
-
break;
|
|
1236
|
-
case observability.AITracingEventType.SPAN_ENDED:
|
|
1237
|
-
const duration = formatDuration(span.startTime, span.endTime);
|
|
1238
|
-
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
1239
|
-
this.logger.info(` Type: ${span.type}`);
|
|
1240
|
-
this.logger.info(` Name: ${span.name}`);
|
|
1241
|
-
this.logger.info(` ID: ${span.id}`);
|
|
1242
|
-
this.logger.info(` Duration: ${duration}`);
|
|
1243
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
1244
|
-
if (span.input !== void 0) {
|
|
1245
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
1246
|
-
}
|
|
1247
|
-
if (span.output !== void 0) {
|
|
1248
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
1249
|
-
}
|
|
1250
|
-
if (span.errorInfo) {
|
|
1251
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
1252
|
-
}
|
|
1253
|
-
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
1254
|
-
this.logger.info("\u2500".repeat(80));
|
|
1255
|
-
break;
|
|
1256
|
-
case observability.AITracingEventType.SPAN_UPDATED:
|
|
1257
|
-
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
1258
|
-
this.logger.info(` Type: ${span.type}`);
|
|
1259
|
-
this.logger.info(` Name: ${span.name}`);
|
|
1260
|
-
this.logger.info(` ID: ${span.id}`);
|
|
1261
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
1262
|
-
if (span.input !== void 0) {
|
|
1263
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
1264
|
-
}
|
|
1265
|
-
if (span.output !== void 0) {
|
|
1266
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
1267
|
-
}
|
|
1268
|
-
if (span.errorInfo) {
|
|
1269
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
1270
|
-
}
|
|
1271
|
-
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
1272
|
-
this.logger.info("\u2500".repeat(80));
|
|
1273
|
-
break;
|
|
1274
|
-
default:
|
|
1275
|
-
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
1411
|
+
error(options) {
|
|
1412
|
+
if (this.isEvent) {
|
|
1413
|
+
return;
|
|
1414
|
+
}
|
|
1415
|
+
const { error: error$1, endSpan = true, attributes, metadata } = options;
|
|
1416
|
+
this.errorInfo = error$1 instanceof error.MastraError ? {
|
|
1417
|
+
id: error$1.id,
|
|
1418
|
+
details: error$1.details,
|
|
1419
|
+
category: error$1.category,
|
|
1420
|
+
domain: error$1.domain,
|
|
1421
|
+
message: error$1.message
|
|
1422
|
+
} : {
|
|
1423
|
+
message: error$1.message
|
|
1424
|
+
};
|
|
1425
|
+
if (attributes) {
|
|
1426
|
+
this.attributes = { ...this.attributes, ...deepClean(attributes) };
|
|
1427
|
+
}
|
|
1428
|
+
if (metadata) {
|
|
1429
|
+
this.metadata = { ...this.metadata, ...deepClean(metadata) };
|
|
1430
|
+
}
|
|
1431
|
+
if (endSpan) {
|
|
1432
|
+
this.end();
|
|
1433
|
+
} else {
|
|
1434
|
+
this.update({});
|
|
1276
1435
|
}
|
|
1277
1436
|
}
|
|
1278
|
-
|
|
1279
|
-
this.
|
|
1437
|
+
update(options) {
|
|
1438
|
+
if (this.isEvent) {
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
if (options.input !== void 0) {
|
|
1442
|
+
this.input = deepClean(options.input);
|
|
1443
|
+
}
|
|
1444
|
+
if (options.output !== void 0) {
|
|
1445
|
+
this.output = deepClean(options.output);
|
|
1446
|
+
}
|
|
1447
|
+
if (options.attributes) {
|
|
1448
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
1449
|
+
}
|
|
1450
|
+
if (options.metadata) {
|
|
1451
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
1452
|
+
}
|
|
1280
1453
|
}
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
fallbackStrategy: hints.preferred
|
|
1454
|
+
get isValid() {
|
|
1455
|
+
return true;
|
|
1456
|
+
}
|
|
1457
|
+
async export() {
|
|
1458
|
+
return JSON.stringify({
|
|
1459
|
+
spanId: this.id,
|
|
1460
|
+
traceId: this.traceId,
|
|
1461
|
+
startTime: this.startTime,
|
|
1462
|
+
endTime: this.endTime,
|
|
1463
|
+
attributes: this.attributes,
|
|
1464
|
+
metadata: this.metadata
|
|
1293
1465
|
});
|
|
1294
1466
|
}
|
|
1295
|
-
|
|
1467
|
+
};
|
|
1468
|
+
function generateSpanId() {
|
|
1469
|
+
const bytes = new Uint8Array(8);
|
|
1470
|
+
if (typeof crypto !== "undefined" && crypto.getRandomValues) {
|
|
1471
|
+
crypto.getRandomValues(bytes);
|
|
1472
|
+
} else {
|
|
1473
|
+
for (let i = 0; i < 8; i++) {
|
|
1474
|
+
bytes[i] = Math.floor(Math.random() * 256);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
1296
1478
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
flushTimer = null;
|
|
1305
|
-
// Track all spans that have been created, persists across flushes
|
|
1306
|
-
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
1307
|
-
constructor(config = {}, logger$1) {
|
|
1308
|
-
if (logger$1) {
|
|
1309
|
-
this.logger = logger$1;
|
|
1310
|
-
} else {
|
|
1311
|
-
this.logger = new logger.ConsoleLogger({ level: logger.LogLevel.INFO });
|
|
1479
|
+
function generateTraceId() {
|
|
1480
|
+
const bytes = new Uint8Array(16);
|
|
1481
|
+
if (typeof crypto !== "undefined" && crypto.getRandomValues) {
|
|
1482
|
+
crypto.getRandomValues(bytes);
|
|
1483
|
+
} else {
|
|
1484
|
+
for (let i = 0; i < 16; i++) {
|
|
1485
|
+
bytes[i] = Math.floor(Math.random() * 256);
|
|
1312
1486
|
}
|
|
1487
|
+
}
|
|
1488
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
1489
|
+
}
|
|
1490
|
+
function isValidTraceId(traceId) {
|
|
1491
|
+
return /^[0-9a-f]{1,32}$/i.test(traceId);
|
|
1492
|
+
}
|
|
1493
|
+
function isValidSpanId(spanId) {
|
|
1494
|
+
return /^[0-9a-f]{1,16}$/i.test(spanId);
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
// src/spans/no-op.ts
|
|
1498
|
+
var NoOpSpan = class extends BaseSpan {
|
|
1499
|
+
id;
|
|
1500
|
+
traceId;
|
|
1501
|
+
constructor(options, observabilityInstance) {
|
|
1502
|
+
super(options, observabilityInstance);
|
|
1503
|
+
this.id = "no-op";
|
|
1504
|
+
this.traceId = "no-op-trace";
|
|
1505
|
+
}
|
|
1506
|
+
end(_options) {
|
|
1507
|
+
}
|
|
1508
|
+
error(_options) {
|
|
1509
|
+
}
|
|
1510
|
+
update(_options) {
|
|
1511
|
+
}
|
|
1512
|
+
get isValid() {
|
|
1513
|
+
return false;
|
|
1514
|
+
}
|
|
1515
|
+
};
|
|
1516
|
+
|
|
1517
|
+
// src/instances/base.ts
|
|
1518
|
+
var BaseObservabilityInstance = class extends base.MastraBase {
|
|
1519
|
+
config;
|
|
1520
|
+
constructor(config) {
|
|
1521
|
+
super({ component: logger.RegisteredLogger.OBSERVABILITY, name: config.serviceName });
|
|
1313
1522
|
this.config = {
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
this.buffer = {
|
|
1322
|
-
creates: [],
|
|
1323
|
-
updates: [],
|
|
1324
|
-
insertOnly: [],
|
|
1325
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
1326
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
1327
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
1328
|
-
outOfOrderCount: 0,
|
|
1329
|
-
totalSize: 0
|
|
1523
|
+
serviceName: config.serviceName,
|
|
1524
|
+
name: config.name,
|
|
1525
|
+
sampling: config.sampling ?? { type: "always" /* ALWAYS */ },
|
|
1526
|
+
exporters: config.exporters ?? [],
|
|
1527
|
+
spanOutputProcessors: config.spanOutputProcessors ?? [],
|
|
1528
|
+
includeInternalSpans: config.includeInternalSpans ?? false,
|
|
1529
|
+
requestContextKeys: config.requestContextKeys ?? []
|
|
1330
1530
|
};
|
|
1331
|
-
this.resolvedStrategy = "batch-with-updates";
|
|
1332
|
-
}
|
|
1333
|
-
strategyInitialized = false;
|
|
1334
|
-
/**
|
|
1335
|
-
* Initialize the exporter (called after all dependencies are ready)
|
|
1336
|
-
*/
|
|
1337
|
-
init(options) {
|
|
1338
|
-
this.storage = options.mastra?.getStorage();
|
|
1339
|
-
if (!this.storage) {
|
|
1340
|
-
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
1341
|
-
return;
|
|
1342
|
-
}
|
|
1343
|
-
this.initializeStrategy(this.storage);
|
|
1344
1531
|
}
|
|
1345
1532
|
/**
|
|
1346
|
-
*
|
|
1533
|
+
* Override setLogger to add Observability specific initialization log
|
|
1534
|
+
* and propagate logger to exporters
|
|
1347
1535
|
*/
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
this.
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
source: this.config.strategy !== "auto" ? "user" : "auto",
|
|
1355
|
-
storageAdapter: storage.constructor.name,
|
|
1356
|
-
maxBatchSize: this.config.maxBatchSize,
|
|
1357
|
-
maxBatchWaitMs: this.config.maxBatchWaitMs
|
|
1536
|
+
__setLogger(logger) {
|
|
1537
|
+
super.__setLogger(logger);
|
|
1538
|
+
this.exporters.forEach((exporter) => {
|
|
1539
|
+
if (typeof exporter.__setLogger === "function") {
|
|
1540
|
+
exporter.__setLogger(logger);
|
|
1541
|
+
}
|
|
1358
1542
|
});
|
|
1543
|
+
this.logger.debug(
|
|
1544
|
+
`[Observability] Initialized [service=${this.config.serviceName}] [instance=${this.config.name}] [sampling=${this.config.sampling.type}]`
|
|
1545
|
+
);
|
|
1359
1546
|
}
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
return
|
|
1547
|
+
// ============================================================================
|
|
1548
|
+
// Protected getters for clean config access
|
|
1549
|
+
// ============================================================================
|
|
1550
|
+
get exporters() {
|
|
1551
|
+
return this.config.exporters || [];
|
|
1365
1552
|
}
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
*/
|
|
1369
|
-
getNextSequence(spanKey) {
|
|
1370
|
-
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
1371
|
-
const next = current + 1;
|
|
1372
|
-
this.buffer.spanSequences.set(spanKey, next);
|
|
1373
|
-
return next;
|
|
1553
|
+
get spanOutputProcessors() {
|
|
1554
|
+
return this.config.spanOutputProcessors || [];
|
|
1374
1555
|
}
|
|
1556
|
+
// ============================================================================
|
|
1557
|
+
// Public API - Single type-safe span creation method
|
|
1558
|
+
// ============================================================================
|
|
1375
1559
|
/**
|
|
1376
|
-
*
|
|
1560
|
+
* Start a new span of a specific SpanType
|
|
1377
1561
|
*/
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1562
|
+
startSpan(options) {
|
|
1563
|
+
const { customSamplerOptions, requestContext, metadata, tracingOptions, ...rest } = options;
|
|
1564
|
+
if (!this.shouldSample(customSamplerOptions)) {
|
|
1565
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
1566
|
+
}
|
|
1567
|
+
let traceState;
|
|
1568
|
+
if (options.parent) {
|
|
1569
|
+
traceState = options.parent.traceState;
|
|
1570
|
+
} else {
|
|
1571
|
+
traceState = this.computeTraceState(tracingOptions);
|
|
1572
|
+
}
|
|
1573
|
+
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, metadata, traceState);
|
|
1574
|
+
const span = this.createSpan({
|
|
1575
|
+
...rest,
|
|
1576
|
+
metadata: enrichedMetadata,
|
|
1577
|
+
traceState
|
|
1384
1578
|
});
|
|
1579
|
+
if (span.isEvent) {
|
|
1580
|
+
this.emitSpanEnded(span);
|
|
1581
|
+
} else {
|
|
1582
|
+
this.wireSpanLifecycle(span);
|
|
1583
|
+
this.emitSpanStarted(span);
|
|
1584
|
+
}
|
|
1585
|
+
return span;
|
|
1385
1586
|
}
|
|
1587
|
+
// ============================================================================
|
|
1588
|
+
// Configuration Management
|
|
1589
|
+
// ============================================================================
|
|
1386
1590
|
/**
|
|
1387
|
-
*
|
|
1388
|
-
*/
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
if (this.buffer.totalSize === 0) {
|
|
1392
|
-
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
1393
|
-
}
|
|
1394
|
-
switch (event.type) {
|
|
1395
|
-
case observability.AITracingEventType.SPAN_STARTED:
|
|
1396
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1397
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1398
|
-
this.buffer.creates.push(createRecord);
|
|
1399
|
-
this.buffer.seenSpans.add(spanKey);
|
|
1400
|
-
this.allCreatedSpans.add(spanKey);
|
|
1401
|
-
}
|
|
1402
|
-
break;
|
|
1403
|
-
case observability.AITracingEventType.SPAN_UPDATED:
|
|
1404
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1405
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
1406
|
-
this.buffer.updates.push({
|
|
1407
|
-
traceId: event.exportedSpan.traceId,
|
|
1408
|
-
spanId: event.exportedSpan.id,
|
|
1409
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
1410
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
1411
|
-
});
|
|
1412
|
-
} else {
|
|
1413
|
-
this.handleOutOfOrderUpdate(event);
|
|
1414
|
-
this.buffer.outOfOrderCount++;
|
|
1415
|
-
}
|
|
1416
|
-
}
|
|
1417
|
-
break;
|
|
1418
|
-
case observability.AITracingEventType.SPAN_ENDED:
|
|
1419
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1420
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
1421
|
-
this.buffer.updates.push({
|
|
1422
|
-
traceId: event.exportedSpan.traceId,
|
|
1423
|
-
spanId: event.exportedSpan.id,
|
|
1424
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
1425
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
1426
|
-
});
|
|
1427
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1428
|
-
} else if (event.exportedSpan.isEvent) {
|
|
1429
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1430
|
-
this.buffer.creates.push(createRecord);
|
|
1431
|
-
this.buffer.seenSpans.add(spanKey);
|
|
1432
|
-
this.allCreatedSpans.add(spanKey);
|
|
1433
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1434
|
-
} else {
|
|
1435
|
-
this.handleOutOfOrderUpdate(event);
|
|
1436
|
-
this.buffer.outOfOrderCount++;
|
|
1437
|
-
}
|
|
1438
|
-
} else if (this.resolvedStrategy === "insert-only") {
|
|
1439
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1440
|
-
this.buffer.insertOnly.push(createRecord);
|
|
1441
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1442
|
-
this.allCreatedSpans.add(spanKey);
|
|
1443
|
-
}
|
|
1444
|
-
break;
|
|
1445
|
-
}
|
|
1446
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
1591
|
+
* Get current configuration
|
|
1592
|
+
*/
|
|
1593
|
+
getConfig() {
|
|
1594
|
+
return { ...this.config };
|
|
1447
1595
|
}
|
|
1596
|
+
// ============================================================================
|
|
1597
|
+
// Plugin Access
|
|
1598
|
+
// ============================================================================
|
|
1448
1599
|
/**
|
|
1449
|
-
*
|
|
1600
|
+
* Get all exporters
|
|
1450
1601
|
*/
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1602
|
+
getExporters() {
|
|
1603
|
+
return [...this.exporters];
|
|
1604
|
+
}
|
|
1605
|
+
/**
|
|
1606
|
+
* Get all span output processors
|
|
1607
|
+
*/
|
|
1608
|
+
getSpanOutputProcessors() {
|
|
1609
|
+
return [...this.spanOutputProcessors];
|
|
1610
|
+
}
|
|
1611
|
+
/**
|
|
1612
|
+
* Get the logger instance (for exporters and other components)
|
|
1613
|
+
*/
|
|
1614
|
+
getLogger() {
|
|
1615
|
+
return this.logger;
|
|
1616
|
+
}
|
|
1617
|
+
// ============================================================================
|
|
1618
|
+
// Span Lifecycle Management
|
|
1619
|
+
// ============================================================================
|
|
1620
|
+
/**
|
|
1621
|
+
* Automatically wires up Observability lifecycle events for any span
|
|
1622
|
+
* This ensures all spans emit events regardless of implementation
|
|
1623
|
+
*/
|
|
1624
|
+
wireSpanLifecycle(span) {
|
|
1625
|
+
if (!this.config.includeInternalSpans && span.isInternal) {
|
|
1626
|
+
return;
|
|
1457
1627
|
}
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1628
|
+
const originalEnd = span.end.bind(span);
|
|
1629
|
+
const originalUpdate = span.update.bind(span);
|
|
1630
|
+
span.end = (options) => {
|
|
1631
|
+
if (span.isEvent) {
|
|
1632
|
+
this.logger.warn(`End event is not available on event spans`);
|
|
1633
|
+
return;
|
|
1462
1634
|
}
|
|
1463
|
-
|
|
1464
|
-
|
|
1635
|
+
originalEnd(options);
|
|
1636
|
+
this.emitSpanEnded(span);
|
|
1637
|
+
};
|
|
1638
|
+
span.update = (options) => {
|
|
1639
|
+
if (span.isEvent) {
|
|
1640
|
+
this.logger.warn(`Update() is not available on event spans`);
|
|
1641
|
+
return;
|
|
1642
|
+
}
|
|
1643
|
+
originalUpdate(options);
|
|
1644
|
+
this.emitSpanUpdated(span);
|
|
1645
|
+
};
|
|
1465
1646
|
}
|
|
1647
|
+
// ============================================================================
|
|
1648
|
+
// Utility Methods
|
|
1649
|
+
// ============================================================================
|
|
1466
1650
|
/**
|
|
1467
|
-
*
|
|
1651
|
+
* Check if a trace should be sampled
|
|
1468
1652
|
*/
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1653
|
+
shouldSample(options) {
|
|
1654
|
+
const { sampling } = this.config;
|
|
1655
|
+
switch (sampling.type) {
|
|
1656
|
+
case "always" /* ALWAYS */:
|
|
1657
|
+
return true;
|
|
1658
|
+
case "never" /* NEVER */:
|
|
1659
|
+
return false;
|
|
1660
|
+
case "ratio" /* RATIO */:
|
|
1661
|
+
if (sampling.probability === void 0 || sampling.probability < 0 || sampling.probability > 1) {
|
|
1662
|
+
this.logger.warn(
|
|
1663
|
+
`Invalid sampling probability: ${sampling.probability}. Expected value between 0 and 1. Defaulting to no sampling.`
|
|
1664
|
+
);
|
|
1665
|
+
return false;
|
|
1666
|
+
}
|
|
1667
|
+
return Math.random() < sampling.probability;
|
|
1668
|
+
case "custom" /* CUSTOM */:
|
|
1669
|
+
return sampling.sampler(options);
|
|
1670
|
+
default:
|
|
1671
|
+
throw new Error(`Sampling strategy type not implemented: ${sampling.type}`);
|
|
1481
1672
|
}
|
|
1482
1673
|
}
|
|
1483
1674
|
/**
|
|
1484
|
-
*
|
|
1675
|
+
* Compute TraceState for a new trace based on configured and per-request keys
|
|
1485
1676
|
*/
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1677
|
+
computeTraceState(tracingOptions) {
|
|
1678
|
+
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
1679
|
+
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
1680
|
+
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
1681
|
+
if (allKeys.length === 0) {
|
|
1682
|
+
return void 0;
|
|
1489
1683
|
}
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1494
|
-
});
|
|
1495
|
-
});
|
|
1496
|
-
}, this.config.maxBatchWaitMs);
|
|
1684
|
+
return {
|
|
1685
|
+
requestContextKeys: allKeys
|
|
1686
|
+
};
|
|
1497
1687
|
}
|
|
1498
1688
|
/**
|
|
1499
|
-
*
|
|
1500
|
-
* Handles all AI span types and their specific attributes
|
|
1689
|
+
* Extract metadata from RequestContext using TraceState
|
|
1501
1690
|
*/
|
|
1502
|
-
|
|
1503
|
-
if (!
|
|
1504
|
-
return
|
|
1691
|
+
extractMetadataFromRequestContext(requestContext, explicitMetadata, traceState) {
|
|
1692
|
+
if (!requestContext || !traceState || traceState.requestContextKeys.length === 0) {
|
|
1693
|
+
return explicitMetadata;
|
|
1505
1694
|
}
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
if (value instanceof Date) {
|
|
1510
|
-
return value.toISOString();
|
|
1511
|
-
}
|
|
1512
|
-
if (typeof value === "object" && value !== null) {
|
|
1513
|
-
return value;
|
|
1514
|
-
}
|
|
1515
|
-
return value;
|
|
1516
|
-
})
|
|
1517
|
-
);
|
|
1518
|
-
} catch (error) {
|
|
1519
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
1520
|
-
spanId: span.id,
|
|
1521
|
-
spanType: span.type,
|
|
1522
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1523
|
-
});
|
|
1524
|
-
return null;
|
|
1695
|
+
const extracted = this.extractKeys(requestContext, traceState.requestContextKeys);
|
|
1696
|
+
if (Object.keys(extracted).length === 0 && !explicitMetadata) {
|
|
1697
|
+
return void 0;
|
|
1525
1698
|
}
|
|
1526
|
-
}
|
|
1527
|
-
buildCreateRecord(span) {
|
|
1528
1699
|
return {
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
name: span.name,
|
|
1533
|
-
scope: null,
|
|
1534
|
-
spanType: span.type,
|
|
1535
|
-
attributes: this.serializeAttributes(span),
|
|
1536
|
-
metadata: span.metadata ?? null,
|
|
1537
|
-
links: null,
|
|
1538
|
-
startedAt: span.startTime,
|
|
1539
|
-
endedAt: span.endTime ?? null,
|
|
1540
|
-
input: span.input,
|
|
1541
|
-
output: span.output,
|
|
1542
|
-
error: span.errorInfo,
|
|
1543
|
-
isEvent: span.isEvent
|
|
1700
|
+
...extracted,
|
|
1701
|
+
...explicitMetadata
|
|
1702
|
+
// Explicit metadata always wins
|
|
1544
1703
|
};
|
|
1545
1704
|
}
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1705
|
+
/**
|
|
1706
|
+
* Extract specific keys from RequestContext
|
|
1707
|
+
*/
|
|
1708
|
+
extractKeys(requestContext, keys) {
|
|
1709
|
+
const result = {};
|
|
1710
|
+
for (const key of keys) {
|
|
1711
|
+
const parts = key.split(".");
|
|
1712
|
+
const rootKey = parts[0];
|
|
1713
|
+
const value = requestContext.get(rootKey);
|
|
1714
|
+
if (value !== void 0) {
|
|
1715
|
+
if (parts.length > 1) {
|
|
1716
|
+
const nestedPath = parts.slice(1).join(".");
|
|
1717
|
+
const nestedValue = utils.getNestedValue(value, nestedPath);
|
|
1718
|
+
if (nestedValue !== void 0) {
|
|
1719
|
+
utils.setNestedValue(result, key, nestedValue);
|
|
1720
|
+
}
|
|
1721
|
+
} else {
|
|
1722
|
+
utils.setNestedValue(result, key, value);
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
return result;
|
|
1558
1727
|
}
|
|
1559
1728
|
/**
|
|
1560
|
-
*
|
|
1729
|
+
* Process a span through all output processors
|
|
1561
1730
|
*/
|
|
1562
|
-
|
|
1563
|
-
const
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
if (event.type === observability.AITracingEventType.SPAN_ENDED) {
|
|
1567
|
-
await storage.createAISpan(this.buildCreateRecord(event.exportedSpan));
|
|
1568
|
-
} else {
|
|
1569
|
-
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
1731
|
+
processSpan(span) {
|
|
1732
|
+
for (const processor of this.spanOutputProcessors) {
|
|
1733
|
+
if (!span) {
|
|
1734
|
+
break;
|
|
1570
1735
|
}
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
this.allCreatedSpans.add(spanKey);
|
|
1576
|
-
break;
|
|
1577
|
-
case observability.AITracingEventType.SPAN_UPDATED:
|
|
1578
|
-
await storage.updateAISpan({
|
|
1579
|
-
traceId: span.traceId,
|
|
1580
|
-
spanId: span.id,
|
|
1581
|
-
updates: this.buildUpdateRecord(span)
|
|
1582
|
-
});
|
|
1583
|
-
break;
|
|
1584
|
-
case observability.AITracingEventType.SPAN_ENDED:
|
|
1585
|
-
await storage.updateAISpan({
|
|
1586
|
-
traceId: span.traceId,
|
|
1587
|
-
spanId: span.id,
|
|
1588
|
-
updates: this.buildUpdateRecord(span)
|
|
1589
|
-
});
|
|
1590
|
-
this.allCreatedSpans.delete(spanKey);
|
|
1591
|
-
break;
|
|
1592
|
-
default:
|
|
1593
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
1736
|
+
try {
|
|
1737
|
+
span = processor.process(span);
|
|
1738
|
+
} catch (error) {
|
|
1739
|
+
this.logger.error(`[Observability] Processor error [name=${processor.name}]`, error);
|
|
1594
1740
|
}
|
|
1595
1741
|
}
|
|
1742
|
+
return span;
|
|
1743
|
+
}
|
|
1744
|
+
// ============================================================================
|
|
1745
|
+
// Event-driven Export Methods
|
|
1746
|
+
// ============================================================================
|
|
1747
|
+
getSpanForExport(span) {
|
|
1748
|
+
if (!span.isValid) return void 0;
|
|
1749
|
+
if (span.isInternal && !this.config.includeInternalSpans) return void 0;
|
|
1750
|
+
const processedSpan = this.processSpan(span);
|
|
1751
|
+
return processedSpan?.exportSpan(this.config.includeInternalSpans);
|
|
1596
1752
|
}
|
|
1597
1753
|
/**
|
|
1598
|
-
*
|
|
1754
|
+
* Emit a span started event
|
|
1599
1755
|
*/
|
|
1600
|
-
|
|
1601
|
-
this.
|
|
1602
|
-
if (
|
|
1603
|
-
this.
|
|
1604
|
-
this.logger.error("
|
|
1605
|
-
|
|
1606
|
-
|
|
1756
|
+
emitSpanStarted(span) {
|
|
1757
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1758
|
+
if (exportedSpan) {
|
|
1759
|
+
this.exportTracingEvent({ type: observability.TracingEventType.SPAN_STARTED, exportedSpan }).catch((error) => {
|
|
1760
|
+
this.logger.error("[Observability] Failed to export span_started event", error);
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
/**
|
|
1765
|
+
* Emit a span ended event (called automatically when spans end)
|
|
1766
|
+
*/
|
|
1767
|
+
emitSpanEnded(span) {
|
|
1768
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1769
|
+
if (exportedSpan) {
|
|
1770
|
+
this.exportTracingEvent({ type: observability.TracingEventType.SPAN_ENDED, exportedSpan }).catch((error) => {
|
|
1771
|
+
this.logger.error("[Observability] Failed to export span_ended event", error);
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
/**
|
|
1776
|
+
* Emit a span updated event
|
|
1777
|
+
*/
|
|
1778
|
+
emitSpanUpdated(span) {
|
|
1779
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1780
|
+
if (exportedSpan) {
|
|
1781
|
+
this.exportTracingEvent({ type: observability.TracingEventType.SPAN_UPDATED, exportedSpan }).catch((error) => {
|
|
1782
|
+
this.logger.error("[Observability] Failed to export span_updated event", error);
|
|
1607
1783
|
});
|
|
1608
|
-
} else if (this.buffer.totalSize === 1) {
|
|
1609
|
-
this.scheduleFlush();
|
|
1610
1784
|
}
|
|
1611
1785
|
}
|
|
1612
1786
|
/**
|
|
1613
|
-
*
|
|
1787
|
+
* Export tracing event through all exporters (realtime mode)
|
|
1614
1788
|
*/
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
this.logger.
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
});
|
|
1624
|
-
} else if (this.buffer.totalSize === 1) {
|
|
1625
|
-
this.scheduleFlush();
|
|
1789
|
+
async exportTracingEvent(event) {
|
|
1790
|
+
const exportPromises = this.exporters.map(async (exporter) => {
|
|
1791
|
+
try {
|
|
1792
|
+
if (exporter.exportTracingEvent) {
|
|
1793
|
+
await exporter.exportTracingEvent(event);
|
|
1794
|
+
this.logger.debug(`[Observability] Event exported [exporter=${exporter.name}] [type=${event.type}]`);
|
|
1795
|
+
}
|
|
1796
|
+
} catch (error) {
|
|
1797
|
+
this.logger.error(`[Observability] Export error [exporter=${exporter.name}]`, error);
|
|
1626
1798
|
}
|
|
1627
|
-
}
|
|
1799
|
+
});
|
|
1800
|
+
await Promise.allSettled(exportPromises);
|
|
1801
|
+
}
|
|
1802
|
+
// ============================================================================
|
|
1803
|
+
// Lifecycle Management
|
|
1804
|
+
// ============================================================================
|
|
1805
|
+
/**
|
|
1806
|
+
* Initialize Observability (called by Mastra during component registration)
|
|
1807
|
+
*/
|
|
1808
|
+
init() {
|
|
1809
|
+
this.logger.debug(`[Observability] Initialization started [name=${this.name}]`);
|
|
1810
|
+
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
1628
1811
|
}
|
|
1629
1812
|
/**
|
|
1630
|
-
*
|
|
1813
|
+
* Shutdown Observability and clean up resources
|
|
1631
1814
|
*/
|
|
1632
|
-
|
|
1633
|
-
|
|
1815
|
+
async shutdown() {
|
|
1816
|
+
this.logger.debug(`[Observability] Shutdown started [name=${this.name}]`);
|
|
1817
|
+
const shutdownPromises = [
|
|
1818
|
+
...this.exporters.map((e) => e.shutdown()),
|
|
1819
|
+
...this.spanOutputProcessors.map((p) => p.shutdown())
|
|
1820
|
+
];
|
|
1821
|
+
await Promise.allSettled(shutdownPromises);
|
|
1822
|
+
this.logger.info(`[Observability] Shutdown completed [name=${this.name}]`);
|
|
1634
1823
|
}
|
|
1824
|
+
};
|
|
1825
|
+
|
|
1826
|
+
// src/instances/default.ts
|
|
1827
|
+
var DefaultObservabilityInstance = class extends BaseObservabilityInstance {
|
|
1828
|
+
constructor(config) {
|
|
1829
|
+
super(config);
|
|
1830
|
+
}
|
|
1831
|
+
createSpan(options) {
|
|
1832
|
+
return new DefaultSpan(options, this);
|
|
1833
|
+
}
|
|
1834
|
+
};
|
|
1835
|
+
|
|
1836
|
+
// src/registry.ts
|
|
1837
|
+
var ObservabilityRegistry = class {
|
|
1838
|
+
#instances = /* @__PURE__ */ new Map();
|
|
1839
|
+
#defaultInstance;
|
|
1840
|
+
#configSelector;
|
|
1635
1841
|
/**
|
|
1636
|
-
*
|
|
1842
|
+
* Register a tracing instance
|
|
1637
1843
|
*/
|
|
1638
|
-
|
|
1639
|
-
if (
|
|
1640
|
-
|
|
1641
|
-
return;
|
|
1642
|
-
}
|
|
1643
|
-
if (this.flushTimer) {
|
|
1644
|
-
clearTimeout(this.flushTimer);
|
|
1645
|
-
this.flushTimer = null;
|
|
1844
|
+
register(name, instance, isDefault = false) {
|
|
1845
|
+
if (this.#instances.has(name)) {
|
|
1846
|
+
throw new Error(`Tracing instance '${name}' already registered`);
|
|
1646
1847
|
}
|
|
1647
|
-
|
|
1648
|
-
|
|
1848
|
+
this.#instances.set(name, instance);
|
|
1849
|
+
if (isDefault || !this.#defaultInstance) {
|
|
1850
|
+
this.#defaultInstance = instance;
|
|
1649
1851
|
}
|
|
1650
|
-
const startTime = Date.now();
|
|
1651
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
1652
|
-
const bufferCopy = {
|
|
1653
|
-
creates: [...this.buffer.creates],
|
|
1654
|
-
updates: [...this.buffer.updates],
|
|
1655
|
-
insertOnly: [...this.buffer.insertOnly],
|
|
1656
|
-
seenSpans: new Set(this.buffer.seenSpans),
|
|
1657
|
-
spanSequences: new Map(this.buffer.spanSequences),
|
|
1658
|
-
completedSpans: new Set(this.buffer.completedSpans),
|
|
1659
|
-
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
1660
|
-
firstEventTime: this.buffer.firstEventTime,
|
|
1661
|
-
totalSize: this.buffer.totalSize
|
|
1662
|
-
};
|
|
1663
|
-
this.resetBuffer();
|
|
1664
|
-
await this.flushWithRetries(this.storage, bufferCopy, 0);
|
|
1665
|
-
const elapsed = Date.now() - startTime;
|
|
1666
|
-
this.logger.debug("Batch flushed", {
|
|
1667
|
-
strategy: this.resolvedStrategy,
|
|
1668
|
-
batchSize: bufferCopy.totalSize,
|
|
1669
|
-
flushReason,
|
|
1670
|
-
durationMs: elapsed,
|
|
1671
|
-
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
1672
|
-
});
|
|
1673
1852
|
}
|
|
1674
1853
|
/**
|
|
1675
|
-
*
|
|
1854
|
+
* Get a tracing instance by name
|
|
1676
1855
|
*/
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
this.
|
|
1700
|
-
}
|
|
1701
|
-
} catch (error) {
|
|
1702
|
-
if (attempt < this.config.maxRetries) {
|
|
1703
|
-
const retryDelay = this.calculateRetryDelay(attempt);
|
|
1704
|
-
this.logger.warn("Batch flush failed, retrying", {
|
|
1705
|
-
attempt: attempt + 1,
|
|
1706
|
-
maxRetries: this.config.maxRetries,
|
|
1707
|
-
nextRetryInMs: retryDelay,
|
|
1708
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1709
|
-
});
|
|
1710
|
-
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
1711
|
-
return this.flushWithRetries(storage, buffer, attempt + 1);
|
|
1712
|
-
} else {
|
|
1713
|
-
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
1714
|
-
finalAttempt: attempt + 1,
|
|
1715
|
-
maxRetries: this.config.maxRetries,
|
|
1716
|
-
droppedBatchSize: buffer.totalSize,
|
|
1717
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1718
|
-
});
|
|
1719
|
-
for (const spanKey of buffer.completedSpans) {
|
|
1720
|
-
this.allCreatedSpans.delete(spanKey);
|
|
1721
|
-
}
|
|
1856
|
+
get(name) {
|
|
1857
|
+
return this.#instances.get(name);
|
|
1858
|
+
}
|
|
1859
|
+
/**
|
|
1860
|
+
* Get the default tracing instance
|
|
1861
|
+
*/
|
|
1862
|
+
getDefault() {
|
|
1863
|
+
return this.#defaultInstance;
|
|
1864
|
+
}
|
|
1865
|
+
/**
|
|
1866
|
+
* Set the tracing selector function
|
|
1867
|
+
*/
|
|
1868
|
+
setSelector(selector) {
|
|
1869
|
+
this.#configSelector = selector;
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* Get the selected tracing instance based on context
|
|
1873
|
+
*/
|
|
1874
|
+
getSelected(options) {
|
|
1875
|
+
if (this.#configSelector) {
|
|
1876
|
+
const selected = this.#configSelector(options, this.#instances);
|
|
1877
|
+
if (selected && this.#instances.has(selected)) {
|
|
1878
|
+
return this.#instances.get(selected);
|
|
1722
1879
|
}
|
|
1723
1880
|
}
|
|
1881
|
+
return this.#defaultInstance;
|
|
1724
1882
|
}
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
case "realtime":
|
|
1735
|
-
await this.handleRealtimeEvent(event, this.storage);
|
|
1736
|
-
break;
|
|
1737
|
-
case "batch-with-updates":
|
|
1738
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
1739
|
-
break;
|
|
1740
|
-
case "insert-only":
|
|
1741
|
-
this.handleInsertOnlyEvent(event);
|
|
1742
|
-
break;
|
|
1883
|
+
/**
|
|
1884
|
+
* Unregister a tracing instance
|
|
1885
|
+
*/
|
|
1886
|
+
unregister(name) {
|
|
1887
|
+
const instance = this.#instances.get(name);
|
|
1888
|
+
const deleted = this.#instances.delete(name);
|
|
1889
|
+
if (deleted && instance === this.#defaultInstance) {
|
|
1890
|
+
const next = this.#instances.values().next();
|
|
1891
|
+
this.#defaultInstance = next.done ? void 0 : next.value;
|
|
1743
1892
|
}
|
|
1893
|
+
return deleted;
|
|
1744
1894
|
}
|
|
1895
|
+
/**
|
|
1896
|
+
* Shutdown all instances and clear the registry
|
|
1897
|
+
*/
|
|
1745
1898
|
async shutdown() {
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1899
|
+
const shutdownPromises = Array.from(this.#instances.values()).map((instance) => instance.shutdown());
|
|
1900
|
+
await Promise.allSettled(shutdownPromises);
|
|
1901
|
+
this.#instances.clear();
|
|
1902
|
+
this.#instances.clear();
|
|
1903
|
+
this.#defaultInstance = void 0;
|
|
1904
|
+
this.#configSelector = void 0;
|
|
1905
|
+
}
|
|
1906
|
+
/**
|
|
1907
|
+
* Clear all instances without shutdown
|
|
1908
|
+
*/
|
|
1909
|
+
clear() {
|
|
1910
|
+
this.#instances.clear();
|
|
1911
|
+
this.#defaultInstance = void 0;
|
|
1912
|
+
this.#configSelector = void 0;
|
|
1913
|
+
}
|
|
1914
|
+
/**
|
|
1915
|
+
* list all registered instances
|
|
1916
|
+
*/
|
|
1917
|
+
list() {
|
|
1918
|
+
return new Map(this.#instances);
|
|
1763
1919
|
}
|
|
1764
1920
|
};
|
|
1765
1921
|
|
|
@@ -1884,173 +2040,154 @@ var SensitiveDataFilter = class {
|
|
|
1884
2040
|
async shutdown() {
|
|
1885
2041
|
}
|
|
1886
2042
|
};
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
2043
|
+
|
|
2044
|
+
// src/default.ts
|
|
2045
|
+
function isInstance(obj) {
|
|
2046
|
+
return obj instanceof BaseObservabilityInstance;
|
|
2047
|
+
}
|
|
2048
|
+
var Observability = class extends base.MastraBase {
|
|
2049
|
+
#registry = new ObservabilityRegistry();
|
|
2050
|
+
constructor(config) {
|
|
2051
|
+
super({
|
|
2052
|
+
component: logger.RegisteredLogger.OBSERVABILITY,
|
|
2053
|
+
name: "Observability"
|
|
2054
|
+
});
|
|
2055
|
+
if (config === void 0) {
|
|
2056
|
+
config = {};
|
|
2057
|
+
}
|
|
2058
|
+
const validationResult = observabilityRegistryConfigSchema.safeParse(config);
|
|
2059
|
+
if (!validationResult.success) {
|
|
2060
|
+
const errorMessages = validationResult.error.errors.map((err) => `${err.path.join(".") || "config"}: ${err.message}`).join("; ");
|
|
2061
|
+
throw new error.MastraError({
|
|
2062
|
+
id: "OBSERVABILITY_INVALID_CONFIG",
|
|
2063
|
+
text: `Invalid observability configuration: ${errorMessages}`,
|
|
2064
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
2065
|
+
category: error.ErrorCategory.USER,
|
|
2066
|
+
details: {
|
|
2067
|
+
validationErrors: errorMessages
|
|
2068
|
+
}
|
|
2069
|
+
});
|
|
2070
|
+
}
|
|
2071
|
+
if (config.configs) {
|
|
2072
|
+
for (const [name, configValue] of Object.entries(config.configs)) {
|
|
2073
|
+
if (!isInstance(configValue)) {
|
|
2074
|
+
const configValidation = observabilityConfigValueSchema.safeParse(configValue);
|
|
2075
|
+
if (!configValidation.success) {
|
|
2076
|
+
const errorMessages = configValidation.error.errors.map((err) => `${err.path.join(".")}: ${err.message}`).join("; ");
|
|
2077
|
+
throw new error.MastraError({
|
|
2078
|
+
id: "OBSERVABILITY_INVALID_INSTANCE_CONFIG",
|
|
2079
|
+
text: `Invalid configuration for observability instance '${name}': ${errorMessages}`,
|
|
2080
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
2081
|
+
category: error.ErrorCategory.USER,
|
|
2082
|
+
details: {
|
|
2083
|
+
instanceName: name,
|
|
2084
|
+
validationErrors: errorMessages
|
|
2085
|
+
}
|
|
2086
|
+
});
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
1897
2090
|
}
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
2091
|
+
if (config.default?.enabled) {
|
|
2092
|
+
const defaultInstance = new DefaultObservabilityInstance({
|
|
2093
|
+
serviceName: "mastra",
|
|
2094
|
+
name: "default",
|
|
2095
|
+
sampling: { type: "always" /* ALWAYS */ },
|
|
2096
|
+
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
2097
|
+
spanOutputProcessors: [new SensitiveDataFilter()]
|
|
2098
|
+
});
|
|
2099
|
+
this.#registry.register("default", defaultInstance, true);
|
|
2100
|
+
}
|
|
2101
|
+
if (config.configs) {
|
|
2102
|
+
const instances = Object.entries(config.configs);
|
|
2103
|
+
instances.forEach(([name, tracingDef], index) => {
|
|
2104
|
+
const instance = isInstance(tracingDef) ? tracingDef : new DefaultObservabilityInstance({ ...tracingDef, name });
|
|
2105
|
+
const isDefault = !config.default?.enabled && index === 0;
|
|
2106
|
+
this.#registry.register(name, instance, isDefault);
|
|
2107
|
+
});
|
|
1901
2108
|
}
|
|
2109
|
+
if (config.configSelector) {
|
|
2110
|
+
this.#registry.setSelector(config.configSelector);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
setMastraContext(options) {
|
|
2114
|
+
const instances = this.listInstances();
|
|
2115
|
+
const { mastra } = options;
|
|
2116
|
+
instances.forEach((instance) => {
|
|
2117
|
+
const config = instance.getConfig();
|
|
2118
|
+
const exporters = instance.getExporters();
|
|
2119
|
+
exporters.forEach((exporter) => {
|
|
2120
|
+
if ("init" in exporter && typeof exporter.init === "function") {
|
|
2121
|
+
try {
|
|
2122
|
+
exporter.init({ mastra, config });
|
|
2123
|
+
} catch (error) {
|
|
2124
|
+
this.logger?.warn("Failed to initialize observability exporter", {
|
|
2125
|
+
exporterName: exporter.name,
|
|
2126
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2127
|
+
});
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
});
|
|
2131
|
+
});
|
|
1902
2132
|
}
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
}
|
|
1909
|
-
/**
|
|
1910
|
-
* Get the default tracing instance
|
|
1911
|
-
*/
|
|
1912
|
-
getDefault() {
|
|
1913
|
-
return this.defaultInstance;
|
|
2133
|
+
setLogger(options) {
|
|
2134
|
+
super.__setLogger(options.logger);
|
|
2135
|
+
this.listInstances().forEach((instance) => {
|
|
2136
|
+
instance.__setLogger(options.logger);
|
|
2137
|
+
});
|
|
1914
2138
|
}
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
*/
|
|
1918
|
-
setSelector(selector) {
|
|
1919
|
-
this.configSelector = selector;
|
|
2139
|
+
getSelectedInstance(options) {
|
|
2140
|
+
return this.#registry.getSelected(options);
|
|
1920
2141
|
}
|
|
1921
2142
|
/**
|
|
1922
|
-
*
|
|
2143
|
+
* Registry management methods
|
|
1923
2144
|
*/
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
const selected = this.configSelector(options, this.instances);
|
|
1927
|
-
if (selected && this.instances.has(selected)) {
|
|
1928
|
-
return this.instances.get(selected);
|
|
1929
|
-
}
|
|
1930
|
-
}
|
|
1931
|
-
return this.defaultInstance;
|
|
2145
|
+
registerInstance(name, instance, isDefault = false) {
|
|
2146
|
+
this.#registry.register(name, instance, isDefault);
|
|
1932
2147
|
}
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
*/
|
|
1936
|
-
unregister(name) {
|
|
1937
|
-
return this.instances.delete(name);
|
|
2148
|
+
getInstance(name) {
|
|
2149
|
+
return this.#registry.get(name);
|
|
1938
2150
|
}
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
*/
|
|
1942
|
-
async shutdown() {
|
|
1943
|
-
const shutdownPromises = Array.from(this.instances.values()).map((instance) => instance.shutdown());
|
|
1944
|
-
await Promise.allSettled(shutdownPromises);
|
|
1945
|
-
this.instances.clear();
|
|
2151
|
+
getDefaultInstance() {
|
|
2152
|
+
return this.#registry.getDefault();
|
|
1946
2153
|
}
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
*/
|
|
1950
|
-
clear() {
|
|
1951
|
-
this.instances.clear();
|
|
1952
|
-
this.defaultInstance = void 0;
|
|
1953
|
-
this.configSelector = void 0;
|
|
2154
|
+
listInstances() {
|
|
2155
|
+
return this.#registry.list();
|
|
1954
2156
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
*/
|
|
1958
|
-
getAll() {
|
|
1959
|
-
return new Map(this.instances);
|
|
2157
|
+
unregisterInstance(name) {
|
|
2158
|
+
return this.#registry.unregister(name);
|
|
1960
2159
|
}
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
function registerAITracing(name, instance, isDefault = false) {
|
|
1964
|
-
aiTracingRegistry.register(name, instance, isDefault);
|
|
1965
|
-
}
|
|
1966
|
-
function getAITracing(name) {
|
|
1967
|
-
return aiTracingRegistry.get(name);
|
|
1968
|
-
}
|
|
1969
|
-
function getDefaultAITracing() {
|
|
1970
|
-
return aiTracingRegistry.getDefault();
|
|
1971
|
-
}
|
|
1972
|
-
function setSelector(selector) {
|
|
1973
|
-
aiTracingRegistry.setSelector(selector);
|
|
1974
|
-
}
|
|
1975
|
-
function getSelectedAITracing(options) {
|
|
1976
|
-
return aiTracingRegistry.getSelected(options);
|
|
1977
|
-
}
|
|
1978
|
-
function unregisterAITracing(name) {
|
|
1979
|
-
return aiTracingRegistry.unregister(name);
|
|
1980
|
-
}
|
|
1981
|
-
async function shutdownAITracingRegistry() {
|
|
1982
|
-
await aiTracingRegistry.shutdown();
|
|
1983
|
-
}
|
|
1984
|
-
function clearAITracingRegistry() {
|
|
1985
|
-
aiTracingRegistry.clear();
|
|
1986
|
-
}
|
|
1987
|
-
function getAllAITracing() {
|
|
1988
|
-
return aiTracingRegistry.getAll();
|
|
1989
|
-
}
|
|
1990
|
-
function hasAITracing(name) {
|
|
1991
|
-
const tracing = getAITracing(name);
|
|
1992
|
-
if (!tracing) return false;
|
|
1993
|
-
const config = tracing.getConfig();
|
|
1994
|
-
const sampling = config.sampling;
|
|
1995
|
-
return sampling.type !== observability.SamplingStrategyType.NEVER;
|
|
1996
|
-
}
|
|
1997
|
-
function isAITracingInstance(obj) {
|
|
1998
|
-
return obj instanceof BaseAITracing;
|
|
1999
|
-
}
|
|
2000
|
-
function setupAITracingRegistry(config) {
|
|
2001
|
-
if (!config) {
|
|
2002
|
-
return;
|
|
2160
|
+
hasInstance(name) {
|
|
2161
|
+
return !!this.#registry.get(name);
|
|
2003
2162
|
}
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
"Cannot use 'default' as a custom config name when default tracing is enabled. Please rename your custom config to avoid conflicts."
|
|
2007
|
-
);
|
|
2163
|
+
setConfigSelector(selector) {
|
|
2164
|
+
this.#registry.setSelector(selector);
|
|
2008
2165
|
}
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
serviceName: "mastra",
|
|
2012
|
-
name: "default",
|
|
2013
|
-
sampling: { type: observability.SamplingStrategyType.ALWAYS },
|
|
2014
|
-
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
2015
|
-
processors: [new SensitiveDataFilter()]
|
|
2016
|
-
});
|
|
2017
|
-
registerAITracing("default", defaultInstance, true);
|
|
2018
|
-
}
|
|
2019
|
-
if (config.configs) {
|
|
2020
|
-
const instances = Object.entries(config.configs);
|
|
2021
|
-
instances.forEach(([name, tracingDef], index) => {
|
|
2022
|
-
const instance = isAITracingInstance(tracingDef) ? tracingDef : new DefaultAITracing({ ...tracingDef, name });
|
|
2023
|
-
const isDefault = !config.default?.enabled && index === 0;
|
|
2024
|
-
registerAITracing(name, instance, isDefault);
|
|
2025
|
-
});
|
|
2166
|
+
clear() {
|
|
2167
|
+
this.#registry.clear();
|
|
2026
2168
|
}
|
|
2027
|
-
|
|
2028
|
-
|
|
2169
|
+
async shutdown() {
|
|
2170
|
+
await this.#registry.shutdown();
|
|
2029
2171
|
}
|
|
2030
|
-
}
|
|
2172
|
+
};
|
|
2031
2173
|
|
|
2032
|
-
exports.BaseAISpan = BaseAISpan;
|
|
2033
|
-
exports.BaseAITracing = BaseAITracing;
|
|
2034
2174
|
exports.BaseExporter = BaseExporter;
|
|
2175
|
+
exports.BaseObservabilityInstance = BaseObservabilityInstance;
|
|
2176
|
+
exports.BaseSpan = BaseSpan;
|
|
2035
2177
|
exports.CloudExporter = CloudExporter;
|
|
2036
2178
|
exports.ConsoleExporter = ConsoleExporter;
|
|
2037
|
-
exports.DefaultAISpan = DefaultAISpan;
|
|
2038
|
-
exports.DefaultAITracing = DefaultAITracing;
|
|
2039
2179
|
exports.DefaultExporter = DefaultExporter;
|
|
2180
|
+
exports.DefaultObservabilityInstance = DefaultObservabilityInstance;
|
|
2181
|
+
exports.DefaultSpan = DefaultSpan;
|
|
2040
2182
|
exports.ModelSpanTracker = ModelSpanTracker;
|
|
2041
|
-
exports.
|
|
2183
|
+
exports.NoOpSpan = NoOpSpan;
|
|
2184
|
+
exports.Observability = Observability;
|
|
2185
|
+
exports.SamplingStrategyType = SamplingStrategyType;
|
|
2042
2186
|
exports.SensitiveDataFilter = SensitiveDataFilter;
|
|
2043
|
-
exports.clearAITracingRegistry = clearAITracingRegistry;
|
|
2044
2187
|
exports.deepClean = deepClean;
|
|
2045
|
-
exports.
|
|
2046
|
-
exports.
|
|
2047
|
-
exports.
|
|
2048
|
-
exports.
|
|
2049
|
-
exports.hasAITracing = hasAITracing;
|
|
2050
|
-
exports.registerAITracing = registerAITracing;
|
|
2051
|
-
exports.setSelector = setSelector;
|
|
2052
|
-
exports.setupAITracingRegistry = setupAITracingRegistry;
|
|
2053
|
-
exports.shutdownAITracingRegistry = shutdownAITracingRegistry;
|
|
2054
|
-
exports.unregisterAITracing = unregisterAITracing;
|
|
2188
|
+
exports.observabilityConfigValueSchema = observabilityConfigValueSchema;
|
|
2189
|
+
exports.observabilityInstanceConfigSchema = observabilityInstanceConfigSchema;
|
|
2190
|
+
exports.observabilityRegistryConfigSchema = observabilityRegistryConfigSchema;
|
|
2191
|
+
exports.samplingStrategySchema = samplingStrategySchema;
|
|
2055
2192
|
//# sourceMappingURL=index.cjs.map
|
|
2056
2193
|
//# sourceMappingURL=index.cjs.map
|