@mastra/observability 0.0.0-vnext-20251104230439 → 0.0.0-vnext-20251119160359
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -3
- package/README.md +18 -20
- package/dist/config.d.ts +343 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/default.d.ts +29 -0
- package/dist/default.d.ts.map +1 -0
- package/dist/exporters/base.d.ts +10 -10
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/cloud.d.ts +2 -2
- package/dist/exporters/cloud.d.ts.map +1 -1
- package/dist/exporters/console.d.ts +2 -2
- package/dist/exporters/console.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +10 -14
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +1 -1
- package/dist/index.cjs +1786 -1649
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1774 -1632
- package/dist/index.js.map +1 -1
- package/dist/{tracers → instances}/base.d.ts +27 -26
- package/dist/instances/base.d.ts.map +1 -0
- package/dist/instances/default.d.ts +8 -0
- package/dist/instances/default.d.ts.map +1 -0
- package/dist/instances/index.d.ts +6 -0
- package/dist/instances/index.d.ts.map +1 -0
- package/dist/model-tracing.d.ts +6 -6
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/registry.d.ts +46 -48
- package/dist/registry.d.ts.map +1 -1
- package/dist/span_processors/index.d.ts +1 -1
- package/dist/span_processors/sensitive-data-filter.d.ts +4 -4
- package/dist/span_processors/sensitive-data-filter.d.ts.map +1 -1
- package/dist/spans/base.d.ts +11 -11
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts +4 -4
- package/dist/spans/default.d.ts.map +1 -1
- package/dist/spans/index.d.ts +1 -1
- package/dist/spans/no-op.d.ts +4 -4
- package/dist/spans/no-op.d.ts.map +1 -1
- package/package.json +16 -11
- package/dist/default-entrypoint.d.ts +0 -16
- package/dist/default-entrypoint.d.ts.map +0 -1
- package/dist/init.d.ts +0 -2
- package/dist/init.d.ts.map +0 -1
- package/dist/tracers/base.d.ts.map +0 -1
- package/dist/tracers/default.d.ts +0 -7
- package/dist/tracers/default.d.ts.map +0 -1
- package/dist/tracers/index.d.ts +0 -6
- package/dist/tracers/index.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,1763 +1,1919 @@
|
|
|
1
1
|
import { MastraBase } from '@mastra/core/base';
|
|
2
|
-
import { RegisteredLogger, ConsoleLogger, LogLevel } from '@mastra/core/logger';
|
|
3
|
-
import { AISpanType, SamplingStrategyType, AITracingEventType, InternalSpans } from '@mastra/core/observability';
|
|
4
|
-
import { getNestedValue, setNestedValue, fetchWithRetry } from '@mastra/core/utils';
|
|
5
|
-
import { TransformStream } from 'stream/web';
|
|
6
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { ConsoleLogger, LogLevel, RegisteredLogger } from '@mastra/core/logger';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { TracingEventType, SpanType, InternalSpans } from '@mastra/core/observability';
|
|
6
|
+
import { fetchWithRetry, getNestedValue, setNestedValue } from '@mastra/core/utils';
|
|
7
|
+
import { TransformStream } from 'stream/web';
|
|
7
8
|
|
|
8
|
-
// src/
|
|
9
|
-
var
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
9
|
+
// src/default.ts
|
|
10
|
+
var SamplingStrategyType = /* @__PURE__ */ ((SamplingStrategyType2) => {
|
|
11
|
+
SamplingStrategyType2["ALWAYS"] = "always";
|
|
12
|
+
SamplingStrategyType2["NEVER"] = "never";
|
|
13
|
+
SamplingStrategyType2["RATIO"] = "ratio";
|
|
14
|
+
SamplingStrategyType2["CUSTOM"] = "custom";
|
|
15
|
+
return SamplingStrategyType2;
|
|
16
|
+
})(SamplingStrategyType || {});
|
|
17
|
+
var samplingStrategySchema = z.discriminatedUnion("type", [
|
|
18
|
+
z.object({
|
|
19
|
+
type: z.literal("always" /* ALWAYS */)
|
|
20
|
+
}),
|
|
21
|
+
z.object({
|
|
22
|
+
type: z.literal("never" /* NEVER */)
|
|
23
|
+
}),
|
|
24
|
+
z.object({
|
|
25
|
+
type: z.literal("ratio" /* RATIO */),
|
|
26
|
+
probability: z.number().min(0, "Probability must be between 0 and 1").max(1, "Probability must be between 0 and 1")
|
|
27
|
+
}),
|
|
28
|
+
z.object({
|
|
29
|
+
type: z.literal("custom" /* CUSTOM */),
|
|
30
|
+
sampler: z.function().args(z.any().optional()).returns(z.boolean())
|
|
31
|
+
})
|
|
32
|
+
]);
|
|
33
|
+
var observabilityInstanceConfigSchema = z.object({
|
|
34
|
+
name: z.string().min(1, "Name is required"),
|
|
35
|
+
serviceName: z.string().min(1, "Service name is required"),
|
|
36
|
+
sampling: samplingStrategySchema.optional(),
|
|
37
|
+
exporters: z.array(z.any()).optional(),
|
|
38
|
+
spanOutputProcessors: z.array(z.any()).optional(),
|
|
39
|
+
includeInternalSpans: z.boolean().optional(),
|
|
40
|
+
requestContextKeys: z.array(z.string()).optional()
|
|
41
|
+
});
|
|
42
|
+
var observabilityConfigValueSchema = z.object({
|
|
43
|
+
serviceName: z.string().min(1, "Service name is required"),
|
|
44
|
+
sampling: samplingStrategySchema.optional(),
|
|
45
|
+
exporters: z.array(z.any()).optional(),
|
|
46
|
+
spanOutputProcessors: z.array(z.any()).optional(),
|
|
47
|
+
includeInternalSpans: z.boolean().optional(),
|
|
48
|
+
requestContextKeys: z.array(z.string()).optional()
|
|
49
|
+
});
|
|
50
|
+
var observabilityRegistryConfigSchema = z.object({
|
|
51
|
+
default: z.object({
|
|
52
|
+
enabled: z.boolean().optional()
|
|
53
|
+
}).optional().nullable(),
|
|
54
|
+
configs: z.union([z.record(z.string(), z.any()), z.array(z.any()), z.null()]).optional(),
|
|
55
|
+
configSelector: z.function().optional()
|
|
56
|
+
}).passthrough().refine(
|
|
57
|
+
(data) => {
|
|
58
|
+
const isDefaultEnabled = data.default?.enabled === true;
|
|
59
|
+
const hasConfigs = data.configs && typeof data.configs === "object" && !Array.isArray(data.configs) ? Object.keys(data.configs).length > 0 : false;
|
|
60
|
+
return !(isDefaultEnabled && hasConfigs);
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
message: 'Cannot specify both "default" (when enabled) and "configs". Use either default observability or custom configs, but not both.'
|
|
64
|
+
}
|
|
65
|
+
).refine(
|
|
66
|
+
(data) => {
|
|
67
|
+
const configCount = data.configs && typeof data.configs === "object" && !Array.isArray(data.configs) ? Object.keys(data.configs).length : 0;
|
|
68
|
+
if (configCount > 1 && !data.configSelector) {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
return true;
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
message: 'A "configSelector" function is required when multiple configs are specified to determine which config to use.'
|
|
27
75
|
}
|
|
76
|
+
);
|
|
77
|
+
var BaseExporter = class {
|
|
78
|
+
/** Mastra logger instance */
|
|
79
|
+
logger;
|
|
80
|
+
/** Whether this exporter is disabled */
|
|
81
|
+
isDisabled = false;
|
|
28
82
|
/**
|
|
29
|
-
*
|
|
83
|
+
* Initialize the base exporter with logger
|
|
30
84
|
*/
|
|
31
|
-
|
|
32
|
-
this
|
|
85
|
+
constructor(config = {}) {
|
|
86
|
+
const logLevel = this.resolveLogLevel(config.logLevel);
|
|
87
|
+
this.logger = config.logger ?? new ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
33
88
|
}
|
|
34
89
|
/**
|
|
35
|
-
*
|
|
90
|
+
* Set the logger for the exporter (called by Mastra/ObservabilityInstance during initialization)
|
|
36
91
|
*/
|
|
37
|
-
|
|
38
|
-
this
|
|
92
|
+
__setLogger(logger) {
|
|
93
|
+
this.logger = logger;
|
|
94
|
+
this.logger.debug(`Logger updated for exporter [name=${this.name}]`);
|
|
39
95
|
}
|
|
40
96
|
/**
|
|
41
|
-
*
|
|
97
|
+
* Convert string log level to LogLevel enum
|
|
42
98
|
*/
|
|
43
|
-
|
|
44
|
-
|
|
99
|
+
resolveLogLevel(logLevel) {
|
|
100
|
+
if (!logLevel) {
|
|
101
|
+
return LogLevel.INFO;
|
|
102
|
+
}
|
|
103
|
+
if (typeof logLevel === "number") {
|
|
104
|
+
return logLevel;
|
|
105
|
+
}
|
|
106
|
+
const logLevelMap = {
|
|
107
|
+
debug: LogLevel.DEBUG,
|
|
108
|
+
info: LogLevel.INFO,
|
|
109
|
+
warn: LogLevel.WARN,
|
|
110
|
+
error: LogLevel.ERROR
|
|
111
|
+
};
|
|
112
|
+
return logLevelMap[logLevel] ?? LogLevel.INFO;
|
|
45
113
|
}
|
|
46
114
|
/**
|
|
47
|
-
*
|
|
115
|
+
* Mark the exporter as disabled and log a message
|
|
116
|
+
*
|
|
117
|
+
* @param reason - Reason why the exporter is disabled
|
|
48
118
|
*/
|
|
49
|
-
|
|
50
|
-
this
|
|
51
|
-
|
|
52
|
-
type: AISpanType.MODEL_STEP,
|
|
53
|
-
attributes: {
|
|
54
|
-
stepIndex: this.#stepIndex,
|
|
55
|
-
...payload?.messageId ? { messageId: payload.messageId } : {},
|
|
56
|
-
...payload?.warnings?.length ? { warnings: payload.warnings } : {}
|
|
57
|
-
},
|
|
58
|
-
input: payload?.request
|
|
59
|
-
});
|
|
60
|
-
this.#chunkSequence = 0;
|
|
119
|
+
setDisabled(reason) {
|
|
120
|
+
this.isDisabled = true;
|
|
121
|
+
this.logger.warn(`${this.name} disabled: ${reason}`);
|
|
61
122
|
}
|
|
62
123
|
/**
|
|
63
|
-
*
|
|
124
|
+
* Export a tracing event
|
|
125
|
+
*
|
|
126
|
+
* This method checks if the exporter is disabled before calling _exportEvent.
|
|
127
|
+
* Subclasses should implement _exportEvent instead of overriding this method.
|
|
64
128
|
*/
|
|
65
|
-
|
|
66
|
-
if (
|
|
67
|
-
|
|
68
|
-
const { usage, ...otherOutput } = output;
|
|
69
|
-
const stepResult = payload.stepResult;
|
|
70
|
-
const metadata = payload.metadata;
|
|
71
|
-
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
72
|
-
if (cleanMetadata?.request) {
|
|
73
|
-
delete cleanMetadata.request;
|
|
129
|
+
async exportTracingEvent(event) {
|
|
130
|
+
if (this.isDisabled) {
|
|
131
|
+
return;
|
|
74
132
|
}
|
|
75
|
-
this
|
|
76
|
-
output: otherOutput,
|
|
77
|
-
attributes: {
|
|
78
|
-
usage,
|
|
79
|
-
isContinued: stepResult.isContinued,
|
|
80
|
-
finishReason: stepResult.reason,
|
|
81
|
-
warnings: stepResult.warnings
|
|
82
|
-
},
|
|
83
|
-
metadata: {
|
|
84
|
-
...cleanMetadata
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
this.#currentStepSpan = void 0;
|
|
88
|
-
this.#stepIndex++;
|
|
133
|
+
await this._exportTracingEvent(event);
|
|
89
134
|
}
|
|
90
135
|
/**
|
|
91
|
-
*
|
|
136
|
+
* Shutdown the exporter and clean up resources
|
|
137
|
+
*
|
|
138
|
+
* Default implementation just logs. Override to add custom cleanup.
|
|
92
139
|
*/
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
this.#startStepSpan();
|
|
96
|
-
}
|
|
97
|
-
this.#currentChunkSpan = this.#currentStepSpan?.createChildSpan({
|
|
98
|
-
name: `chunk: '${chunkType}'`,
|
|
99
|
-
type: AISpanType.MODEL_CHUNK,
|
|
100
|
-
attributes: {
|
|
101
|
-
chunkType,
|
|
102
|
-
sequenceNumber: this.#chunkSequence
|
|
103
|
-
}
|
|
104
|
-
});
|
|
105
|
-
this.#accumulator = initialData || {};
|
|
140
|
+
async shutdown() {
|
|
141
|
+
this.logger.info(`${this.name} shutdown complete`);
|
|
106
142
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
143
|
+
};
|
|
144
|
+
var CloudExporter = class extends BaseExporter {
|
|
145
|
+
name = "mastra-cloud-observability-exporter";
|
|
146
|
+
config;
|
|
147
|
+
buffer;
|
|
148
|
+
flushTimer = null;
|
|
149
|
+
constructor(config = {}) {
|
|
150
|
+
super(config);
|
|
151
|
+
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
152
|
+
if (!accessToken) {
|
|
153
|
+
this.setDisabled(
|
|
154
|
+
"MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.\n\u{1F680} Sign up at https://cloud.mastra.ai to see your AI traces online and obtain your access token."
|
|
155
|
+
);
|
|
115
156
|
}
|
|
157
|
+
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
158
|
+
this.config = {
|
|
159
|
+
logger: this.logger,
|
|
160
|
+
logLevel: config.logLevel ?? LogLevel.INFO,
|
|
161
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
162
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
163
|
+
maxRetries: config.maxRetries ?? 3,
|
|
164
|
+
accessToken: accessToken || "",
|
|
165
|
+
endpoint
|
|
166
|
+
};
|
|
167
|
+
this.buffer = {
|
|
168
|
+
spans: [],
|
|
169
|
+
totalSize: 0
|
|
170
|
+
};
|
|
116
171
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
*/
|
|
121
|
-
#endChunkSpan(output) {
|
|
122
|
-
if (!this.#currentChunkSpan) return;
|
|
123
|
-
this.#currentChunkSpan.end({
|
|
124
|
-
output: output !== void 0 ? output : this.#accumulator
|
|
125
|
-
});
|
|
126
|
-
this.#currentChunkSpan = void 0;
|
|
127
|
-
this.#accumulator = {};
|
|
128
|
-
this.#chunkSequence++;
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* Create an event span (for single chunks like tool-call)
|
|
132
|
-
*/
|
|
133
|
-
#createEventSpan(chunkType, output) {
|
|
134
|
-
if (!this.#currentStepSpan) {
|
|
135
|
-
this.#startStepSpan();
|
|
172
|
+
async _exportTracingEvent(event) {
|
|
173
|
+
if (event.type !== TracingEventType.SPAN_ENDED) {
|
|
174
|
+
return;
|
|
136
175
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if (span) {
|
|
147
|
-
this.#chunkSequence++;
|
|
176
|
+
this.addToBuffer(event);
|
|
177
|
+
if (this.shouldFlush()) {
|
|
178
|
+
this.flush().catch((error) => {
|
|
179
|
+
this.logger.error("Batch flush failed", {
|
|
180
|
+
error: error instanceof Error ? error.message : String(error)
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
} else if (this.buffer.totalSize === 1) {
|
|
184
|
+
this.scheduleFlush();
|
|
148
185
|
}
|
|
149
186
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
187
|
+
addToBuffer(event) {
|
|
188
|
+
if (this.buffer.totalSize === 0) {
|
|
189
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
190
|
+
}
|
|
191
|
+
const spanRecord = this.formatSpan(event.exportedSpan);
|
|
192
|
+
this.buffer.spans.push(spanRecord);
|
|
193
|
+
this.buffer.totalSize++;
|
|
155
194
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
195
|
+
formatSpan(span) {
|
|
196
|
+
const spanRecord = {
|
|
197
|
+
traceId: span.traceId,
|
|
198
|
+
spanId: span.id,
|
|
199
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
200
|
+
name: span.name,
|
|
201
|
+
spanType: span.type,
|
|
202
|
+
attributes: span.attributes ?? null,
|
|
203
|
+
metadata: span.metadata ?? null,
|
|
204
|
+
startedAt: span.startTime,
|
|
205
|
+
endedAt: span.endTime ?? null,
|
|
206
|
+
input: span.input ?? null,
|
|
207
|
+
output: span.output ?? null,
|
|
208
|
+
error: span.errorInfo,
|
|
209
|
+
isEvent: span.isEvent,
|
|
210
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
211
|
+
updatedAt: null
|
|
212
|
+
};
|
|
213
|
+
return spanRecord;
|
|
161
214
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
case "text-delta":
|
|
171
|
-
this.#appendToAccumulator("text", chunk.payload.text);
|
|
172
|
-
break;
|
|
173
|
-
case "text-end": {
|
|
174
|
-
this.#endChunkSpan();
|
|
175
|
-
break;
|
|
215
|
+
shouldFlush() {
|
|
216
|
+
if (this.buffer.totalSize >= this.config.maxBatchSize) {
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
220
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
221
|
+
if (elapsed >= this.config.maxBatchWaitMs) {
|
|
222
|
+
return true;
|
|
176
223
|
}
|
|
177
224
|
}
|
|
225
|
+
return false;
|
|
178
226
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
#handleReasoningChunk(chunk) {
|
|
183
|
-
switch (chunk.type) {
|
|
184
|
-
case "reasoning-start":
|
|
185
|
-
this.#startChunkSpan("reasoning");
|
|
186
|
-
break;
|
|
187
|
-
case "reasoning-delta":
|
|
188
|
-
this.#appendToAccumulator("text", chunk.payload.text);
|
|
189
|
-
break;
|
|
190
|
-
case "reasoning-end": {
|
|
191
|
-
this.#endChunkSpan();
|
|
192
|
-
break;
|
|
193
|
-
}
|
|
227
|
+
scheduleFlush() {
|
|
228
|
+
if (this.flushTimer) {
|
|
229
|
+
clearTimeout(this.flushTimer);
|
|
194
230
|
}
|
|
231
|
+
this.flushTimer = setTimeout(() => {
|
|
232
|
+
this.flush().catch((error) => {
|
|
233
|
+
const mastraError = new MastraError(
|
|
234
|
+
{
|
|
235
|
+
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
236
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
237
|
+
category: ErrorCategory.USER
|
|
238
|
+
},
|
|
239
|
+
error
|
|
240
|
+
);
|
|
241
|
+
this.logger.trackException(mastraError);
|
|
242
|
+
this.logger.error("Scheduled flush failed", mastraError);
|
|
243
|
+
});
|
|
244
|
+
}, this.config.maxBatchWaitMs);
|
|
195
245
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
switch (chunk.type) {
|
|
201
|
-
case "tool-call-input-streaming-start":
|
|
202
|
-
this.#startChunkSpan("tool-call", {
|
|
203
|
-
toolName: chunk.payload.toolName,
|
|
204
|
-
toolCallId: chunk.payload.toolCallId
|
|
205
|
-
});
|
|
206
|
-
break;
|
|
207
|
-
case "tool-call-delta":
|
|
208
|
-
this.#appendToAccumulator("toolInput", chunk.payload.argsTextDelta);
|
|
209
|
-
break;
|
|
210
|
-
case "tool-call-input-streaming-end":
|
|
211
|
-
case "tool-call": {
|
|
212
|
-
const acc = this.#getAccumulator();
|
|
213
|
-
let toolInput;
|
|
214
|
-
try {
|
|
215
|
-
toolInput = acc.toolInput ? JSON.parse(acc.toolInput) : {};
|
|
216
|
-
} catch {
|
|
217
|
-
toolInput = acc.toolInput;
|
|
218
|
-
}
|
|
219
|
-
this.#endChunkSpan({
|
|
220
|
-
toolName: acc.toolName,
|
|
221
|
-
toolCallId: acc.toolCallId,
|
|
222
|
-
toolInput
|
|
223
|
-
});
|
|
224
|
-
break;
|
|
225
|
-
}
|
|
246
|
+
async flush() {
|
|
247
|
+
if (this.flushTimer) {
|
|
248
|
+
clearTimeout(this.flushTimer);
|
|
249
|
+
this.flushTimer = null;
|
|
226
250
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
* Handle object chunk spans (object, object-result)
|
|
230
|
-
*/
|
|
231
|
-
#handleObjectChunk(chunk) {
|
|
232
|
-
switch (chunk.type) {
|
|
233
|
-
case "object":
|
|
234
|
-
if (!this.#hasActiveChunkSpan()) {
|
|
235
|
-
this.#startChunkSpan("object");
|
|
236
|
-
}
|
|
237
|
-
break;
|
|
238
|
-
case "object-result":
|
|
239
|
-
this.#endChunkSpan(chunk.object);
|
|
240
|
-
break;
|
|
251
|
+
if (this.buffer.totalSize === 0) {
|
|
252
|
+
return;
|
|
241
253
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
case "tool-call-input-streaming-end":
|
|
263
|
-
case "tool-call":
|
|
264
|
-
this.#handleToolCallChunk(chunk);
|
|
265
|
-
break;
|
|
266
|
-
case "reasoning-start":
|
|
267
|
-
case "reasoning-delta":
|
|
268
|
-
case "reasoning-end":
|
|
269
|
-
this.#handleReasoningChunk(chunk);
|
|
270
|
-
break;
|
|
271
|
-
case "object":
|
|
272
|
-
case "object-result":
|
|
273
|
-
this.#handleObjectChunk(chunk);
|
|
274
|
-
break;
|
|
275
|
-
case "step-start":
|
|
276
|
-
this.#startStepSpan(chunk.payload);
|
|
277
|
-
break;
|
|
278
|
-
case "step-finish":
|
|
279
|
-
this.#endStepSpan(chunk.payload);
|
|
280
|
-
break;
|
|
281
|
-
case "raw":
|
|
282
|
-
// Skip raw chunks as they're redundant
|
|
283
|
-
case "start":
|
|
284
|
-
case "finish":
|
|
285
|
-
break;
|
|
286
|
-
// Default: auto-create event span for all other chunk types
|
|
287
|
-
default: {
|
|
288
|
-
let outputPayload = chunk.payload;
|
|
289
|
-
if (outputPayload && typeof outputPayload === "object" && "data" in outputPayload) {
|
|
290
|
-
const typedPayload = outputPayload;
|
|
291
|
-
outputPayload = { ...typedPayload };
|
|
292
|
-
if (typedPayload.data) {
|
|
293
|
-
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
294
|
-
delete outputPayload.data;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
this.#createEventSpan(chunk.type, outputPayload);
|
|
298
|
-
break;
|
|
299
|
-
}
|
|
254
|
+
const startTime = Date.now();
|
|
255
|
+
const spansCopy = [...this.buffer.spans];
|
|
256
|
+
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
257
|
+
this.resetBuffer();
|
|
258
|
+
try {
|
|
259
|
+
await this.batchUpload(spansCopy);
|
|
260
|
+
const elapsed = Date.now() - startTime;
|
|
261
|
+
this.logger.debug("Batch flushed successfully", {
|
|
262
|
+
batchSize: spansCopy.length,
|
|
263
|
+
flushReason,
|
|
264
|
+
durationMs: elapsed
|
|
265
|
+
});
|
|
266
|
+
} catch (error) {
|
|
267
|
+
const mastraError = new MastraError(
|
|
268
|
+
{
|
|
269
|
+
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
270
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
271
|
+
category: ErrorCategory.USER,
|
|
272
|
+
details: {
|
|
273
|
+
droppedBatchSize: spansCopy.length
|
|
300
274
|
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
// src/spans/base.ts
|
|
308
|
-
function isSpanInternal(spanType, flags) {
|
|
309
|
-
if (flags === void 0 || flags === InternalSpans.NONE) {
|
|
310
|
-
return false;
|
|
311
|
-
}
|
|
312
|
-
switch (spanType) {
|
|
313
|
-
// Workflow-related spans
|
|
314
|
-
case AISpanType.WORKFLOW_RUN:
|
|
315
|
-
case AISpanType.WORKFLOW_STEP:
|
|
316
|
-
case AISpanType.WORKFLOW_CONDITIONAL:
|
|
317
|
-
case AISpanType.WORKFLOW_CONDITIONAL_EVAL:
|
|
318
|
-
case AISpanType.WORKFLOW_PARALLEL:
|
|
319
|
-
case AISpanType.WORKFLOW_LOOP:
|
|
320
|
-
case AISpanType.WORKFLOW_SLEEP:
|
|
321
|
-
case AISpanType.WORKFLOW_WAIT_EVENT:
|
|
322
|
-
return (flags & InternalSpans.WORKFLOW) !== 0;
|
|
323
|
-
// Agent-related spans
|
|
324
|
-
case AISpanType.AGENT_RUN:
|
|
325
|
-
return (flags & InternalSpans.AGENT) !== 0;
|
|
326
|
-
// Tool-related spans
|
|
327
|
-
case AISpanType.TOOL_CALL:
|
|
328
|
-
case AISpanType.MCP_TOOL_CALL:
|
|
329
|
-
return (flags & InternalSpans.TOOL) !== 0;
|
|
330
|
-
// Model-related spans
|
|
331
|
-
case AISpanType.MODEL_GENERATION:
|
|
332
|
-
case AISpanType.MODEL_STEP:
|
|
333
|
-
case AISpanType.MODEL_CHUNK:
|
|
334
|
-
return (flags & InternalSpans.MODEL) !== 0;
|
|
335
|
-
// Default: never internal
|
|
336
|
-
default:
|
|
337
|
-
return false;
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
var BaseAISpan = class {
|
|
341
|
-
name;
|
|
342
|
-
type;
|
|
343
|
-
attributes;
|
|
344
|
-
parent;
|
|
345
|
-
startTime;
|
|
346
|
-
endTime;
|
|
347
|
-
isEvent;
|
|
348
|
-
isInternal;
|
|
349
|
-
aiTracing;
|
|
350
|
-
input;
|
|
351
|
-
output;
|
|
352
|
-
errorInfo;
|
|
353
|
-
metadata;
|
|
354
|
-
traceState;
|
|
355
|
-
/** Parent span ID (for root spans that are children of external spans) */
|
|
356
|
-
parentSpanId;
|
|
357
|
-
constructor(options, aiTracing) {
|
|
358
|
-
this.name = options.name;
|
|
359
|
-
this.type = options.type;
|
|
360
|
-
this.attributes = deepClean(options.attributes) || {};
|
|
361
|
-
this.metadata = deepClean(options.metadata);
|
|
362
|
-
this.parent = options.parent;
|
|
363
|
-
this.startTime = /* @__PURE__ */ new Date();
|
|
364
|
-
this.aiTracing = aiTracing;
|
|
365
|
-
this.isEvent = options.isEvent ?? false;
|
|
366
|
-
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
367
|
-
this.traceState = options.traceState;
|
|
368
|
-
if (this.isEvent) {
|
|
369
|
-
this.output = deepClean(options.output);
|
|
370
|
-
} else {
|
|
371
|
-
this.input = deepClean(options.input);
|
|
275
|
+
},
|
|
276
|
+
error
|
|
277
|
+
);
|
|
278
|
+
this.logger.trackException(mastraError);
|
|
279
|
+
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
372
280
|
}
|
|
373
281
|
}
|
|
374
|
-
createChildSpan(options) {
|
|
375
|
-
return this.aiTracing.startSpan({ ...options, parent: this, isEvent: false });
|
|
376
|
-
}
|
|
377
|
-
createEventSpan(options) {
|
|
378
|
-
return this.aiTracing.startSpan({ ...options, parent: this, isEvent: true });
|
|
379
|
-
}
|
|
380
282
|
/**
|
|
381
|
-
*
|
|
382
|
-
* Returns undefined for non-MODEL_GENERATION spans
|
|
283
|
+
* Uploads spans to cloud API using fetchWithRetry for all retry logic
|
|
383
284
|
*/
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
return new ModelSpanTracker(this);
|
|
389
|
-
}
|
|
390
|
-
/** Returns `TRUE` if the span is the root span of a trace */
|
|
391
|
-
get isRootSpan() {
|
|
392
|
-
return !this.parent;
|
|
393
|
-
}
|
|
394
|
-
/** Get the closest parent spanId that isn't an internal span */
|
|
395
|
-
getParentSpanId(includeInternalSpans) {
|
|
396
|
-
if (!this.parent) {
|
|
397
|
-
return this.parentSpanId;
|
|
398
|
-
}
|
|
399
|
-
if (includeInternalSpans) return this.parent.id;
|
|
400
|
-
if (this.parent.isInternal) return this.parent.getParentSpanId(includeInternalSpans);
|
|
401
|
-
return this.parent.id;
|
|
402
|
-
}
|
|
403
|
-
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
404
|
-
findParent(spanType) {
|
|
405
|
-
let current = this.parent;
|
|
406
|
-
while (current) {
|
|
407
|
-
if (current.type === spanType) {
|
|
408
|
-
return current;
|
|
409
|
-
}
|
|
410
|
-
current = current.parent;
|
|
411
|
-
}
|
|
412
|
-
return void 0;
|
|
413
|
-
}
|
|
414
|
-
/** Returns a lightweight span ready for export */
|
|
415
|
-
exportSpan(includeInternalSpans) {
|
|
416
|
-
return {
|
|
417
|
-
id: this.id,
|
|
418
|
-
traceId: this.traceId,
|
|
419
|
-
name: this.name,
|
|
420
|
-
type: this.type,
|
|
421
|
-
attributes: this.attributes,
|
|
422
|
-
metadata: this.metadata,
|
|
423
|
-
startTime: this.startTime,
|
|
424
|
-
endTime: this.endTime,
|
|
425
|
-
input: this.input,
|
|
426
|
-
output: this.output,
|
|
427
|
-
errorInfo: this.errorInfo,
|
|
428
|
-
isEvent: this.isEvent,
|
|
429
|
-
isRootSpan: this.isRootSpan,
|
|
430
|
-
parentSpanId: this.getParentSpanId(includeInternalSpans)
|
|
285
|
+
async batchUpload(spans) {
|
|
286
|
+
const headers = {
|
|
287
|
+
Authorization: `Bearer ${this.config.accessToken}`,
|
|
288
|
+
"Content-Type": "application/json"
|
|
431
289
|
};
|
|
290
|
+
const options = {
|
|
291
|
+
method: "POST",
|
|
292
|
+
headers,
|
|
293
|
+
body: JSON.stringify({ spans })
|
|
294
|
+
};
|
|
295
|
+
await fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
432
296
|
}
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
438
|
-
"logger",
|
|
439
|
-
"experimental_providerMetadata",
|
|
440
|
-
"providerMetadata",
|
|
441
|
-
"steps",
|
|
442
|
-
"tracingContext"
|
|
443
|
-
]);
|
|
444
|
-
function deepClean(value, options = {}, _seen = /* @__PURE__ */ new WeakSet(), _depth = 0) {
|
|
445
|
-
const { keysToStrip = DEFAULT_KEYS_TO_STRIP, maxDepth = 10 } = options;
|
|
446
|
-
if (_depth > maxDepth) {
|
|
447
|
-
return "[MaxDepth]";
|
|
448
|
-
}
|
|
449
|
-
if (value === null || typeof value !== "object") {
|
|
450
|
-
try {
|
|
451
|
-
JSON.stringify(value);
|
|
452
|
-
return value;
|
|
453
|
-
} catch (error) {
|
|
454
|
-
return `[${error instanceof Error ? error.message : String(error)}]`;
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
if (_seen.has(value)) {
|
|
458
|
-
return "[Circular]";
|
|
459
|
-
}
|
|
460
|
-
_seen.add(value);
|
|
461
|
-
if (Array.isArray(value)) {
|
|
462
|
-
return value.map((item) => deepClean(item, options, _seen, _depth + 1));
|
|
297
|
+
resetBuffer() {
|
|
298
|
+
this.buffer.spans = [];
|
|
299
|
+
this.buffer.firstEventTime = void 0;
|
|
300
|
+
this.buffer.totalSize = 0;
|
|
463
301
|
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
continue;
|
|
468
|
-
}
|
|
469
|
-
try {
|
|
470
|
-
cleaned[key] = deepClean(val, options, _seen, _depth + 1);
|
|
471
|
-
} catch (error) {
|
|
472
|
-
cleaned[key] = `[${error instanceof Error ? error.message : String(error)}]`;
|
|
302
|
+
async shutdown() {
|
|
303
|
+
if (this.isDisabled) {
|
|
304
|
+
return;
|
|
473
305
|
}
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
var DefaultAISpan = class extends BaseAISpan {
|
|
478
|
-
id;
|
|
479
|
-
traceId;
|
|
480
|
-
constructor(options, aiTracing) {
|
|
481
|
-
super(options, aiTracing);
|
|
482
|
-
this.id = generateSpanId();
|
|
483
|
-
if (options.parent) {
|
|
484
|
-
this.traceId = options.parent.traceId;
|
|
485
|
-
} else if (options.traceId) {
|
|
486
|
-
if (isValidTraceId(options.traceId)) {
|
|
487
|
-
this.traceId = options.traceId;
|
|
488
|
-
} else {
|
|
489
|
-
console.error(
|
|
490
|
-
`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`
|
|
491
|
-
);
|
|
492
|
-
this.traceId = generateTraceId();
|
|
493
|
-
}
|
|
494
|
-
} else {
|
|
495
|
-
this.traceId = generateTraceId();
|
|
306
|
+
if (this.flushTimer) {
|
|
307
|
+
clearTimeout(this.flushTimer);
|
|
308
|
+
this.flushTimer = null;
|
|
496
309
|
}
|
|
497
|
-
if (
|
|
498
|
-
|
|
499
|
-
this.
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
|
|
310
|
+
if (this.buffer.totalSize > 0) {
|
|
311
|
+
this.logger.info("Flushing remaining events on shutdown", {
|
|
312
|
+
remainingEvents: this.buffer.totalSize
|
|
313
|
+
});
|
|
314
|
+
try {
|
|
315
|
+
await this.flush();
|
|
316
|
+
} catch (error) {
|
|
317
|
+
const mastraError = new MastraError(
|
|
318
|
+
{
|
|
319
|
+
id: `CLOUD_EXPORTER_FAILED_TO_FLUSH_REMAINING_EVENTS_DURING_SHUTDOWN`,
|
|
320
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
321
|
+
category: ErrorCategory.USER,
|
|
322
|
+
details: {
|
|
323
|
+
remainingEvents: this.buffer.totalSize
|
|
324
|
+
}
|
|
325
|
+
},
|
|
326
|
+
error
|
|
503
327
|
);
|
|
328
|
+
this.logger.trackException(mastraError);
|
|
329
|
+
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
504
330
|
}
|
|
505
331
|
}
|
|
332
|
+
this.logger.info("CloudExporter shutdown complete");
|
|
506
333
|
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
if (options?.output !== void 0) {
|
|
513
|
-
this.output = deepClean(options.output);
|
|
514
|
-
}
|
|
515
|
-
if (options?.attributes) {
|
|
516
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
517
|
-
}
|
|
518
|
-
if (options?.metadata) {
|
|
519
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
520
|
-
}
|
|
334
|
+
};
|
|
335
|
+
var ConsoleExporter = class extends BaseExporter {
|
|
336
|
+
name = "tracing-console-exporter";
|
|
337
|
+
constructor(config = {}) {
|
|
338
|
+
super(config);
|
|
521
339
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
domain: error.domain,
|
|
532
|
-
message: error.message
|
|
533
|
-
} : {
|
|
534
|
-
message: error.message
|
|
340
|
+
async _exportTracingEvent(event) {
|
|
341
|
+
const span = event.exportedSpan;
|
|
342
|
+
const formatAttributes = (attributes) => {
|
|
343
|
+
try {
|
|
344
|
+
return JSON.stringify(attributes, null, 2);
|
|
345
|
+
} catch (error) {
|
|
346
|
+
const errMsg = error instanceof Error ? error.message : "Unknown formatting error";
|
|
347
|
+
return `[Unable to serialize attributes: ${errMsg}]`;
|
|
348
|
+
}
|
|
535
349
|
};
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
350
|
+
const formatDuration = (startTime, endTime) => {
|
|
351
|
+
if (!endTime) return "N/A";
|
|
352
|
+
const duration = endTime.getTime() - startTime.getTime();
|
|
353
|
+
return `${duration}ms`;
|
|
354
|
+
};
|
|
355
|
+
switch (event.type) {
|
|
356
|
+
case TracingEventType.SPAN_STARTED:
|
|
357
|
+
this.logger.info(`\u{1F680} SPAN_STARTED`);
|
|
358
|
+
this.logger.info(` Type: ${span.type}`);
|
|
359
|
+
this.logger.info(` Name: ${span.name}`);
|
|
360
|
+
this.logger.info(` ID: ${span.id}`);
|
|
361
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
362
|
+
if (span.input !== void 0) {
|
|
363
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
364
|
+
}
|
|
365
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
366
|
+
this.logger.info("\u2500".repeat(80));
|
|
367
|
+
break;
|
|
368
|
+
case TracingEventType.SPAN_ENDED:
|
|
369
|
+
const duration = formatDuration(span.startTime, span.endTime);
|
|
370
|
+
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
371
|
+
this.logger.info(` Type: ${span.type}`);
|
|
372
|
+
this.logger.info(` Name: ${span.name}`);
|
|
373
|
+
this.logger.info(` ID: ${span.id}`);
|
|
374
|
+
this.logger.info(` Duration: ${duration}`);
|
|
375
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
376
|
+
if (span.input !== void 0) {
|
|
377
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
378
|
+
}
|
|
379
|
+
if (span.output !== void 0) {
|
|
380
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
381
|
+
}
|
|
382
|
+
if (span.errorInfo) {
|
|
383
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
384
|
+
}
|
|
385
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
386
|
+
this.logger.info("\u2500".repeat(80));
|
|
387
|
+
break;
|
|
388
|
+
case TracingEventType.SPAN_UPDATED:
|
|
389
|
+
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
390
|
+
this.logger.info(` Type: ${span.type}`);
|
|
391
|
+
this.logger.info(` Name: ${span.name}`);
|
|
392
|
+
this.logger.info(` ID: ${span.id}`);
|
|
393
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
394
|
+
if (span.input !== void 0) {
|
|
395
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
396
|
+
}
|
|
397
|
+
if (span.output !== void 0) {
|
|
398
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
399
|
+
}
|
|
400
|
+
if (span.errorInfo) {
|
|
401
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
402
|
+
}
|
|
403
|
+
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
404
|
+
this.logger.info("\u2500".repeat(80));
|
|
405
|
+
break;
|
|
406
|
+
default:
|
|
407
|
+
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
563
408
|
}
|
|
564
409
|
}
|
|
565
|
-
|
|
566
|
-
|
|
410
|
+
async shutdown() {
|
|
411
|
+
this.logger.info("ConsoleExporter shutdown");
|
|
567
412
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
413
|
+
};
|
|
414
|
+
function resolveTracingStorageStrategy(config, storage, logger) {
|
|
415
|
+
if (config.strategy && config.strategy !== "auto") {
|
|
416
|
+
const hints = storage.tracingStrategy;
|
|
417
|
+
if (hints.supported.includes(config.strategy)) {
|
|
418
|
+
return config.strategy;
|
|
419
|
+
}
|
|
420
|
+
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
421
|
+
userStrategy: config.strategy,
|
|
422
|
+
storageAdapter: storage.constructor.name,
|
|
423
|
+
supportedStrategies: hints.supported,
|
|
424
|
+
fallbackStrategy: hints.preferred
|
|
576
425
|
});
|
|
577
426
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
427
|
+
return storage.tracingStrategy.preferred;
|
|
428
|
+
}
|
|
429
|
+
var DefaultExporter = class extends BaseExporter {
|
|
430
|
+
name = "mastra-default-observability-exporter";
|
|
431
|
+
#storage;
|
|
432
|
+
#config;
|
|
433
|
+
#resolvedStrategy;
|
|
434
|
+
buffer;
|
|
435
|
+
#flushTimer = null;
|
|
436
|
+
// Track all spans that have been created, persists across flushes
|
|
437
|
+
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
438
|
+
constructor(config = {}) {
|
|
439
|
+
super(config);
|
|
440
|
+
if (config === void 0) {
|
|
441
|
+
config = {};
|
|
586
442
|
}
|
|
443
|
+
this.#config = {
|
|
444
|
+
...config,
|
|
445
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
446
|
+
maxBufferSize: config.maxBufferSize ?? 1e4,
|
|
447
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
448
|
+
maxRetries: config.maxRetries ?? 4,
|
|
449
|
+
retryDelayMs: config.retryDelayMs ?? 500,
|
|
450
|
+
strategy: config.strategy ?? "auto"
|
|
451
|
+
};
|
|
452
|
+
this.buffer = {
|
|
453
|
+
creates: [],
|
|
454
|
+
updates: [],
|
|
455
|
+
insertOnly: [],
|
|
456
|
+
seenSpans: /* @__PURE__ */ new Set(),
|
|
457
|
+
spanSequences: /* @__PURE__ */ new Map(),
|
|
458
|
+
completedSpans: /* @__PURE__ */ new Set(),
|
|
459
|
+
outOfOrderCount: 0,
|
|
460
|
+
totalSize: 0
|
|
461
|
+
};
|
|
462
|
+
this.#resolvedStrategy = "batch-with-updates";
|
|
587
463
|
}
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
464
|
+
#strategyInitialized = false;
|
|
465
|
+
/**
|
|
466
|
+
* Initialize the exporter (called after all dependencies are ready)
|
|
467
|
+
*/
|
|
468
|
+
init(options) {
|
|
469
|
+
this.#storage = options.mastra?.getStorage();
|
|
470
|
+
if (!this.#storage) {
|
|
471
|
+
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
472
|
+
return;
|
|
597
473
|
}
|
|
474
|
+
this.initializeStrategy(this.#storage);
|
|
598
475
|
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
super(options, aiTracing);
|
|
614
|
-
this.id = "no-op";
|
|
615
|
-
this.traceId = "no-op-trace";
|
|
476
|
+
/**
|
|
477
|
+
* Initialize the resolved strategy once storage is available
|
|
478
|
+
*/
|
|
479
|
+
initializeStrategy(storage) {
|
|
480
|
+
if (this.#strategyInitialized) return;
|
|
481
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, storage, this.logger);
|
|
482
|
+
this.#strategyInitialized = true;
|
|
483
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
484
|
+
strategy: this.#resolvedStrategy,
|
|
485
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
486
|
+
storageAdapter: storage.constructor.name,
|
|
487
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
488
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
489
|
+
});
|
|
616
490
|
}
|
|
617
|
-
|
|
491
|
+
/**
|
|
492
|
+
* Builds a unique span key for tracking
|
|
493
|
+
*/
|
|
494
|
+
buildSpanKey(traceId, spanId) {
|
|
495
|
+
return `${traceId}:${spanId}`;
|
|
618
496
|
}
|
|
619
|
-
|
|
497
|
+
/**
|
|
498
|
+
* Gets the next sequence number for a span
|
|
499
|
+
*/
|
|
500
|
+
getNextSequence(spanKey) {
|
|
501
|
+
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
502
|
+
const next = current + 1;
|
|
503
|
+
this.buffer.spanSequences.set(spanKey, next);
|
|
504
|
+
return next;
|
|
620
505
|
}
|
|
621
|
-
|
|
506
|
+
/**
|
|
507
|
+
* Handles out-of-order span updates by logging and skipping
|
|
508
|
+
*/
|
|
509
|
+
handleOutOfOrderUpdate(event) {
|
|
510
|
+
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
511
|
+
spanId: event.exportedSpan.id,
|
|
512
|
+
traceId: event.exportedSpan.traceId,
|
|
513
|
+
spanName: event.exportedSpan.name,
|
|
514
|
+
eventType: event.type
|
|
515
|
+
});
|
|
622
516
|
}
|
|
623
|
-
|
|
517
|
+
/**
|
|
518
|
+
* Adds an event to the appropriate buffer based on strategy
|
|
519
|
+
*/
|
|
520
|
+
addToBuffer(event) {
|
|
521
|
+
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
522
|
+
if (this.buffer.totalSize === 0) {
|
|
523
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
524
|
+
}
|
|
525
|
+
switch (event.type) {
|
|
526
|
+
case TracingEventType.SPAN_STARTED:
|
|
527
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
528
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
529
|
+
this.buffer.creates.push(createRecord);
|
|
530
|
+
this.buffer.seenSpans.add(spanKey);
|
|
531
|
+
this.allCreatedSpans.add(spanKey);
|
|
532
|
+
}
|
|
533
|
+
break;
|
|
534
|
+
case TracingEventType.SPAN_UPDATED:
|
|
535
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
536
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
537
|
+
this.buffer.updates.push({
|
|
538
|
+
traceId: event.exportedSpan.traceId,
|
|
539
|
+
spanId: event.exportedSpan.id,
|
|
540
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
541
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
542
|
+
});
|
|
543
|
+
} else {
|
|
544
|
+
this.handleOutOfOrderUpdate(event);
|
|
545
|
+
this.buffer.outOfOrderCount++;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
break;
|
|
549
|
+
case TracingEventType.SPAN_ENDED:
|
|
550
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
551
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
552
|
+
this.buffer.updates.push({
|
|
553
|
+
traceId: event.exportedSpan.traceId,
|
|
554
|
+
spanId: event.exportedSpan.id,
|
|
555
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
556
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
557
|
+
});
|
|
558
|
+
this.buffer.completedSpans.add(spanKey);
|
|
559
|
+
} else if (event.exportedSpan.isEvent) {
|
|
560
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
561
|
+
this.buffer.creates.push(createRecord);
|
|
562
|
+
this.buffer.seenSpans.add(spanKey);
|
|
563
|
+
this.allCreatedSpans.add(spanKey);
|
|
564
|
+
this.buffer.completedSpans.add(spanKey);
|
|
565
|
+
} else {
|
|
566
|
+
this.handleOutOfOrderUpdate(event);
|
|
567
|
+
this.buffer.outOfOrderCount++;
|
|
568
|
+
}
|
|
569
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
570
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
571
|
+
this.buffer.insertOnly.push(createRecord);
|
|
572
|
+
this.buffer.completedSpans.add(spanKey);
|
|
573
|
+
this.allCreatedSpans.add(spanKey);
|
|
574
|
+
}
|
|
575
|
+
break;
|
|
576
|
+
}
|
|
577
|
+
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Checks if buffer should be flushed based on size or time triggers
|
|
581
|
+
*/
|
|
582
|
+
shouldFlush() {
|
|
583
|
+
if (this.buffer.totalSize >= this.#config.maxBufferSize) {
|
|
584
|
+
return true;
|
|
585
|
+
}
|
|
586
|
+
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
587
|
+
return true;
|
|
588
|
+
}
|
|
589
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
590
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
591
|
+
if (elapsed >= this.#config.maxBatchWaitMs) {
|
|
592
|
+
return true;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
624
595
|
return false;
|
|
625
596
|
}
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
this.
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
}
|
|
597
|
+
/**
|
|
598
|
+
* Resets the buffer after successful flush
|
|
599
|
+
*/
|
|
600
|
+
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
601
|
+
this.buffer.creates = [];
|
|
602
|
+
this.buffer.updates = [];
|
|
603
|
+
this.buffer.insertOnly = [];
|
|
604
|
+
this.buffer.seenSpans.clear();
|
|
605
|
+
this.buffer.spanSequences.clear();
|
|
606
|
+
this.buffer.completedSpans.clear();
|
|
607
|
+
this.buffer.outOfOrderCount = 0;
|
|
608
|
+
this.buffer.firstEventTime = void 0;
|
|
609
|
+
this.buffer.totalSize = 0;
|
|
610
|
+
for (const spanKey of completedSpansToCleanup) {
|
|
611
|
+
this.allCreatedSpans.delete(spanKey);
|
|
612
|
+
}
|
|
642
613
|
}
|
|
643
614
|
/**
|
|
644
|
-
*
|
|
645
|
-
* and propagate logger to exporters
|
|
615
|
+
* Schedules a flush using setTimeout
|
|
646
616
|
*/
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
617
|
+
scheduleFlush() {
|
|
618
|
+
if (this.#flushTimer) {
|
|
619
|
+
clearTimeout(this.#flushTimer);
|
|
620
|
+
}
|
|
621
|
+
this.#flushTimer = setTimeout(() => {
|
|
622
|
+
this.flush().catch((error) => {
|
|
623
|
+
this.logger.error("Scheduled flush failed", {
|
|
624
|
+
error: error instanceof Error ? error.message : String(error)
|
|
625
|
+
});
|
|
626
|
+
});
|
|
627
|
+
}, this.#config.maxBatchWaitMs);
|
|
657
628
|
}
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
629
|
+
/**
|
|
630
|
+
* Serializes span attributes to storage record format
|
|
631
|
+
* Handles all Span types and their specific attributes
|
|
632
|
+
*/
|
|
633
|
+
serializeAttributes(span) {
|
|
634
|
+
if (!span.attributes) {
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
try {
|
|
638
|
+
return JSON.parse(
|
|
639
|
+
JSON.stringify(span.attributes, (_key, value) => {
|
|
640
|
+
if (value instanceof Date) {
|
|
641
|
+
return value.toISOString();
|
|
642
|
+
}
|
|
643
|
+
if (typeof value === "object" && value !== null) {
|
|
644
|
+
return value;
|
|
645
|
+
}
|
|
646
|
+
return value;
|
|
647
|
+
})
|
|
648
|
+
);
|
|
649
|
+
} catch (error) {
|
|
650
|
+
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
651
|
+
spanId: span.id,
|
|
652
|
+
spanType: span.type,
|
|
653
|
+
error: error instanceof Error ? error.message : String(error)
|
|
654
|
+
});
|
|
655
|
+
return null;
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
buildCreateRecord(span) {
|
|
659
|
+
return {
|
|
660
|
+
traceId: span.traceId,
|
|
661
|
+
spanId: span.id,
|
|
662
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
663
|
+
name: span.name,
|
|
664
|
+
scope: null,
|
|
665
|
+
spanType: span.type,
|
|
666
|
+
attributes: this.serializeAttributes(span),
|
|
667
|
+
metadata: span.metadata ?? null,
|
|
668
|
+
links: null,
|
|
669
|
+
startedAt: span.startTime,
|
|
670
|
+
endedAt: span.endTime ?? null,
|
|
671
|
+
input: span.input,
|
|
672
|
+
output: span.output,
|
|
673
|
+
error: span.errorInfo,
|
|
674
|
+
isEvent: span.isEvent
|
|
675
|
+
};
|
|
663
676
|
}
|
|
664
|
-
|
|
665
|
-
return
|
|
677
|
+
buildUpdateRecord(span) {
|
|
678
|
+
return {
|
|
679
|
+
name: span.name,
|
|
680
|
+
scope: null,
|
|
681
|
+
attributes: this.serializeAttributes(span),
|
|
682
|
+
metadata: span.metadata ?? null,
|
|
683
|
+
links: null,
|
|
684
|
+
endedAt: span.endTime ?? null,
|
|
685
|
+
input: span.input,
|
|
686
|
+
output: span.output,
|
|
687
|
+
error: span.errorInfo
|
|
688
|
+
};
|
|
666
689
|
}
|
|
667
|
-
// ============================================================================
|
|
668
|
-
// Public API - Single type-safe span creation method
|
|
669
|
-
// ============================================================================
|
|
670
690
|
/**
|
|
671
|
-
*
|
|
691
|
+
* Handles realtime strategy - processes each event immediately
|
|
672
692
|
*/
|
|
673
|
-
|
|
674
|
-
const
|
|
675
|
-
|
|
676
|
-
return new NoOpAISpan({ ...rest, metadata }, this);
|
|
677
|
-
}
|
|
678
|
-
let traceState;
|
|
679
|
-
if (options.parent) {
|
|
680
|
-
traceState = options.parent.traceState;
|
|
681
|
-
} else {
|
|
682
|
-
traceState = this.computeTraceState(tracingOptions);
|
|
683
|
-
}
|
|
684
|
-
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, metadata, traceState);
|
|
685
|
-
const span = this.createSpan({
|
|
686
|
-
...rest,
|
|
687
|
-
metadata: enrichedMetadata,
|
|
688
|
-
traceState
|
|
689
|
-
});
|
|
693
|
+
async handleRealtimeEvent(event, storage) {
|
|
694
|
+
const span = event.exportedSpan;
|
|
695
|
+
const spanKey = this.buildSpanKey(span.traceId, span.id);
|
|
690
696
|
if (span.isEvent) {
|
|
691
|
-
|
|
697
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
698
|
+
await storage.createSpan(this.buildCreateRecord(event.exportedSpan));
|
|
699
|
+
} else {
|
|
700
|
+
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
701
|
+
}
|
|
692
702
|
} else {
|
|
693
|
-
|
|
694
|
-
|
|
703
|
+
switch (event.type) {
|
|
704
|
+
case TracingEventType.SPAN_STARTED:
|
|
705
|
+
await storage.createSpan(this.buildCreateRecord(event.exportedSpan));
|
|
706
|
+
this.allCreatedSpans.add(spanKey);
|
|
707
|
+
break;
|
|
708
|
+
case TracingEventType.SPAN_UPDATED:
|
|
709
|
+
await storage.updateSpan({
|
|
710
|
+
traceId: span.traceId,
|
|
711
|
+
spanId: span.id,
|
|
712
|
+
updates: this.buildUpdateRecord(span)
|
|
713
|
+
});
|
|
714
|
+
break;
|
|
715
|
+
case TracingEventType.SPAN_ENDED:
|
|
716
|
+
await storage.updateSpan({
|
|
717
|
+
traceId: span.traceId,
|
|
718
|
+
spanId: span.id,
|
|
719
|
+
updates: this.buildUpdateRecord(span)
|
|
720
|
+
});
|
|
721
|
+
this.allCreatedSpans.delete(spanKey);
|
|
722
|
+
break;
|
|
723
|
+
default:
|
|
724
|
+
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
725
|
+
}
|
|
695
726
|
}
|
|
696
|
-
return span;
|
|
697
|
-
}
|
|
698
|
-
// ============================================================================
|
|
699
|
-
// Configuration Management
|
|
700
|
-
// ============================================================================
|
|
701
|
-
/**
|
|
702
|
-
* Get current configuration
|
|
703
|
-
*/
|
|
704
|
-
getConfig() {
|
|
705
|
-
return { ...this.config };
|
|
706
727
|
}
|
|
707
|
-
// ============================================================================
|
|
708
|
-
// Plugin Access
|
|
709
|
-
// ============================================================================
|
|
710
728
|
/**
|
|
711
|
-
*
|
|
729
|
+
* Handles batch-with-updates strategy - buffers events and processes in batches
|
|
712
730
|
*/
|
|
713
|
-
|
|
714
|
-
|
|
731
|
+
handleBatchWithUpdatesEvent(event) {
|
|
732
|
+
this.addToBuffer(event);
|
|
733
|
+
if (this.shouldFlush()) {
|
|
734
|
+
this.flush().catch((error) => {
|
|
735
|
+
this.logger.error("Batch flush failed", {
|
|
736
|
+
error: error instanceof Error ? error.message : String(error)
|
|
737
|
+
});
|
|
738
|
+
});
|
|
739
|
+
} else if (this.buffer.totalSize === 1) {
|
|
740
|
+
this.scheduleFlush();
|
|
741
|
+
}
|
|
715
742
|
}
|
|
716
743
|
/**
|
|
717
|
-
*
|
|
744
|
+
* Handles insert-only strategy - only processes SPAN_ENDED events in batches
|
|
718
745
|
*/
|
|
719
|
-
|
|
720
|
-
|
|
746
|
+
handleInsertOnlyEvent(event) {
|
|
747
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
748
|
+
this.addToBuffer(event);
|
|
749
|
+
if (this.shouldFlush()) {
|
|
750
|
+
this.flush().catch((error) => {
|
|
751
|
+
this.logger.error("Batch flush failed", {
|
|
752
|
+
error: error instanceof Error ? error.message : String(error)
|
|
753
|
+
});
|
|
754
|
+
});
|
|
755
|
+
} else if (this.buffer.totalSize === 1) {
|
|
756
|
+
this.scheduleFlush();
|
|
757
|
+
}
|
|
758
|
+
}
|
|
721
759
|
}
|
|
722
760
|
/**
|
|
723
|
-
*
|
|
761
|
+
* Calculates retry delay using exponential backoff
|
|
724
762
|
*/
|
|
725
|
-
|
|
726
|
-
return this.
|
|
763
|
+
calculateRetryDelay(attempt) {
|
|
764
|
+
return this.#config.retryDelayMs * Math.pow(2, attempt);
|
|
727
765
|
}
|
|
728
|
-
// ============================================================================
|
|
729
|
-
// Span Lifecycle Management
|
|
730
|
-
// ============================================================================
|
|
731
766
|
/**
|
|
732
|
-
*
|
|
733
|
-
* This ensures all spans emit events regardless of implementation
|
|
767
|
+
* Flushes the current buffer to storage with retry logic
|
|
734
768
|
*/
|
|
735
|
-
|
|
736
|
-
if (!this
|
|
769
|
+
async flush() {
|
|
770
|
+
if (!this.#storage) {
|
|
771
|
+
this.logger.debug("Cannot flush traces. Mastra storage is not initialized");
|
|
737
772
|
return;
|
|
738
773
|
}
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
this.
|
|
774
|
+
if (this.#flushTimer) {
|
|
775
|
+
clearTimeout(this.#flushTimer);
|
|
776
|
+
this.#flushTimer = null;
|
|
777
|
+
}
|
|
778
|
+
if (this.buffer.totalSize === 0) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
const startTime = Date.now();
|
|
782
|
+
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
783
|
+
const bufferCopy = {
|
|
784
|
+
creates: [...this.buffer.creates],
|
|
785
|
+
updates: [...this.buffer.updates],
|
|
786
|
+
insertOnly: [...this.buffer.insertOnly],
|
|
787
|
+
seenSpans: new Set(this.buffer.seenSpans),
|
|
788
|
+
spanSequences: new Map(this.buffer.spanSequences),
|
|
789
|
+
completedSpans: new Set(this.buffer.completedSpans),
|
|
790
|
+
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
791
|
+
firstEventTime: this.buffer.firstEventTime,
|
|
792
|
+
totalSize: this.buffer.totalSize
|
|
756
793
|
};
|
|
794
|
+
this.resetBuffer();
|
|
795
|
+
await this.flushWithRetries(this.#storage, bufferCopy, 0);
|
|
796
|
+
const elapsed = Date.now() - startTime;
|
|
797
|
+
this.logger.debug("Batch flushed", {
|
|
798
|
+
strategy: this.#resolvedStrategy,
|
|
799
|
+
batchSize: bufferCopy.totalSize,
|
|
800
|
+
flushReason,
|
|
801
|
+
durationMs: elapsed,
|
|
802
|
+
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
803
|
+
});
|
|
757
804
|
}
|
|
758
|
-
// ============================================================================
|
|
759
|
-
// Utility Methods
|
|
760
|
-
// ============================================================================
|
|
761
805
|
/**
|
|
762
|
-
*
|
|
806
|
+
* Attempts to flush with exponential backoff retry logic
|
|
763
807
|
*/
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
case SamplingStrategyType.NEVER:
|
|
770
|
-
return false;
|
|
771
|
-
case SamplingStrategyType.RATIO:
|
|
772
|
-
if (sampling.probability === void 0 || sampling.probability < 0 || sampling.probability > 1) {
|
|
773
|
-
this.logger.warn(
|
|
774
|
-
`Invalid sampling probability: ${sampling.probability}. Expected value between 0 and 1. Defaulting to no sampling.`
|
|
775
|
-
);
|
|
776
|
-
return false;
|
|
808
|
+
async flushWithRetries(storage, buffer, attempt) {
|
|
809
|
+
try {
|
|
810
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
811
|
+
if (buffer.creates.length > 0) {
|
|
812
|
+
await storage.batchCreateSpans({ records: buffer.creates });
|
|
777
813
|
}
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
814
|
+
if (buffer.updates.length > 0) {
|
|
815
|
+
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
816
|
+
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
817
|
+
this.buildSpanKey(b.traceId, b.spanId)
|
|
818
|
+
);
|
|
819
|
+
if (spanCompare !== 0) return spanCompare;
|
|
820
|
+
return a.sequenceNumber - b.sequenceNumber;
|
|
821
|
+
});
|
|
822
|
+
await storage.batchUpdateSpans({ records: sortedUpdates });
|
|
823
|
+
}
|
|
824
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
825
|
+
if (buffer.insertOnly.length > 0) {
|
|
826
|
+
await storage.batchCreateSpans({ records: buffer.insertOnly });
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
for (const spanKey of buffer.completedSpans) {
|
|
830
|
+
this.allCreatedSpans.delete(spanKey);
|
|
831
|
+
}
|
|
832
|
+
} catch (error) {
|
|
833
|
+
if (attempt < this.#config.maxRetries) {
|
|
834
|
+
const retryDelay = this.calculateRetryDelay(attempt);
|
|
835
|
+
this.logger.warn("Batch flush failed, retrying", {
|
|
836
|
+
attempt: attempt + 1,
|
|
837
|
+
maxRetries: this.#config.maxRetries,
|
|
838
|
+
nextRetryInMs: retryDelay,
|
|
839
|
+
error: error instanceof Error ? error.message : String(error)
|
|
840
|
+
});
|
|
841
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
842
|
+
return this.flushWithRetries(storage, buffer, attempt + 1);
|
|
843
|
+
} else {
|
|
844
|
+
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
845
|
+
finalAttempt: attempt + 1,
|
|
846
|
+
maxRetries: this.#config.maxRetries,
|
|
847
|
+
droppedBatchSize: buffer.totalSize,
|
|
848
|
+
error: error instanceof Error ? error.message : String(error)
|
|
849
|
+
});
|
|
850
|
+
for (const spanKey of buffer.completedSpans) {
|
|
851
|
+
this.allCreatedSpans.delete(spanKey);
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
async _exportTracingEvent(event) {
|
|
857
|
+
if (!this.#storage) {
|
|
858
|
+
this.logger.debug("Cannot store traces. Mastra storage is not initialized");
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
if (!this.#strategyInitialized) {
|
|
862
|
+
this.initializeStrategy(this.#storage);
|
|
863
|
+
}
|
|
864
|
+
switch (this.#resolvedStrategy) {
|
|
865
|
+
case "realtime":
|
|
866
|
+
await this.handleRealtimeEvent(event, this.#storage);
|
|
867
|
+
break;
|
|
868
|
+
case "batch-with-updates":
|
|
869
|
+
this.handleBatchWithUpdatesEvent(event);
|
|
870
|
+
break;
|
|
871
|
+
case "insert-only":
|
|
872
|
+
this.handleInsertOnlyEvent(event);
|
|
873
|
+
break;
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
async shutdown() {
|
|
877
|
+
if (this.#flushTimer) {
|
|
878
|
+
clearTimeout(this.#flushTimer);
|
|
879
|
+
this.#flushTimer = null;
|
|
880
|
+
}
|
|
881
|
+
if (this.buffer.totalSize > 0) {
|
|
882
|
+
this.logger.info("Flushing remaining events on shutdown", {
|
|
883
|
+
remainingEvents: this.buffer.totalSize
|
|
884
|
+
});
|
|
885
|
+
try {
|
|
886
|
+
await this.flush();
|
|
887
|
+
} catch (error) {
|
|
888
|
+
this.logger.error("Failed to flush remaining events during shutdown", {
|
|
889
|
+
error: error instanceof Error ? error.message : String(error)
|
|
890
|
+
});
|
|
891
|
+
}
|
|
783
892
|
}
|
|
893
|
+
this.logger.info("DefaultExporter shutdown complete");
|
|
894
|
+
}
|
|
895
|
+
};
|
|
896
|
+
var ModelSpanTracker = class {
|
|
897
|
+
#modelSpan;
|
|
898
|
+
#currentStepSpan;
|
|
899
|
+
#currentChunkSpan;
|
|
900
|
+
#accumulator = {};
|
|
901
|
+
#stepIndex = 0;
|
|
902
|
+
#chunkSequence = 0;
|
|
903
|
+
constructor(modelSpan) {
|
|
904
|
+
this.#modelSpan = modelSpan;
|
|
784
905
|
}
|
|
785
906
|
/**
|
|
786
|
-
*
|
|
907
|
+
* Get the tracing context for creating child spans.
|
|
908
|
+
* Returns the current step span if active, otherwise the model span.
|
|
787
909
|
*/
|
|
788
|
-
|
|
789
|
-
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
790
|
-
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
791
|
-
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
792
|
-
if (allKeys.length === 0) {
|
|
793
|
-
return void 0;
|
|
794
|
-
}
|
|
910
|
+
getTracingContext() {
|
|
795
911
|
return {
|
|
796
|
-
|
|
912
|
+
currentSpan: this.#currentStepSpan ?? this.#modelSpan
|
|
797
913
|
};
|
|
798
914
|
}
|
|
799
915
|
/**
|
|
800
|
-
*
|
|
916
|
+
* Report an error on the generation span
|
|
801
917
|
*/
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
return explicitMetadata;
|
|
805
|
-
}
|
|
806
|
-
const extracted = this.extractKeys(requestContext, traceState.requestContextKeys);
|
|
807
|
-
if (Object.keys(extracted).length === 0 && !explicitMetadata) {
|
|
808
|
-
return void 0;
|
|
809
|
-
}
|
|
810
|
-
return {
|
|
811
|
-
...extracted,
|
|
812
|
-
...explicitMetadata
|
|
813
|
-
// Explicit metadata always wins
|
|
814
|
-
};
|
|
918
|
+
reportGenerationError(options) {
|
|
919
|
+
this.#modelSpan?.error(options);
|
|
815
920
|
}
|
|
816
921
|
/**
|
|
817
|
-
*
|
|
922
|
+
* End the generation span
|
|
818
923
|
*/
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
for (const key of keys) {
|
|
822
|
-
const parts = key.split(".");
|
|
823
|
-
const rootKey = parts[0];
|
|
824
|
-
const value = requestContext.get(rootKey);
|
|
825
|
-
if (value !== void 0) {
|
|
826
|
-
if (parts.length > 1) {
|
|
827
|
-
const nestedPath = parts.slice(1).join(".");
|
|
828
|
-
const nestedValue = getNestedValue(value, nestedPath);
|
|
829
|
-
if (nestedValue !== void 0) {
|
|
830
|
-
setNestedValue(result, key, nestedValue);
|
|
831
|
-
}
|
|
832
|
-
} else {
|
|
833
|
-
setNestedValue(result, key, value);
|
|
834
|
-
}
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
return result;
|
|
924
|
+
endGeneration(options) {
|
|
925
|
+
this.#modelSpan?.end(options);
|
|
838
926
|
}
|
|
839
927
|
/**
|
|
840
|
-
*
|
|
928
|
+
* Update the generation span
|
|
841
929
|
*/
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
if (!span) {
|
|
845
|
-
break;
|
|
846
|
-
}
|
|
847
|
-
try {
|
|
848
|
-
span = processor.process(span);
|
|
849
|
-
} catch (error) {
|
|
850
|
-
this.logger.error(`[AI Tracing] Processor error [name=${processor.name}]`, error);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
return span;
|
|
930
|
+
updateGeneration(options) {
|
|
931
|
+
this.#modelSpan?.update(options);
|
|
854
932
|
}
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
933
|
+
/**
|
|
934
|
+
* Start a new Model execution step
|
|
935
|
+
*/
|
|
936
|
+
#startStepSpan(payload) {
|
|
937
|
+
this.#currentStepSpan = this.#modelSpan?.createChildSpan({
|
|
938
|
+
name: `step: ${this.#stepIndex}`,
|
|
939
|
+
type: SpanType.MODEL_STEP,
|
|
940
|
+
attributes: {
|
|
941
|
+
stepIndex: this.#stepIndex,
|
|
942
|
+
...payload?.messageId ? { messageId: payload.messageId } : {},
|
|
943
|
+
...payload?.warnings?.length ? { warnings: payload.warnings } : {}
|
|
944
|
+
},
|
|
945
|
+
input: payload?.request
|
|
946
|
+
});
|
|
947
|
+
this.#chunkSequence = 0;
|
|
863
948
|
}
|
|
864
949
|
/**
|
|
865
|
-
*
|
|
950
|
+
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
866
951
|
*/
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
952
|
+
#endStepSpan(payload) {
|
|
953
|
+
if (!this.#currentStepSpan) return;
|
|
954
|
+
const output = payload.output;
|
|
955
|
+
const { usage, ...otherOutput } = output;
|
|
956
|
+
const stepResult = payload.stepResult;
|
|
957
|
+
const metadata = payload.metadata;
|
|
958
|
+
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
959
|
+
if (cleanMetadata?.request) {
|
|
960
|
+
delete cleanMetadata.request;
|
|
873
961
|
}
|
|
962
|
+
this.#currentStepSpan.end({
|
|
963
|
+
output: otherOutput,
|
|
964
|
+
attributes: {
|
|
965
|
+
usage,
|
|
966
|
+
isContinued: stepResult.isContinued,
|
|
967
|
+
finishReason: stepResult.reason,
|
|
968
|
+
warnings: stepResult.warnings
|
|
969
|
+
},
|
|
970
|
+
metadata: {
|
|
971
|
+
...cleanMetadata
|
|
972
|
+
}
|
|
973
|
+
});
|
|
974
|
+
this.#currentStepSpan = void 0;
|
|
975
|
+
this.#stepIndex++;
|
|
874
976
|
}
|
|
875
977
|
/**
|
|
876
|
-
*
|
|
978
|
+
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
877
979
|
*/
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
this.exportEvent({ type: AITracingEventType.SPAN_ENDED, exportedSpan }).catch((error) => {
|
|
882
|
-
this.logger.error("[AI Tracing] Failed to export span_ended event", error);
|
|
883
|
-
});
|
|
980
|
+
#startChunkSpan(chunkType, initialData) {
|
|
981
|
+
if (!this.#currentStepSpan) {
|
|
982
|
+
this.#startStepSpan();
|
|
884
983
|
}
|
|
984
|
+
this.#currentChunkSpan = this.#currentStepSpan?.createChildSpan({
|
|
985
|
+
name: `chunk: '${chunkType}'`,
|
|
986
|
+
type: SpanType.MODEL_CHUNK,
|
|
987
|
+
attributes: {
|
|
988
|
+
chunkType,
|
|
989
|
+
sequenceNumber: this.#chunkSequence
|
|
990
|
+
}
|
|
991
|
+
});
|
|
992
|
+
this.#accumulator = initialData || {};
|
|
885
993
|
}
|
|
886
994
|
/**
|
|
887
|
-
*
|
|
995
|
+
* Append string content to a specific field in the accumulator
|
|
888
996
|
*/
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
});
|
|
997
|
+
#appendToAccumulator(field, text) {
|
|
998
|
+
if (this.#accumulator[field] === void 0) {
|
|
999
|
+
this.#accumulator[field] = text;
|
|
1000
|
+
} else {
|
|
1001
|
+
this.#accumulator[field] += text;
|
|
895
1002
|
}
|
|
896
1003
|
}
|
|
897
1004
|
/**
|
|
898
|
-
*
|
|
1005
|
+
* End the current chunk span.
|
|
1006
|
+
* Safe to call multiple times - will no-op if span already ended.
|
|
899
1007
|
*/
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
await exporter.exportEvent(event);
|
|
905
|
-
this.logger.debug(`[AI Tracing] Event exported [exporter=${exporter.name}] [type=${event.type}]`);
|
|
906
|
-
}
|
|
907
|
-
} catch (error) {
|
|
908
|
-
this.logger.error(`[AI Tracing] Export error [exporter=${exporter.name}]`, error);
|
|
909
|
-
}
|
|
1008
|
+
#endChunkSpan(output) {
|
|
1009
|
+
if (!this.#currentChunkSpan) return;
|
|
1010
|
+
this.#currentChunkSpan.end({
|
|
1011
|
+
output: output !== void 0 ? output : this.#accumulator
|
|
910
1012
|
});
|
|
911
|
-
|
|
1013
|
+
this.#currentChunkSpan = void 0;
|
|
1014
|
+
this.#accumulator = {};
|
|
1015
|
+
this.#chunkSequence++;
|
|
912
1016
|
}
|
|
913
|
-
// ============================================================================
|
|
914
|
-
// Lifecycle Management
|
|
915
|
-
// ============================================================================
|
|
916
1017
|
/**
|
|
917
|
-
*
|
|
1018
|
+
* Create an event span (for single chunks like tool-call)
|
|
918
1019
|
*/
|
|
919
|
-
|
|
920
|
-
this
|
|
921
|
-
|
|
1020
|
+
#createEventSpan(chunkType, output) {
|
|
1021
|
+
if (!this.#currentStepSpan) {
|
|
1022
|
+
this.#startStepSpan();
|
|
1023
|
+
}
|
|
1024
|
+
const span = this.#currentStepSpan?.createEventSpan({
|
|
1025
|
+
name: `chunk: '${chunkType}'`,
|
|
1026
|
+
type: SpanType.MODEL_CHUNK,
|
|
1027
|
+
attributes: {
|
|
1028
|
+
chunkType,
|
|
1029
|
+
sequenceNumber: this.#chunkSequence
|
|
1030
|
+
},
|
|
1031
|
+
output
|
|
1032
|
+
});
|
|
1033
|
+
if (span) {
|
|
1034
|
+
this.#chunkSequence++;
|
|
1035
|
+
}
|
|
922
1036
|
}
|
|
923
1037
|
/**
|
|
924
|
-
*
|
|
1038
|
+
* Check if there is currently an active chunk span
|
|
925
1039
|
*/
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
const shutdownPromises = [...this.exporters.map((e) => e.shutdown()), ...this.processors.map((p) => p.shutdown())];
|
|
929
|
-
await Promise.allSettled(shutdownPromises);
|
|
930
|
-
this.logger.info(`[AI Tracing] Shutdown completed [name=${this.name}]`);
|
|
931
|
-
}
|
|
932
|
-
};
|
|
933
|
-
|
|
934
|
-
// src/tracers/default.ts
|
|
935
|
-
var DefaultAITracing = class extends BaseAITracing {
|
|
936
|
-
constructor(config) {
|
|
937
|
-
super(config);
|
|
938
|
-
}
|
|
939
|
-
createSpan(options) {
|
|
940
|
-
return new DefaultAISpan(options, this);
|
|
1040
|
+
#hasActiveChunkSpan() {
|
|
1041
|
+
return !!this.#currentChunkSpan;
|
|
941
1042
|
}
|
|
942
|
-
};
|
|
943
|
-
var BaseExporter = class {
|
|
944
|
-
/** Mastra logger instance */
|
|
945
|
-
logger;
|
|
946
|
-
/** Whether this exporter is disabled */
|
|
947
|
-
isDisabled = false;
|
|
948
1043
|
/**
|
|
949
|
-
*
|
|
1044
|
+
* Get the current accumulator value
|
|
950
1045
|
*/
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
this.logger = config.logger ?? new ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
1046
|
+
#getAccumulator() {
|
|
1047
|
+
return this.#accumulator;
|
|
954
1048
|
}
|
|
955
1049
|
/**
|
|
956
|
-
*
|
|
1050
|
+
* Handle text chunk spans (text-start/delta/end)
|
|
957
1051
|
*/
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
1052
|
+
#handleTextChunk(chunk) {
|
|
1053
|
+
switch (chunk.type) {
|
|
1054
|
+
case "text-start":
|
|
1055
|
+
this.#startChunkSpan("text");
|
|
1056
|
+
break;
|
|
1057
|
+
case "text-delta":
|
|
1058
|
+
this.#appendToAccumulator("text", chunk.payload.text);
|
|
1059
|
+
break;
|
|
1060
|
+
case "text-end": {
|
|
1061
|
+
this.#endChunkSpan();
|
|
1062
|
+
break;
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
961
1065
|
}
|
|
962
1066
|
/**
|
|
963
|
-
*
|
|
1067
|
+
* Handle reasoning chunk spans (reasoning-start/delta/end)
|
|
964
1068
|
*/
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
1069
|
+
#handleReasoningChunk(chunk) {
|
|
1070
|
+
switch (chunk.type) {
|
|
1071
|
+
case "reasoning-start":
|
|
1072
|
+
this.#startChunkSpan("reasoning");
|
|
1073
|
+
break;
|
|
1074
|
+
case "reasoning-delta":
|
|
1075
|
+
this.#appendToAccumulator("text", chunk.payload.text);
|
|
1076
|
+
break;
|
|
1077
|
+
case "reasoning-end": {
|
|
1078
|
+
this.#endChunkSpan();
|
|
1079
|
+
break;
|
|
1080
|
+
}
|
|
971
1081
|
}
|
|
972
|
-
const logLevelMap = {
|
|
973
|
-
debug: LogLevel.DEBUG,
|
|
974
|
-
info: LogLevel.INFO,
|
|
975
|
-
warn: LogLevel.WARN,
|
|
976
|
-
error: LogLevel.ERROR
|
|
977
|
-
};
|
|
978
|
-
return logLevelMap[logLevel] ?? LogLevel.INFO;
|
|
979
1082
|
}
|
|
980
1083
|
/**
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
* @param reason - Reason why the exporter is disabled
|
|
1084
|
+
* Handle tool call chunk spans (tool-call-input-streaming-start/delta/end, tool-call)
|
|
984
1085
|
*/
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1086
|
+
#handleToolCallChunk(chunk) {
|
|
1087
|
+
switch (chunk.type) {
|
|
1088
|
+
case "tool-call-input-streaming-start":
|
|
1089
|
+
this.#startChunkSpan("tool-call", {
|
|
1090
|
+
toolName: chunk.payload.toolName,
|
|
1091
|
+
toolCallId: chunk.payload.toolCallId
|
|
1092
|
+
});
|
|
1093
|
+
break;
|
|
1094
|
+
case "tool-call-delta":
|
|
1095
|
+
this.#appendToAccumulator("toolInput", chunk.payload.argsTextDelta);
|
|
1096
|
+
break;
|
|
1097
|
+
case "tool-call-input-streaming-end":
|
|
1098
|
+
case "tool-call": {
|
|
1099
|
+
const acc = this.#getAccumulator();
|
|
1100
|
+
let toolInput;
|
|
1101
|
+
try {
|
|
1102
|
+
toolInput = acc.toolInput ? JSON.parse(acc.toolInput) : {};
|
|
1103
|
+
} catch {
|
|
1104
|
+
toolInput = acc.toolInput;
|
|
1105
|
+
}
|
|
1106
|
+
this.#endChunkSpan({
|
|
1107
|
+
toolName: acc.toolName,
|
|
1108
|
+
toolCallId: acc.toolCallId,
|
|
1109
|
+
toolInput
|
|
1110
|
+
});
|
|
1111
|
+
break;
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
988
1114
|
}
|
|
989
1115
|
/**
|
|
990
|
-
*
|
|
991
|
-
*
|
|
992
|
-
* This method checks if the exporter is disabled before calling _exportEvent.
|
|
993
|
-
* Subclasses should implement _exportEvent instead of overriding this method.
|
|
1116
|
+
* Handle object chunk spans (object, object-result)
|
|
994
1117
|
*/
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1118
|
+
#handleObjectChunk(chunk) {
|
|
1119
|
+
switch (chunk.type) {
|
|
1120
|
+
case "object":
|
|
1121
|
+
if (!this.#hasActiveChunkSpan()) {
|
|
1122
|
+
this.#startChunkSpan("object");
|
|
1123
|
+
}
|
|
1124
|
+
break;
|
|
1125
|
+
case "object-result":
|
|
1126
|
+
this.#endChunkSpan(chunk.object);
|
|
1127
|
+
break;
|
|
998
1128
|
}
|
|
999
|
-
await this._exportEvent(event);
|
|
1000
1129
|
}
|
|
1001
1130
|
/**
|
|
1002
|
-
*
|
|
1131
|
+
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
1003
1132
|
*
|
|
1004
|
-
*
|
|
1133
|
+
* This should be added to the stream pipeline to automatically
|
|
1134
|
+
* create MODEL_STEP and MODEL_CHUNK spans for each semantic unit in the stream.
|
|
1005
1135
|
*/
|
|
1006
|
-
|
|
1007
|
-
|
|
1136
|
+
wrapStream(stream) {
|
|
1137
|
+
return stream.pipeThrough(
|
|
1138
|
+
new TransformStream({
|
|
1139
|
+
transform: (chunk, controller) => {
|
|
1140
|
+
controller.enqueue(chunk);
|
|
1141
|
+
switch (chunk.type) {
|
|
1142
|
+
case "text-start":
|
|
1143
|
+
case "text-delta":
|
|
1144
|
+
case "text-end":
|
|
1145
|
+
this.#handleTextChunk(chunk);
|
|
1146
|
+
break;
|
|
1147
|
+
case "tool-call-input-streaming-start":
|
|
1148
|
+
case "tool-call-delta":
|
|
1149
|
+
case "tool-call-input-streaming-end":
|
|
1150
|
+
case "tool-call":
|
|
1151
|
+
this.#handleToolCallChunk(chunk);
|
|
1152
|
+
break;
|
|
1153
|
+
case "reasoning-start":
|
|
1154
|
+
case "reasoning-delta":
|
|
1155
|
+
case "reasoning-end":
|
|
1156
|
+
this.#handleReasoningChunk(chunk);
|
|
1157
|
+
break;
|
|
1158
|
+
case "object":
|
|
1159
|
+
case "object-result":
|
|
1160
|
+
this.#handleObjectChunk(chunk);
|
|
1161
|
+
break;
|
|
1162
|
+
case "step-start":
|
|
1163
|
+
this.#startStepSpan(chunk.payload);
|
|
1164
|
+
break;
|
|
1165
|
+
case "step-finish":
|
|
1166
|
+
this.#endStepSpan(chunk.payload);
|
|
1167
|
+
break;
|
|
1168
|
+
case "raw":
|
|
1169
|
+
// Skip raw chunks as they're redundant
|
|
1170
|
+
case "start":
|
|
1171
|
+
case "finish":
|
|
1172
|
+
break;
|
|
1173
|
+
// Default: auto-create event span for all other chunk types
|
|
1174
|
+
default: {
|
|
1175
|
+
let outputPayload = chunk.payload;
|
|
1176
|
+
if (outputPayload && typeof outputPayload === "object" && "data" in outputPayload) {
|
|
1177
|
+
const typedPayload = outputPayload;
|
|
1178
|
+
outputPayload = { ...typedPayload };
|
|
1179
|
+
if (typedPayload.data) {
|
|
1180
|
+
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
1181
|
+
delete outputPayload.data;
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
this.#createEventSpan(chunk.type, outputPayload);
|
|
1185
|
+
break;
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
})
|
|
1190
|
+
);
|
|
1008
1191
|
}
|
|
1009
1192
|
};
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
constructor(config = {}) {
|
|
1016
|
-
super(config);
|
|
1017
|
-
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
1018
|
-
if (!accessToken) {
|
|
1019
|
-
this.setDisabled(
|
|
1020
|
-
"MASTRA_CLOUD_ACCESS_TOKEN environment variable not set. \u{1F680} Sign up for Mastra Cloud at https://cloud.mastra.ai to see your AI traces online and obtain your access token."
|
|
1021
|
-
);
|
|
1022
|
-
}
|
|
1023
|
-
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_AI_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
1024
|
-
this.config = {
|
|
1025
|
-
logger: this.logger,
|
|
1026
|
-
logLevel: config.logLevel ?? LogLevel.INFO,
|
|
1027
|
-
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
1028
|
-
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
1029
|
-
maxRetries: config.maxRetries ?? 3,
|
|
1030
|
-
accessToken: accessToken || "",
|
|
1031
|
-
endpoint
|
|
1032
|
-
};
|
|
1033
|
-
this.buffer = {
|
|
1034
|
-
spans: [],
|
|
1035
|
-
totalSize: 0
|
|
1036
|
-
};
|
|
1193
|
+
|
|
1194
|
+
// src/spans/base.ts
|
|
1195
|
+
function isSpanInternal(spanType, flags) {
|
|
1196
|
+
if (flags === void 0 || flags === InternalSpans.NONE) {
|
|
1197
|
+
return false;
|
|
1037
1198
|
}
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1199
|
+
switch (spanType) {
|
|
1200
|
+
// Workflow-related spans
|
|
1201
|
+
case SpanType.WORKFLOW_RUN:
|
|
1202
|
+
case SpanType.WORKFLOW_STEP:
|
|
1203
|
+
case SpanType.WORKFLOW_CONDITIONAL:
|
|
1204
|
+
case SpanType.WORKFLOW_CONDITIONAL_EVAL:
|
|
1205
|
+
case SpanType.WORKFLOW_PARALLEL:
|
|
1206
|
+
case SpanType.WORKFLOW_LOOP:
|
|
1207
|
+
case SpanType.WORKFLOW_SLEEP:
|
|
1208
|
+
case SpanType.WORKFLOW_WAIT_EVENT:
|
|
1209
|
+
return (flags & InternalSpans.WORKFLOW) !== 0;
|
|
1210
|
+
// Agent-related spans
|
|
1211
|
+
case SpanType.AGENT_RUN:
|
|
1212
|
+
return (flags & InternalSpans.AGENT) !== 0;
|
|
1213
|
+
// Tool-related spans
|
|
1214
|
+
case SpanType.TOOL_CALL:
|
|
1215
|
+
case SpanType.MCP_TOOL_CALL:
|
|
1216
|
+
return (flags & InternalSpans.TOOL) !== 0;
|
|
1217
|
+
// Model-related spans
|
|
1218
|
+
case SpanType.MODEL_GENERATION:
|
|
1219
|
+
case SpanType.MODEL_STEP:
|
|
1220
|
+
case SpanType.MODEL_CHUNK:
|
|
1221
|
+
return (flags & InternalSpans.MODEL) !== 0;
|
|
1222
|
+
// Default: never internal
|
|
1223
|
+
default:
|
|
1224
|
+
return false;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
var BaseSpan = class {
|
|
1228
|
+
name;
|
|
1229
|
+
type;
|
|
1230
|
+
attributes;
|
|
1231
|
+
parent;
|
|
1232
|
+
startTime;
|
|
1233
|
+
endTime;
|
|
1234
|
+
isEvent;
|
|
1235
|
+
isInternal;
|
|
1236
|
+
observabilityInstance;
|
|
1237
|
+
input;
|
|
1238
|
+
output;
|
|
1239
|
+
errorInfo;
|
|
1240
|
+
metadata;
|
|
1241
|
+
traceState;
|
|
1242
|
+
/** Parent span ID (for root spans that are children of external spans) */
|
|
1243
|
+
parentSpanId;
|
|
1244
|
+
constructor(options, observabilityInstance) {
|
|
1245
|
+
this.name = options.name;
|
|
1246
|
+
this.type = options.type;
|
|
1247
|
+
this.attributes = deepClean(options.attributes) || {};
|
|
1248
|
+
this.metadata = deepClean(options.metadata);
|
|
1249
|
+
this.parent = options.parent;
|
|
1250
|
+
this.startTime = /* @__PURE__ */ new Date();
|
|
1251
|
+
this.observabilityInstance = observabilityInstance;
|
|
1252
|
+
this.isEvent = options.isEvent ?? false;
|
|
1253
|
+
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
1254
|
+
this.traceState = options.traceState;
|
|
1255
|
+
if (this.isEvent) {
|
|
1256
|
+
this.output = deepClean(options.output);
|
|
1257
|
+
} else {
|
|
1258
|
+
this.input = deepClean(options.input);
|
|
1051
1259
|
}
|
|
1052
1260
|
}
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1261
|
+
createChildSpan(options) {
|
|
1262
|
+
return this.observabilityInstance.startSpan({ ...options, parent: this, isEvent: false });
|
|
1263
|
+
}
|
|
1264
|
+
createEventSpan(options) {
|
|
1265
|
+
return this.observabilityInstance.startSpan({ ...options, parent: this, isEvent: true });
|
|
1266
|
+
}
|
|
1267
|
+
/**
|
|
1268
|
+
* Create a ModelSpanTracker for this span (only works if this is a MODEL_GENERATION span)
|
|
1269
|
+
* Returns undefined for non-MODEL_GENERATION spans
|
|
1270
|
+
*/
|
|
1271
|
+
createTracker() {
|
|
1272
|
+
if (this.type !== SpanType.MODEL_GENERATION) {
|
|
1273
|
+
return void 0;
|
|
1056
1274
|
}
|
|
1057
|
-
|
|
1058
|
-
this.buffer.spans.push(spanRecord);
|
|
1059
|
-
this.buffer.totalSize++;
|
|
1275
|
+
return new ModelSpanTracker(this);
|
|
1060
1276
|
}
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
spanId: span.id,
|
|
1065
|
-
parentSpanId: span.parentSpanId ?? null,
|
|
1066
|
-
name: span.name,
|
|
1067
|
-
spanType: span.type,
|
|
1068
|
-
attributes: span.attributes ?? null,
|
|
1069
|
-
metadata: span.metadata ?? null,
|
|
1070
|
-
startedAt: span.startTime,
|
|
1071
|
-
endedAt: span.endTime ?? null,
|
|
1072
|
-
input: span.input ?? null,
|
|
1073
|
-
output: span.output ?? null,
|
|
1074
|
-
error: span.errorInfo,
|
|
1075
|
-
isEvent: span.isEvent,
|
|
1076
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
1077
|
-
updatedAt: null
|
|
1078
|
-
};
|
|
1079
|
-
return spanRecord;
|
|
1277
|
+
/** Returns `TRUE` if the span is the root span of a trace */
|
|
1278
|
+
get isRootSpan() {
|
|
1279
|
+
return !this.parent;
|
|
1080
1280
|
}
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1281
|
+
/** Get the closest parent spanId that isn't an internal span */
|
|
1282
|
+
getParentSpanId(includeInternalSpans) {
|
|
1283
|
+
if (!this.parent) {
|
|
1284
|
+
return this.parentSpanId;
|
|
1084
1285
|
}
|
|
1085
|
-
if (
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1286
|
+
if (includeInternalSpans) return this.parent.id;
|
|
1287
|
+
if (this.parent.isInternal) return this.parent.getParentSpanId(includeInternalSpans);
|
|
1288
|
+
return this.parent.id;
|
|
1289
|
+
}
|
|
1290
|
+
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
1291
|
+
findParent(spanType) {
|
|
1292
|
+
let current = this.parent;
|
|
1293
|
+
while (current) {
|
|
1294
|
+
if (current.type === spanType) {
|
|
1295
|
+
return current;
|
|
1089
1296
|
}
|
|
1297
|
+
current = current.parent;
|
|
1090
1298
|
}
|
|
1091
|
-
return
|
|
1299
|
+
return void 0;
|
|
1092
1300
|
}
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
this.
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
}
|
|
1301
|
+
/** Returns a lightweight span ready for export */
|
|
1302
|
+
exportSpan(includeInternalSpans) {
|
|
1303
|
+
return {
|
|
1304
|
+
id: this.id,
|
|
1305
|
+
traceId: this.traceId,
|
|
1306
|
+
name: this.name,
|
|
1307
|
+
type: this.type,
|
|
1308
|
+
attributes: this.attributes,
|
|
1309
|
+
metadata: this.metadata,
|
|
1310
|
+
startTime: this.startTime,
|
|
1311
|
+
endTime: this.endTime,
|
|
1312
|
+
input: this.input,
|
|
1313
|
+
output: this.output,
|
|
1314
|
+
errorInfo: this.errorInfo,
|
|
1315
|
+
isEvent: this.isEvent,
|
|
1316
|
+
isRootSpan: this.isRootSpan,
|
|
1317
|
+
parentSpanId: this.getParentSpanId(includeInternalSpans)
|
|
1318
|
+
};
|
|
1111
1319
|
}
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1320
|
+
get externalTraceId() {
|
|
1321
|
+
return this.isValid ? this.traceId : void 0;
|
|
1322
|
+
}
|
|
1323
|
+
};
|
|
1324
|
+
var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
1325
|
+
"logger",
|
|
1326
|
+
"experimental_providerMetadata",
|
|
1327
|
+
"providerMetadata",
|
|
1328
|
+
"steps",
|
|
1329
|
+
"tracingContext"
|
|
1330
|
+
]);
|
|
1331
|
+
function deepClean(value, options = {}, _seen = /* @__PURE__ */ new WeakSet(), _depth = 0) {
|
|
1332
|
+
const { keysToStrip = DEFAULT_KEYS_TO_STRIP, maxDepth = 10 } = options;
|
|
1333
|
+
if (_depth > maxDepth) {
|
|
1334
|
+
return "[MaxDepth]";
|
|
1335
|
+
}
|
|
1336
|
+
if (value === null || typeof value !== "object") {
|
|
1124
1337
|
try {
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
this.logger.debug("Batch flushed successfully", {
|
|
1128
|
-
batchSize: spansCopy.length,
|
|
1129
|
-
flushReason,
|
|
1130
|
-
durationMs: elapsed
|
|
1131
|
-
});
|
|
1338
|
+
JSON.stringify(value);
|
|
1339
|
+
return value;
|
|
1132
1340
|
} catch (error) {
|
|
1133
|
-
|
|
1134
|
-
{
|
|
1135
|
-
id: `CLOUD_AI_TRACING_FAILED_TO_BATCH_UPLOAD`,
|
|
1136
|
-
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
1137
|
-
category: ErrorCategory.USER,
|
|
1138
|
-
details: {
|
|
1139
|
-
droppedBatchSize: spansCopy.length
|
|
1140
|
-
}
|
|
1141
|
-
},
|
|
1142
|
-
error
|
|
1143
|
-
);
|
|
1144
|
-
this.logger.trackException(mastraError);
|
|
1145
|
-
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
1341
|
+
return `[${error instanceof Error ? error.message : String(error)}]`;
|
|
1146
1342
|
}
|
|
1147
1343
|
}
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
*/
|
|
1151
|
-
async batchUpload(spans) {
|
|
1152
|
-
const headers = {
|
|
1153
|
-
Authorization: `Bearer ${this.config.accessToken}`,
|
|
1154
|
-
"Content-Type": "application/json"
|
|
1155
|
-
};
|
|
1156
|
-
const options = {
|
|
1157
|
-
method: "POST",
|
|
1158
|
-
headers,
|
|
1159
|
-
body: JSON.stringify({ spans })
|
|
1160
|
-
};
|
|
1161
|
-
await fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
1344
|
+
if (_seen.has(value)) {
|
|
1345
|
+
return "[Circular]";
|
|
1162
1346
|
}
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
this.buffer.totalSize = 0;
|
|
1347
|
+
_seen.add(value);
|
|
1348
|
+
if (Array.isArray(value)) {
|
|
1349
|
+
return value.map((item) => deepClean(item, options, _seen, _depth + 1));
|
|
1167
1350
|
}
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1351
|
+
const cleaned = {};
|
|
1352
|
+
for (const [key, val] of Object.entries(value)) {
|
|
1353
|
+
if (keysToStrip.has(key)) {
|
|
1354
|
+
continue;
|
|
1171
1355
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1356
|
+
try {
|
|
1357
|
+
cleaned[key] = deepClean(val, options, _seen, _depth + 1);
|
|
1358
|
+
} catch (error) {
|
|
1359
|
+
cleaned[key] = `[${error instanceof Error ? error.message : String(error)}]`;
|
|
1175
1360
|
}
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1361
|
+
}
|
|
1362
|
+
return cleaned;
|
|
1363
|
+
}
|
|
1364
|
+
var DefaultSpan = class extends BaseSpan {
|
|
1365
|
+
id;
|
|
1366
|
+
traceId;
|
|
1367
|
+
constructor(options, observabilityInstance) {
|
|
1368
|
+
super(options, observabilityInstance);
|
|
1369
|
+
this.id = generateSpanId();
|
|
1370
|
+
if (options.parent) {
|
|
1371
|
+
this.traceId = options.parent.traceId;
|
|
1372
|
+
} else if (options.traceId) {
|
|
1373
|
+
if (isValidTraceId(options.traceId)) {
|
|
1374
|
+
this.traceId = options.traceId;
|
|
1375
|
+
} else {
|
|
1376
|
+
console.error(
|
|
1377
|
+
`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`
|
|
1378
|
+
);
|
|
1379
|
+
this.traceId = generateTraceId();
|
|
1380
|
+
}
|
|
1381
|
+
} else {
|
|
1382
|
+
this.traceId = generateTraceId();
|
|
1383
|
+
}
|
|
1384
|
+
if (!options.parent && options.parentSpanId) {
|
|
1385
|
+
if (isValidSpanId(options.parentSpanId)) {
|
|
1386
|
+
this.parentSpanId = options.parentSpanId;
|
|
1387
|
+
} else {
|
|
1388
|
+
console.error(
|
|
1389
|
+
`[Mastra Tracing] Invalid parentSpanId: must be 1-16 hexadecimal characters, got "${options.parentSpanId}". Ignoring parent span ID.`
|
|
1193
1390
|
);
|
|
1194
|
-
this.logger.trackException(mastraError);
|
|
1195
|
-
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
1196
1391
|
}
|
|
1197
1392
|
}
|
|
1198
|
-
this.logger.info("CloudExporter shutdown complete");
|
|
1199
1393
|
}
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1394
|
+
end(options) {
|
|
1395
|
+
if (this.isEvent) {
|
|
1396
|
+
return;
|
|
1397
|
+
}
|
|
1398
|
+
this.endTime = /* @__PURE__ */ new Date();
|
|
1399
|
+
if (options?.output !== void 0) {
|
|
1400
|
+
this.output = deepClean(options.output);
|
|
1401
|
+
}
|
|
1402
|
+
if (options?.attributes) {
|
|
1403
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
1404
|
+
}
|
|
1405
|
+
if (options?.metadata) {
|
|
1406
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
1407
|
+
}
|
|
1205
1408
|
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
}
|
|
1231
|
-
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
1232
|
-
this.logger.info("\u2500".repeat(80));
|
|
1233
|
-
break;
|
|
1234
|
-
case AITracingEventType.SPAN_ENDED:
|
|
1235
|
-
const duration = formatDuration(span.startTime, span.endTime);
|
|
1236
|
-
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
1237
|
-
this.logger.info(` Type: ${span.type}`);
|
|
1238
|
-
this.logger.info(` Name: ${span.name}`);
|
|
1239
|
-
this.logger.info(` ID: ${span.id}`);
|
|
1240
|
-
this.logger.info(` Duration: ${duration}`);
|
|
1241
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
1242
|
-
if (span.input !== void 0) {
|
|
1243
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
1244
|
-
}
|
|
1245
|
-
if (span.output !== void 0) {
|
|
1246
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
1247
|
-
}
|
|
1248
|
-
if (span.errorInfo) {
|
|
1249
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
1250
|
-
}
|
|
1251
|
-
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
1252
|
-
this.logger.info("\u2500".repeat(80));
|
|
1253
|
-
break;
|
|
1254
|
-
case AITracingEventType.SPAN_UPDATED:
|
|
1255
|
-
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
1256
|
-
this.logger.info(` Type: ${span.type}`);
|
|
1257
|
-
this.logger.info(` Name: ${span.name}`);
|
|
1258
|
-
this.logger.info(` ID: ${span.id}`);
|
|
1259
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
1260
|
-
if (span.input !== void 0) {
|
|
1261
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
1262
|
-
}
|
|
1263
|
-
if (span.output !== void 0) {
|
|
1264
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
1265
|
-
}
|
|
1266
|
-
if (span.errorInfo) {
|
|
1267
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
1268
|
-
}
|
|
1269
|
-
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
1270
|
-
this.logger.info("\u2500".repeat(80));
|
|
1271
|
-
break;
|
|
1272
|
-
default:
|
|
1273
|
-
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
1409
|
+
error(options) {
|
|
1410
|
+
if (this.isEvent) {
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
const { error, endSpan = true, attributes, metadata } = options;
|
|
1414
|
+
this.errorInfo = error instanceof MastraError ? {
|
|
1415
|
+
id: error.id,
|
|
1416
|
+
details: error.details,
|
|
1417
|
+
category: error.category,
|
|
1418
|
+
domain: error.domain,
|
|
1419
|
+
message: error.message
|
|
1420
|
+
} : {
|
|
1421
|
+
message: error.message
|
|
1422
|
+
};
|
|
1423
|
+
if (attributes) {
|
|
1424
|
+
this.attributes = { ...this.attributes, ...deepClean(attributes) };
|
|
1425
|
+
}
|
|
1426
|
+
if (metadata) {
|
|
1427
|
+
this.metadata = { ...this.metadata, ...deepClean(metadata) };
|
|
1428
|
+
}
|
|
1429
|
+
if (endSpan) {
|
|
1430
|
+
this.end();
|
|
1431
|
+
} else {
|
|
1432
|
+
this.update({});
|
|
1274
1433
|
}
|
|
1275
1434
|
}
|
|
1276
|
-
|
|
1277
|
-
this.
|
|
1435
|
+
update(options) {
|
|
1436
|
+
if (this.isEvent) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
if (options.input !== void 0) {
|
|
1440
|
+
this.input = deepClean(options.input);
|
|
1441
|
+
}
|
|
1442
|
+
if (options.output !== void 0) {
|
|
1443
|
+
this.output = deepClean(options.output);
|
|
1444
|
+
}
|
|
1445
|
+
if (options.attributes) {
|
|
1446
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
1447
|
+
}
|
|
1448
|
+
if (options.metadata) {
|
|
1449
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
1450
|
+
}
|
|
1278
1451
|
}
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
fallbackStrategy: hints.preferred
|
|
1452
|
+
get isValid() {
|
|
1453
|
+
return true;
|
|
1454
|
+
}
|
|
1455
|
+
async export() {
|
|
1456
|
+
return JSON.stringify({
|
|
1457
|
+
spanId: this.id,
|
|
1458
|
+
traceId: this.traceId,
|
|
1459
|
+
startTime: this.startTime,
|
|
1460
|
+
endTime: this.endTime,
|
|
1461
|
+
attributes: this.attributes,
|
|
1462
|
+
metadata: this.metadata
|
|
1291
1463
|
});
|
|
1292
1464
|
}
|
|
1293
|
-
|
|
1465
|
+
};
|
|
1466
|
+
function generateSpanId() {
|
|
1467
|
+
const bytes = new Uint8Array(8);
|
|
1468
|
+
if (typeof crypto !== "undefined" && crypto.getRandomValues) {
|
|
1469
|
+
crypto.getRandomValues(bytes);
|
|
1470
|
+
} else {
|
|
1471
|
+
for (let i = 0; i < 8; i++) {
|
|
1472
|
+
bytes[i] = Math.floor(Math.random() * 256);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
1294
1476
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
flushTimer = null;
|
|
1303
|
-
// Track all spans that have been created, persists across flushes
|
|
1304
|
-
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
1305
|
-
constructor(config = {}, logger) {
|
|
1306
|
-
if (logger) {
|
|
1307
|
-
this.logger = logger;
|
|
1308
|
-
} else {
|
|
1309
|
-
this.logger = new ConsoleLogger({ level: LogLevel.INFO });
|
|
1477
|
+
function generateTraceId() {
|
|
1478
|
+
const bytes = new Uint8Array(16);
|
|
1479
|
+
if (typeof crypto !== "undefined" && crypto.getRandomValues) {
|
|
1480
|
+
crypto.getRandomValues(bytes);
|
|
1481
|
+
} else {
|
|
1482
|
+
for (let i = 0; i < 16; i++) {
|
|
1483
|
+
bytes[i] = Math.floor(Math.random() * 256);
|
|
1310
1484
|
}
|
|
1485
|
+
}
|
|
1486
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
1487
|
+
}
|
|
1488
|
+
function isValidTraceId(traceId) {
|
|
1489
|
+
return /^[0-9a-f]{1,32}$/i.test(traceId);
|
|
1490
|
+
}
|
|
1491
|
+
function isValidSpanId(spanId) {
|
|
1492
|
+
return /^[0-9a-f]{1,16}$/i.test(spanId);
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
// src/spans/no-op.ts
|
|
1496
|
+
var NoOpSpan = class extends BaseSpan {
|
|
1497
|
+
id;
|
|
1498
|
+
traceId;
|
|
1499
|
+
constructor(options, observabilityInstance) {
|
|
1500
|
+
super(options, observabilityInstance);
|
|
1501
|
+
this.id = "no-op";
|
|
1502
|
+
this.traceId = "no-op-trace";
|
|
1503
|
+
}
|
|
1504
|
+
end(_options) {
|
|
1505
|
+
}
|
|
1506
|
+
error(_options) {
|
|
1507
|
+
}
|
|
1508
|
+
update(_options) {
|
|
1509
|
+
}
|
|
1510
|
+
get isValid() {
|
|
1511
|
+
return false;
|
|
1512
|
+
}
|
|
1513
|
+
};
|
|
1514
|
+
|
|
1515
|
+
// src/instances/base.ts
|
|
1516
|
+
var BaseObservabilityInstance = class extends MastraBase {
|
|
1517
|
+
config;
|
|
1518
|
+
constructor(config) {
|
|
1519
|
+
super({ component: RegisteredLogger.OBSERVABILITY, name: config.serviceName });
|
|
1311
1520
|
this.config = {
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
this.buffer = {
|
|
1320
|
-
creates: [],
|
|
1321
|
-
updates: [],
|
|
1322
|
-
insertOnly: [],
|
|
1323
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
1324
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
1325
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
1326
|
-
outOfOrderCount: 0,
|
|
1327
|
-
totalSize: 0
|
|
1521
|
+
serviceName: config.serviceName,
|
|
1522
|
+
name: config.name,
|
|
1523
|
+
sampling: config.sampling ?? { type: "always" /* ALWAYS */ },
|
|
1524
|
+
exporters: config.exporters ?? [],
|
|
1525
|
+
spanOutputProcessors: config.spanOutputProcessors ?? [],
|
|
1526
|
+
includeInternalSpans: config.includeInternalSpans ?? false,
|
|
1527
|
+
requestContextKeys: config.requestContextKeys ?? []
|
|
1328
1528
|
};
|
|
1329
|
-
this.resolvedStrategy = "batch-with-updates";
|
|
1330
|
-
}
|
|
1331
|
-
strategyInitialized = false;
|
|
1332
|
-
/**
|
|
1333
|
-
* Initialize the exporter (called after all dependencies are ready)
|
|
1334
|
-
*/
|
|
1335
|
-
init(options) {
|
|
1336
|
-
this.storage = options.mastra?.getStorage();
|
|
1337
|
-
if (!this.storage) {
|
|
1338
|
-
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
1339
|
-
return;
|
|
1340
|
-
}
|
|
1341
|
-
this.initializeStrategy(this.storage);
|
|
1342
1529
|
}
|
|
1343
1530
|
/**
|
|
1344
|
-
*
|
|
1531
|
+
* Override setLogger to add Observability specific initialization log
|
|
1532
|
+
* and propagate logger to exporters
|
|
1345
1533
|
*/
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
this.
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
source: this.config.strategy !== "auto" ? "user" : "auto",
|
|
1353
|
-
storageAdapter: storage.constructor.name,
|
|
1354
|
-
maxBatchSize: this.config.maxBatchSize,
|
|
1355
|
-
maxBatchWaitMs: this.config.maxBatchWaitMs
|
|
1534
|
+
__setLogger(logger) {
|
|
1535
|
+
super.__setLogger(logger);
|
|
1536
|
+
this.exporters.forEach((exporter) => {
|
|
1537
|
+
if (typeof exporter.__setLogger === "function") {
|
|
1538
|
+
exporter.__setLogger(logger);
|
|
1539
|
+
}
|
|
1356
1540
|
});
|
|
1541
|
+
this.logger.debug(
|
|
1542
|
+
`[Observability] Initialized [service=${this.config.serviceName}] [instance=${this.config.name}] [sampling=${this.config.sampling.type}]`
|
|
1543
|
+
);
|
|
1357
1544
|
}
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
return
|
|
1545
|
+
// ============================================================================
|
|
1546
|
+
// Protected getters for clean config access
|
|
1547
|
+
// ============================================================================
|
|
1548
|
+
get exporters() {
|
|
1549
|
+
return this.config.exporters || [];
|
|
1363
1550
|
}
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
*/
|
|
1367
|
-
getNextSequence(spanKey) {
|
|
1368
|
-
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
1369
|
-
const next = current + 1;
|
|
1370
|
-
this.buffer.spanSequences.set(spanKey, next);
|
|
1371
|
-
return next;
|
|
1551
|
+
get spanOutputProcessors() {
|
|
1552
|
+
return this.config.spanOutputProcessors || [];
|
|
1372
1553
|
}
|
|
1554
|
+
// ============================================================================
|
|
1555
|
+
// Public API - Single type-safe span creation method
|
|
1556
|
+
// ============================================================================
|
|
1373
1557
|
/**
|
|
1374
|
-
*
|
|
1558
|
+
* Start a new span of a specific SpanType
|
|
1375
1559
|
*/
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1560
|
+
startSpan(options) {
|
|
1561
|
+
const { customSamplerOptions, requestContext, metadata, tracingOptions, ...rest } = options;
|
|
1562
|
+
if (!this.shouldSample(customSamplerOptions)) {
|
|
1563
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
1564
|
+
}
|
|
1565
|
+
let traceState;
|
|
1566
|
+
if (options.parent) {
|
|
1567
|
+
traceState = options.parent.traceState;
|
|
1568
|
+
} else {
|
|
1569
|
+
traceState = this.computeTraceState(tracingOptions);
|
|
1570
|
+
}
|
|
1571
|
+
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, metadata, traceState);
|
|
1572
|
+
const span = this.createSpan({
|
|
1573
|
+
...rest,
|
|
1574
|
+
metadata: enrichedMetadata,
|
|
1575
|
+
traceState
|
|
1382
1576
|
});
|
|
1577
|
+
if (span.isEvent) {
|
|
1578
|
+
this.emitSpanEnded(span);
|
|
1579
|
+
} else {
|
|
1580
|
+
this.wireSpanLifecycle(span);
|
|
1581
|
+
this.emitSpanStarted(span);
|
|
1582
|
+
}
|
|
1583
|
+
return span;
|
|
1383
1584
|
}
|
|
1585
|
+
// ============================================================================
|
|
1586
|
+
// Configuration Management
|
|
1587
|
+
// ============================================================================
|
|
1384
1588
|
/**
|
|
1385
|
-
*
|
|
1386
|
-
*/
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
if (this.buffer.totalSize === 0) {
|
|
1390
|
-
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
1391
|
-
}
|
|
1392
|
-
switch (event.type) {
|
|
1393
|
-
case AITracingEventType.SPAN_STARTED:
|
|
1394
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1395
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1396
|
-
this.buffer.creates.push(createRecord);
|
|
1397
|
-
this.buffer.seenSpans.add(spanKey);
|
|
1398
|
-
this.allCreatedSpans.add(spanKey);
|
|
1399
|
-
}
|
|
1400
|
-
break;
|
|
1401
|
-
case AITracingEventType.SPAN_UPDATED:
|
|
1402
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1403
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
1404
|
-
this.buffer.updates.push({
|
|
1405
|
-
traceId: event.exportedSpan.traceId,
|
|
1406
|
-
spanId: event.exportedSpan.id,
|
|
1407
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
1408
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
1409
|
-
});
|
|
1410
|
-
} else {
|
|
1411
|
-
this.handleOutOfOrderUpdate(event);
|
|
1412
|
-
this.buffer.outOfOrderCount++;
|
|
1413
|
-
}
|
|
1414
|
-
}
|
|
1415
|
-
break;
|
|
1416
|
-
case AITracingEventType.SPAN_ENDED:
|
|
1417
|
-
if (this.resolvedStrategy === "batch-with-updates") {
|
|
1418
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
1419
|
-
this.buffer.updates.push({
|
|
1420
|
-
traceId: event.exportedSpan.traceId,
|
|
1421
|
-
spanId: event.exportedSpan.id,
|
|
1422
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
1423
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
1424
|
-
});
|
|
1425
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1426
|
-
} else if (event.exportedSpan.isEvent) {
|
|
1427
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1428
|
-
this.buffer.creates.push(createRecord);
|
|
1429
|
-
this.buffer.seenSpans.add(spanKey);
|
|
1430
|
-
this.allCreatedSpans.add(spanKey);
|
|
1431
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1432
|
-
} else {
|
|
1433
|
-
this.handleOutOfOrderUpdate(event);
|
|
1434
|
-
this.buffer.outOfOrderCount++;
|
|
1435
|
-
}
|
|
1436
|
-
} else if (this.resolvedStrategy === "insert-only") {
|
|
1437
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
1438
|
-
this.buffer.insertOnly.push(createRecord);
|
|
1439
|
-
this.buffer.completedSpans.add(spanKey);
|
|
1440
|
-
this.allCreatedSpans.add(spanKey);
|
|
1441
|
-
}
|
|
1442
|
-
break;
|
|
1443
|
-
}
|
|
1444
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
1589
|
+
* Get current configuration
|
|
1590
|
+
*/
|
|
1591
|
+
getConfig() {
|
|
1592
|
+
return { ...this.config };
|
|
1445
1593
|
}
|
|
1594
|
+
// ============================================================================
|
|
1595
|
+
// Plugin Access
|
|
1596
|
+
// ============================================================================
|
|
1446
1597
|
/**
|
|
1447
|
-
*
|
|
1598
|
+
* Get all exporters
|
|
1448
1599
|
*/
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1600
|
+
getExporters() {
|
|
1601
|
+
return [...this.exporters];
|
|
1602
|
+
}
|
|
1603
|
+
/**
|
|
1604
|
+
* Get all span output processors
|
|
1605
|
+
*/
|
|
1606
|
+
getSpanOutputProcessors() {
|
|
1607
|
+
return [...this.spanOutputProcessors];
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* Get the logger instance (for exporters and other components)
|
|
1611
|
+
*/
|
|
1612
|
+
getLogger() {
|
|
1613
|
+
return this.logger;
|
|
1614
|
+
}
|
|
1615
|
+
// ============================================================================
|
|
1616
|
+
// Span Lifecycle Management
|
|
1617
|
+
// ============================================================================
|
|
1618
|
+
/**
|
|
1619
|
+
* Automatically wires up Observability lifecycle events for any span
|
|
1620
|
+
* This ensures all spans emit events regardless of implementation
|
|
1621
|
+
*/
|
|
1622
|
+
wireSpanLifecycle(span) {
|
|
1623
|
+
if (!this.config.includeInternalSpans && span.isInternal) {
|
|
1624
|
+
return;
|
|
1455
1625
|
}
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1626
|
+
const originalEnd = span.end.bind(span);
|
|
1627
|
+
const originalUpdate = span.update.bind(span);
|
|
1628
|
+
span.end = (options) => {
|
|
1629
|
+
if (span.isEvent) {
|
|
1630
|
+
this.logger.warn(`End event is not available on event spans`);
|
|
1631
|
+
return;
|
|
1460
1632
|
}
|
|
1461
|
-
|
|
1462
|
-
|
|
1633
|
+
originalEnd(options);
|
|
1634
|
+
this.emitSpanEnded(span);
|
|
1635
|
+
};
|
|
1636
|
+
span.update = (options) => {
|
|
1637
|
+
if (span.isEvent) {
|
|
1638
|
+
this.logger.warn(`Update() is not available on event spans`);
|
|
1639
|
+
return;
|
|
1640
|
+
}
|
|
1641
|
+
originalUpdate(options);
|
|
1642
|
+
this.emitSpanUpdated(span);
|
|
1643
|
+
};
|
|
1463
1644
|
}
|
|
1645
|
+
// ============================================================================
|
|
1646
|
+
// Utility Methods
|
|
1647
|
+
// ============================================================================
|
|
1464
1648
|
/**
|
|
1465
|
-
*
|
|
1649
|
+
* Check if a trace should be sampled
|
|
1466
1650
|
*/
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1651
|
+
shouldSample(options) {
|
|
1652
|
+
const { sampling } = this.config;
|
|
1653
|
+
switch (sampling.type) {
|
|
1654
|
+
case "always" /* ALWAYS */:
|
|
1655
|
+
return true;
|
|
1656
|
+
case "never" /* NEVER */:
|
|
1657
|
+
return false;
|
|
1658
|
+
case "ratio" /* RATIO */:
|
|
1659
|
+
if (sampling.probability === void 0 || sampling.probability < 0 || sampling.probability > 1) {
|
|
1660
|
+
this.logger.warn(
|
|
1661
|
+
`Invalid sampling probability: ${sampling.probability}. Expected value between 0 and 1. Defaulting to no sampling.`
|
|
1662
|
+
);
|
|
1663
|
+
return false;
|
|
1664
|
+
}
|
|
1665
|
+
return Math.random() < sampling.probability;
|
|
1666
|
+
case "custom" /* CUSTOM */:
|
|
1667
|
+
return sampling.sampler(options);
|
|
1668
|
+
default:
|
|
1669
|
+
throw new Error(`Sampling strategy type not implemented: ${sampling.type}`);
|
|
1479
1670
|
}
|
|
1480
1671
|
}
|
|
1481
1672
|
/**
|
|
1482
|
-
*
|
|
1673
|
+
* Compute TraceState for a new trace based on configured and per-request keys
|
|
1483
1674
|
*/
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1675
|
+
computeTraceState(tracingOptions) {
|
|
1676
|
+
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
1677
|
+
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
1678
|
+
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
1679
|
+
if (allKeys.length === 0) {
|
|
1680
|
+
return void 0;
|
|
1487
1681
|
}
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1492
|
-
});
|
|
1493
|
-
});
|
|
1494
|
-
}, this.config.maxBatchWaitMs);
|
|
1682
|
+
return {
|
|
1683
|
+
requestContextKeys: allKeys
|
|
1684
|
+
};
|
|
1495
1685
|
}
|
|
1496
1686
|
/**
|
|
1497
|
-
*
|
|
1498
|
-
* Handles all AI span types and their specific attributes
|
|
1687
|
+
* Extract metadata from RequestContext using TraceState
|
|
1499
1688
|
*/
|
|
1500
|
-
|
|
1501
|
-
if (!
|
|
1502
|
-
return
|
|
1689
|
+
extractMetadataFromRequestContext(requestContext, explicitMetadata, traceState) {
|
|
1690
|
+
if (!requestContext || !traceState || traceState.requestContextKeys.length === 0) {
|
|
1691
|
+
return explicitMetadata;
|
|
1503
1692
|
}
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
if (value instanceof Date) {
|
|
1508
|
-
return value.toISOString();
|
|
1509
|
-
}
|
|
1510
|
-
if (typeof value === "object" && value !== null) {
|
|
1511
|
-
return value;
|
|
1512
|
-
}
|
|
1513
|
-
return value;
|
|
1514
|
-
})
|
|
1515
|
-
);
|
|
1516
|
-
} catch (error) {
|
|
1517
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
1518
|
-
spanId: span.id,
|
|
1519
|
-
spanType: span.type,
|
|
1520
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1521
|
-
});
|
|
1522
|
-
return null;
|
|
1693
|
+
const extracted = this.extractKeys(requestContext, traceState.requestContextKeys);
|
|
1694
|
+
if (Object.keys(extracted).length === 0 && !explicitMetadata) {
|
|
1695
|
+
return void 0;
|
|
1523
1696
|
}
|
|
1524
|
-
}
|
|
1525
|
-
buildCreateRecord(span) {
|
|
1526
1697
|
return {
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
name: span.name,
|
|
1531
|
-
scope: null,
|
|
1532
|
-
spanType: span.type,
|
|
1533
|
-
attributes: this.serializeAttributes(span),
|
|
1534
|
-
metadata: span.metadata ?? null,
|
|
1535
|
-
links: null,
|
|
1536
|
-
startedAt: span.startTime,
|
|
1537
|
-
endedAt: span.endTime ?? null,
|
|
1538
|
-
input: span.input,
|
|
1539
|
-
output: span.output,
|
|
1540
|
-
error: span.errorInfo,
|
|
1541
|
-
isEvent: span.isEvent
|
|
1698
|
+
...extracted,
|
|
1699
|
+
...explicitMetadata
|
|
1700
|
+
// Explicit metadata always wins
|
|
1542
1701
|
};
|
|
1543
1702
|
}
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1703
|
+
/**
|
|
1704
|
+
* Extract specific keys from RequestContext
|
|
1705
|
+
*/
|
|
1706
|
+
extractKeys(requestContext, keys) {
|
|
1707
|
+
const result = {};
|
|
1708
|
+
for (const key of keys) {
|
|
1709
|
+
const parts = key.split(".");
|
|
1710
|
+
const rootKey = parts[0];
|
|
1711
|
+
const value = requestContext.get(rootKey);
|
|
1712
|
+
if (value !== void 0) {
|
|
1713
|
+
if (parts.length > 1) {
|
|
1714
|
+
const nestedPath = parts.slice(1).join(".");
|
|
1715
|
+
const nestedValue = getNestedValue(value, nestedPath);
|
|
1716
|
+
if (nestedValue !== void 0) {
|
|
1717
|
+
setNestedValue(result, key, nestedValue);
|
|
1718
|
+
}
|
|
1719
|
+
} else {
|
|
1720
|
+
setNestedValue(result, key, value);
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
return result;
|
|
1556
1725
|
}
|
|
1557
1726
|
/**
|
|
1558
|
-
*
|
|
1727
|
+
* Process a span through all output processors
|
|
1559
1728
|
*/
|
|
1560
|
-
|
|
1561
|
-
const
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
if (event.type === AITracingEventType.SPAN_ENDED) {
|
|
1565
|
-
await storage.createAISpan(this.buildCreateRecord(event.exportedSpan));
|
|
1566
|
-
} else {
|
|
1567
|
-
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
1729
|
+
processSpan(span) {
|
|
1730
|
+
for (const processor of this.spanOutputProcessors) {
|
|
1731
|
+
if (!span) {
|
|
1732
|
+
break;
|
|
1568
1733
|
}
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
this.allCreatedSpans.add(spanKey);
|
|
1574
|
-
break;
|
|
1575
|
-
case AITracingEventType.SPAN_UPDATED:
|
|
1576
|
-
await storage.updateAISpan({
|
|
1577
|
-
traceId: span.traceId,
|
|
1578
|
-
spanId: span.id,
|
|
1579
|
-
updates: this.buildUpdateRecord(span)
|
|
1580
|
-
});
|
|
1581
|
-
break;
|
|
1582
|
-
case AITracingEventType.SPAN_ENDED:
|
|
1583
|
-
await storage.updateAISpan({
|
|
1584
|
-
traceId: span.traceId,
|
|
1585
|
-
spanId: span.id,
|
|
1586
|
-
updates: this.buildUpdateRecord(span)
|
|
1587
|
-
});
|
|
1588
|
-
this.allCreatedSpans.delete(spanKey);
|
|
1589
|
-
break;
|
|
1590
|
-
default:
|
|
1591
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
1734
|
+
try {
|
|
1735
|
+
span = processor.process(span);
|
|
1736
|
+
} catch (error) {
|
|
1737
|
+
this.logger.error(`[Observability] Processor error [name=${processor.name}]`, error);
|
|
1592
1738
|
}
|
|
1593
1739
|
}
|
|
1740
|
+
return span;
|
|
1741
|
+
}
|
|
1742
|
+
// ============================================================================
|
|
1743
|
+
// Event-driven Export Methods
|
|
1744
|
+
// ============================================================================
|
|
1745
|
+
getSpanForExport(span) {
|
|
1746
|
+
if (!span.isValid) return void 0;
|
|
1747
|
+
if (span.isInternal && !this.config.includeInternalSpans) return void 0;
|
|
1748
|
+
const processedSpan = this.processSpan(span);
|
|
1749
|
+
return processedSpan?.exportSpan(this.config.includeInternalSpans);
|
|
1594
1750
|
}
|
|
1595
1751
|
/**
|
|
1596
|
-
*
|
|
1752
|
+
* Emit a span started event
|
|
1597
1753
|
*/
|
|
1598
|
-
|
|
1599
|
-
this.
|
|
1600
|
-
if (
|
|
1601
|
-
this.
|
|
1602
|
-
this.logger.error("
|
|
1603
|
-
|
|
1604
|
-
|
|
1754
|
+
emitSpanStarted(span) {
|
|
1755
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1756
|
+
if (exportedSpan) {
|
|
1757
|
+
this.exportTracingEvent({ type: TracingEventType.SPAN_STARTED, exportedSpan }).catch((error) => {
|
|
1758
|
+
this.logger.error("[Observability] Failed to export span_started event", error);
|
|
1759
|
+
});
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
/**
|
|
1763
|
+
* Emit a span ended event (called automatically when spans end)
|
|
1764
|
+
*/
|
|
1765
|
+
emitSpanEnded(span) {
|
|
1766
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1767
|
+
if (exportedSpan) {
|
|
1768
|
+
this.exportTracingEvent({ type: TracingEventType.SPAN_ENDED, exportedSpan }).catch((error) => {
|
|
1769
|
+
this.logger.error("[Observability] Failed to export span_ended event", error);
|
|
1770
|
+
});
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
/**
|
|
1774
|
+
* Emit a span updated event
|
|
1775
|
+
*/
|
|
1776
|
+
emitSpanUpdated(span) {
|
|
1777
|
+
const exportedSpan = this.getSpanForExport(span);
|
|
1778
|
+
if (exportedSpan) {
|
|
1779
|
+
this.exportTracingEvent({ type: TracingEventType.SPAN_UPDATED, exportedSpan }).catch((error) => {
|
|
1780
|
+
this.logger.error("[Observability] Failed to export span_updated event", error);
|
|
1605
1781
|
});
|
|
1606
|
-
} else if (this.buffer.totalSize === 1) {
|
|
1607
|
-
this.scheduleFlush();
|
|
1608
1782
|
}
|
|
1609
1783
|
}
|
|
1610
1784
|
/**
|
|
1611
|
-
*
|
|
1785
|
+
* Export tracing event through all exporters (realtime mode)
|
|
1612
1786
|
*/
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
this.logger.
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
});
|
|
1622
|
-
} else if (this.buffer.totalSize === 1) {
|
|
1623
|
-
this.scheduleFlush();
|
|
1787
|
+
async exportTracingEvent(event) {
|
|
1788
|
+
const exportPromises = this.exporters.map(async (exporter) => {
|
|
1789
|
+
try {
|
|
1790
|
+
if (exporter.exportTracingEvent) {
|
|
1791
|
+
await exporter.exportTracingEvent(event);
|
|
1792
|
+
this.logger.debug(`[Observability] Event exported [exporter=${exporter.name}] [type=${event.type}]`);
|
|
1793
|
+
}
|
|
1794
|
+
} catch (error) {
|
|
1795
|
+
this.logger.error(`[Observability] Export error [exporter=${exporter.name}]`, error);
|
|
1624
1796
|
}
|
|
1625
|
-
}
|
|
1797
|
+
});
|
|
1798
|
+
await Promise.allSettled(exportPromises);
|
|
1799
|
+
}
|
|
1800
|
+
// ============================================================================
|
|
1801
|
+
// Lifecycle Management
|
|
1802
|
+
// ============================================================================
|
|
1803
|
+
/**
|
|
1804
|
+
* Initialize Observability (called by Mastra during component registration)
|
|
1805
|
+
*/
|
|
1806
|
+
init() {
|
|
1807
|
+
this.logger.debug(`[Observability] Initialization started [name=${this.name}]`);
|
|
1808
|
+
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
1626
1809
|
}
|
|
1627
1810
|
/**
|
|
1628
|
-
*
|
|
1811
|
+
* Shutdown Observability and clean up resources
|
|
1629
1812
|
*/
|
|
1630
|
-
|
|
1631
|
-
|
|
1813
|
+
async shutdown() {
|
|
1814
|
+
this.logger.debug(`[Observability] Shutdown started [name=${this.name}]`);
|
|
1815
|
+
const shutdownPromises = [
|
|
1816
|
+
...this.exporters.map((e) => e.shutdown()),
|
|
1817
|
+
...this.spanOutputProcessors.map((p) => p.shutdown())
|
|
1818
|
+
];
|
|
1819
|
+
await Promise.allSettled(shutdownPromises);
|
|
1820
|
+
this.logger.info(`[Observability] Shutdown completed [name=${this.name}]`);
|
|
1632
1821
|
}
|
|
1822
|
+
};
|
|
1823
|
+
|
|
1824
|
+
// src/instances/default.ts
|
|
1825
|
+
var DefaultObservabilityInstance = class extends BaseObservabilityInstance {
|
|
1826
|
+
constructor(config) {
|
|
1827
|
+
super(config);
|
|
1828
|
+
}
|
|
1829
|
+
createSpan(options) {
|
|
1830
|
+
return new DefaultSpan(options, this);
|
|
1831
|
+
}
|
|
1832
|
+
};
|
|
1833
|
+
|
|
1834
|
+
// src/registry.ts
|
|
1835
|
+
var ObservabilityRegistry = class {
|
|
1836
|
+
#instances = /* @__PURE__ */ new Map();
|
|
1837
|
+
#defaultInstance;
|
|
1838
|
+
#configSelector;
|
|
1633
1839
|
/**
|
|
1634
|
-
*
|
|
1840
|
+
* Register a tracing instance
|
|
1635
1841
|
*/
|
|
1636
|
-
|
|
1637
|
-
if (
|
|
1638
|
-
|
|
1639
|
-
return;
|
|
1640
|
-
}
|
|
1641
|
-
if (this.flushTimer) {
|
|
1642
|
-
clearTimeout(this.flushTimer);
|
|
1643
|
-
this.flushTimer = null;
|
|
1842
|
+
register(name, instance, isDefault = false) {
|
|
1843
|
+
if (this.#instances.has(name)) {
|
|
1844
|
+
throw new Error(`Tracing instance '${name}' already registered`);
|
|
1644
1845
|
}
|
|
1645
|
-
|
|
1646
|
-
|
|
1846
|
+
this.#instances.set(name, instance);
|
|
1847
|
+
if (isDefault || !this.#defaultInstance) {
|
|
1848
|
+
this.#defaultInstance = instance;
|
|
1647
1849
|
}
|
|
1648
|
-
const startTime = Date.now();
|
|
1649
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
1650
|
-
const bufferCopy = {
|
|
1651
|
-
creates: [...this.buffer.creates],
|
|
1652
|
-
updates: [...this.buffer.updates],
|
|
1653
|
-
insertOnly: [...this.buffer.insertOnly],
|
|
1654
|
-
seenSpans: new Set(this.buffer.seenSpans),
|
|
1655
|
-
spanSequences: new Map(this.buffer.spanSequences),
|
|
1656
|
-
completedSpans: new Set(this.buffer.completedSpans),
|
|
1657
|
-
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
1658
|
-
firstEventTime: this.buffer.firstEventTime,
|
|
1659
|
-
totalSize: this.buffer.totalSize
|
|
1660
|
-
};
|
|
1661
|
-
this.resetBuffer();
|
|
1662
|
-
await this.flushWithRetries(this.storage, bufferCopy, 0);
|
|
1663
|
-
const elapsed = Date.now() - startTime;
|
|
1664
|
-
this.logger.debug("Batch flushed", {
|
|
1665
|
-
strategy: this.resolvedStrategy,
|
|
1666
|
-
batchSize: bufferCopy.totalSize,
|
|
1667
|
-
flushReason,
|
|
1668
|
-
durationMs: elapsed,
|
|
1669
|
-
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
1670
|
-
});
|
|
1671
1850
|
}
|
|
1672
1851
|
/**
|
|
1673
|
-
*
|
|
1852
|
+
* Get a tracing instance by name
|
|
1674
1853
|
*/
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
this.
|
|
1698
|
-
}
|
|
1699
|
-
} catch (error) {
|
|
1700
|
-
if (attempt < this.config.maxRetries) {
|
|
1701
|
-
const retryDelay = this.calculateRetryDelay(attempt);
|
|
1702
|
-
this.logger.warn("Batch flush failed, retrying", {
|
|
1703
|
-
attempt: attempt + 1,
|
|
1704
|
-
maxRetries: this.config.maxRetries,
|
|
1705
|
-
nextRetryInMs: retryDelay,
|
|
1706
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1707
|
-
});
|
|
1708
|
-
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
1709
|
-
return this.flushWithRetries(storage, buffer, attempt + 1);
|
|
1710
|
-
} else {
|
|
1711
|
-
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
1712
|
-
finalAttempt: attempt + 1,
|
|
1713
|
-
maxRetries: this.config.maxRetries,
|
|
1714
|
-
droppedBatchSize: buffer.totalSize,
|
|
1715
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1716
|
-
});
|
|
1717
|
-
for (const spanKey of buffer.completedSpans) {
|
|
1718
|
-
this.allCreatedSpans.delete(spanKey);
|
|
1719
|
-
}
|
|
1854
|
+
get(name) {
|
|
1855
|
+
return this.#instances.get(name);
|
|
1856
|
+
}
|
|
1857
|
+
/**
|
|
1858
|
+
* Get the default tracing instance
|
|
1859
|
+
*/
|
|
1860
|
+
getDefault() {
|
|
1861
|
+
return this.#defaultInstance;
|
|
1862
|
+
}
|
|
1863
|
+
/**
|
|
1864
|
+
* Set the tracing selector function
|
|
1865
|
+
*/
|
|
1866
|
+
setSelector(selector) {
|
|
1867
|
+
this.#configSelector = selector;
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* Get the selected tracing instance based on context
|
|
1871
|
+
*/
|
|
1872
|
+
getSelected(options) {
|
|
1873
|
+
if (this.#configSelector) {
|
|
1874
|
+
const selected = this.#configSelector(options, this.#instances);
|
|
1875
|
+
if (selected && this.#instances.has(selected)) {
|
|
1876
|
+
return this.#instances.get(selected);
|
|
1720
1877
|
}
|
|
1721
1878
|
}
|
|
1879
|
+
return this.#defaultInstance;
|
|
1722
1880
|
}
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
case "realtime":
|
|
1733
|
-
await this.handleRealtimeEvent(event, this.storage);
|
|
1734
|
-
break;
|
|
1735
|
-
case "batch-with-updates":
|
|
1736
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
1737
|
-
break;
|
|
1738
|
-
case "insert-only":
|
|
1739
|
-
this.handleInsertOnlyEvent(event);
|
|
1740
|
-
break;
|
|
1881
|
+
/**
|
|
1882
|
+
* Unregister a tracing instance
|
|
1883
|
+
*/
|
|
1884
|
+
unregister(name) {
|
|
1885
|
+
const instance = this.#instances.get(name);
|
|
1886
|
+
const deleted = this.#instances.delete(name);
|
|
1887
|
+
if (deleted && instance === this.#defaultInstance) {
|
|
1888
|
+
const next = this.#instances.values().next();
|
|
1889
|
+
this.#defaultInstance = next.done ? void 0 : next.value;
|
|
1741
1890
|
}
|
|
1891
|
+
return deleted;
|
|
1742
1892
|
}
|
|
1893
|
+
/**
|
|
1894
|
+
* Shutdown all instances and clear the registry
|
|
1895
|
+
*/
|
|
1743
1896
|
async shutdown() {
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1897
|
+
const shutdownPromises = Array.from(this.#instances.values()).map((instance) => instance.shutdown());
|
|
1898
|
+
await Promise.allSettled(shutdownPromises);
|
|
1899
|
+
this.#instances.clear();
|
|
1900
|
+
this.#instances.clear();
|
|
1901
|
+
this.#defaultInstance = void 0;
|
|
1902
|
+
this.#configSelector = void 0;
|
|
1903
|
+
}
|
|
1904
|
+
/**
|
|
1905
|
+
* Clear all instances without shutdown
|
|
1906
|
+
*/
|
|
1907
|
+
clear() {
|
|
1908
|
+
this.#instances.clear();
|
|
1909
|
+
this.#defaultInstance = void 0;
|
|
1910
|
+
this.#configSelector = void 0;
|
|
1911
|
+
}
|
|
1912
|
+
/**
|
|
1913
|
+
* list all registered instances
|
|
1914
|
+
*/
|
|
1915
|
+
list() {
|
|
1916
|
+
return new Map(this.#instances);
|
|
1761
1917
|
}
|
|
1762
1918
|
};
|
|
1763
1919
|
|
|
@@ -1882,151 +2038,137 @@ var SensitiveDataFilter = class {
|
|
|
1882
2038
|
async shutdown() {
|
|
1883
2039
|
}
|
|
1884
2040
|
};
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
2041
|
+
|
|
2042
|
+
// src/default.ts
|
|
2043
|
+
function isInstance(obj) {
|
|
2044
|
+
return obj instanceof BaseObservabilityInstance;
|
|
2045
|
+
}
|
|
2046
|
+
var Observability = class extends MastraBase {
|
|
2047
|
+
#registry = new ObservabilityRegistry();
|
|
2048
|
+
constructor(config) {
|
|
2049
|
+
super({
|
|
2050
|
+
component: RegisteredLogger.OBSERVABILITY,
|
|
2051
|
+
name: "Observability"
|
|
2052
|
+
});
|
|
2053
|
+
if (config === void 0) {
|
|
2054
|
+
config = {};
|
|
2055
|
+
}
|
|
2056
|
+
const validationResult = observabilityRegistryConfigSchema.safeParse(config);
|
|
2057
|
+
if (!validationResult.success) {
|
|
2058
|
+
const errorMessages = validationResult.error.errors.map((err) => `${err.path.join(".") || "config"}: ${err.message}`).join("; ");
|
|
2059
|
+
throw new MastraError({
|
|
2060
|
+
id: "OBSERVABILITY_INVALID_CONFIG",
|
|
2061
|
+
text: `Invalid observability configuration: ${errorMessages}`,
|
|
2062
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
2063
|
+
category: ErrorCategory.USER,
|
|
2064
|
+
details: {
|
|
2065
|
+
validationErrors: errorMessages
|
|
2066
|
+
}
|
|
2067
|
+
});
|
|
2068
|
+
}
|
|
2069
|
+
if (config.configs) {
|
|
2070
|
+
for (const [name, configValue] of Object.entries(config.configs)) {
|
|
2071
|
+
if (!isInstance(configValue)) {
|
|
2072
|
+
const configValidation = observabilityConfigValueSchema.safeParse(configValue);
|
|
2073
|
+
if (!configValidation.success) {
|
|
2074
|
+
const errorMessages = configValidation.error.errors.map((err) => `${err.path.join(".")}: ${err.message}`).join("; ");
|
|
2075
|
+
throw new MastraError({
|
|
2076
|
+
id: "OBSERVABILITY_INVALID_INSTANCE_CONFIG",
|
|
2077
|
+
text: `Invalid configuration for observability instance '${name}': ${errorMessages}`,
|
|
2078
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
2079
|
+
category: ErrorCategory.USER,
|
|
2080
|
+
details: {
|
|
2081
|
+
instanceName: name,
|
|
2082
|
+
validationErrors: errorMessages
|
|
2083
|
+
}
|
|
2084
|
+
});
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
1895
2088
|
}
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
2089
|
+
if (config.default?.enabled) {
|
|
2090
|
+
const defaultInstance = new DefaultObservabilityInstance({
|
|
2091
|
+
serviceName: "mastra",
|
|
2092
|
+
name: "default",
|
|
2093
|
+
sampling: { type: "always" /* ALWAYS */ },
|
|
2094
|
+
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
2095
|
+
spanOutputProcessors: [new SensitiveDataFilter()]
|
|
2096
|
+
});
|
|
2097
|
+
this.#registry.register("default", defaultInstance, true);
|
|
2098
|
+
}
|
|
2099
|
+
if (config.configs) {
|
|
2100
|
+
const instances = Object.entries(config.configs);
|
|
2101
|
+
instances.forEach(([name, tracingDef], index) => {
|
|
2102
|
+
const instance = isInstance(tracingDef) ? tracingDef : new DefaultObservabilityInstance({ ...tracingDef, name });
|
|
2103
|
+
const isDefault = !config.default?.enabled && index === 0;
|
|
2104
|
+
this.#registry.register(name, instance, isDefault);
|
|
2105
|
+
});
|
|
1899
2106
|
}
|
|
2107
|
+
if (config.configSelector) {
|
|
2108
|
+
this.#registry.setSelector(config.configSelector);
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
setMastraContext(options) {
|
|
2112
|
+
const instances = this.listInstances();
|
|
2113
|
+
const { mastra } = options;
|
|
2114
|
+
instances.forEach((instance) => {
|
|
2115
|
+
const config = instance.getConfig();
|
|
2116
|
+
const exporters = instance.getExporters();
|
|
2117
|
+
exporters.forEach((exporter) => {
|
|
2118
|
+
if ("init" in exporter && typeof exporter.init === "function") {
|
|
2119
|
+
try {
|
|
2120
|
+
exporter.init({ mastra, config });
|
|
2121
|
+
} catch (error) {
|
|
2122
|
+
this.logger?.warn("Failed to initialize observability exporter", {
|
|
2123
|
+
exporterName: exporter.name,
|
|
2124
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2125
|
+
});
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
});
|
|
2129
|
+
});
|
|
1900
2130
|
}
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
}
|
|
1907
|
-
/**
|
|
1908
|
-
* Get the default tracing instance
|
|
1909
|
-
*/
|
|
1910
|
-
getDefault() {
|
|
1911
|
-
return this.defaultInstance;
|
|
2131
|
+
setLogger(options) {
|
|
2132
|
+
super.__setLogger(options.logger);
|
|
2133
|
+
this.listInstances().forEach((instance) => {
|
|
2134
|
+
instance.__setLogger(options.logger);
|
|
2135
|
+
});
|
|
1912
2136
|
}
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
*/
|
|
1916
|
-
setSelector(selector) {
|
|
1917
|
-
this.configSelector = selector;
|
|
2137
|
+
getSelectedInstance(options) {
|
|
2138
|
+
return this.#registry.getSelected(options);
|
|
1918
2139
|
}
|
|
1919
2140
|
/**
|
|
1920
|
-
*
|
|
2141
|
+
* Registry management methods
|
|
1921
2142
|
*/
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
const selected = this.configSelector(options, this.instances);
|
|
1925
|
-
if (selected && this.instances.has(selected)) {
|
|
1926
|
-
return this.instances.get(selected);
|
|
1927
|
-
}
|
|
1928
|
-
}
|
|
1929
|
-
return this.defaultInstance;
|
|
2143
|
+
registerInstance(name, instance, isDefault = false) {
|
|
2144
|
+
this.#registry.register(name, instance, isDefault);
|
|
1930
2145
|
}
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
*/
|
|
1934
|
-
unregister(name) {
|
|
1935
|
-
return this.instances.delete(name);
|
|
2146
|
+
getInstance(name) {
|
|
2147
|
+
return this.#registry.get(name);
|
|
1936
2148
|
}
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
*/
|
|
1940
|
-
async shutdown() {
|
|
1941
|
-
const shutdownPromises = Array.from(this.instances.values()).map((instance) => instance.shutdown());
|
|
1942
|
-
await Promise.allSettled(shutdownPromises);
|
|
1943
|
-
this.instances.clear();
|
|
2149
|
+
getDefaultInstance() {
|
|
2150
|
+
return this.#registry.getDefault();
|
|
1944
2151
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
*/
|
|
1948
|
-
clear() {
|
|
1949
|
-
this.instances.clear();
|
|
1950
|
-
this.defaultInstance = void 0;
|
|
1951
|
-
this.configSelector = void 0;
|
|
2152
|
+
listInstances() {
|
|
2153
|
+
return this.#registry.list();
|
|
1952
2154
|
}
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
*/
|
|
1956
|
-
getAll() {
|
|
1957
|
-
return new Map(this.instances);
|
|
2155
|
+
unregisterInstance(name) {
|
|
2156
|
+
return this.#registry.unregister(name);
|
|
1958
2157
|
}
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
function registerAITracing(name, instance, isDefault = false) {
|
|
1962
|
-
aiTracingRegistry.register(name, instance, isDefault);
|
|
1963
|
-
}
|
|
1964
|
-
function getAITracing(name) {
|
|
1965
|
-
return aiTracingRegistry.get(name);
|
|
1966
|
-
}
|
|
1967
|
-
function getDefaultAITracing() {
|
|
1968
|
-
return aiTracingRegistry.getDefault();
|
|
1969
|
-
}
|
|
1970
|
-
function setSelector(selector) {
|
|
1971
|
-
aiTracingRegistry.setSelector(selector);
|
|
1972
|
-
}
|
|
1973
|
-
function getSelectedAITracing(options) {
|
|
1974
|
-
return aiTracingRegistry.getSelected(options);
|
|
1975
|
-
}
|
|
1976
|
-
function unregisterAITracing(name) {
|
|
1977
|
-
return aiTracingRegistry.unregister(name);
|
|
1978
|
-
}
|
|
1979
|
-
async function shutdownAITracingRegistry() {
|
|
1980
|
-
await aiTracingRegistry.shutdown();
|
|
1981
|
-
}
|
|
1982
|
-
function clearAITracingRegistry() {
|
|
1983
|
-
aiTracingRegistry.clear();
|
|
1984
|
-
}
|
|
1985
|
-
function getAllAITracing() {
|
|
1986
|
-
return aiTracingRegistry.getAll();
|
|
1987
|
-
}
|
|
1988
|
-
function hasAITracing(name) {
|
|
1989
|
-
const tracing = getAITracing(name);
|
|
1990
|
-
if (!tracing) return false;
|
|
1991
|
-
const config = tracing.getConfig();
|
|
1992
|
-
const sampling = config.sampling;
|
|
1993
|
-
return sampling.type !== SamplingStrategyType.NEVER;
|
|
1994
|
-
}
|
|
1995
|
-
function isAITracingInstance(obj) {
|
|
1996
|
-
return obj instanceof BaseAITracing;
|
|
1997
|
-
}
|
|
1998
|
-
function setupAITracingRegistry(config) {
|
|
1999
|
-
if (!config) {
|
|
2000
|
-
return;
|
|
2158
|
+
hasInstance(name) {
|
|
2159
|
+
return !!this.#registry.get(name);
|
|
2001
2160
|
}
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
"Cannot use 'default' as a custom config name when default tracing is enabled. Please rename your custom config to avoid conflicts."
|
|
2005
|
-
);
|
|
2161
|
+
setConfigSelector(selector) {
|
|
2162
|
+
this.#registry.setSelector(selector);
|
|
2006
2163
|
}
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
serviceName: "mastra",
|
|
2010
|
-
name: "default",
|
|
2011
|
-
sampling: { type: SamplingStrategyType.ALWAYS },
|
|
2012
|
-
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
2013
|
-
processors: [new SensitiveDataFilter()]
|
|
2014
|
-
});
|
|
2015
|
-
registerAITracing("default", defaultInstance, true);
|
|
2016
|
-
}
|
|
2017
|
-
if (config.configs) {
|
|
2018
|
-
const instances = Object.entries(config.configs);
|
|
2019
|
-
instances.forEach(([name, tracingDef], index) => {
|
|
2020
|
-
const instance = isAITracingInstance(tracingDef) ? tracingDef : new DefaultAITracing({ ...tracingDef, name });
|
|
2021
|
-
const isDefault = !config.default?.enabled && index === 0;
|
|
2022
|
-
registerAITracing(name, instance, isDefault);
|
|
2023
|
-
});
|
|
2164
|
+
clear() {
|
|
2165
|
+
this.#registry.clear();
|
|
2024
2166
|
}
|
|
2025
|
-
|
|
2026
|
-
|
|
2167
|
+
async shutdown() {
|
|
2168
|
+
await this.#registry.shutdown();
|
|
2027
2169
|
}
|
|
2028
|
-
}
|
|
2170
|
+
};
|
|
2029
2171
|
|
|
2030
|
-
export {
|
|
2172
|
+
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, deepClean, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema };
|
|
2031
2173
|
//# sourceMappingURL=index.js.map
|
|
2032
2174
|
//# sourceMappingURL=index.js.map
|