@mastra/observability 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/LICENSE.md +15 -0
- package/README.md +62 -1
- package/dist/bus/base.d.ts +43 -0
- package/dist/bus/base.d.ts.map +1 -0
- package/dist/bus/index.d.ts +7 -0
- package/dist/bus/index.d.ts.map +1 -0
- package/dist/bus/observability-bus.d.ts +97 -0
- package/dist/bus/observability-bus.d.ts.map +1 -0
- package/dist/bus/route-event.d.ts +31 -0
- package/dist/bus/route-event.d.ts.map +1 -0
- package/dist/context/index.d.ts +6 -0
- package/dist/context/index.d.ts.map +1 -0
- package/dist/context/logger.d.ts +45 -0
- package/dist/context/logger.d.ts.map +1 -0
- package/dist/context/metrics.d.ts +47 -0
- package/dist/context/metrics.d.ts.map +1 -0
- package/dist/exporters/base.d.ts +11 -0
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +0 -1
- package/dist/exporters/index.d.ts.map +1 -1
- package/dist/exporters/test.d.ts +538 -3
- package/dist/exporters/test.d.ts.map +1 -1
- package/dist/index.cjs +1165 -104
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1155 -101
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +77 -10
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/metrics/auto-extract.d.ts +47 -0
- package/dist/metrics/auto-extract.d.ts.map +1 -0
- package/dist/metrics/cardinality.d.ts +24 -0
- package/dist/metrics/cardinality.d.ts.map +1 -0
- package/dist/metrics/index.d.ts +6 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/package.json +6 -6
- package/dist/exporters/json.d.ts +0 -386
- package/dist/exporters/json.d.ts.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -5,9 +5,6 @@ var error = require('@mastra/core/error');
|
|
|
5
5
|
var logger = require('@mastra/core/logger');
|
|
6
6
|
var observability = require('@mastra/core/observability');
|
|
7
7
|
var utils = require('@mastra/core/utils');
|
|
8
|
-
var promises = require('fs/promises');
|
|
9
|
-
var path = require('path');
|
|
10
|
-
var url = require('url');
|
|
11
8
|
var web = require('stream/web');
|
|
12
9
|
|
|
13
10
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
@@ -4245,6 +4242,19 @@ var BaseExporter = class {
|
|
|
4245
4242
|
}
|
|
4246
4243
|
return event;
|
|
4247
4244
|
}
|
|
4245
|
+
/**
|
|
4246
|
+
* Default onTracingEvent handler that delegates to exportTracingEvent.
|
|
4247
|
+
*
|
|
4248
|
+
* This provides backward compatibility: existing exporters that only implement
|
|
4249
|
+
* _exportTracingEvent will automatically receive tracing events routed through
|
|
4250
|
+
* the ObservabilityBus. Subclasses can override this if they need different
|
|
4251
|
+
* routing behavior for bus-delivered events.
|
|
4252
|
+
*
|
|
4253
|
+
* Handler presence on ObservabilityExporter = signal support.
|
|
4254
|
+
*/
|
|
4255
|
+
onTracingEvent(event) {
|
|
4256
|
+
return this.exportTracingEvent(event);
|
|
4257
|
+
}
|
|
4248
4258
|
/**
|
|
4249
4259
|
* Export a tracing event
|
|
4250
4260
|
*
|
|
@@ -6073,59 +6083,58 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6073
6083
|
this.logger.info("DefaultExporter shutdown complete");
|
|
6074
6084
|
}
|
|
6075
6085
|
};
|
|
6076
|
-
|
|
6077
|
-
// src/exporters/test.ts
|
|
6078
|
-
var TestExporter = class extends BaseExporter {
|
|
6079
|
-
name = "tracing-test-exporter";
|
|
6080
|
-
#events = [];
|
|
6081
|
-
constructor(config = {}) {
|
|
6082
|
-
super(config);
|
|
6083
|
-
}
|
|
6084
|
-
async _exportTracingEvent(event) {
|
|
6085
|
-
this.#events.push(event);
|
|
6086
|
-
}
|
|
6087
|
-
clearEvents() {
|
|
6088
|
-
this.#events = [];
|
|
6089
|
-
}
|
|
6090
|
-
get events() {
|
|
6091
|
-
return this.#events;
|
|
6092
|
-
}
|
|
6093
|
-
async shutdown() {
|
|
6094
|
-
this.logger.info("TestExporter shutdown");
|
|
6095
|
-
}
|
|
6096
|
-
};
|
|
6097
6086
|
var _snapshotsDir;
|
|
6098
|
-
function getSnapshotsDir() {
|
|
6087
|
+
async function getSnapshotsDir() {
|
|
6099
6088
|
if (!_snapshotsDir) {
|
|
6100
6089
|
if (typeof (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)) !== "string") {
|
|
6101
6090
|
throw new Error(
|
|
6102
6091
|
"Snapshot functionality requires a Node.js environment. import.meta.url is not available in this runtime."
|
|
6103
6092
|
);
|
|
6104
6093
|
}
|
|
6105
|
-
const
|
|
6106
|
-
const
|
|
6107
|
-
|
|
6094
|
+
const { fileURLToPath } = await import('url');
|
|
6095
|
+
const { dirname, join } = await import('path');
|
|
6096
|
+
const __filename = fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
6097
|
+
const __dirname = dirname(__filename);
|
|
6098
|
+
_snapshotsDir = join(__dirname, "..", "__snapshots__");
|
|
6108
6099
|
}
|
|
6109
6100
|
return _snapshotsDir;
|
|
6110
6101
|
}
|
|
6111
|
-
var
|
|
6112
|
-
name = "
|
|
6113
|
-
/** All collected events */
|
|
6114
|
-
#
|
|
6102
|
+
var TestExporter = class extends BaseExporter {
|
|
6103
|
+
name = "test-exporter";
|
|
6104
|
+
/** All collected tracing events */
|
|
6105
|
+
#tracingEvents = [];
|
|
6115
6106
|
/** Per-span state tracking */
|
|
6116
6107
|
#spanStates = /* @__PURE__ */ new Map();
|
|
6117
|
-
/**
|
|
6118
|
-
#
|
|
6108
|
+
/** All collected log events */
|
|
6109
|
+
#logEvents = [];
|
|
6110
|
+
/** All collected metric events */
|
|
6111
|
+
#metricEvents = [];
|
|
6112
|
+
/** All collected score events */
|
|
6113
|
+
#scoreEvents = [];
|
|
6114
|
+
/** All collected feedback events */
|
|
6115
|
+
#feedbackEvents = [];
|
|
6116
|
+
/** Debug logs for the exporter itself */
|
|
6117
|
+
#debugLogs = [];
|
|
6119
6118
|
/** Configuration */
|
|
6120
6119
|
#config;
|
|
6120
|
+
/** Internal metrics tracking */
|
|
6121
|
+
#internalMetrics;
|
|
6121
6122
|
constructor(config = {}) {
|
|
6122
6123
|
super(config);
|
|
6123
6124
|
this.#config = {
|
|
6124
6125
|
validateLifecycle: true,
|
|
6125
6126
|
storeLogs: true,
|
|
6126
6127
|
jsonIndent: 2,
|
|
6128
|
+
logMetricsOnFlush: true,
|
|
6127
6129
|
...config
|
|
6128
6130
|
};
|
|
6131
|
+
this.#internalMetrics = {
|
|
6132
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
6133
|
+
lastEventAt: null,
|
|
6134
|
+
totalEventsReceived: 0,
|
|
6135
|
+
bySignal: { tracing: 0, log: 0, metric: 0, score: 0, feedback: 0 },
|
|
6136
|
+
flushCount: 0
|
|
6137
|
+
};
|
|
6129
6138
|
}
|
|
6130
6139
|
/**
|
|
6131
6140
|
* Process incoming tracing events with lifecycle tracking
|
|
@@ -6133,9 +6142,10 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6133
6142
|
async _exportTracingEvent(event) {
|
|
6134
6143
|
const span = event.exportedSpan;
|
|
6135
6144
|
const spanId = span.id;
|
|
6136
|
-
|
|
6145
|
+
this.#trackEvent("tracing");
|
|
6146
|
+
const logMessage = `[TestExporter] ${event.type}: ${span.type} "${span.name}" (entity: ${span.entityName ?? span.entityId ?? "unknown"}, trace: ${span.traceId.slice(-8)}, span: ${spanId.slice(-8)})`;
|
|
6137
6147
|
if (this.#config.storeLogs) {
|
|
6138
|
-
this.#
|
|
6148
|
+
this.#debugLogs.push(logMessage);
|
|
6139
6149
|
}
|
|
6140
6150
|
const state = this.#spanStates.get(spanId) || {
|
|
6141
6151
|
hasStart: false,
|
|
@@ -6158,7 +6168,67 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6158
6168
|
}
|
|
6159
6169
|
state.events.push(event);
|
|
6160
6170
|
this.#spanStates.set(spanId, state);
|
|
6161
|
-
this.#
|
|
6171
|
+
this.#tracingEvents.push(event);
|
|
6172
|
+
}
|
|
6173
|
+
// ============================================================================
|
|
6174
|
+
// Signal Handlers (Logs, Metrics, Scores, Feedback)
|
|
6175
|
+
// ============================================================================
|
|
6176
|
+
/**
|
|
6177
|
+
* Process incoming log events
|
|
6178
|
+
*/
|
|
6179
|
+
async onLogEvent(event) {
|
|
6180
|
+
this.#trackEvent("log");
|
|
6181
|
+
if (this.#config.storeLogs) {
|
|
6182
|
+
const log = event.log;
|
|
6183
|
+
const logMessage = `[TestExporter] log.${log.level}: "${log.message}"${log.traceId ? ` (trace: ${log.traceId.slice(-8)})` : ""}`;
|
|
6184
|
+
this.#debugLogs.push(logMessage);
|
|
6185
|
+
}
|
|
6186
|
+
this.#logEvents.push(event);
|
|
6187
|
+
}
|
|
6188
|
+
/**
|
|
6189
|
+
* Process incoming metric events
|
|
6190
|
+
*/
|
|
6191
|
+
async onMetricEvent(event) {
|
|
6192
|
+
this.#trackEvent("metric");
|
|
6193
|
+
if (this.#config.storeLogs) {
|
|
6194
|
+
const metric = event.metric;
|
|
6195
|
+
const labelsStr = Object.entries(metric.labels).map(([k, v]) => `${k}=${v}`).join(", ");
|
|
6196
|
+
const logMessage = `[TestExporter] metric.${metric.metricType}: ${metric.name}=${metric.value}${labelsStr ? ` {${labelsStr}}` : ""}`;
|
|
6197
|
+
this.#debugLogs.push(logMessage);
|
|
6198
|
+
}
|
|
6199
|
+
this.#metricEvents.push(event);
|
|
6200
|
+
}
|
|
6201
|
+
/**
|
|
6202
|
+
* Process incoming score events
|
|
6203
|
+
*/
|
|
6204
|
+
async onScoreEvent(event) {
|
|
6205
|
+
this.#trackEvent("score");
|
|
6206
|
+
if (this.#config.storeLogs) {
|
|
6207
|
+
const score = event.score;
|
|
6208
|
+
const logMessage = `[TestExporter] score: ${score.scorerName}=${score.score} (trace: ${score.traceId.slice(-8)}${score.spanId ? `, span: ${score.spanId.slice(-8)}` : ""})`;
|
|
6209
|
+
this.#debugLogs.push(logMessage);
|
|
6210
|
+
}
|
|
6211
|
+
this.#scoreEvents.push(event);
|
|
6212
|
+
}
|
|
6213
|
+
/**
|
|
6214
|
+
* Process incoming feedback events
|
|
6215
|
+
*/
|
|
6216
|
+
async onFeedbackEvent(event) {
|
|
6217
|
+
this.#trackEvent("feedback");
|
|
6218
|
+
if (this.#config.storeLogs) {
|
|
6219
|
+
const fb = event.feedback;
|
|
6220
|
+
const logMessage = `[TestExporter] feedback: ${fb.feedbackType} from ${fb.source}=${fb.value} (trace: ${fb.traceId.slice(-8)}${fb.spanId ? `, span: ${fb.spanId.slice(-8)}` : ""})`;
|
|
6221
|
+
this.#debugLogs.push(logMessage);
|
|
6222
|
+
}
|
|
6223
|
+
this.#feedbackEvents.push(event);
|
|
6224
|
+
}
|
|
6225
|
+
/**
|
|
6226
|
+
* Track an event for internal metrics
|
|
6227
|
+
*/
|
|
6228
|
+
#trackEvent(signal) {
|
|
6229
|
+
this.#internalMetrics.lastEventAt = /* @__PURE__ */ new Date();
|
|
6230
|
+
this.#internalMetrics.totalEventsReceived++;
|
|
6231
|
+
this.#internalMetrics.bySignal[signal]++;
|
|
6162
6232
|
}
|
|
6163
6233
|
/**
|
|
6164
6234
|
* Validate span lifecycle rules
|
|
@@ -6185,13 +6255,13 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6185
6255
|
}
|
|
6186
6256
|
}
|
|
6187
6257
|
// ============================================================================
|
|
6188
|
-
// Query Methods
|
|
6258
|
+
// Tracing Query Methods
|
|
6189
6259
|
// ============================================================================
|
|
6190
6260
|
/**
|
|
6191
|
-
* Get all collected events
|
|
6261
|
+
* Get all collected tracing events
|
|
6192
6262
|
*/
|
|
6193
6263
|
get events() {
|
|
6194
|
-
return [...this.#
|
|
6264
|
+
return [...this.#tracingEvents];
|
|
6195
6265
|
}
|
|
6196
6266
|
/**
|
|
6197
6267
|
* Get completed spans by SpanType (e.g., 'agent_run', 'tool_call')
|
|
@@ -6216,18 +6286,21 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6216
6286
|
* @returns Array of events of the specified type
|
|
6217
6287
|
*/
|
|
6218
6288
|
getByEventType(type) {
|
|
6219
|
-
return this.#
|
|
6289
|
+
return this.#tracingEvents.filter((e) => e.type === type);
|
|
6220
6290
|
}
|
|
6221
6291
|
/**
|
|
6222
6292
|
* Get all events and spans for a specific trace
|
|
6223
6293
|
*
|
|
6224
6294
|
* @param traceId - The trace ID to filter by
|
|
6225
|
-
* @returns Object containing events
|
|
6295
|
+
* @returns Object containing tracing events, final spans, plus logs/scores/feedback for the trace
|
|
6226
6296
|
*/
|
|
6227
6297
|
getByTraceId(traceId) {
|
|
6228
|
-
const events = this.#
|
|
6298
|
+
const events = this.#tracingEvents.filter((e) => e.exportedSpan.traceId === traceId);
|
|
6229
6299
|
const spans = this.#getUniqueSpansFromEvents(events);
|
|
6230
|
-
|
|
6300
|
+
const logs = this.#logEvents.filter((e) => e.log.traceId === traceId).map((e) => e.log);
|
|
6301
|
+
const scores = this.#scoreEvents.filter((e) => e.score.traceId === traceId).map((e) => e.score);
|
|
6302
|
+
const feedback = this.#feedbackEvents.filter((e) => e.feedback.traceId === traceId).map((e) => e.feedback);
|
|
6303
|
+
return { events, spans, logs, scores, feedback };
|
|
6231
6304
|
}
|
|
6232
6305
|
/**
|
|
6233
6306
|
* Get all events for a specific span
|
|
@@ -6283,20 +6356,137 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6283
6356
|
}));
|
|
6284
6357
|
}
|
|
6285
6358
|
/**
|
|
6286
|
-
* Get unique trace IDs from all collected
|
|
6359
|
+
* Get unique trace IDs from all collected signals
|
|
6287
6360
|
*/
|
|
6288
6361
|
getTraceIds() {
|
|
6289
6362
|
const traceIds = /* @__PURE__ */ new Set();
|
|
6290
|
-
for (const event of this.#
|
|
6363
|
+
for (const event of this.#tracingEvents) {
|
|
6291
6364
|
traceIds.add(event.exportedSpan.traceId);
|
|
6292
6365
|
}
|
|
6366
|
+
for (const event of this.#logEvents) {
|
|
6367
|
+
if (event.log.traceId) traceIds.add(event.log.traceId);
|
|
6368
|
+
}
|
|
6369
|
+
for (const event of this.#scoreEvents) {
|
|
6370
|
+
traceIds.add(event.score.traceId);
|
|
6371
|
+
}
|
|
6372
|
+
for (const event of this.#feedbackEvents) {
|
|
6373
|
+
traceIds.add(event.feedback.traceId);
|
|
6374
|
+
}
|
|
6293
6375
|
return Array.from(traceIds);
|
|
6294
6376
|
}
|
|
6295
6377
|
// ============================================================================
|
|
6378
|
+
// Log Query Methods
|
|
6379
|
+
// ============================================================================
|
|
6380
|
+
/**
|
|
6381
|
+
* Get all collected log events
|
|
6382
|
+
*/
|
|
6383
|
+
getLogEvents() {
|
|
6384
|
+
return [...this.#logEvents];
|
|
6385
|
+
}
|
|
6386
|
+
/**
|
|
6387
|
+
* Get all collected logs (unwrapped from events)
|
|
6388
|
+
*/
|
|
6389
|
+
getAllLogs() {
|
|
6390
|
+
return this.#logEvents.map((e) => e.log);
|
|
6391
|
+
}
|
|
6392
|
+
/**
|
|
6393
|
+
* Get logs filtered by level
|
|
6394
|
+
*/
|
|
6395
|
+
getLogsByLevel(level) {
|
|
6396
|
+
return this.#logEvents.filter((e) => e.log.level === level).map((e) => e.log);
|
|
6397
|
+
}
|
|
6398
|
+
/**
|
|
6399
|
+
* Get logs for a specific trace
|
|
6400
|
+
*/
|
|
6401
|
+
getLogsByTraceId(traceId) {
|
|
6402
|
+
return this.#logEvents.filter((e) => e.log.traceId === traceId).map((e) => e.log);
|
|
6403
|
+
}
|
|
6404
|
+
// ============================================================================
|
|
6405
|
+
// Metric Query Methods
|
|
6406
|
+
// ============================================================================
|
|
6407
|
+
/**
|
|
6408
|
+
* Get all collected metric events
|
|
6409
|
+
*/
|
|
6410
|
+
getMetricEvents() {
|
|
6411
|
+
return [...this.#metricEvents];
|
|
6412
|
+
}
|
|
6413
|
+
/**
|
|
6414
|
+
* Get all collected metrics (unwrapped from events)
|
|
6415
|
+
*/
|
|
6416
|
+
getAllMetrics() {
|
|
6417
|
+
return this.#metricEvents.map((e) => e.metric);
|
|
6418
|
+
}
|
|
6419
|
+
/**
|
|
6420
|
+
* Get metrics filtered by name
|
|
6421
|
+
*/
|
|
6422
|
+
getMetricsByName(name) {
|
|
6423
|
+
return this.#metricEvents.filter((e) => e.metric.name === name).map((e) => e.metric);
|
|
6424
|
+
}
|
|
6425
|
+
/**
|
|
6426
|
+
* Get metrics filtered by type
|
|
6427
|
+
*/
|
|
6428
|
+
getMetricsByType(metricType) {
|
|
6429
|
+
return this.#metricEvents.filter((e) => e.metric.metricType === metricType).map((e) => e.metric);
|
|
6430
|
+
}
|
|
6431
|
+
// ============================================================================
|
|
6432
|
+
// Score Query Methods
|
|
6433
|
+
// ============================================================================
|
|
6434
|
+
/**
|
|
6435
|
+
* Get all collected score events
|
|
6436
|
+
*/
|
|
6437
|
+
getScoreEvents() {
|
|
6438
|
+
return [...this.#scoreEvents];
|
|
6439
|
+
}
|
|
6440
|
+
/**
|
|
6441
|
+
* Get all collected scores (unwrapped from events)
|
|
6442
|
+
*/
|
|
6443
|
+
getAllScores() {
|
|
6444
|
+
return this.#scoreEvents.map((e) => e.score);
|
|
6445
|
+
}
|
|
6446
|
+
/**
|
|
6447
|
+
* Get scores filtered by scorer name
|
|
6448
|
+
*/
|
|
6449
|
+
getScoresByScorer(scorerName) {
|
|
6450
|
+
return this.#scoreEvents.filter((e) => e.score.scorerName === scorerName).map((e) => e.score);
|
|
6451
|
+
}
|
|
6452
|
+
/**
|
|
6453
|
+
* Get scores for a specific trace
|
|
6454
|
+
*/
|
|
6455
|
+
getScoresByTraceId(traceId) {
|
|
6456
|
+
return this.#scoreEvents.filter((e) => e.score.traceId === traceId).map((e) => e.score);
|
|
6457
|
+
}
|
|
6458
|
+
// ============================================================================
|
|
6459
|
+
// Feedback Query Methods
|
|
6460
|
+
// ============================================================================
|
|
6461
|
+
/**
|
|
6462
|
+
* Get all collected feedback events
|
|
6463
|
+
*/
|
|
6464
|
+
getFeedbackEvents() {
|
|
6465
|
+
return [...this.#feedbackEvents];
|
|
6466
|
+
}
|
|
6467
|
+
/**
|
|
6468
|
+
* Get all collected feedback (unwrapped from events)
|
|
6469
|
+
*/
|
|
6470
|
+
getAllFeedback() {
|
|
6471
|
+
return this.#feedbackEvents.map((e) => e.feedback);
|
|
6472
|
+
}
|
|
6473
|
+
/**
|
|
6474
|
+
* Get feedback filtered by type
|
|
6475
|
+
*/
|
|
6476
|
+
getFeedbackByType(feedbackType) {
|
|
6477
|
+
return this.#feedbackEvents.filter((e) => e.feedback.feedbackType === feedbackType).map((e) => e.feedback);
|
|
6478
|
+
}
|
|
6479
|
+
/**
|
|
6480
|
+
* Get feedback for a specific trace
|
|
6481
|
+
*/
|
|
6482
|
+
getFeedbackByTraceId(traceId) {
|
|
6483
|
+
return this.#feedbackEvents.filter((e) => e.feedback.traceId === traceId).map((e) => e.feedback);
|
|
6484
|
+
}
|
|
6485
|
+
// ============================================================================
|
|
6296
6486
|
// Statistics
|
|
6297
6487
|
// ============================================================================
|
|
6298
6488
|
/**
|
|
6299
|
-
* Get comprehensive statistics about collected
|
|
6489
|
+
* Get comprehensive statistics about all collected signals
|
|
6300
6490
|
*/
|
|
6301
6491
|
getStatistics() {
|
|
6302
6492
|
const bySpanType = {};
|
|
@@ -6314,18 +6504,52 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6314
6504
|
incompleteSpans++;
|
|
6315
6505
|
}
|
|
6316
6506
|
}
|
|
6507
|
+
const logsByLevel = {};
|
|
6508
|
+
for (const event of this.#logEvents) {
|
|
6509
|
+
const level = event.log.level;
|
|
6510
|
+
logsByLevel[level] = (logsByLevel[level] || 0) + 1;
|
|
6511
|
+
}
|
|
6512
|
+
const metricsByType = {};
|
|
6513
|
+
const metricsByName = {};
|
|
6514
|
+
for (const event of this.#metricEvents) {
|
|
6515
|
+
const mType = event.metric.metricType;
|
|
6516
|
+
metricsByType[mType] = (metricsByType[mType] || 0) + 1;
|
|
6517
|
+
const mName = event.metric.name;
|
|
6518
|
+
metricsByName[mName] = (metricsByName[mName] || 0) + 1;
|
|
6519
|
+
}
|
|
6520
|
+
const scoresByScorer = {};
|
|
6521
|
+
for (const event of this.#scoreEvents) {
|
|
6522
|
+
const scorer = event.score.scorerName;
|
|
6523
|
+
scoresByScorer[scorer] = (scoresByScorer[scorer] || 0) + 1;
|
|
6524
|
+
}
|
|
6525
|
+
const feedbackByType = {};
|
|
6526
|
+
for (const event of this.#feedbackEvents) {
|
|
6527
|
+
const fbType = event.feedback.feedbackType;
|
|
6528
|
+
feedbackByType[fbType] = (feedbackByType[fbType] || 0) + 1;
|
|
6529
|
+
}
|
|
6317
6530
|
return {
|
|
6318
|
-
|
|
6531
|
+
totalTracingEvents: this.#tracingEvents.length,
|
|
6532
|
+
totalEvents: this.#tracingEvents.length,
|
|
6533
|
+
// deprecated alias
|
|
6319
6534
|
totalSpans: this.#spanStates.size,
|
|
6320
6535
|
totalTraces: this.getTraceIds().length,
|
|
6321
6536
|
completedSpans,
|
|
6322
6537
|
incompleteSpans,
|
|
6323
6538
|
byEventType: {
|
|
6324
|
-
started: this.#
|
|
6325
|
-
updated: this.#
|
|
6326
|
-
ended: this.#
|
|
6539
|
+
started: this.#tracingEvents.filter((e) => e.type === observability.TracingEventType.SPAN_STARTED).length,
|
|
6540
|
+
updated: this.#tracingEvents.filter((e) => e.type === observability.TracingEventType.SPAN_UPDATED).length,
|
|
6541
|
+
ended: this.#tracingEvents.filter((e) => e.type === observability.TracingEventType.SPAN_ENDED).length
|
|
6327
6542
|
},
|
|
6328
|
-
bySpanType
|
|
6543
|
+
bySpanType,
|
|
6544
|
+
totalLogs: this.#logEvents.length,
|
|
6545
|
+
logsByLevel,
|
|
6546
|
+
totalMetrics: this.#metricEvents.length,
|
|
6547
|
+
metricsByType,
|
|
6548
|
+
metricsByName,
|
|
6549
|
+
totalScores: this.#scoreEvents.length,
|
|
6550
|
+
scoresByScorer,
|
|
6551
|
+
totalFeedback: this.#feedbackEvents.length,
|
|
6552
|
+
feedbackByType
|
|
6329
6553
|
};
|
|
6330
6554
|
}
|
|
6331
6555
|
// ============================================================================
|
|
@@ -6344,8 +6568,20 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6344
6568
|
const data = {
|
|
6345
6569
|
spans: this.getAllSpans()
|
|
6346
6570
|
};
|
|
6571
|
+
if (this.#logEvents.length > 0) {
|
|
6572
|
+
data.logs = this.getAllLogs();
|
|
6573
|
+
}
|
|
6574
|
+
if (this.#metricEvents.length > 0) {
|
|
6575
|
+
data.metrics = this.getAllMetrics();
|
|
6576
|
+
}
|
|
6577
|
+
if (this.#scoreEvents.length > 0) {
|
|
6578
|
+
data.scores = this.getAllScores();
|
|
6579
|
+
}
|
|
6580
|
+
if (this.#feedbackEvents.length > 0) {
|
|
6581
|
+
data.feedback = this.getAllFeedback();
|
|
6582
|
+
}
|
|
6347
6583
|
if (includeEvents) {
|
|
6348
|
-
data.events = this.#
|
|
6584
|
+
data.events = this.#tracingEvents;
|
|
6349
6585
|
}
|
|
6350
6586
|
if (includeStats) {
|
|
6351
6587
|
data.statistics = this.getStatistics();
|
|
@@ -6419,6 +6655,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6419
6655
|
const uuidRegex2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6420
6656
|
const hexId32Regex = /^[0-9a-f]{32}$/i;
|
|
6421
6657
|
const prefixedUuidRegex = /^([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
|
|
6658
|
+
const embeddedPrefixedUuidTest = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i;
|
|
6422
6659
|
const embeddedPrefixedUuidRegex = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
|
|
6423
6660
|
const normalizeUuid = (uuid, key) => {
|
|
6424
6661
|
if (!uuidMapsByKey.has(key)) {
|
|
@@ -6453,8 +6690,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6453
6690
|
const uuid = prefixMatch[2];
|
|
6454
6691
|
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6455
6692
|
}
|
|
6456
|
-
if (
|
|
6457
|
-
embeddedPrefixedUuidRegex.lastIndex = 0;
|
|
6693
|
+
if (embeddedPrefixedUuidTest.test(value)) {
|
|
6458
6694
|
return value.replace(embeddedPrefixedUuidRegex, (_match, prefix, uuid) => {
|
|
6459
6695
|
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6460
6696
|
});
|
|
@@ -6516,7 +6752,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6516
6752
|
normalizedSpan.output = normalizeValue(span.output);
|
|
6517
6753
|
}
|
|
6518
6754
|
if (span.errorInfo) {
|
|
6519
|
-
normalizedSpan.errorInfo = span.errorInfo;
|
|
6755
|
+
normalizedSpan.errorInfo = normalizeValue(span.errorInfo);
|
|
6520
6756
|
}
|
|
6521
6757
|
if (span.tags && span.tags.length > 0) {
|
|
6522
6758
|
normalizedSpan.tags = span.tags;
|
|
@@ -6603,8 +6839,9 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6603
6839
|
} else {
|
|
6604
6840
|
json = this.toJSON(options);
|
|
6605
6841
|
}
|
|
6606
|
-
|
|
6607
|
-
|
|
6842
|
+
const { writeFile } = await import('fs/promises');
|
|
6843
|
+
await writeFile(filePath, json, "utf-8");
|
|
6844
|
+
this.logger.info(`TestExporter: wrote ${this.#tracingEvents.length} tracing events to ${filePath}`);
|
|
6608
6845
|
}
|
|
6609
6846
|
/**
|
|
6610
6847
|
* Assert that the current normalized tree matches a snapshot file.
|
|
@@ -6632,7 +6869,8 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6632
6869
|
* @throws Error if the snapshot doesn't match (and updateSnapshot is false)
|
|
6633
6870
|
*/
|
|
6634
6871
|
async assertMatchesSnapshot(snapshotName, options) {
|
|
6635
|
-
const
|
|
6872
|
+
const { join } = await import('path');
|
|
6873
|
+
const snapshotPath = join(await getSnapshotsDir(), snapshotName);
|
|
6636
6874
|
const normalizedTree = this.buildNormalizedTree();
|
|
6637
6875
|
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6638
6876
|
const currentData = {
|
|
@@ -6642,17 +6880,29 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6642
6880
|
const currentJson = JSON.stringify(currentData, null, this.#config.jsonIndent);
|
|
6643
6881
|
const shouldUpdate = options?.updateSnapshot;
|
|
6644
6882
|
if (shouldUpdate) {
|
|
6645
|
-
|
|
6646
|
-
|
|
6883
|
+
const { writeFile } = await import('fs/promises');
|
|
6884
|
+
await writeFile(snapshotPath, currentJson, "utf-8");
|
|
6885
|
+
this.logger.info(`TestExporter: updated snapshot ${snapshotPath}`);
|
|
6647
6886
|
return;
|
|
6648
6887
|
}
|
|
6649
6888
|
let snapshotData;
|
|
6889
|
+
let snapshotContent;
|
|
6890
|
+
try {
|
|
6891
|
+
const { readFile } = await import('fs/promises');
|
|
6892
|
+
snapshotContent = await readFile(snapshotPath, "utf-8");
|
|
6893
|
+
} catch (err) {
|
|
6894
|
+
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
|
6895
|
+
throw new Error(
|
|
6896
|
+
`Snapshot file not found: ${snapshotPath}
|
|
6897
|
+
Run with { updateSnapshot: true } to create it.`
|
|
6898
|
+
);
|
|
6899
|
+
}
|
|
6900
|
+
throw err;
|
|
6901
|
+
}
|
|
6650
6902
|
try {
|
|
6651
|
-
const snapshotContent = await promises.readFile(snapshotPath, "utf-8");
|
|
6652
6903
|
snapshotData = JSON.parse(snapshotContent);
|
|
6653
|
-
} catch {
|
|
6654
|
-
throw new Error(`
|
|
6655
|
-
Run with { updateSnapshot: true } to create it.`);
|
|
6904
|
+
} catch (err) {
|
|
6905
|
+
throw new Error(`Failed to parse snapshot ${snapshotPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
6656
6906
|
}
|
|
6657
6907
|
let expectedSpans;
|
|
6658
6908
|
let expectedStructure;
|
|
@@ -6853,17 +7103,19 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6853
7103
|
// Debugging Helpers
|
|
6854
7104
|
// ============================================================================
|
|
6855
7105
|
/**
|
|
6856
|
-
* Get all stored logs
|
|
7106
|
+
* Get all stored debug logs (internal exporter logging, not signal logs)
|
|
6857
7107
|
*/
|
|
6858
7108
|
getLogs() {
|
|
6859
|
-
return [...this.#
|
|
7109
|
+
return [...this.#debugLogs];
|
|
6860
7110
|
}
|
|
6861
7111
|
/**
|
|
6862
|
-
* Dump logs to console for debugging (uses console.error for visibility in test output)
|
|
7112
|
+
* Dump debug logs to console for debugging (uses console.error for visibility in test output)
|
|
6863
7113
|
*/
|
|
6864
7114
|
dumpLogs() {
|
|
6865
|
-
console.error("\n===
|
|
6866
|
-
this.#
|
|
7115
|
+
console.error("\n=== TestExporter Logs ===");
|
|
7116
|
+
this.#debugLogs.forEach((log) => {
|
|
7117
|
+
console.error(log);
|
|
7118
|
+
});
|
|
6867
7119
|
console.error("=== End Logs ===\n");
|
|
6868
7120
|
}
|
|
6869
7121
|
/**
|
|
@@ -6888,12 +7140,16 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6888
7140
|
// Reset & Lifecycle
|
|
6889
7141
|
// ============================================================================
|
|
6890
7142
|
/**
|
|
6891
|
-
* Clear all collected events and state
|
|
7143
|
+
* Clear all collected events and state across all signals
|
|
6892
7144
|
*/
|
|
6893
7145
|
clearEvents() {
|
|
6894
|
-
this.#
|
|
7146
|
+
this.#tracingEvents = [];
|
|
6895
7147
|
this.#spanStates.clear();
|
|
6896
|
-
this.#
|
|
7148
|
+
this.#logEvents = [];
|
|
7149
|
+
this.#metricEvents = [];
|
|
7150
|
+
this.#scoreEvents = [];
|
|
7151
|
+
this.#feedbackEvents = [];
|
|
7152
|
+
this.#debugLogs = [];
|
|
6897
7153
|
}
|
|
6898
7154
|
/**
|
|
6899
7155
|
* Alias for clearEvents (compatibility with TestExporter)
|
|
@@ -6901,8 +7157,42 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6901
7157
|
reset() {
|
|
6902
7158
|
this.clearEvents();
|
|
6903
7159
|
}
|
|
7160
|
+
/**
|
|
7161
|
+
* Get internal metrics about the exporter's own activity.
|
|
7162
|
+
*/
|
|
7163
|
+
getInternalMetrics() {
|
|
7164
|
+
const json = this.toJSON({ includeEvents: false, includeStats: false });
|
|
7165
|
+
return {
|
|
7166
|
+
startedAt: this.#internalMetrics.startedAt,
|
|
7167
|
+
lastEventAt: this.#internalMetrics.lastEventAt,
|
|
7168
|
+
totalEventsReceived: this.#internalMetrics.totalEventsReceived,
|
|
7169
|
+
bySignal: { ...this.#internalMetrics.bySignal },
|
|
7170
|
+
flushCount: this.#internalMetrics.flushCount,
|
|
7171
|
+
estimatedJsonBytes: new TextEncoder().encode(json).byteLength
|
|
7172
|
+
};
|
|
7173
|
+
}
|
|
7174
|
+
/**
|
|
7175
|
+
* Flush buffered data and log internal metrics summary.
|
|
7176
|
+
*/
|
|
7177
|
+
async flush() {
|
|
7178
|
+
this.#internalMetrics.flushCount++;
|
|
7179
|
+
if (this.#config.logMetricsOnFlush) {
|
|
7180
|
+
const metrics = this.getInternalMetrics();
|
|
7181
|
+
const uptimeMs = Date.now() - metrics.startedAt.getTime();
|
|
7182
|
+
const summary = [
|
|
7183
|
+
`[TestExporter] flush #${metrics.flushCount} summary:`,
|
|
7184
|
+
` uptime: ${(uptimeMs / 1e3).toFixed(1)}s`,
|
|
7185
|
+
` total events received: ${metrics.totalEventsReceived}`,
|
|
7186
|
+
` by signal: tracing=${metrics.bySignal.tracing}, log=${metrics.bySignal.log}, metric=${metrics.bySignal.metric}, score=${metrics.bySignal.score}, feedback=${metrics.bySignal.feedback}`,
|
|
7187
|
+
` buffered: spans=${this.#spanStates.size}, logs=${this.#logEvents.length}, metrics=${this.#metricEvents.length}, scores=${this.#scoreEvents.length}, feedback=${this.#feedbackEvents.length}`,
|
|
7188
|
+
` estimated JSON size: ${(metrics.estimatedJsonBytes / 1024).toFixed(1)}KB`
|
|
7189
|
+
].join("\n");
|
|
7190
|
+
this.logger.info(summary);
|
|
7191
|
+
}
|
|
7192
|
+
}
|
|
6904
7193
|
async shutdown() {
|
|
6905
|
-
this.
|
|
7194
|
+
await this.flush();
|
|
7195
|
+
this.logger.info("TestExporter shutdown");
|
|
6906
7196
|
}
|
|
6907
7197
|
// ============================================================================
|
|
6908
7198
|
// Private Helpers
|
|
@@ -6921,6 +7211,634 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6921
7211
|
return Array.from(spanMap.values());
|
|
6922
7212
|
}
|
|
6923
7213
|
};
|
|
7214
|
+
var JsonExporter = TestExporter;
|
|
7215
|
+
var BaseObservabilityEventBus = class _BaseObservabilityEventBus extends base.MastraBase {
|
|
7216
|
+
subscribers = /* @__PURE__ */ new Set();
|
|
7217
|
+
/** In-flight async subscriber promises. Self-cleaning via .finally(). */
|
|
7218
|
+
pendingSubscribers = /* @__PURE__ */ new Set();
|
|
7219
|
+
constructor({ name } = {}) {
|
|
7220
|
+
super({ component: logger.RegisteredLogger.OBSERVABILITY, name: name ?? "EventBus" });
|
|
7221
|
+
}
|
|
7222
|
+
/**
|
|
7223
|
+
* Dispatch an event to all subscribers synchronously.
|
|
7224
|
+
* Async handler promises are tracked internally and drained by {@link flush}.
|
|
7225
|
+
*
|
|
7226
|
+
* @param event - The event to broadcast to subscribers.
|
|
7227
|
+
*/
|
|
7228
|
+
emit(event) {
|
|
7229
|
+
for (const handler of this.subscribers) {
|
|
7230
|
+
try {
|
|
7231
|
+
const result = handler(event);
|
|
7232
|
+
if (result && typeof result.then === "function") {
|
|
7233
|
+
const promise = result.catch((err) => {
|
|
7234
|
+
this.logger.error("[ObservabilityEventBus] Handler error:", err);
|
|
7235
|
+
});
|
|
7236
|
+
this.pendingSubscribers.add(promise);
|
|
7237
|
+
void promise.finally(() => this.pendingSubscribers.delete(promise));
|
|
7238
|
+
}
|
|
7239
|
+
} catch (err) {
|
|
7240
|
+
this.logger.error("[ObservabilityEventBus] Handler error:", err);
|
|
7241
|
+
}
|
|
7242
|
+
}
|
|
7243
|
+
}
|
|
7244
|
+
/**
|
|
7245
|
+
* Register a handler to receive future events.
|
|
7246
|
+
*
|
|
7247
|
+
* @param handler - Callback invoked synchronously on each {@link emit}.
|
|
7248
|
+
* @returns An unsubscribe function that removes the handler.
|
|
7249
|
+
*/
|
|
7250
|
+
subscribe(handler) {
|
|
7251
|
+
this.subscribers.add(handler);
|
|
7252
|
+
return () => {
|
|
7253
|
+
this.subscribers.delete(handler);
|
|
7254
|
+
};
|
|
7255
|
+
}
|
|
7256
|
+
/** Max flush drain iterations before bailing — prevents infinite loops when handlers re-emit. */
|
|
7257
|
+
static MAX_FLUSH_ITERATIONS = 3;
|
|
7258
|
+
/** Await all in-flight async subscriber promises, draining until empty. */
|
|
7259
|
+
async flush() {
|
|
7260
|
+
let iterations = 0;
|
|
7261
|
+
while (this.pendingSubscribers.size > 0) {
|
|
7262
|
+
await Promise.allSettled([...this.pendingSubscribers]);
|
|
7263
|
+
iterations++;
|
|
7264
|
+
if (iterations >= _BaseObservabilityEventBus.MAX_FLUSH_ITERATIONS) {
|
|
7265
|
+
this.logger.error(
|
|
7266
|
+
`[ObservabilityEventBus] flush() exceeded ${_BaseObservabilityEventBus.MAX_FLUSH_ITERATIONS} drain iterations \u2014 ${this.pendingSubscribers.size} promises still pending. Handlers may be re-emitting during flush.`
|
|
7267
|
+
);
|
|
7268
|
+
break;
|
|
7269
|
+
}
|
|
7270
|
+
}
|
|
7271
|
+
}
|
|
7272
|
+
/** Flush pending promises, then clear all subscribers. */
|
|
7273
|
+
async shutdown() {
|
|
7274
|
+
await this.flush();
|
|
7275
|
+
this.subscribers.clear();
|
|
7276
|
+
}
|
|
7277
|
+
};
|
|
7278
|
+
var AutoExtractedMetrics = class {
|
|
7279
|
+
/**
|
|
7280
|
+
* @param observabilityBus - Bus used to emit derived MetricEvents.
|
|
7281
|
+
* @param cardinalityFilter - Optional filter applied to metric labels before emission.
|
|
7282
|
+
*/
|
|
7283
|
+
constructor(observabilityBus, cardinalityFilter) {
|
|
7284
|
+
this.observabilityBus = observabilityBus;
|
|
7285
|
+
this.cardinalityFilter = cardinalityFilter;
|
|
7286
|
+
}
|
|
7287
|
+
/**
|
|
7288
|
+
* Route a tracing event to the appropriate span lifecycle handler.
|
|
7289
|
+
* SPAN_STARTED increments a started counter; SPAN_ENDED emits ended counter,
|
|
7290
|
+
* duration histogram, and (for model spans) token counters.
|
|
7291
|
+
*/
|
|
7292
|
+
processTracingEvent(event) {
|
|
7293
|
+
switch (event.type) {
|
|
7294
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
7295
|
+
this.onSpanStarted(event.exportedSpan);
|
|
7296
|
+
break;
|
|
7297
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
7298
|
+
this.onSpanEnded(event.exportedSpan);
|
|
7299
|
+
break;
|
|
7300
|
+
}
|
|
7301
|
+
}
|
|
7302
|
+
/** Emit a `mastra_scores_total` counter for a score event. */
|
|
7303
|
+
processScoreEvent(event) {
|
|
7304
|
+
const labels = {
|
|
7305
|
+
scorer: event.score.scorerName
|
|
7306
|
+
};
|
|
7307
|
+
if (event.score.metadata?.entityType) {
|
|
7308
|
+
labels.entity_type = String(event.score.metadata.entityType);
|
|
7309
|
+
}
|
|
7310
|
+
if (event.score.experimentId) {
|
|
7311
|
+
labels.experiment = event.score.experimentId;
|
|
7312
|
+
}
|
|
7313
|
+
this.emit("mastra_scores_total", "counter", 1, labels);
|
|
7314
|
+
}
|
|
7315
|
+
/** Emit a `mastra_feedback_total` counter for a feedback event. */
|
|
7316
|
+
processFeedbackEvent(event) {
|
|
7317
|
+
const labels = {
|
|
7318
|
+
feedback_type: event.feedback.feedbackType,
|
|
7319
|
+
source: event.feedback.source
|
|
7320
|
+
};
|
|
7321
|
+
if (event.feedback.metadata?.entityType) {
|
|
7322
|
+
labels.entity_type = String(event.feedback.metadata.entityType);
|
|
7323
|
+
}
|
|
7324
|
+
if (event.feedback.experimentId) {
|
|
7325
|
+
labels.experiment = event.feedback.experimentId;
|
|
7326
|
+
}
|
|
7327
|
+
this.emit("mastra_feedback_total", "counter", 1, labels);
|
|
7328
|
+
}
|
|
7329
|
+
/** Emit a started counter (e.g. `mastra_agent_runs_started`) for the span type. */
|
|
7330
|
+
onSpanStarted(span) {
|
|
7331
|
+
const labels = this.extractLabels(span);
|
|
7332
|
+
const metricName = this.getStartedMetricName(span);
|
|
7333
|
+
if (metricName) {
|
|
7334
|
+
this.emit(metricName, "counter", 1, labels);
|
|
7335
|
+
}
|
|
7336
|
+
}
|
|
7337
|
+
/** Emit ended counter, duration histogram, and token counters (for model spans). */
|
|
7338
|
+
onSpanEnded(span) {
|
|
7339
|
+
const labels = this.extractLabels(span);
|
|
7340
|
+
const endedMetricName = this.getEndedMetricName(span);
|
|
7341
|
+
if (endedMetricName) {
|
|
7342
|
+
const endedLabels = { ...labels };
|
|
7343
|
+
if (span.errorInfo) {
|
|
7344
|
+
endedLabels.status = "error";
|
|
7345
|
+
} else {
|
|
7346
|
+
endedLabels.status = "ok";
|
|
7347
|
+
}
|
|
7348
|
+
this.emit(endedMetricName, "counter", 1, endedLabels);
|
|
7349
|
+
}
|
|
7350
|
+
const durationMetricName = this.getDurationMetricName(span);
|
|
7351
|
+
if (durationMetricName && span.startTime && span.endTime) {
|
|
7352
|
+
const durationMs = Math.max(0, span.endTime.getTime() - span.startTime.getTime());
|
|
7353
|
+
const durationLabels = { ...labels };
|
|
7354
|
+
if (span.errorInfo) {
|
|
7355
|
+
durationLabels.status = "error";
|
|
7356
|
+
} else {
|
|
7357
|
+
durationLabels.status = "ok";
|
|
7358
|
+
}
|
|
7359
|
+
this.emit(durationMetricName, "histogram", durationMs, durationLabels);
|
|
7360
|
+
}
|
|
7361
|
+
if (span.type === observability.SpanType.MODEL_GENERATION) {
|
|
7362
|
+
this.extractTokenMetrics(span, labels);
|
|
7363
|
+
}
|
|
7364
|
+
}
|
|
7365
|
+
/** Build base metric labels from a span's entity and model attributes. */
|
|
7366
|
+
extractLabels(span) {
|
|
7367
|
+
const labels = {};
|
|
7368
|
+
if (span.entityType) labels.entity_type = span.entityType;
|
|
7369
|
+
const entityName = span.entityName ?? span.entityId;
|
|
7370
|
+
if (entityName) labels.entity_name = entityName;
|
|
7371
|
+
if (span.type === observability.SpanType.MODEL_GENERATION) {
|
|
7372
|
+
const attrs = span.attributes;
|
|
7373
|
+
if (attrs?.model) labels.model = String(attrs.model);
|
|
7374
|
+
if (attrs?.provider) labels.provider = String(attrs.provider);
|
|
7375
|
+
}
|
|
7376
|
+
return labels;
|
|
7377
|
+
}
|
|
7378
|
+
/** Emit token usage counters from a MODEL_GENERATION span's `usage` attributes. Negative and non-finite values are skipped. */
|
|
7379
|
+
extractTokenMetrics(span, labels) {
|
|
7380
|
+
const attrs = span.attributes;
|
|
7381
|
+
const usage = attrs?.usage;
|
|
7382
|
+
if (!usage) return;
|
|
7383
|
+
const inputTokens = Number(usage.inputTokens);
|
|
7384
|
+
if (Number.isFinite(inputTokens) && inputTokens >= 0) {
|
|
7385
|
+
this.emit("mastra_model_input_tokens", "counter", inputTokens, labels);
|
|
7386
|
+
}
|
|
7387
|
+
const outputTokens = Number(usage.outputTokens);
|
|
7388
|
+
if (Number.isFinite(outputTokens) && outputTokens >= 0) {
|
|
7389
|
+
this.emit("mastra_model_output_tokens", "counter", outputTokens, labels);
|
|
7390
|
+
}
|
|
7391
|
+
const inputDetails = usage.inputDetails;
|
|
7392
|
+
const cacheRead = Number(inputDetails?.cacheRead);
|
|
7393
|
+
if (Number.isFinite(cacheRead) && cacheRead >= 0) {
|
|
7394
|
+
this.emit("mastra_model_cache_read_tokens", "counter", cacheRead, labels);
|
|
7395
|
+
}
|
|
7396
|
+
const cacheWrite = Number(inputDetails?.cacheWrite);
|
|
7397
|
+
if (Number.isFinite(cacheWrite) && cacheWrite >= 0) {
|
|
7398
|
+
this.emit("mastra_model_cache_write_tokens", "counter", cacheWrite, labels);
|
|
7399
|
+
}
|
|
7400
|
+
}
|
|
7401
|
+
/** Map a span type to its `*_started` counter metric name, or `null` for unsupported types. */
|
|
7402
|
+
getStartedMetricName(span) {
|
|
7403
|
+
switch (span.type) {
|
|
7404
|
+
case observability.SpanType.AGENT_RUN:
|
|
7405
|
+
return "mastra_agent_runs_started";
|
|
7406
|
+
case observability.SpanType.TOOL_CALL:
|
|
7407
|
+
return "mastra_tool_calls_started";
|
|
7408
|
+
case observability.SpanType.WORKFLOW_RUN:
|
|
7409
|
+
return "mastra_workflow_runs_started";
|
|
7410
|
+
case observability.SpanType.MODEL_GENERATION:
|
|
7411
|
+
return "mastra_model_requests_started";
|
|
7412
|
+
default:
|
|
7413
|
+
return null;
|
|
7414
|
+
}
|
|
7415
|
+
}
|
|
7416
|
+
/** Map a span type to its `*_ended` counter metric name, or `null` for unsupported types. */
|
|
7417
|
+
getEndedMetricName(span) {
|
|
7418
|
+
switch (span.type) {
|
|
7419
|
+
case observability.SpanType.AGENT_RUN:
|
|
7420
|
+
return "mastra_agent_runs_ended";
|
|
7421
|
+
case observability.SpanType.TOOL_CALL:
|
|
7422
|
+
return "mastra_tool_calls_ended";
|
|
7423
|
+
case observability.SpanType.WORKFLOW_RUN:
|
|
7424
|
+
return "mastra_workflow_runs_ended";
|
|
7425
|
+
case observability.SpanType.MODEL_GENERATION:
|
|
7426
|
+
return "mastra_model_requests_ended";
|
|
7427
|
+
default:
|
|
7428
|
+
return null;
|
|
7429
|
+
}
|
|
7430
|
+
}
|
|
7431
|
+
/** Map a span type to its `*_duration_ms` histogram metric name, or `null` for unsupported types. */
|
|
7432
|
+
getDurationMetricName(span) {
|
|
7433
|
+
switch (span.type) {
|
|
7434
|
+
case observability.SpanType.AGENT_RUN:
|
|
7435
|
+
return "mastra_agent_duration_ms";
|
|
7436
|
+
case observability.SpanType.TOOL_CALL:
|
|
7437
|
+
return "mastra_tool_duration_ms";
|
|
7438
|
+
case observability.SpanType.WORKFLOW_RUN:
|
|
7439
|
+
return "mastra_workflow_duration_ms";
|
|
7440
|
+
case observability.SpanType.MODEL_GENERATION:
|
|
7441
|
+
return "mastra_model_duration_ms";
|
|
7442
|
+
default:
|
|
7443
|
+
return null;
|
|
7444
|
+
}
|
|
7445
|
+
}
|
|
7446
|
+
/** Build an ExportedMetric, apply cardinality filtering, and emit it through the bus. */
|
|
7447
|
+
emit(name, metricType, value, labels) {
|
|
7448
|
+
const filteredLabels = this.cardinalityFilter ? this.cardinalityFilter.filterLabels(labels) : labels;
|
|
7449
|
+
const exportedMetric = {
|
|
7450
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7451
|
+
name,
|
|
7452
|
+
metricType,
|
|
7453
|
+
value,
|
|
7454
|
+
labels: filteredLabels
|
|
7455
|
+
};
|
|
7456
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
7457
|
+
this.observabilityBus.emit(event);
|
|
7458
|
+
}
|
|
7459
|
+
};
|
|
7460
|
+
function routeToHandler(handler, event, logger) {
|
|
7461
|
+
try {
|
|
7462
|
+
switch (event.type) {
|
|
7463
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
7464
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
7465
|
+
case observability.TracingEventType.SPAN_ENDED: {
|
|
7466
|
+
const fn = handler.onTracingEvent ? handler.onTracingEvent.bind(handler) : handler.exportTracingEvent.bind(handler);
|
|
7467
|
+
return catchAsyncResult(fn(event), handler.name, "tracing", logger);
|
|
7468
|
+
}
|
|
7469
|
+
case "log":
|
|
7470
|
+
if (handler.onLogEvent) {
|
|
7471
|
+
return catchAsyncResult(handler.onLogEvent(event), handler.name, "log", logger);
|
|
7472
|
+
}
|
|
7473
|
+
break;
|
|
7474
|
+
case "metric":
|
|
7475
|
+
if (handler.onMetricEvent) {
|
|
7476
|
+
return catchAsyncResult(handler.onMetricEvent(event), handler.name, "metric", logger);
|
|
7477
|
+
}
|
|
7478
|
+
break;
|
|
7479
|
+
case "score":
|
|
7480
|
+
if (handler.onScoreEvent) {
|
|
7481
|
+
return catchAsyncResult(handler.onScoreEvent(event), handler.name, "score", logger);
|
|
7482
|
+
}
|
|
7483
|
+
break;
|
|
7484
|
+
case "feedback":
|
|
7485
|
+
if (handler.onFeedbackEvent) {
|
|
7486
|
+
return catchAsyncResult(handler.onFeedbackEvent(event), handler.name, "feedback", logger);
|
|
7487
|
+
}
|
|
7488
|
+
break;
|
|
7489
|
+
}
|
|
7490
|
+
} catch (err) {
|
|
7491
|
+
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
7492
|
+
}
|
|
7493
|
+
}
|
|
7494
|
+
function catchAsyncResult(result, handlerName, signal, logger) {
|
|
7495
|
+
if (result && typeof result.then === "function") {
|
|
7496
|
+
return result.catch((err) => {
|
|
7497
|
+
logger.error(`[Observability] ${signal} handler error [handler=${handlerName}]:`, err);
|
|
7498
|
+
});
|
|
7499
|
+
}
|
|
7500
|
+
}
|
|
7501
|
+
|
|
7502
|
+
// src/bus/observability-bus.ts
|
|
7503
|
+
var MAX_FLUSH_ITERATIONS = 3;
|
|
7504
|
+
function isTracingEvent(event) {
|
|
7505
|
+
return event.type === observability.TracingEventType.SPAN_STARTED || event.type === observability.TracingEventType.SPAN_UPDATED || event.type === observability.TracingEventType.SPAN_ENDED;
|
|
7506
|
+
}
|
|
7507
|
+
var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
7508
|
+
exporters = [];
|
|
7509
|
+
bridge;
|
|
7510
|
+
autoExtractor;
|
|
7511
|
+
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
7512
|
+
pendingHandlers = /* @__PURE__ */ new Set();
|
|
7513
|
+
constructor() {
|
|
7514
|
+
super({ name: "ObservabilityBus" });
|
|
7515
|
+
}
|
|
7516
|
+
/**
|
|
7517
|
+
* Enable auto-extraction of metrics from tracing, score, and feedback events.
|
|
7518
|
+
* When enabled, span lifecycle events automatically generate counter/histogram
|
|
7519
|
+
* metrics (e.g., mastra_agent_runs_started, mastra_model_duration_ms).
|
|
7520
|
+
*
|
|
7521
|
+
* No-ops if auto-extraction is already enabled.
|
|
7522
|
+
*
|
|
7523
|
+
* @param cardinalityFilter - Optional filter applied to auto-extracted metric labels.
|
|
7524
|
+
*/
|
|
7525
|
+
enableAutoExtractedMetrics(cardinalityFilter) {
|
|
7526
|
+
if (this.autoExtractor) {
|
|
7527
|
+
return;
|
|
7528
|
+
}
|
|
7529
|
+
this.autoExtractor = new AutoExtractedMetrics(this, cardinalityFilter);
|
|
7530
|
+
}
|
|
7531
|
+
/**
|
|
7532
|
+
* Register an exporter to receive routed events.
|
|
7533
|
+
* Duplicate registrations (same instance) are silently ignored.
|
|
7534
|
+
*
|
|
7535
|
+
* @param exporter - The exporter to register.
|
|
7536
|
+
*/
|
|
7537
|
+
registerExporter(exporter) {
|
|
7538
|
+
if (this.exporters.includes(exporter)) {
|
|
7539
|
+
return;
|
|
7540
|
+
}
|
|
7541
|
+
this.exporters.push(exporter);
|
|
7542
|
+
}
|
|
7543
|
+
/**
|
|
7544
|
+
* Unregister an exporter.
|
|
7545
|
+
*
|
|
7546
|
+
* @param exporter - The exporter instance to remove.
|
|
7547
|
+
* @returns `true` if the exporter was found and removed, `false` otherwise.
|
|
7548
|
+
*/
|
|
7549
|
+
unregisterExporter(exporter) {
|
|
7550
|
+
const index = this.exporters.indexOf(exporter);
|
|
7551
|
+
if (index !== -1) {
|
|
7552
|
+
this.exporters.splice(index, 1);
|
|
7553
|
+
return true;
|
|
7554
|
+
}
|
|
7555
|
+
return false;
|
|
7556
|
+
}
|
|
7557
|
+
/**
|
|
7558
|
+
* Get registered exporters (read-only snapshot).
|
|
7559
|
+
*/
|
|
7560
|
+
getExporters() {
|
|
7561
|
+
return [...this.exporters];
|
|
7562
|
+
}
|
|
7563
|
+
/**
|
|
7564
|
+
* Register a bridge to receive all routed events alongside exporters.
|
|
7565
|
+
* Only one bridge can be registered at a time; replacing an existing bridge
|
|
7566
|
+
* logs a warning.
|
|
7567
|
+
*
|
|
7568
|
+
* @param bridge - The bridge to register.
|
|
7569
|
+
*/
|
|
7570
|
+
registerBridge(bridge) {
|
|
7571
|
+
if (this.bridge) {
|
|
7572
|
+
this.logger.warn(`[ObservabilityBus] Replacing existing bridge with new bridge`);
|
|
7573
|
+
}
|
|
7574
|
+
this.bridge = bridge;
|
|
7575
|
+
}
|
|
7576
|
+
/**
|
|
7577
|
+
* Unregister the bridge.
|
|
7578
|
+
*
|
|
7579
|
+
* @returns `true` if a bridge was registered and removed, `false` otherwise.
|
|
7580
|
+
*/
|
|
7581
|
+
unregisterBridge() {
|
|
7582
|
+
if (this.bridge) {
|
|
7583
|
+
this.bridge = void 0;
|
|
7584
|
+
return true;
|
|
7585
|
+
}
|
|
7586
|
+
return false;
|
|
7587
|
+
}
|
|
7588
|
+
/**
|
|
7589
|
+
* Get the registered bridge, if any.
|
|
7590
|
+
*/
|
|
7591
|
+
getBridge() {
|
|
7592
|
+
return this.bridge;
|
|
7593
|
+
}
|
|
7594
|
+
/**
|
|
7595
|
+
* Emit an event: route to exporter/bridge handlers, run auto-extraction,
|
|
7596
|
+
* then forward to base class for subscriber delivery.
|
|
7597
|
+
*
|
|
7598
|
+
* emit() is synchronous — async handler promises are tracked internally
|
|
7599
|
+
* and can be drained via flush().
|
|
7600
|
+
*/
|
|
7601
|
+
emit(event) {
|
|
7602
|
+
for (const exporter of this.exporters) {
|
|
7603
|
+
this.trackPromise(routeToHandler(exporter, event, this.logger));
|
|
7604
|
+
}
|
|
7605
|
+
if (this.bridge) {
|
|
7606
|
+
this.trackPromise(routeToHandler(this.bridge, event, this.logger));
|
|
7607
|
+
}
|
|
7608
|
+
if (this.autoExtractor) {
|
|
7609
|
+
try {
|
|
7610
|
+
if (isTracingEvent(event)) {
|
|
7611
|
+
this.autoExtractor.processTracingEvent(event);
|
|
7612
|
+
} else if (event.type === "score") {
|
|
7613
|
+
this.autoExtractor.processScoreEvent(event);
|
|
7614
|
+
} else if (event.type === "feedback") {
|
|
7615
|
+
this.autoExtractor.processFeedbackEvent(event);
|
|
7616
|
+
}
|
|
7617
|
+
} catch (err) {
|
|
7618
|
+
this.logger.error("[ObservabilityBus] Auto-extraction error:", err);
|
|
7619
|
+
}
|
|
7620
|
+
}
|
|
7621
|
+
super.emit(event);
|
|
7622
|
+
}
|
|
7623
|
+
/**
|
|
7624
|
+
* Track an async handler promise so flush() can await it.
|
|
7625
|
+
* No-ops for sync (void) results.
|
|
7626
|
+
*/
|
|
7627
|
+
trackPromise(result) {
|
|
7628
|
+
if (result && typeof result.then === "function") {
|
|
7629
|
+
const promise = result;
|
|
7630
|
+
this.pendingHandlers.add(promise);
|
|
7631
|
+
void promise.finally(() => this.pendingHandlers.delete(promise));
|
|
7632
|
+
}
|
|
7633
|
+
}
|
|
7634
|
+
/**
|
|
7635
|
+
* Two-phase flush to ensure all observability data is fully exported.
|
|
7636
|
+
*
|
|
7637
|
+
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
7638
|
+
* bridge, and base-class subscribers). After this resolves, all event data
|
|
7639
|
+
* has been delivered to handler methods.
|
|
7640
|
+
*
|
|
7641
|
+
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
7642
|
+
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
7643
|
+
* client queue). Phases are sequential — Phase 2 must not start until
|
|
7644
|
+
* Phase 1 completes, otherwise exporters would flush empty buffers.
|
|
7645
|
+
*/
|
|
7646
|
+
async flush() {
|
|
7647
|
+
let iterations = 0;
|
|
7648
|
+
while (this.pendingHandlers.size > 0) {
|
|
7649
|
+
await Promise.allSettled([...this.pendingHandlers]);
|
|
7650
|
+
iterations++;
|
|
7651
|
+
if (iterations >= MAX_FLUSH_ITERATIONS) {
|
|
7652
|
+
this.logger.error(
|
|
7653
|
+
`[ObservabilityBus] flush() exceeded ${MAX_FLUSH_ITERATIONS} drain iterations \u2014 ${this.pendingHandlers.size} promises still pending. Handlers may be re-emitting during flush.`
|
|
7654
|
+
);
|
|
7655
|
+
if (this.pendingHandlers.size > 0) {
|
|
7656
|
+
await Promise.allSettled([...this.pendingHandlers]);
|
|
7657
|
+
}
|
|
7658
|
+
break;
|
|
7659
|
+
}
|
|
7660
|
+
}
|
|
7661
|
+
await super.flush();
|
|
7662
|
+
const bufferFlushPromises = this.exporters.map((e) => e.flush());
|
|
7663
|
+
if (this.bridge) {
|
|
7664
|
+
bufferFlushPromises.push(this.bridge.flush());
|
|
7665
|
+
}
|
|
7666
|
+
if (bufferFlushPromises.length > 0) {
|
|
7667
|
+
await Promise.allSettled(bufferFlushPromises);
|
|
7668
|
+
}
|
|
7669
|
+
}
|
|
7670
|
+
/** Flush all pending events and exporter buffers, then clear subscribers. */
|
|
7671
|
+
async shutdown() {
|
|
7672
|
+
await this.flush();
|
|
7673
|
+
await super.shutdown();
|
|
7674
|
+
}
|
|
7675
|
+
};
|
|
7676
|
+
|
|
7677
|
+
// src/context/logger.ts
|
|
7678
|
+
var LOG_LEVEL_PRIORITY = {
|
|
7679
|
+
debug: 0,
|
|
7680
|
+
info: 1,
|
|
7681
|
+
warn: 2,
|
|
7682
|
+
error: 3,
|
|
7683
|
+
fatal: 4
|
|
7684
|
+
};
|
|
7685
|
+
var LoggerContextImpl = class {
|
|
7686
|
+
config;
|
|
7687
|
+
/**
|
|
7688
|
+
* Create a logger context. Tags and metadata are defensively copied so
|
|
7689
|
+
* mutations after construction do not affect emitted logs.
|
|
7690
|
+
*/
|
|
7691
|
+
constructor(config) {
|
|
7692
|
+
this.config = {
|
|
7693
|
+
...config,
|
|
7694
|
+
tags: config.tags ? [...config.tags] : void 0,
|
|
7695
|
+
metadata: config.metadata ? structuredClone(config.metadata) : void 0
|
|
7696
|
+
};
|
|
7697
|
+
}
|
|
7698
|
+
/** Log at DEBUG level. */
|
|
7699
|
+
debug(message, data) {
|
|
7700
|
+
this.log("debug", message, data);
|
|
7701
|
+
}
|
|
7702
|
+
/** Log at INFO level. */
|
|
7703
|
+
info(message, data) {
|
|
7704
|
+
this.log("info", message, data);
|
|
7705
|
+
}
|
|
7706
|
+
/** Log at WARN level. */
|
|
7707
|
+
warn(message, data) {
|
|
7708
|
+
this.log("warn", message, data);
|
|
7709
|
+
}
|
|
7710
|
+
/** Log at ERROR level. */
|
|
7711
|
+
error(message, data) {
|
|
7712
|
+
this.log("error", message, data);
|
|
7713
|
+
}
|
|
7714
|
+
/** Log at FATAL level. */
|
|
7715
|
+
fatal(message, data) {
|
|
7716
|
+
this.log("fatal", message, data);
|
|
7717
|
+
}
|
|
7718
|
+
/**
|
|
7719
|
+
* Build an ExportedLog, check against the minimum level, and emit it through the bus.
|
|
7720
|
+
*/
|
|
7721
|
+
log(level, message, data) {
|
|
7722
|
+
const minLevel = this.config.minLevel ?? "debug";
|
|
7723
|
+
if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[minLevel]) {
|
|
7724
|
+
return;
|
|
7725
|
+
}
|
|
7726
|
+
const exportedLog = {
|
|
7727
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7728
|
+
level,
|
|
7729
|
+
message,
|
|
7730
|
+
data,
|
|
7731
|
+
traceId: this.config.traceId,
|
|
7732
|
+
spanId: this.config.spanId,
|
|
7733
|
+
tags: this.config.tags,
|
|
7734
|
+
metadata: this.config.metadata
|
|
7735
|
+
};
|
|
7736
|
+
const event = { type: "log", log: exportedLog };
|
|
7737
|
+
this.config.observabilityBus.emit(event);
|
|
7738
|
+
}
|
|
7739
|
+
};
|
|
7740
|
+
|
|
7741
|
+
// src/context/metrics.ts
|
|
7742
|
+
var MetricsContextImpl = class {
|
|
7743
|
+
config;
|
|
7744
|
+
/**
|
|
7745
|
+
* Create a metrics context. Base labels are defensively copied so
|
|
7746
|
+
* mutations after construction do not affect emitted metrics.
|
|
7747
|
+
*/
|
|
7748
|
+
constructor(config) {
|
|
7749
|
+
this.config = {
|
|
7750
|
+
...config,
|
|
7751
|
+
labels: config.labels ? { ...config.labels } : void 0
|
|
7752
|
+
};
|
|
7753
|
+
}
|
|
7754
|
+
/**
|
|
7755
|
+
* Create a counter instrument. Call `.add(value)` to increment.
|
|
7756
|
+
*
|
|
7757
|
+
* @param name - Metric name (e.g. `mastra_custom_requests_total`).
|
|
7758
|
+
*/
|
|
7759
|
+
counter(name) {
|
|
7760
|
+
return {
|
|
7761
|
+
add: (value, additionalLabels) => {
|
|
7762
|
+
this.emit(name, "counter", value, additionalLabels);
|
|
7763
|
+
}
|
|
7764
|
+
};
|
|
7765
|
+
}
|
|
7766
|
+
/**
|
|
7767
|
+
* Create a gauge instrument. Call `.set(value)` to record a point-in-time value.
|
|
7768
|
+
*
|
|
7769
|
+
* @param name - Metric name (e.g. `mastra_queue_depth`).
|
|
7770
|
+
*/
|
|
7771
|
+
gauge(name) {
|
|
7772
|
+
return {
|
|
7773
|
+
set: (value, additionalLabels) => {
|
|
7774
|
+
this.emit(name, "gauge", value, additionalLabels);
|
|
7775
|
+
}
|
|
7776
|
+
};
|
|
7777
|
+
}
|
|
7778
|
+
/**
|
|
7779
|
+
* Create a histogram instrument. Call `.record(value)` to observe a measurement.
|
|
7780
|
+
*
|
|
7781
|
+
* @param name - Metric name (e.g. `mastra_request_duration_ms`).
|
|
7782
|
+
*/
|
|
7783
|
+
histogram(name) {
|
|
7784
|
+
return {
|
|
7785
|
+
record: (value, additionalLabels) => {
|
|
7786
|
+
this.emit(name, "histogram", value, additionalLabels);
|
|
7787
|
+
}
|
|
7788
|
+
};
|
|
7789
|
+
}
|
|
7790
|
+
/** Merge base + additional labels, apply cardinality filtering, and emit a MetricEvent. Non-finite values are silently dropped. */
|
|
7791
|
+
emit(name, metricType, value, additionalLabels) {
|
|
7792
|
+
if (!Number.isFinite(value)) return;
|
|
7793
|
+
const allLabels = {
|
|
7794
|
+
...this.config.labels,
|
|
7795
|
+
...additionalLabels
|
|
7796
|
+
};
|
|
7797
|
+
const filteredLabels = this.config.cardinalityFilter.filterLabels(allLabels);
|
|
7798
|
+
const exportedMetric = {
|
|
7799
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7800
|
+
name,
|
|
7801
|
+
metricType,
|
|
7802
|
+
value,
|
|
7803
|
+
labels: filteredLabels
|
|
7804
|
+
};
|
|
7805
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
7806
|
+
this.config.observabilityBus.emit(event);
|
|
7807
|
+
}
|
|
7808
|
+
};
|
|
7809
|
+
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
7810
|
+
var CardinalityFilter = class {
|
|
7811
|
+
blockedLabels;
|
|
7812
|
+
blockUUIDs;
|
|
7813
|
+
/**
|
|
7814
|
+
* @param config - Optional configuration. When omitted, uses
|
|
7815
|
+
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
7816
|
+
*/
|
|
7817
|
+
constructor(config) {
|
|
7818
|
+
const blocked = config?.blockedLabels ?? [...observability.DEFAULT_BLOCKED_LABELS];
|
|
7819
|
+
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
7820
|
+
this.blockUUIDs = config?.blockUUIDs ?? true;
|
|
7821
|
+
}
|
|
7822
|
+
/**
|
|
7823
|
+
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
7824
|
+
*
|
|
7825
|
+
* @param labels - Raw metric labels to filter.
|
|
7826
|
+
* @returns A new object containing only the allowed labels.
|
|
7827
|
+
*/
|
|
7828
|
+
filterLabels(labels) {
|
|
7829
|
+
const filtered = {};
|
|
7830
|
+
for (const [key, value] of Object.entries(labels)) {
|
|
7831
|
+
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
7832
|
+
continue;
|
|
7833
|
+
}
|
|
7834
|
+
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
7835
|
+
continue;
|
|
7836
|
+
}
|
|
7837
|
+
filtered[key] = value;
|
|
7838
|
+
}
|
|
7839
|
+
return filtered;
|
|
7840
|
+
}
|
|
7841
|
+
};
|
|
6924
7842
|
|
|
6925
7843
|
// src/usage.ts
|
|
6926
7844
|
function extractUsageMetrics(usage, providerMetadata) {
|
|
@@ -7943,6 +8861,15 @@ var NoOpSpan = class extends BaseSpan {
|
|
|
7943
8861
|
// src/instances/base.ts
|
|
7944
8862
|
var BaseObservabilityInstance = class extends base.MastraBase {
|
|
7945
8863
|
config;
|
|
8864
|
+
/**
|
|
8865
|
+
* Unified event bus for all observability signals.
|
|
8866
|
+
* Routes events to registered exporters based on event type.
|
|
8867
|
+
*/
|
|
8868
|
+
observabilityBus;
|
|
8869
|
+
/**
|
|
8870
|
+
* Cardinality filter for metrics label protection.
|
|
8871
|
+
*/
|
|
8872
|
+
cardinalityFilter;
|
|
7946
8873
|
constructor(config) {
|
|
7947
8874
|
super({ component: logger.RegisteredLogger.OBSERVABILITY, name: config.serviceName });
|
|
7948
8875
|
this.config = {
|
|
@@ -7956,6 +8883,15 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
7956
8883
|
requestContextKeys: config.requestContextKeys ?? [],
|
|
7957
8884
|
serializationOptions: config.serializationOptions
|
|
7958
8885
|
};
|
|
8886
|
+
this.cardinalityFilter = new CardinalityFilter();
|
|
8887
|
+
this.observabilityBus = new ObservabilityBus();
|
|
8888
|
+
for (const exporter of this.exporters) {
|
|
8889
|
+
this.observabilityBus.registerExporter(exporter);
|
|
8890
|
+
}
|
|
8891
|
+
if (this.config.bridge) {
|
|
8892
|
+
this.observabilityBus.registerBridge(this.config.bridge);
|
|
8893
|
+
}
|
|
8894
|
+
this.observabilityBus.enableAutoExtractedMetrics(this.cardinalityFilter);
|
|
7959
8895
|
if (this.config.bridge?.init) {
|
|
7960
8896
|
this.config.bridge.init({ config: this.config });
|
|
7961
8897
|
}
|
|
@@ -8108,6 +9044,108 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
8108
9044
|
getLogger() {
|
|
8109
9045
|
return this.logger;
|
|
8110
9046
|
}
|
|
9047
|
+
/**
|
|
9048
|
+
* Get the ObservabilityBus for this instance.
|
|
9049
|
+
* The bus routes all observability events (tracing, logs, metrics, scores, feedback)
|
|
9050
|
+
* to registered exporters based on event type.
|
|
9051
|
+
*/
|
|
9052
|
+
getObservabilityBus() {
|
|
9053
|
+
return this.observabilityBus;
|
|
9054
|
+
}
|
|
9055
|
+
// ============================================================================
|
|
9056
|
+
// Context-factory bridge methods
|
|
9057
|
+
// ============================================================================
|
|
9058
|
+
/**
|
|
9059
|
+
* Extract entity context labels from a span's entity hierarchy by
|
|
9060
|
+
* walking the parent chain.
|
|
9061
|
+
*
|
|
9062
|
+
* Returns labels for: entity_type/name, parent_type/name, root_type/name.
|
|
9063
|
+
* Internal spans are skipped when resolving parent and root entities.
|
|
9064
|
+
*/
|
|
9065
|
+
extractEntityLabels(span) {
|
|
9066
|
+
const labels = {};
|
|
9067
|
+
if (span.entityType) labels.entity_type = span.entityType;
|
|
9068
|
+
if (span.entityName) labels.entity_name = span.entityName;
|
|
9069
|
+
let parentSpan = span.parent;
|
|
9070
|
+
while (parentSpan && parentSpan.isInternal) {
|
|
9071
|
+
parentSpan = parentSpan.parent;
|
|
9072
|
+
}
|
|
9073
|
+
if (parentSpan?.entityType && parentSpan.entityName) {
|
|
9074
|
+
labels.parent_type = parentSpan.entityType;
|
|
9075
|
+
labels.parent_name = parentSpan.entityName;
|
|
9076
|
+
let rootEntity = parentSpan;
|
|
9077
|
+
let current = parentSpan.parent;
|
|
9078
|
+
while (current) {
|
|
9079
|
+
if (!current.isInternal && current.entityType && current.entityName) {
|
|
9080
|
+
rootEntity = current;
|
|
9081
|
+
}
|
|
9082
|
+
current = current.parent;
|
|
9083
|
+
}
|
|
9084
|
+
if (rootEntity !== parentSpan) {
|
|
9085
|
+
labels.root_type = rootEntity.entityType;
|
|
9086
|
+
labels.root_name = rootEntity.entityName;
|
|
9087
|
+
}
|
|
9088
|
+
}
|
|
9089
|
+
return labels;
|
|
9090
|
+
}
|
|
9091
|
+
/**
|
|
9092
|
+
* Resolve tags for a span. Uses the span's own tags if present,
|
|
9093
|
+
* otherwise walks to the root span to inherit its tags.
|
|
9094
|
+
*/
|
|
9095
|
+
resolveSpanTags(span) {
|
|
9096
|
+
if (span.tags) return span.tags;
|
|
9097
|
+
let root = span;
|
|
9098
|
+
while (root.parent) {
|
|
9099
|
+
root = root.parent;
|
|
9100
|
+
}
|
|
9101
|
+
return root.tags;
|
|
9102
|
+
}
|
|
9103
|
+
/**
|
|
9104
|
+
* Get a LoggerContext correlated to a span.
|
|
9105
|
+
* Called by the context-factory in core (deriveLoggerContext) so that
|
|
9106
|
+
* `observabilityContext.loggerVNext` is a real logger instead of no-op.
|
|
9107
|
+
*/
|
|
9108
|
+
getLoggerContext(span) {
|
|
9109
|
+
const entityLabels = span ? this.extractEntityLabels(span) : void 0;
|
|
9110
|
+
const hasEntityLabels = entityLabels && Object.keys(entityLabels).length > 0;
|
|
9111
|
+
const metadata = hasEntityLabels || span?.metadata || this.config.serviceName ? {
|
|
9112
|
+
...hasEntityLabels ? entityLabels : void 0,
|
|
9113
|
+
...span?.metadata,
|
|
9114
|
+
...this.config.serviceName ? { serviceName: this.config.serviceName } : void 0
|
|
9115
|
+
} : void 0;
|
|
9116
|
+
return new LoggerContextImpl({
|
|
9117
|
+
traceId: span?.traceId,
|
|
9118
|
+
spanId: span?.id,
|
|
9119
|
+
tags: span ? this.resolveSpanTags(span) : void 0,
|
|
9120
|
+
metadata,
|
|
9121
|
+
observabilityBus: this.observabilityBus
|
|
9122
|
+
});
|
|
9123
|
+
}
|
|
9124
|
+
/**
|
|
9125
|
+
* Get a MetricsContext, optionally tagged from a span's entity info.
|
|
9126
|
+
* Called by the context-factory in core (deriveMetricsContext) so that
|
|
9127
|
+
* `observabilityContext.metrics` is a real metrics context instead of no-op.
|
|
9128
|
+
*/
|
|
9129
|
+
getMetricsContext(span) {
|
|
9130
|
+
const labels = span ? this.extractEntityLabels(span) : {};
|
|
9131
|
+
const attrs = span?.attributes;
|
|
9132
|
+
if (attrs?.model && typeof attrs.model === "string") labels.model = attrs.model;
|
|
9133
|
+
if (attrs?.provider && typeof attrs.provider === "string") labels.provider = attrs.provider;
|
|
9134
|
+
if (this.config.serviceName) labels.service_name = this.config.serviceName;
|
|
9135
|
+
return new MetricsContextImpl({
|
|
9136
|
+
labels: Object.keys(labels).length > 0 ? labels : void 0,
|
|
9137
|
+
observabilityBus: this.observabilityBus,
|
|
9138
|
+
cardinalityFilter: this.cardinalityFilter
|
|
9139
|
+
});
|
|
9140
|
+
}
|
|
9141
|
+
/**
|
|
9142
|
+
* Emit any observability event through the bus.
|
|
9143
|
+
* The bus routes the event to the appropriate handler on each registered exporter,
|
|
9144
|
+
* and for tracing events triggers auto-extracted metrics.
|
|
9145
|
+
*/
|
|
9146
|
+
emitObservabilityEvent(event) {
|
|
9147
|
+
this.observabilityBus.emit(event);
|
|
9148
|
+
}
|
|
8111
9149
|
// ============================================================================
|
|
8112
9150
|
// Span Lifecycle Management
|
|
8113
9151
|
// ============================================================================
|
|
@@ -8251,40 +9289,56 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
8251
9289
|
return processedSpan?.exportSpan(this.config.includeInternalSpans);
|
|
8252
9290
|
}
|
|
8253
9291
|
/**
|
|
8254
|
-
* Emit a span started event
|
|
9292
|
+
* Emit a span started event.
|
|
9293
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9294
|
+
* and auto-extracted metrics are generated.
|
|
8255
9295
|
*/
|
|
8256
9296
|
emitSpanStarted(span) {
|
|
8257
9297
|
const exportedSpan = this.getSpanForExport(span);
|
|
8258
9298
|
if (exportedSpan) {
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
});
|
|
9299
|
+
const event = { type: observability.TracingEventType.SPAN_STARTED, exportedSpan };
|
|
9300
|
+
this.emitTracingEvent(event);
|
|
8262
9301
|
}
|
|
8263
9302
|
}
|
|
8264
9303
|
/**
|
|
8265
|
-
* Emit a span ended event (called automatically when spans end)
|
|
9304
|
+
* Emit a span ended event (called automatically when spans end).
|
|
9305
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9306
|
+
* and auto-extracted metrics are generated.
|
|
8266
9307
|
*/
|
|
8267
9308
|
emitSpanEnded(span) {
|
|
8268
9309
|
const exportedSpan = this.getSpanForExport(span);
|
|
8269
9310
|
if (exportedSpan) {
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
});
|
|
9311
|
+
const event = { type: observability.TracingEventType.SPAN_ENDED, exportedSpan };
|
|
9312
|
+
this.emitTracingEvent(event);
|
|
8273
9313
|
}
|
|
8274
9314
|
}
|
|
8275
9315
|
/**
|
|
8276
|
-
* Emit a span updated event
|
|
9316
|
+
* Emit a span updated event.
|
|
9317
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9318
|
+
* and auto-extracted metrics are generated.
|
|
8277
9319
|
*/
|
|
8278
9320
|
emitSpanUpdated(span) {
|
|
8279
9321
|
const exportedSpan = this.getSpanForExport(span);
|
|
8280
9322
|
if (exportedSpan) {
|
|
8281
|
-
|
|
8282
|
-
|
|
8283
|
-
});
|
|
9323
|
+
const event = { type: observability.TracingEventType.SPAN_UPDATED, exportedSpan };
|
|
9324
|
+
this.emitTracingEvent(event);
|
|
8284
9325
|
}
|
|
8285
9326
|
}
|
|
8286
9327
|
/**
|
|
8287
|
-
*
|
|
9328
|
+
* Emit a tracing event through the bus.
|
|
9329
|
+
*
|
|
9330
|
+
* The bus routes the event to each registered exporter's and bridge's
|
|
9331
|
+
* onTracingEvent handler and triggers auto-extracted metrics (e.g.,
|
|
9332
|
+
* mastra_agent_runs_started, mastra_model_duration_ms).
|
|
9333
|
+
*/
|
|
9334
|
+
emitTracingEvent(event) {
|
|
9335
|
+
this.observabilityBus.emit(event);
|
|
9336
|
+
}
|
|
9337
|
+
/**
|
|
9338
|
+
* Export tracing event through all exporters and bridge.
|
|
9339
|
+
*
|
|
9340
|
+
* @deprecated Prefer emitTracingEvent() which routes through the bus.
|
|
9341
|
+
* Kept for backward compatibility with subclasses that may override it.
|
|
8288
9342
|
*/
|
|
8289
9343
|
async exportTracingEvent(event) {
|
|
8290
9344
|
const targets = [
|
|
@@ -8314,26 +9368,18 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
8314
9368
|
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
8315
9369
|
}
|
|
8316
9370
|
/**
|
|
8317
|
-
*
|
|
8318
|
-
*
|
|
9371
|
+
* Flush all observability data: awaits in-flight handler promises, then
|
|
9372
|
+
* drains exporter and bridge SDK-internal buffers.
|
|
9373
|
+
*
|
|
9374
|
+
* Delegates to ObservabilityBus.flush() which owns the two-phase logic.
|
|
8319
9375
|
*
|
|
8320
|
-
* This is
|
|
8321
|
-
*
|
|
8322
|
-
*
|
|
9376
|
+
* This is critical for durable execution engines (e.g., Inngest) where
|
|
9377
|
+
* the process may be interrupted after a step completes. Calling flush()
|
|
9378
|
+
* outside the durable step ensures all span data reaches external systems.
|
|
8323
9379
|
*/
|
|
8324
9380
|
async flush() {
|
|
8325
9381
|
this.logger.debug(`[Observability] Flush started [name=${this.name}]`);
|
|
8326
|
-
|
|
8327
|
-
if (this.config.bridge) {
|
|
8328
|
-
flushPromises.push(this.config.bridge.flush());
|
|
8329
|
-
}
|
|
8330
|
-
const results = await Promise.allSettled(flushPromises);
|
|
8331
|
-
results.forEach((result, index) => {
|
|
8332
|
-
if (result.status === "rejected") {
|
|
8333
|
-
const targetName = index < this.exporters.length ? this.exporters[index]?.name : "bridge";
|
|
8334
|
-
this.logger.error(`[Observability] Flush error [target=${targetName}]`, result.reason);
|
|
8335
|
-
}
|
|
8336
|
-
});
|
|
9382
|
+
await this.observabilityBus.flush();
|
|
8337
9383
|
this.logger.debug(`[Observability] Flush completed [name=${this.name}]`);
|
|
8338
9384
|
}
|
|
8339
9385
|
/**
|
|
@@ -8341,6 +9387,7 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
8341
9387
|
*/
|
|
8342
9388
|
async shutdown() {
|
|
8343
9389
|
this.logger.debug(`[Observability] Shutdown started [name=${this.name}]`);
|
|
9390
|
+
await this.observabilityBus.shutdown();
|
|
8344
9391
|
const shutdownPromises = [
|
|
8345
9392
|
...this.exporters.map((e) => e.shutdown()),
|
|
8346
9393
|
...this.spanOutputProcessors.map((p) => p.shutdown())
|
|
@@ -8348,7 +9395,14 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
8348
9395
|
if (this.config.bridge) {
|
|
8349
9396
|
shutdownPromises.push(this.config.bridge.shutdown());
|
|
8350
9397
|
}
|
|
8351
|
-
|
|
9398
|
+
if (shutdownPromises.length > 0) {
|
|
9399
|
+
const results = await Promise.allSettled(shutdownPromises);
|
|
9400
|
+
for (const result of results) {
|
|
9401
|
+
if (result.status === "rejected") {
|
|
9402
|
+
this.logger.error(`[Observability] Component shutdown failed [name=${this.name}]:`, result.reason);
|
|
9403
|
+
}
|
|
9404
|
+
}
|
|
9405
|
+
}
|
|
8352
9406
|
this.logger.info(`[Observability] Shutdown completed [name=${this.name}]`);
|
|
8353
9407
|
}
|
|
8354
9408
|
};
|
|
@@ -8735,9 +9789,12 @@ function buildTracingOptions(...updaters) {
|
|
|
8735
9789
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
8736
9790
|
}
|
|
8737
9791
|
|
|
9792
|
+
exports.AutoExtractedMetrics = AutoExtractedMetrics;
|
|
8738
9793
|
exports.BaseExporter = BaseExporter;
|
|
9794
|
+
exports.BaseObservabilityEventBus = BaseObservabilityEventBus;
|
|
8739
9795
|
exports.BaseObservabilityInstance = BaseObservabilityInstance;
|
|
8740
9796
|
exports.BaseSpan = BaseSpan;
|
|
9797
|
+
exports.CardinalityFilter = CardinalityFilter;
|
|
8741
9798
|
exports.CloudExporter = CloudExporter;
|
|
8742
9799
|
exports.ConsoleExporter = ConsoleExporter;
|
|
8743
9800
|
exports.DEFAULT_DEEP_CLEAN_OPTIONS = DEFAULT_DEEP_CLEAN_OPTIONS;
|
|
@@ -8746,9 +9803,12 @@ exports.DefaultExporter = DefaultExporter;
|
|
|
8746
9803
|
exports.DefaultObservabilityInstance = DefaultObservabilityInstance;
|
|
8747
9804
|
exports.DefaultSpan = DefaultSpan;
|
|
8748
9805
|
exports.JsonExporter = JsonExporter;
|
|
9806
|
+
exports.LoggerContextImpl = LoggerContextImpl;
|
|
9807
|
+
exports.MetricsContextImpl = MetricsContextImpl;
|
|
8749
9808
|
exports.ModelSpanTracker = ModelSpanTracker;
|
|
8750
9809
|
exports.NoOpSpan = NoOpSpan;
|
|
8751
9810
|
exports.Observability = Observability;
|
|
9811
|
+
exports.ObservabilityBus = ObservabilityBus;
|
|
8752
9812
|
exports.SamplingStrategyType = SamplingStrategyType;
|
|
8753
9813
|
exports.SensitiveDataFilter = SensitiveDataFilter;
|
|
8754
9814
|
exports.TestExporter = TestExporter;
|
|
@@ -8762,6 +9822,7 @@ exports.mergeSerializationOptions = mergeSerializationOptions;
|
|
|
8762
9822
|
exports.observabilityConfigValueSchema = observabilityConfigValueSchema;
|
|
8763
9823
|
exports.observabilityInstanceConfigSchema = observabilityInstanceConfigSchema;
|
|
8764
9824
|
exports.observabilityRegistryConfigSchema = observabilityRegistryConfigSchema;
|
|
9825
|
+
exports.routeToHandler = routeToHandler;
|
|
8765
9826
|
exports.samplingStrategySchema = samplingStrategySchema;
|
|
8766
9827
|
exports.serializationOptionsSchema = serializationOptionsSchema;
|
|
8767
9828
|
exports.truncateString = truncateString;
|