@mastra/observability 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/LICENSE.md +15 -0
- package/README.md +62 -1
- package/dist/bus/base.d.ts +43 -0
- package/dist/bus/base.d.ts.map +1 -0
- package/dist/bus/index.d.ts +7 -0
- package/dist/bus/index.d.ts.map +1 -0
- package/dist/bus/observability-bus.d.ts +97 -0
- package/dist/bus/observability-bus.d.ts.map +1 -0
- package/dist/bus/route-event.d.ts +31 -0
- package/dist/bus/route-event.d.ts.map +1 -0
- package/dist/context/index.d.ts +6 -0
- package/dist/context/index.d.ts.map +1 -0
- package/dist/context/logger.d.ts +45 -0
- package/dist/context/logger.d.ts.map +1 -0
- package/dist/context/metrics.d.ts +47 -0
- package/dist/context/metrics.d.ts.map +1 -0
- package/dist/exporters/base.d.ts +11 -0
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +0 -1
- package/dist/exporters/index.d.ts.map +1 -1
- package/dist/exporters/test.d.ts +538 -3
- package/dist/exporters/test.d.ts.map +1 -1
- package/dist/index.cjs +1165 -104
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1155 -101
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +77 -10
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/metrics/auto-extract.d.ts +47 -0
- package/dist/metrics/auto-extract.d.ts.map +1 -0
- package/dist/metrics/cardinality.d.ts +24 -0
- package/dist/metrics/cardinality.d.ts.map +1 -0
- package/dist/metrics/index.d.ts +6 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/package.json +6 -6
- package/dist/exporters/json.d.ts +0 -386
- package/dist/exporters/json.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import { MastraBase } from '@mastra/core/base';
|
|
2
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
3
|
import { ConsoleLogger, LogLevel, RegisteredLogger } from '@mastra/core/logger';
|
|
4
|
-
import { TracingEventType, SpanType, InternalSpans } from '@mastra/core/observability';
|
|
4
|
+
import { TracingEventType, SpanType, DEFAULT_BLOCKED_LABELS, InternalSpans } from '@mastra/core/observability';
|
|
5
5
|
import { fetchWithRetry, getNestedValue, setNestedValue } from '@mastra/core/utils';
|
|
6
|
-
import { writeFile, readFile } from 'fs/promises';
|
|
7
|
-
import { join, dirname } from 'path';
|
|
8
|
-
import { fileURLToPath } from 'url';
|
|
9
6
|
import { TransformStream } from 'stream/web';
|
|
10
7
|
|
|
11
8
|
var __defProp = Object.defineProperty;
|
|
@@ -4242,6 +4239,19 @@ var BaseExporter = class {
|
|
|
4242
4239
|
}
|
|
4243
4240
|
return event;
|
|
4244
4241
|
}
|
|
4242
|
+
/**
|
|
4243
|
+
* Default onTracingEvent handler that delegates to exportTracingEvent.
|
|
4244
|
+
*
|
|
4245
|
+
* This provides backward compatibility: existing exporters that only implement
|
|
4246
|
+
* _exportTracingEvent will automatically receive tracing events routed through
|
|
4247
|
+
* the ObservabilityBus. Subclasses can override this if they need different
|
|
4248
|
+
* routing behavior for bus-delivered events.
|
|
4249
|
+
*
|
|
4250
|
+
* Handler presence on ObservabilityExporter = signal support.
|
|
4251
|
+
*/
|
|
4252
|
+
onTracingEvent(event) {
|
|
4253
|
+
return this.exportTracingEvent(event);
|
|
4254
|
+
}
|
|
4245
4255
|
/**
|
|
4246
4256
|
* Export a tracing event
|
|
4247
4257
|
*
|
|
@@ -6070,59 +6080,58 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6070
6080
|
this.logger.info("DefaultExporter shutdown complete");
|
|
6071
6081
|
}
|
|
6072
6082
|
};
|
|
6073
|
-
|
|
6074
|
-
// src/exporters/test.ts
|
|
6075
|
-
var TestExporter = class extends BaseExporter {
|
|
6076
|
-
name = "tracing-test-exporter";
|
|
6077
|
-
#events = [];
|
|
6078
|
-
constructor(config = {}) {
|
|
6079
|
-
super(config);
|
|
6080
|
-
}
|
|
6081
|
-
async _exportTracingEvent(event) {
|
|
6082
|
-
this.#events.push(event);
|
|
6083
|
-
}
|
|
6084
|
-
clearEvents() {
|
|
6085
|
-
this.#events = [];
|
|
6086
|
-
}
|
|
6087
|
-
get events() {
|
|
6088
|
-
return this.#events;
|
|
6089
|
-
}
|
|
6090
|
-
async shutdown() {
|
|
6091
|
-
this.logger.info("TestExporter shutdown");
|
|
6092
|
-
}
|
|
6093
|
-
};
|
|
6094
6083
|
var _snapshotsDir;
|
|
6095
|
-
function getSnapshotsDir() {
|
|
6084
|
+
async function getSnapshotsDir() {
|
|
6096
6085
|
if (!_snapshotsDir) {
|
|
6097
6086
|
if (typeof import.meta.url !== "string") {
|
|
6098
6087
|
throw new Error(
|
|
6099
6088
|
"Snapshot functionality requires a Node.js environment. import.meta.url is not available in this runtime."
|
|
6100
6089
|
);
|
|
6101
6090
|
}
|
|
6091
|
+
const { fileURLToPath } = await import('url');
|
|
6092
|
+
const { dirname, join } = await import('path');
|
|
6102
6093
|
const __filename = fileURLToPath(import.meta.url);
|
|
6103
6094
|
const __dirname = dirname(__filename);
|
|
6104
6095
|
_snapshotsDir = join(__dirname, "..", "__snapshots__");
|
|
6105
6096
|
}
|
|
6106
6097
|
return _snapshotsDir;
|
|
6107
6098
|
}
|
|
6108
|
-
var
|
|
6109
|
-
name = "
|
|
6110
|
-
/** All collected events */
|
|
6111
|
-
#
|
|
6099
|
+
var TestExporter = class extends BaseExporter {
|
|
6100
|
+
name = "test-exporter";
|
|
6101
|
+
/** All collected tracing events */
|
|
6102
|
+
#tracingEvents = [];
|
|
6112
6103
|
/** Per-span state tracking */
|
|
6113
6104
|
#spanStates = /* @__PURE__ */ new Map();
|
|
6114
|
-
/**
|
|
6115
|
-
#
|
|
6105
|
+
/** All collected log events */
|
|
6106
|
+
#logEvents = [];
|
|
6107
|
+
/** All collected metric events */
|
|
6108
|
+
#metricEvents = [];
|
|
6109
|
+
/** All collected score events */
|
|
6110
|
+
#scoreEvents = [];
|
|
6111
|
+
/** All collected feedback events */
|
|
6112
|
+
#feedbackEvents = [];
|
|
6113
|
+
/** Debug logs for the exporter itself */
|
|
6114
|
+
#debugLogs = [];
|
|
6116
6115
|
/** Configuration */
|
|
6117
6116
|
#config;
|
|
6117
|
+
/** Internal metrics tracking */
|
|
6118
|
+
#internalMetrics;
|
|
6118
6119
|
constructor(config = {}) {
|
|
6119
6120
|
super(config);
|
|
6120
6121
|
this.#config = {
|
|
6121
6122
|
validateLifecycle: true,
|
|
6122
6123
|
storeLogs: true,
|
|
6123
6124
|
jsonIndent: 2,
|
|
6125
|
+
logMetricsOnFlush: true,
|
|
6124
6126
|
...config
|
|
6125
6127
|
};
|
|
6128
|
+
this.#internalMetrics = {
|
|
6129
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
6130
|
+
lastEventAt: null,
|
|
6131
|
+
totalEventsReceived: 0,
|
|
6132
|
+
bySignal: { tracing: 0, log: 0, metric: 0, score: 0, feedback: 0 },
|
|
6133
|
+
flushCount: 0
|
|
6134
|
+
};
|
|
6126
6135
|
}
|
|
6127
6136
|
/**
|
|
6128
6137
|
* Process incoming tracing events with lifecycle tracking
|
|
@@ -6130,9 +6139,10 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6130
6139
|
async _exportTracingEvent(event) {
|
|
6131
6140
|
const span = event.exportedSpan;
|
|
6132
6141
|
const spanId = span.id;
|
|
6133
|
-
|
|
6142
|
+
this.#trackEvent("tracing");
|
|
6143
|
+
const logMessage = `[TestExporter] ${event.type}: ${span.type} "${span.name}" (entity: ${span.entityName ?? span.entityId ?? "unknown"}, trace: ${span.traceId.slice(-8)}, span: ${spanId.slice(-8)})`;
|
|
6134
6144
|
if (this.#config.storeLogs) {
|
|
6135
|
-
this.#
|
|
6145
|
+
this.#debugLogs.push(logMessage);
|
|
6136
6146
|
}
|
|
6137
6147
|
const state = this.#spanStates.get(spanId) || {
|
|
6138
6148
|
hasStart: false,
|
|
@@ -6155,7 +6165,67 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6155
6165
|
}
|
|
6156
6166
|
state.events.push(event);
|
|
6157
6167
|
this.#spanStates.set(spanId, state);
|
|
6158
|
-
this.#
|
|
6168
|
+
this.#tracingEvents.push(event);
|
|
6169
|
+
}
|
|
6170
|
+
// ============================================================================
|
|
6171
|
+
// Signal Handlers (Logs, Metrics, Scores, Feedback)
|
|
6172
|
+
// ============================================================================
|
|
6173
|
+
/**
|
|
6174
|
+
* Process incoming log events
|
|
6175
|
+
*/
|
|
6176
|
+
async onLogEvent(event) {
|
|
6177
|
+
this.#trackEvent("log");
|
|
6178
|
+
if (this.#config.storeLogs) {
|
|
6179
|
+
const log = event.log;
|
|
6180
|
+
const logMessage = `[TestExporter] log.${log.level}: "${log.message}"${log.traceId ? ` (trace: ${log.traceId.slice(-8)})` : ""}`;
|
|
6181
|
+
this.#debugLogs.push(logMessage);
|
|
6182
|
+
}
|
|
6183
|
+
this.#logEvents.push(event);
|
|
6184
|
+
}
|
|
6185
|
+
/**
|
|
6186
|
+
* Process incoming metric events
|
|
6187
|
+
*/
|
|
6188
|
+
async onMetricEvent(event) {
|
|
6189
|
+
this.#trackEvent("metric");
|
|
6190
|
+
if (this.#config.storeLogs) {
|
|
6191
|
+
const metric = event.metric;
|
|
6192
|
+
const labelsStr = Object.entries(metric.labels).map(([k, v]) => `${k}=${v}`).join(", ");
|
|
6193
|
+
const logMessage = `[TestExporter] metric.${metric.metricType}: ${metric.name}=${metric.value}${labelsStr ? ` {${labelsStr}}` : ""}`;
|
|
6194
|
+
this.#debugLogs.push(logMessage);
|
|
6195
|
+
}
|
|
6196
|
+
this.#metricEvents.push(event);
|
|
6197
|
+
}
|
|
6198
|
+
/**
|
|
6199
|
+
* Process incoming score events
|
|
6200
|
+
*/
|
|
6201
|
+
async onScoreEvent(event) {
|
|
6202
|
+
this.#trackEvent("score");
|
|
6203
|
+
if (this.#config.storeLogs) {
|
|
6204
|
+
const score = event.score;
|
|
6205
|
+
const logMessage = `[TestExporter] score: ${score.scorerName}=${score.score} (trace: ${score.traceId.slice(-8)}${score.spanId ? `, span: ${score.spanId.slice(-8)}` : ""})`;
|
|
6206
|
+
this.#debugLogs.push(logMessage);
|
|
6207
|
+
}
|
|
6208
|
+
this.#scoreEvents.push(event);
|
|
6209
|
+
}
|
|
6210
|
+
/**
|
|
6211
|
+
* Process incoming feedback events
|
|
6212
|
+
*/
|
|
6213
|
+
async onFeedbackEvent(event) {
|
|
6214
|
+
this.#trackEvent("feedback");
|
|
6215
|
+
if (this.#config.storeLogs) {
|
|
6216
|
+
const fb = event.feedback;
|
|
6217
|
+
const logMessage = `[TestExporter] feedback: ${fb.feedbackType} from ${fb.source}=${fb.value} (trace: ${fb.traceId.slice(-8)}${fb.spanId ? `, span: ${fb.spanId.slice(-8)}` : ""})`;
|
|
6218
|
+
this.#debugLogs.push(logMessage);
|
|
6219
|
+
}
|
|
6220
|
+
this.#feedbackEvents.push(event);
|
|
6221
|
+
}
|
|
6222
|
+
/**
|
|
6223
|
+
* Track an event for internal metrics
|
|
6224
|
+
*/
|
|
6225
|
+
#trackEvent(signal) {
|
|
6226
|
+
this.#internalMetrics.lastEventAt = /* @__PURE__ */ new Date();
|
|
6227
|
+
this.#internalMetrics.totalEventsReceived++;
|
|
6228
|
+
this.#internalMetrics.bySignal[signal]++;
|
|
6159
6229
|
}
|
|
6160
6230
|
/**
|
|
6161
6231
|
* Validate span lifecycle rules
|
|
@@ -6182,13 +6252,13 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6182
6252
|
}
|
|
6183
6253
|
}
|
|
6184
6254
|
// ============================================================================
|
|
6185
|
-
// Query Methods
|
|
6255
|
+
// Tracing Query Methods
|
|
6186
6256
|
// ============================================================================
|
|
6187
6257
|
/**
|
|
6188
|
-
* Get all collected events
|
|
6258
|
+
* Get all collected tracing events
|
|
6189
6259
|
*/
|
|
6190
6260
|
get events() {
|
|
6191
|
-
return [...this.#
|
|
6261
|
+
return [...this.#tracingEvents];
|
|
6192
6262
|
}
|
|
6193
6263
|
/**
|
|
6194
6264
|
* Get completed spans by SpanType (e.g., 'agent_run', 'tool_call')
|
|
@@ -6213,18 +6283,21 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6213
6283
|
* @returns Array of events of the specified type
|
|
6214
6284
|
*/
|
|
6215
6285
|
getByEventType(type) {
|
|
6216
|
-
return this.#
|
|
6286
|
+
return this.#tracingEvents.filter((e) => e.type === type);
|
|
6217
6287
|
}
|
|
6218
6288
|
/**
|
|
6219
6289
|
* Get all events and spans for a specific trace
|
|
6220
6290
|
*
|
|
6221
6291
|
* @param traceId - The trace ID to filter by
|
|
6222
|
-
* @returns Object containing events
|
|
6292
|
+
* @returns Object containing tracing events, final spans, plus logs/scores/feedback for the trace
|
|
6223
6293
|
*/
|
|
6224
6294
|
getByTraceId(traceId) {
|
|
6225
|
-
const events = this.#
|
|
6295
|
+
const events = this.#tracingEvents.filter((e) => e.exportedSpan.traceId === traceId);
|
|
6226
6296
|
const spans = this.#getUniqueSpansFromEvents(events);
|
|
6227
|
-
|
|
6297
|
+
const logs = this.#logEvents.filter((e) => e.log.traceId === traceId).map((e) => e.log);
|
|
6298
|
+
const scores = this.#scoreEvents.filter((e) => e.score.traceId === traceId).map((e) => e.score);
|
|
6299
|
+
const feedback = this.#feedbackEvents.filter((e) => e.feedback.traceId === traceId).map((e) => e.feedback);
|
|
6300
|
+
return { events, spans, logs, scores, feedback };
|
|
6228
6301
|
}
|
|
6229
6302
|
/**
|
|
6230
6303
|
* Get all events for a specific span
|
|
@@ -6280,20 +6353,137 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6280
6353
|
}));
|
|
6281
6354
|
}
|
|
6282
6355
|
/**
|
|
6283
|
-
* Get unique trace IDs from all collected
|
|
6356
|
+
* Get unique trace IDs from all collected signals
|
|
6284
6357
|
*/
|
|
6285
6358
|
getTraceIds() {
|
|
6286
6359
|
const traceIds = /* @__PURE__ */ new Set();
|
|
6287
|
-
for (const event of this.#
|
|
6360
|
+
for (const event of this.#tracingEvents) {
|
|
6288
6361
|
traceIds.add(event.exportedSpan.traceId);
|
|
6289
6362
|
}
|
|
6363
|
+
for (const event of this.#logEvents) {
|
|
6364
|
+
if (event.log.traceId) traceIds.add(event.log.traceId);
|
|
6365
|
+
}
|
|
6366
|
+
for (const event of this.#scoreEvents) {
|
|
6367
|
+
traceIds.add(event.score.traceId);
|
|
6368
|
+
}
|
|
6369
|
+
for (const event of this.#feedbackEvents) {
|
|
6370
|
+
traceIds.add(event.feedback.traceId);
|
|
6371
|
+
}
|
|
6290
6372
|
return Array.from(traceIds);
|
|
6291
6373
|
}
|
|
6292
6374
|
// ============================================================================
|
|
6375
|
+
// Log Query Methods
|
|
6376
|
+
// ============================================================================
|
|
6377
|
+
/**
|
|
6378
|
+
* Get all collected log events
|
|
6379
|
+
*/
|
|
6380
|
+
getLogEvents() {
|
|
6381
|
+
return [...this.#logEvents];
|
|
6382
|
+
}
|
|
6383
|
+
/**
|
|
6384
|
+
* Get all collected logs (unwrapped from events)
|
|
6385
|
+
*/
|
|
6386
|
+
getAllLogs() {
|
|
6387
|
+
return this.#logEvents.map((e) => e.log);
|
|
6388
|
+
}
|
|
6389
|
+
/**
|
|
6390
|
+
* Get logs filtered by level
|
|
6391
|
+
*/
|
|
6392
|
+
getLogsByLevel(level) {
|
|
6393
|
+
return this.#logEvents.filter((e) => e.log.level === level).map((e) => e.log);
|
|
6394
|
+
}
|
|
6395
|
+
/**
|
|
6396
|
+
* Get logs for a specific trace
|
|
6397
|
+
*/
|
|
6398
|
+
getLogsByTraceId(traceId) {
|
|
6399
|
+
return this.#logEvents.filter((e) => e.log.traceId === traceId).map((e) => e.log);
|
|
6400
|
+
}
|
|
6401
|
+
// ============================================================================
|
|
6402
|
+
// Metric Query Methods
|
|
6403
|
+
// ============================================================================
|
|
6404
|
+
/**
|
|
6405
|
+
* Get all collected metric events
|
|
6406
|
+
*/
|
|
6407
|
+
getMetricEvents() {
|
|
6408
|
+
return [...this.#metricEvents];
|
|
6409
|
+
}
|
|
6410
|
+
/**
|
|
6411
|
+
* Get all collected metrics (unwrapped from events)
|
|
6412
|
+
*/
|
|
6413
|
+
getAllMetrics() {
|
|
6414
|
+
return this.#metricEvents.map((e) => e.metric);
|
|
6415
|
+
}
|
|
6416
|
+
/**
|
|
6417
|
+
* Get metrics filtered by name
|
|
6418
|
+
*/
|
|
6419
|
+
getMetricsByName(name) {
|
|
6420
|
+
return this.#metricEvents.filter((e) => e.metric.name === name).map((e) => e.metric);
|
|
6421
|
+
}
|
|
6422
|
+
/**
|
|
6423
|
+
* Get metrics filtered by type
|
|
6424
|
+
*/
|
|
6425
|
+
getMetricsByType(metricType) {
|
|
6426
|
+
return this.#metricEvents.filter((e) => e.metric.metricType === metricType).map((e) => e.metric);
|
|
6427
|
+
}
|
|
6428
|
+
// ============================================================================
|
|
6429
|
+
// Score Query Methods
|
|
6430
|
+
// ============================================================================
|
|
6431
|
+
/**
|
|
6432
|
+
* Get all collected score events
|
|
6433
|
+
*/
|
|
6434
|
+
getScoreEvents() {
|
|
6435
|
+
return [...this.#scoreEvents];
|
|
6436
|
+
}
|
|
6437
|
+
/**
|
|
6438
|
+
* Get all collected scores (unwrapped from events)
|
|
6439
|
+
*/
|
|
6440
|
+
getAllScores() {
|
|
6441
|
+
return this.#scoreEvents.map((e) => e.score);
|
|
6442
|
+
}
|
|
6443
|
+
/**
|
|
6444
|
+
* Get scores filtered by scorer name
|
|
6445
|
+
*/
|
|
6446
|
+
getScoresByScorer(scorerName) {
|
|
6447
|
+
return this.#scoreEvents.filter((e) => e.score.scorerName === scorerName).map((e) => e.score);
|
|
6448
|
+
}
|
|
6449
|
+
/**
|
|
6450
|
+
* Get scores for a specific trace
|
|
6451
|
+
*/
|
|
6452
|
+
getScoresByTraceId(traceId) {
|
|
6453
|
+
return this.#scoreEvents.filter((e) => e.score.traceId === traceId).map((e) => e.score);
|
|
6454
|
+
}
|
|
6455
|
+
// ============================================================================
|
|
6456
|
+
// Feedback Query Methods
|
|
6457
|
+
// ============================================================================
|
|
6458
|
+
/**
|
|
6459
|
+
* Get all collected feedback events
|
|
6460
|
+
*/
|
|
6461
|
+
getFeedbackEvents() {
|
|
6462
|
+
return [...this.#feedbackEvents];
|
|
6463
|
+
}
|
|
6464
|
+
/**
|
|
6465
|
+
* Get all collected feedback (unwrapped from events)
|
|
6466
|
+
*/
|
|
6467
|
+
getAllFeedback() {
|
|
6468
|
+
return this.#feedbackEvents.map((e) => e.feedback);
|
|
6469
|
+
}
|
|
6470
|
+
/**
|
|
6471
|
+
* Get feedback filtered by type
|
|
6472
|
+
*/
|
|
6473
|
+
getFeedbackByType(feedbackType) {
|
|
6474
|
+
return this.#feedbackEvents.filter((e) => e.feedback.feedbackType === feedbackType).map((e) => e.feedback);
|
|
6475
|
+
}
|
|
6476
|
+
/**
|
|
6477
|
+
* Get feedback for a specific trace
|
|
6478
|
+
*/
|
|
6479
|
+
getFeedbackByTraceId(traceId) {
|
|
6480
|
+
return this.#feedbackEvents.filter((e) => e.feedback.traceId === traceId).map((e) => e.feedback);
|
|
6481
|
+
}
|
|
6482
|
+
// ============================================================================
|
|
6293
6483
|
// Statistics
|
|
6294
6484
|
// ============================================================================
|
|
6295
6485
|
/**
|
|
6296
|
-
* Get comprehensive statistics about collected
|
|
6486
|
+
* Get comprehensive statistics about all collected signals
|
|
6297
6487
|
*/
|
|
6298
6488
|
getStatistics() {
|
|
6299
6489
|
const bySpanType = {};
|
|
@@ -6311,18 +6501,52 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6311
6501
|
incompleteSpans++;
|
|
6312
6502
|
}
|
|
6313
6503
|
}
|
|
6504
|
+
const logsByLevel = {};
|
|
6505
|
+
for (const event of this.#logEvents) {
|
|
6506
|
+
const level = event.log.level;
|
|
6507
|
+
logsByLevel[level] = (logsByLevel[level] || 0) + 1;
|
|
6508
|
+
}
|
|
6509
|
+
const metricsByType = {};
|
|
6510
|
+
const metricsByName = {};
|
|
6511
|
+
for (const event of this.#metricEvents) {
|
|
6512
|
+
const mType = event.metric.metricType;
|
|
6513
|
+
metricsByType[mType] = (metricsByType[mType] || 0) + 1;
|
|
6514
|
+
const mName = event.metric.name;
|
|
6515
|
+
metricsByName[mName] = (metricsByName[mName] || 0) + 1;
|
|
6516
|
+
}
|
|
6517
|
+
const scoresByScorer = {};
|
|
6518
|
+
for (const event of this.#scoreEvents) {
|
|
6519
|
+
const scorer = event.score.scorerName;
|
|
6520
|
+
scoresByScorer[scorer] = (scoresByScorer[scorer] || 0) + 1;
|
|
6521
|
+
}
|
|
6522
|
+
const feedbackByType = {};
|
|
6523
|
+
for (const event of this.#feedbackEvents) {
|
|
6524
|
+
const fbType = event.feedback.feedbackType;
|
|
6525
|
+
feedbackByType[fbType] = (feedbackByType[fbType] || 0) + 1;
|
|
6526
|
+
}
|
|
6314
6527
|
return {
|
|
6315
|
-
|
|
6528
|
+
totalTracingEvents: this.#tracingEvents.length,
|
|
6529
|
+
totalEvents: this.#tracingEvents.length,
|
|
6530
|
+
// deprecated alias
|
|
6316
6531
|
totalSpans: this.#spanStates.size,
|
|
6317
6532
|
totalTraces: this.getTraceIds().length,
|
|
6318
6533
|
completedSpans,
|
|
6319
6534
|
incompleteSpans,
|
|
6320
6535
|
byEventType: {
|
|
6321
|
-
started: this.#
|
|
6322
|
-
updated: this.#
|
|
6323
|
-
ended: this.#
|
|
6536
|
+
started: this.#tracingEvents.filter((e) => e.type === TracingEventType.SPAN_STARTED).length,
|
|
6537
|
+
updated: this.#tracingEvents.filter((e) => e.type === TracingEventType.SPAN_UPDATED).length,
|
|
6538
|
+
ended: this.#tracingEvents.filter((e) => e.type === TracingEventType.SPAN_ENDED).length
|
|
6324
6539
|
},
|
|
6325
|
-
bySpanType
|
|
6540
|
+
bySpanType,
|
|
6541
|
+
totalLogs: this.#logEvents.length,
|
|
6542
|
+
logsByLevel,
|
|
6543
|
+
totalMetrics: this.#metricEvents.length,
|
|
6544
|
+
metricsByType,
|
|
6545
|
+
metricsByName,
|
|
6546
|
+
totalScores: this.#scoreEvents.length,
|
|
6547
|
+
scoresByScorer,
|
|
6548
|
+
totalFeedback: this.#feedbackEvents.length,
|
|
6549
|
+
feedbackByType
|
|
6326
6550
|
};
|
|
6327
6551
|
}
|
|
6328
6552
|
// ============================================================================
|
|
@@ -6341,8 +6565,20 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6341
6565
|
const data = {
|
|
6342
6566
|
spans: this.getAllSpans()
|
|
6343
6567
|
};
|
|
6568
|
+
if (this.#logEvents.length > 0) {
|
|
6569
|
+
data.logs = this.getAllLogs();
|
|
6570
|
+
}
|
|
6571
|
+
if (this.#metricEvents.length > 0) {
|
|
6572
|
+
data.metrics = this.getAllMetrics();
|
|
6573
|
+
}
|
|
6574
|
+
if (this.#scoreEvents.length > 0) {
|
|
6575
|
+
data.scores = this.getAllScores();
|
|
6576
|
+
}
|
|
6577
|
+
if (this.#feedbackEvents.length > 0) {
|
|
6578
|
+
data.feedback = this.getAllFeedback();
|
|
6579
|
+
}
|
|
6344
6580
|
if (includeEvents) {
|
|
6345
|
-
data.events = this.#
|
|
6581
|
+
data.events = this.#tracingEvents;
|
|
6346
6582
|
}
|
|
6347
6583
|
if (includeStats) {
|
|
6348
6584
|
data.statistics = this.getStatistics();
|
|
@@ -6416,6 +6652,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6416
6652
|
const uuidRegex2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6417
6653
|
const hexId32Regex = /^[0-9a-f]{32}$/i;
|
|
6418
6654
|
const prefixedUuidRegex = /^([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
|
|
6655
|
+
const embeddedPrefixedUuidTest = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i;
|
|
6419
6656
|
const embeddedPrefixedUuidRegex = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
|
|
6420
6657
|
const normalizeUuid = (uuid, key) => {
|
|
6421
6658
|
if (!uuidMapsByKey.has(key)) {
|
|
@@ -6450,8 +6687,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6450
6687
|
const uuid = prefixMatch[2];
|
|
6451
6688
|
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6452
6689
|
}
|
|
6453
|
-
if (
|
|
6454
|
-
embeddedPrefixedUuidRegex.lastIndex = 0;
|
|
6690
|
+
if (embeddedPrefixedUuidTest.test(value)) {
|
|
6455
6691
|
return value.replace(embeddedPrefixedUuidRegex, (_match, prefix, uuid) => {
|
|
6456
6692
|
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6457
6693
|
});
|
|
@@ -6513,7 +6749,7 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6513
6749
|
normalizedSpan.output = normalizeValue(span.output);
|
|
6514
6750
|
}
|
|
6515
6751
|
if (span.errorInfo) {
|
|
6516
|
-
normalizedSpan.errorInfo = span.errorInfo;
|
|
6752
|
+
normalizedSpan.errorInfo = normalizeValue(span.errorInfo);
|
|
6517
6753
|
}
|
|
6518
6754
|
if (span.tags && span.tags.length > 0) {
|
|
6519
6755
|
normalizedSpan.tags = span.tags;
|
|
@@ -6600,8 +6836,9 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6600
6836
|
} else {
|
|
6601
6837
|
json = this.toJSON(options);
|
|
6602
6838
|
}
|
|
6839
|
+
const { writeFile } = await import('fs/promises');
|
|
6603
6840
|
await writeFile(filePath, json, "utf-8");
|
|
6604
|
-
this.logger.info(`
|
|
6841
|
+
this.logger.info(`TestExporter: wrote ${this.#tracingEvents.length} tracing events to ${filePath}`);
|
|
6605
6842
|
}
|
|
6606
6843
|
/**
|
|
6607
6844
|
* Assert that the current normalized tree matches a snapshot file.
|
|
@@ -6629,7 +6866,8 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6629
6866
|
* @throws Error if the snapshot doesn't match (and updateSnapshot is false)
|
|
6630
6867
|
*/
|
|
6631
6868
|
async assertMatchesSnapshot(snapshotName, options) {
|
|
6632
|
-
const
|
|
6869
|
+
const { join } = await import('path');
|
|
6870
|
+
const snapshotPath = join(await getSnapshotsDir(), snapshotName);
|
|
6633
6871
|
const normalizedTree = this.buildNormalizedTree();
|
|
6634
6872
|
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6635
6873
|
const currentData = {
|
|
@@ -6639,17 +6877,29 @@ var JsonExporter = class extends BaseExporter {
|
|
|
6639
6877
|
const currentJson = JSON.stringify(currentData, null, this.#config.jsonIndent);
|
|
6640
6878
|
const shouldUpdate = options?.updateSnapshot;
|
|
6641
6879
|
if (shouldUpdate) {
|
|
6880
|
+
const { writeFile } = await import('fs/promises');
|
|
6642
6881
|
await writeFile(snapshotPath, currentJson, "utf-8");
|
|
6643
|
-
this.logger.info(`
|
|
6882
|
+
this.logger.info(`TestExporter: updated snapshot ${snapshotPath}`);
|
|
6644
6883
|
return;
|
|
6645
6884
|
}
|
|
6646
6885
|
let snapshotData;
|
|
6886
|
+
let snapshotContent;
|
|
6887
|
+
try {
|
|
6888
|
+
const { readFile } = await import('fs/promises');
|
|
6889
|
+
snapshotContent = await readFile(snapshotPath, "utf-8");
|
|
6890
|
+
} catch (err) {
|
|
6891
|
+
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
|
6892
|
+
throw new Error(
|
|
6893
|
+
`Snapshot file not found: ${snapshotPath}
|
|
6894
|
+
Run with { updateSnapshot: true } to create it.`
|
|
6895
|
+
);
|
|
6896
|
+
}
|
|
6897
|
+
throw err;
|
|
6898
|
+
}
|
|
6647
6899
|
try {
|
|
6648
|
-
const snapshotContent = await readFile(snapshotPath, "utf-8");
|
|
6649
6900
|
snapshotData = JSON.parse(snapshotContent);
|
|
6650
|
-
} catch {
|
|
6651
|
-
throw new Error(`
|
|
6652
|
-
Run with { updateSnapshot: true } to create it.`);
|
|
6901
|
+
} catch (err) {
|
|
6902
|
+
throw new Error(`Failed to parse snapshot ${snapshotPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
6653
6903
|
}
|
|
6654
6904
|
let expectedSpans;
|
|
6655
6905
|
let expectedStructure;
|
|
@@ -6850,17 +7100,19 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6850
7100
|
// Debugging Helpers
|
|
6851
7101
|
// ============================================================================
|
|
6852
7102
|
/**
|
|
6853
|
-
* Get all stored logs
|
|
7103
|
+
* Get all stored debug logs (internal exporter logging, not signal logs)
|
|
6854
7104
|
*/
|
|
6855
7105
|
getLogs() {
|
|
6856
|
-
return [...this.#
|
|
7106
|
+
return [...this.#debugLogs];
|
|
6857
7107
|
}
|
|
6858
7108
|
/**
|
|
6859
|
-
* Dump logs to console for debugging (uses console.error for visibility in test output)
|
|
7109
|
+
* Dump debug logs to console for debugging (uses console.error for visibility in test output)
|
|
6860
7110
|
*/
|
|
6861
7111
|
dumpLogs() {
|
|
6862
|
-
console.error("\n===
|
|
6863
|
-
this.#
|
|
7112
|
+
console.error("\n=== TestExporter Logs ===");
|
|
7113
|
+
this.#debugLogs.forEach((log) => {
|
|
7114
|
+
console.error(log);
|
|
7115
|
+
});
|
|
6864
7116
|
console.error("=== End Logs ===\n");
|
|
6865
7117
|
}
|
|
6866
7118
|
/**
|
|
@@ -6885,12 +7137,16 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6885
7137
|
// Reset & Lifecycle
|
|
6886
7138
|
// ============================================================================
|
|
6887
7139
|
/**
|
|
6888
|
-
* Clear all collected events and state
|
|
7140
|
+
* Clear all collected events and state across all signals
|
|
6889
7141
|
*/
|
|
6890
7142
|
clearEvents() {
|
|
6891
|
-
this.#
|
|
7143
|
+
this.#tracingEvents = [];
|
|
6892
7144
|
this.#spanStates.clear();
|
|
6893
|
-
this.#
|
|
7145
|
+
this.#logEvents = [];
|
|
7146
|
+
this.#metricEvents = [];
|
|
7147
|
+
this.#scoreEvents = [];
|
|
7148
|
+
this.#feedbackEvents = [];
|
|
7149
|
+
this.#debugLogs = [];
|
|
6894
7150
|
}
|
|
6895
7151
|
/**
|
|
6896
7152
|
* Alias for clearEvents (compatibility with TestExporter)
|
|
@@ -6898,8 +7154,42 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6898
7154
|
reset() {
|
|
6899
7155
|
this.clearEvents();
|
|
6900
7156
|
}
|
|
7157
|
+
/**
|
|
7158
|
+
* Get internal metrics about the exporter's own activity.
|
|
7159
|
+
*/
|
|
7160
|
+
getInternalMetrics() {
|
|
7161
|
+
const json = this.toJSON({ includeEvents: false, includeStats: false });
|
|
7162
|
+
return {
|
|
7163
|
+
startedAt: this.#internalMetrics.startedAt,
|
|
7164
|
+
lastEventAt: this.#internalMetrics.lastEventAt,
|
|
7165
|
+
totalEventsReceived: this.#internalMetrics.totalEventsReceived,
|
|
7166
|
+
bySignal: { ...this.#internalMetrics.bySignal },
|
|
7167
|
+
flushCount: this.#internalMetrics.flushCount,
|
|
7168
|
+
estimatedJsonBytes: new TextEncoder().encode(json).byteLength
|
|
7169
|
+
};
|
|
7170
|
+
}
|
|
7171
|
+
/**
|
|
7172
|
+
* Flush buffered data and log internal metrics summary.
|
|
7173
|
+
*/
|
|
7174
|
+
async flush() {
|
|
7175
|
+
this.#internalMetrics.flushCount++;
|
|
7176
|
+
if (this.#config.logMetricsOnFlush) {
|
|
7177
|
+
const metrics = this.getInternalMetrics();
|
|
7178
|
+
const uptimeMs = Date.now() - metrics.startedAt.getTime();
|
|
7179
|
+
const summary = [
|
|
7180
|
+
`[TestExporter] flush #${metrics.flushCount} summary:`,
|
|
7181
|
+
` uptime: ${(uptimeMs / 1e3).toFixed(1)}s`,
|
|
7182
|
+
` total events received: ${metrics.totalEventsReceived}`,
|
|
7183
|
+
` by signal: tracing=${metrics.bySignal.tracing}, log=${metrics.bySignal.log}, metric=${metrics.bySignal.metric}, score=${metrics.bySignal.score}, feedback=${metrics.bySignal.feedback}`,
|
|
7184
|
+
` buffered: spans=${this.#spanStates.size}, logs=${this.#logEvents.length}, metrics=${this.#metricEvents.length}, scores=${this.#scoreEvents.length}, feedback=${this.#feedbackEvents.length}`,
|
|
7185
|
+
` estimated JSON size: ${(metrics.estimatedJsonBytes / 1024).toFixed(1)}KB`
|
|
7186
|
+
].join("\n");
|
|
7187
|
+
this.logger.info(summary);
|
|
7188
|
+
}
|
|
7189
|
+
}
|
|
6901
7190
|
async shutdown() {
|
|
6902
|
-
this.
|
|
7191
|
+
await this.flush();
|
|
7192
|
+
this.logger.info("TestExporter shutdown");
|
|
6903
7193
|
}
|
|
6904
7194
|
// ============================================================================
|
|
6905
7195
|
// Private Helpers
|
|
@@ -6918,6 +7208,634 @@ Run with { updateSnapshot: true } to update.`
|
|
|
6918
7208
|
return Array.from(spanMap.values());
|
|
6919
7209
|
}
|
|
6920
7210
|
};
|
|
7211
|
+
var JsonExporter = TestExporter;
|
|
7212
|
+
var BaseObservabilityEventBus = class _BaseObservabilityEventBus extends MastraBase {
|
|
7213
|
+
subscribers = /* @__PURE__ */ new Set();
|
|
7214
|
+
/** In-flight async subscriber promises. Self-cleaning via .finally(). */
|
|
7215
|
+
pendingSubscribers = /* @__PURE__ */ new Set();
|
|
7216
|
+
constructor({ name } = {}) {
|
|
7217
|
+
super({ component: RegisteredLogger.OBSERVABILITY, name: name ?? "EventBus" });
|
|
7218
|
+
}
|
|
7219
|
+
/**
|
|
7220
|
+
* Dispatch an event to all subscribers synchronously.
|
|
7221
|
+
* Async handler promises are tracked internally and drained by {@link flush}.
|
|
7222
|
+
*
|
|
7223
|
+
* @param event - The event to broadcast to subscribers.
|
|
7224
|
+
*/
|
|
7225
|
+
emit(event) {
|
|
7226
|
+
for (const handler of this.subscribers) {
|
|
7227
|
+
try {
|
|
7228
|
+
const result = handler(event);
|
|
7229
|
+
if (result && typeof result.then === "function") {
|
|
7230
|
+
const promise = result.catch((err) => {
|
|
7231
|
+
this.logger.error("[ObservabilityEventBus] Handler error:", err);
|
|
7232
|
+
});
|
|
7233
|
+
this.pendingSubscribers.add(promise);
|
|
7234
|
+
void promise.finally(() => this.pendingSubscribers.delete(promise));
|
|
7235
|
+
}
|
|
7236
|
+
} catch (err) {
|
|
7237
|
+
this.logger.error("[ObservabilityEventBus] Handler error:", err);
|
|
7238
|
+
}
|
|
7239
|
+
}
|
|
7240
|
+
}
|
|
7241
|
+
/**
|
|
7242
|
+
* Register a handler to receive future events.
|
|
7243
|
+
*
|
|
7244
|
+
* @param handler - Callback invoked synchronously on each {@link emit}.
|
|
7245
|
+
* @returns An unsubscribe function that removes the handler.
|
|
7246
|
+
*/
|
|
7247
|
+
subscribe(handler) {
|
|
7248
|
+
this.subscribers.add(handler);
|
|
7249
|
+
return () => {
|
|
7250
|
+
this.subscribers.delete(handler);
|
|
7251
|
+
};
|
|
7252
|
+
}
|
|
7253
|
+
/** Max flush drain iterations before bailing — prevents infinite loops when handlers re-emit. */
|
|
7254
|
+
static MAX_FLUSH_ITERATIONS = 3;
|
|
7255
|
+
/** Await all in-flight async subscriber promises, draining until empty. */
|
|
7256
|
+
async flush() {
|
|
7257
|
+
let iterations = 0;
|
|
7258
|
+
while (this.pendingSubscribers.size > 0) {
|
|
7259
|
+
await Promise.allSettled([...this.pendingSubscribers]);
|
|
7260
|
+
iterations++;
|
|
7261
|
+
if (iterations >= _BaseObservabilityEventBus.MAX_FLUSH_ITERATIONS) {
|
|
7262
|
+
this.logger.error(
|
|
7263
|
+
`[ObservabilityEventBus] flush() exceeded ${_BaseObservabilityEventBus.MAX_FLUSH_ITERATIONS} drain iterations \u2014 ${this.pendingSubscribers.size} promises still pending. Handlers may be re-emitting during flush.`
|
|
7264
|
+
);
|
|
7265
|
+
break;
|
|
7266
|
+
}
|
|
7267
|
+
}
|
|
7268
|
+
}
|
|
7269
|
+
/** Flush pending promises, then clear all subscribers. */
|
|
7270
|
+
async shutdown() {
|
|
7271
|
+
await this.flush();
|
|
7272
|
+
this.subscribers.clear();
|
|
7273
|
+
}
|
|
7274
|
+
};
|
|
7275
|
+
var AutoExtractedMetrics = class {
|
|
7276
|
+
/**
|
|
7277
|
+
* @param observabilityBus - Bus used to emit derived MetricEvents.
|
|
7278
|
+
* @param cardinalityFilter - Optional filter applied to metric labels before emission.
|
|
7279
|
+
*/
|
|
7280
|
+
constructor(observabilityBus, cardinalityFilter) {
|
|
7281
|
+
this.observabilityBus = observabilityBus;
|
|
7282
|
+
this.cardinalityFilter = cardinalityFilter;
|
|
7283
|
+
}
|
|
7284
|
+
/**
|
|
7285
|
+
* Route a tracing event to the appropriate span lifecycle handler.
|
|
7286
|
+
* SPAN_STARTED increments a started counter; SPAN_ENDED emits ended counter,
|
|
7287
|
+
* duration histogram, and (for model spans) token counters.
|
|
7288
|
+
*/
|
|
7289
|
+
processTracingEvent(event) {
|
|
7290
|
+
switch (event.type) {
|
|
7291
|
+
case TracingEventType.SPAN_STARTED:
|
|
7292
|
+
this.onSpanStarted(event.exportedSpan);
|
|
7293
|
+
break;
|
|
7294
|
+
case TracingEventType.SPAN_ENDED:
|
|
7295
|
+
this.onSpanEnded(event.exportedSpan);
|
|
7296
|
+
break;
|
|
7297
|
+
}
|
|
7298
|
+
}
|
|
7299
|
+
/** Emit a `mastra_scores_total` counter for a score event. */
|
|
7300
|
+
processScoreEvent(event) {
|
|
7301
|
+
const labels = {
|
|
7302
|
+
scorer: event.score.scorerName
|
|
7303
|
+
};
|
|
7304
|
+
if (event.score.metadata?.entityType) {
|
|
7305
|
+
labels.entity_type = String(event.score.metadata.entityType);
|
|
7306
|
+
}
|
|
7307
|
+
if (event.score.experimentId) {
|
|
7308
|
+
labels.experiment = event.score.experimentId;
|
|
7309
|
+
}
|
|
7310
|
+
this.emit("mastra_scores_total", "counter", 1, labels);
|
|
7311
|
+
}
|
|
7312
|
+
/** Emit a `mastra_feedback_total` counter for a feedback event. */
|
|
7313
|
+
processFeedbackEvent(event) {
|
|
7314
|
+
const labels = {
|
|
7315
|
+
feedback_type: event.feedback.feedbackType,
|
|
7316
|
+
source: event.feedback.source
|
|
7317
|
+
};
|
|
7318
|
+
if (event.feedback.metadata?.entityType) {
|
|
7319
|
+
labels.entity_type = String(event.feedback.metadata.entityType);
|
|
7320
|
+
}
|
|
7321
|
+
if (event.feedback.experimentId) {
|
|
7322
|
+
labels.experiment = event.feedback.experimentId;
|
|
7323
|
+
}
|
|
7324
|
+
this.emit("mastra_feedback_total", "counter", 1, labels);
|
|
7325
|
+
}
|
|
7326
|
+
/** Emit a started counter (e.g. `mastra_agent_runs_started`) for the span type. */
|
|
7327
|
+
onSpanStarted(span) {
|
|
7328
|
+
const labels = this.extractLabels(span);
|
|
7329
|
+
const metricName = this.getStartedMetricName(span);
|
|
7330
|
+
if (metricName) {
|
|
7331
|
+
this.emit(metricName, "counter", 1, labels);
|
|
7332
|
+
}
|
|
7333
|
+
}
|
|
7334
|
+
/** Emit ended counter, duration histogram, and token counters (for model spans). */
|
|
7335
|
+
onSpanEnded(span) {
|
|
7336
|
+
const labels = this.extractLabels(span);
|
|
7337
|
+
const endedMetricName = this.getEndedMetricName(span);
|
|
7338
|
+
if (endedMetricName) {
|
|
7339
|
+
const endedLabels = { ...labels };
|
|
7340
|
+
if (span.errorInfo) {
|
|
7341
|
+
endedLabels.status = "error";
|
|
7342
|
+
} else {
|
|
7343
|
+
endedLabels.status = "ok";
|
|
7344
|
+
}
|
|
7345
|
+
this.emit(endedMetricName, "counter", 1, endedLabels);
|
|
7346
|
+
}
|
|
7347
|
+
const durationMetricName = this.getDurationMetricName(span);
|
|
7348
|
+
if (durationMetricName && span.startTime && span.endTime) {
|
|
7349
|
+
const durationMs = Math.max(0, span.endTime.getTime() - span.startTime.getTime());
|
|
7350
|
+
const durationLabels = { ...labels };
|
|
7351
|
+
if (span.errorInfo) {
|
|
7352
|
+
durationLabels.status = "error";
|
|
7353
|
+
} else {
|
|
7354
|
+
durationLabels.status = "ok";
|
|
7355
|
+
}
|
|
7356
|
+
this.emit(durationMetricName, "histogram", durationMs, durationLabels);
|
|
7357
|
+
}
|
|
7358
|
+
if (span.type === SpanType.MODEL_GENERATION) {
|
|
7359
|
+
this.extractTokenMetrics(span, labels);
|
|
7360
|
+
}
|
|
7361
|
+
}
|
|
7362
|
+
/** Build base metric labels from a span's entity and model attributes. */
|
|
7363
|
+
extractLabels(span) {
|
|
7364
|
+
const labels = {};
|
|
7365
|
+
if (span.entityType) labels.entity_type = span.entityType;
|
|
7366
|
+
const entityName = span.entityName ?? span.entityId;
|
|
7367
|
+
if (entityName) labels.entity_name = entityName;
|
|
7368
|
+
if (span.type === SpanType.MODEL_GENERATION) {
|
|
7369
|
+
const attrs = span.attributes;
|
|
7370
|
+
if (attrs?.model) labels.model = String(attrs.model);
|
|
7371
|
+
if (attrs?.provider) labels.provider = String(attrs.provider);
|
|
7372
|
+
}
|
|
7373
|
+
return labels;
|
|
7374
|
+
}
|
|
7375
|
+
/** Emit token usage counters from a MODEL_GENERATION span's `usage` attributes. Negative and non-finite values are skipped. */
|
|
7376
|
+
extractTokenMetrics(span, labels) {
|
|
7377
|
+
const attrs = span.attributes;
|
|
7378
|
+
const usage = attrs?.usage;
|
|
7379
|
+
if (!usage) return;
|
|
7380
|
+
const inputTokens = Number(usage.inputTokens);
|
|
7381
|
+
if (Number.isFinite(inputTokens) && inputTokens >= 0) {
|
|
7382
|
+
this.emit("mastra_model_input_tokens", "counter", inputTokens, labels);
|
|
7383
|
+
}
|
|
7384
|
+
const outputTokens = Number(usage.outputTokens);
|
|
7385
|
+
if (Number.isFinite(outputTokens) && outputTokens >= 0) {
|
|
7386
|
+
this.emit("mastra_model_output_tokens", "counter", outputTokens, labels);
|
|
7387
|
+
}
|
|
7388
|
+
const inputDetails = usage.inputDetails;
|
|
7389
|
+
const cacheRead = Number(inputDetails?.cacheRead);
|
|
7390
|
+
if (Number.isFinite(cacheRead) && cacheRead >= 0) {
|
|
7391
|
+
this.emit("mastra_model_cache_read_tokens", "counter", cacheRead, labels);
|
|
7392
|
+
}
|
|
7393
|
+
const cacheWrite = Number(inputDetails?.cacheWrite);
|
|
7394
|
+
if (Number.isFinite(cacheWrite) && cacheWrite >= 0) {
|
|
7395
|
+
this.emit("mastra_model_cache_write_tokens", "counter", cacheWrite, labels);
|
|
7396
|
+
}
|
|
7397
|
+
}
|
|
7398
|
+
/** Map a span type to its `*_started` counter metric name, or `null` for unsupported types. */
|
|
7399
|
+
getStartedMetricName(span) {
|
|
7400
|
+
switch (span.type) {
|
|
7401
|
+
case SpanType.AGENT_RUN:
|
|
7402
|
+
return "mastra_agent_runs_started";
|
|
7403
|
+
case SpanType.TOOL_CALL:
|
|
7404
|
+
return "mastra_tool_calls_started";
|
|
7405
|
+
case SpanType.WORKFLOW_RUN:
|
|
7406
|
+
return "mastra_workflow_runs_started";
|
|
7407
|
+
case SpanType.MODEL_GENERATION:
|
|
7408
|
+
return "mastra_model_requests_started";
|
|
7409
|
+
default:
|
|
7410
|
+
return null;
|
|
7411
|
+
}
|
|
7412
|
+
}
|
|
7413
|
+
/** Map a span type to its `*_ended` counter metric name, or `null` for unsupported types. */
|
|
7414
|
+
getEndedMetricName(span) {
|
|
7415
|
+
switch (span.type) {
|
|
7416
|
+
case SpanType.AGENT_RUN:
|
|
7417
|
+
return "mastra_agent_runs_ended";
|
|
7418
|
+
case SpanType.TOOL_CALL:
|
|
7419
|
+
return "mastra_tool_calls_ended";
|
|
7420
|
+
case SpanType.WORKFLOW_RUN:
|
|
7421
|
+
return "mastra_workflow_runs_ended";
|
|
7422
|
+
case SpanType.MODEL_GENERATION:
|
|
7423
|
+
return "mastra_model_requests_ended";
|
|
7424
|
+
default:
|
|
7425
|
+
return null;
|
|
7426
|
+
}
|
|
7427
|
+
}
|
|
7428
|
+
/** Map a span type to its `*_duration_ms` histogram metric name, or `null` for unsupported types. */
|
|
7429
|
+
getDurationMetricName(span) {
|
|
7430
|
+
switch (span.type) {
|
|
7431
|
+
case SpanType.AGENT_RUN:
|
|
7432
|
+
return "mastra_agent_duration_ms";
|
|
7433
|
+
case SpanType.TOOL_CALL:
|
|
7434
|
+
return "mastra_tool_duration_ms";
|
|
7435
|
+
case SpanType.WORKFLOW_RUN:
|
|
7436
|
+
return "mastra_workflow_duration_ms";
|
|
7437
|
+
case SpanType.MODEL_GENERATION:
|
|
7438
|
+
return "mastra_model_duration_ms";
|
|
7439
|
+
default:
|
|
7440
|
+
return null;
|
|
7441
|
+
}
|
|
7442
|
+
}
|
|
7443
|
+
/** Build an ExportedMetric, apply cardinality filtering, and emit it through the bus. */
|
|
7444
|
+
emit(name, metricType, value, labels) {
|
|
7445
|
+
const filteredLabels = this.cardinalityFilter ? this.cardinalityFilter.filterLabels(labels) : labels;
|
|
7446
|
+
const exportedMetric = {
|
|
7447
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7448
|
+
name,
|
|
7449
|
+
metricType,
|
|
7450
|
+
value,
|
|
7451
|
+
labels: filteredLabels
|
|
7452
|
+
};
|
|
7453
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
7454
|
+
this.observabilityBus.emit(event);
|
|
7455
|
+
}
|
|
7456
|
+
};
|
|
7457
|
+
function routeToHandler(handler, event, logger) {
|
|
7458
|
+
try {
|
|
7459
|
+
switch (event.type) {
|
|
7460
|
+
case TracingEventType.SPAN_STARTED:
|
|
7461
|
+
case TracingEventType.SPAN_UPDATED:
|
|
7462
|
+
case TracingEventType.SPAN_ENDED: {
|
|
7463
|
+
const fn = handler.onTracingEvent ? handler.onTracingEvent.bind(handler) : handler.exportTracingEvent.bind(handler);
|
|
7464
|
+
return catchAsyncResult(fn(event), handler.name, "tracing", logger);
|
|
7465
|
+
}
|
|
7466
|
+
case "log":
|
|
7467
|
+
if (handler.onLogEvent) {
|
|
7468
|
+
return catchAsyncResult(handler.onLogEvent(event), handler.name, "log", logger);
|
|
7469
|
+
}
|
|
7470
|
+
break;
|
|
7471
|
+
case "metric":
|
|
7472
|
+
if (handler.onMetricEvent) {
|
|
7473
|
+
return catchAsyncResult(handler.onMetricEvent(event), handler.name, "metric", logger);
|
|
7474
|
+
}
|
|
7475
|
+
break;
|
|
7476
|
+
case "score":
|
|
7477
|
+
if (handler.onScoreEvent) {
|
|
7478
|
+
return catchAsyncResult(handler.onScoreEvent(event), handler.name, "score", logger);
|
|
7479
|
+
}
|
|
7480
|
+
break;
|
|
7481
|
+
case "feedback":
|
|
7482
|
+
if (handler.onFeedbackEvent) {
|
|
7483
|
+
return catchAsyncResult(handler.onFeedbackEvent(event), handler.name, "feedback", logger);
|
|
7484
|
+
}
|
|
7485
|
+
break;
|
|
7486
|
+
}
|
|
7487
|
+
} catch (err) {
|
|
7488
|
+
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
7489
|
+
}
|
|
7490
|
+
}
|
|
7491
|
+
function catchAsyncResult(result, handlerName, signal, logger) {
|
|
7492
|
+
if (result && typeof result.then === "function") {
|
|
7493
|
+
return result.catch((err) => {
|
|
7494
|
+
logger.error(`[Observability] ${signal} handler error [handler=${handlerName}]:`, err);
|
|
7495
|
+
});
|
|
7496
|
+
}
|
|
7497
|
+
}
|
|
7498
|
+
|
|
7499
|
+
// src/bus/observability-bus.ts
|
|
7500
|
+
var MAX_FLUSH_ITERATIONS = 3;
|
|
7501
|
+
function isTracingEvent(event) {
|
|
7502
|
+
return event.type === TracingEventType.SPAN_STARTED || event.type === TracingEventType.SPAN_UPDATED || event.type === TracingEventType.SPAN_ENDED;
|
|
7503
|
+
}
|
|
7504
|
+
var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
7505
|
+
exporters = [];
|
|
7506
|
+
bridge;
|
|
7507
|
+
autoExtractor;
|
|
7508
|
+
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
7509
|
+
pendingHandlers = /* @__PURE__ */ new Set();
|
|
7510
|
+
constructor() {
|
|
7511
|
+
super({ name: "ObservabilityBus" });
|
|
7512
|
+
}
|
|
7513
|
+
/**
|
|
7514
|
+
* Enable auto-extraction of metrics from tracing, score, and feedback events.
|
|
7515
|
+
* When enabled, span lifecycle events automatically generate counter/histogram
|
|
7516
|
+
* metrics (e.g., mastra_agent_runs_started, mastra_model_duration_ms).
|
|
7517
|
+
*
|
|
7518
|
+
* No-ops if auto-extraction is already enabled.
|
|
7519
|
+
*
|
|
7520
|
+
* @param cardinalityFilter - Optional filter applied to auto-extracted metric labels.
|
|
7521
|
+
*/
|
|
7522
|
+
enableAutoExtractedMetrics(cardinalityFilter) {
|
|
7523
|
+
if (this.autoExtractor) {
|
|
7524
|
+
return;
|
|
7525
|
+
}
|
|
7526
|
+
this.autoExtractor = new AutoExtractedMetrics(this, cardinalityFilter);
|
|
7527
|
+
}
|
|
7528
|
+
/**
|
|
7529
|
+
* Register an exporter to receive routed events.
|
|
7530
|
+
* Duplicate registrations (same instance) are silently ignored.
|
|
7531
|
+
*
|
|
7532
|
+
* @param exporter - The exporter to register.
|
|
7533
|
+
*/
|
|
7534
|
+
registerExporter(exporter) {
|
|
7535
|
+
if (this.exporters.includes(exporter)) {
|
|
7536
|
+
return;
|
|
7537
|
+
}
|
|
7538
|
+
this.exporters.push(exporter);
|
|
7539
|
+
}
|
|
7540
|
+
/**
|
|
7541
|
+
* Unregister an exporter.
|
|
7542
|
+
*
|
|
7543
|
+
* @param exporter - The exporter instance to remove.
|
|
7544
|
+
* @returns `true` if the exporter was found and removed, `false` otherwise.
|
|
7545
|
+
*/
|
|
7546
|
+
unregisterExporter(exporter) {
|
|
7547
|
+
const index = this.exporters.indexOf(exporter);
|
|
7548
|
+
if (index !== -1) {
|
|
7549
|
+
this.exporters.splice(index, 1);
|
|
7550
|
+
return true;
|
|
7551
|
+
}
|
|
7552
|
+
return false;
|
|
7553
|
+
}
|
|
7554
|
+
/**
|
|
7555
|
+
* Get registered exporters (read-only snapshot).
|
|
7556
|
+
*/
|
|
7557
|
+
getExporters() {
|
|
7558
|
+
return [...this.exporters];
|
|
7559
|
+
}
|
|
7560
|
+
/**
|
|
7561
|
+
* Register a bridge to receive all routed events alongside exporters.
|
|
7562
|
+
* Only one bridge can be registered at a time; replacing an existing bridge
|
|
7563
|
+
* logs a warning.
|
|
7564
|
+
*
|
|
7565
|
+
* @param bridge - The bridge to register.
|
|
7566
|
+
*/
|
|
7567
|
+
registerBridge(bridge) {
|
|
7568
|
+
if (this.bridge) {
|
|
7569
|
+
this.logger.warn(`[ObservabilityBus] Replacing existing bridge with new bridge`);
|
|
7570
|
+
}
|
|
7571
|
+
this.bridge = bridge;
|
|
7572
|
+
}
|
|
7573
|
+
/**
|
|
7574
|
+
* Unregister the bridge.
|
|
7575
|
+
*
|
|
7576
|
+
* @returns `true` if a bridge was registered and removed, `false` otherwise.
|
|
7577
|
+
*/
|
|
7578
|
+
unregisterBridge() {
|
|
7579
|
+
if (this.bridge) {
|
|
7580
|
+
this.bridge = void 0;
|
|
7581
|
+
return true;
|
|
7582
|
+
}
|
|
7583
|
+
return false;
|
|
7584
|
+
}
|
|
7585
|
+
/**
|
|
7586
|
+
* Get the registered bridge, if any.
|
|
7587
|
+
*/
|
|
7588
|
+
getBridge() {
|
|
7589
|
+
return this.bridge;
|
|
7590
|
+
}
|
|
7591
|
+
/**
|
|
7592
|
+
* Emit an event: route to exporter/bridge handlers, run auto-extraction,
|
|
7593
|
+
* then forward to base class for subscriber delivery.
|
|
7594
|
+
*
|
|
7595
|
+
* emit() is synchronous — async handler promises are tracked internally
|
|
7596
|
+
* and can be drained via flush().
|
|
7597
|
+
*/
|
|
7598
|
+
emit(event) {
|
|
7599
|
+
for (const exporter of this.exporters) {
|
|
7600
|
+
this.trackPromise(routeToHandler(exporter, event, this.logger));
|
|
7601
|
+
}
|
|
7602
|
+
if (this.bridge) {
|
|
7603
|
+
this.trackPromise(routeToHandler(this.bridge, event, this.logger));
|
|
7604
|
+
}
|
|
7605
|
+
if (this.autoExtractor) {
|
|
7606
|
+
try {
|
|
7607
|
+
if (isTracingEvent(event)) {
|
|
7608
|
+
this.autoExtractor.processTracingEvent(event);
|
|
7609
|
+
} else if (event.type === "score") {
|
|
7610
|
+
this.autoExtractor.processScoreEvent(event);
|
|
7611
|
+
} else if (event.type === "feedback") {
|
|
7612
|
+
this.autoExtractor.processFeedbackEvent(event);
|
|
7613
|
+
}
|
|
7614
|
+
} catch (err) {
|
|
7615
|
+
this.logger.error("[ObservabilityBus] Auto-extraction error:", err);
|
|
7616
|
+
}
|
|
7617
|
+
}
|
|
7618
|
+
super.emit(event);
|
|
7619
|
+
}
|
|
7620
|
+
/**
|
|
7621
|
+
* Track an async handler promise so flush() can await it.
|
|
7622
|
+
* No-ops for sync (void) results.
|
|
7623
|
+
*/
|
|
7624
|
+
trackPromise(result) {
|
|
7625
|
+
if (result && typeof result.then === "function") {
|
|
7626
|
+
const promise = result;
|
|
7627
|
+
this.pendingHandlers.add(promise);
|
|
7628
|
+
void promise.finally(() => this.pendingHandlers.delete(promise));
|
|
7629
|
+
}
|
|
7630
|
+
}
|
|
7631
|
+
/**
|
|
7632
|
+
* Two-phase flush to ensure all observability data is fully exported.
|
|
7633
|
+
*
|
|
7634
|
+
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
7635
|
+
* bridge, and base-class subscribers). After this resolves, all event data
|
|
7636
|
+
* has been delivered to handler methods.
|
|
7637
|
+
*
|
|
7638
|
+
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
7639
|
+
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
7640
|
+
* client queue). Phases are sequential — Phase 2 must not start until
|
|
7641
|
+
* Phase 1 completes, otherwise exporters would flush empty buffers.
|
|
7642
|
+
*/
|
|
7643
|
+
async flush() {
|
|
7644
|
+
let iterations = 0;
|
|
7645
|
+
while (this.pendingHandlers.size > 0) {
|
|
7646
|
+
await Promise.allSettled([...this.pendingHandlers]);
|
|
7647
|
+
iterations++;
|
|
7648
|
+
if (iterations >= MAX_FLUSH_ITERATIONS) {
|
|
7649
|
+
this.logger.error(
|
|
7650
|
+
`[ObservabilityBus] flush() exceeded ${MAX_FLUSH_ITERATIONS} drain iterations \u2014 ${this.pendingHandlers.size} promises still pending. Handlers may be re-emitting during flush.`
|
|
7651
|
+
);
|
|
7652
|
+
if (this.pendingHandlers.size > 0) {
|
|
7653
|
+
await Promise.allSettled([...this.pendingHandlers]);
|
|
7654
|
+
}
|
|
7655
|
+
break;
|
|
7656
|
+
}
|
|
7657
|
+
}
|
|
7658
|
+
await super.flush();
|
|
7659
|
+
const bufferFlushPromises = this.exporters.map((e) => e.flush());
|
|
7660
|
+
if (this.bridge) {
|
|
7661
|
+
bufferFlushPromises.push(this.bridge.flush());
|
|
7662
|
+
}
|
|
7663
|
+
if (bufferFlushPromises.length > 0) {
|
|
7664
|
+
await Promise.allSettled(bufferFlushPromises);
|
|
7665
|
+
}
|
|
7666
|
+
}
|
|
7667
|
+
/** Flush all pending events and exporter buffers, then clear subscribers. */
|
|
7668
|
+
async shutdown() {
|
|
7669
|
+
await this.flush();
|
|
7670
|
+
await super.shutdown();
|
|
7671
|
+
}
|
|
7672
|
+
};
|
|
7673
|
+
|
|
7674
|
+
// src/context/logger.ts
|
|
7675
|
+
var LOG_LEVEL_PRIORITY = {
|
|
7676
|
+
debug: 0,
|
|
7677
|
+
info: 1,
|
|
7678
|
+
warn: 2,
|
|
7679
|
+
error: 3,
|
|
7680
|
+
fatal: 4
|
|
7681
|
+
};
|
|
7682
|
+
var LoggerContextImpl = class {
|
|
7683
|
+
config;
|
|
7684
|
+
/**
|
|
7685
|
+
* Create a logger context. Tags and metadata are defensively copied so
|
|
7686
|
+
* mutations after construction do not affect emitted logs.
|
|
7687
|
+
*/
|
|
7688
|
+
constructor(config) {
|
|
7689
|
+
this.config = {
|
|
7690
|
+
...config,
|
|
7691
|
+
tags: config.tags ? [...config.tags] : void 0,
|
|
7692
|
+
metadata: config.metadata ? structuredClone(config.metadata) : void 0
|
|
7693
|
+
};
|
|
7694
|
+
}
|
|
7695
|
+
/** Log at DEBUG level. */
|
|
7696
|
+
debug(message, data) {
|
|
7697
|
+
this.log("debug", message, data);
|
|
7698
|
+
}
|
|
7699
|
+
/** Log at INFO level. */
|
|
7700
|
+
info(message, data) {
|
|
7701
|
+
this.log("info", message, data);
|
|
7702
|
+
}
|
|
7703
|
+
/** Log at WARN level. */
|
|
7704
|
+
warn(message, data) {
|
|
7705
|
+
this.log("warn", message, data);
|
|
7706
|
+
}
|
|
7707
|
+
/** Log at ERROR level. */
|
|
7708
|
+
error(message, data) {
|
|
7709
|
+
this.log("error", message, data);
|
|
7710
|
+
}
|
|
7711
|
+
/** Log at FATAL level. */
|
|
7712
|
+
fatal(message, data) {
|
|
7713
|
+
this.log("fatal", message, data);
|
|
7714
|
+
}
|
|
7715
|
+
/**
|
|
7716
|
+
* Build an ExportedLog, check against the minimum level, and emit it through the bus.
|
|
7717
|
+
*/
|
|
7718
|
+
log(level, message, data) {
|
|
7719
|
+
const minLevel = this.config.minLevel ?? "debug";
|
|
7720
|
+
if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[minLevel]) {
|
|
7721
|
+
return;
|
|
7722
|
+
}
|
|
7723
|
+
const exportedLog = {
|
|
7724
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7725
|
+
level,
|
|
7726
|
+
message,
|
|
7727
|
+
data,
|
|
7728
|
+
traceId: this.config.traceId,
|
|
7729
|
+
spanId: this.config.spanId,
|
|
7730
|
+
tags: this.config.tags,
|
|
7731
|
+
metadata: this.config.metadata
|
|
7732
|
+
};
|
|
7733
|
+
const event = { type: "log", log: exportedLog };
|
|
7734
|
+
this.config.observabilityBus.emit(event);
|
|
7735
|
+
}
|
|
7736
|
+
};
|
|
7737
|
+
|
|
7738
|
+
// src/context/metrics.ts
|
|
7739
|
+
var MetricsContextImpl = class {
|
|
7740
|
+
config;
|
|
7741
|
+
/**
|
|
7742
|
+
* Create a metrics context. Base labels are defensively copied so
|
|
7743
|
+
* mutations after construction do not affect emitted metrics.
|
|
7744
|
+
*/
|
|
7745
|
+
constructor(config) {
|
|
7746
|
+
this.config = {
|
|
7747
|
+
...config,
|
|
7748
|
+
labels: config.labels ? { ...config.labels } : void 0
|
|
7749
|
+
};
|
|
7750
|
+
}
|
|
7751
|
+
/**
|
|
7752
|
+
* Create a counter instrument. Call `.add(value)` to increment.
|
|
7753
|
+
*
|
|
7754
|
+
* @param name - Metric name (e.g. `mastra_custom_requests_total`).
|
|
7755
|
+
*/
|
|
7756
|
+
counter(name) {
|
|
7757
|
+
return {
|
|
7758
|
+
add: (value, additionalLabels) => {
|
|
7759
|
+
this.emit(name, "counter", value, additionalLabels);
|
|
7760
|
+
}
|
|
7761
|
+
};
|
|
7762
|
+
}
|
|
7763
|
+
/**
|
|
7764
|
+
* Create a gauge instrument. Call `.set(value)` to record a point-in-time value.
|
|
7765
|
+
*
|
|
7766
|
+
* @param name - Metric name (e.g. `mastra_queue_depth`).
|
|
7767
|
+
*/
|
|
7768
|
+
gauge(name) {
|
|
7769
|
+
return {
|
|
7770
|
+
set: (value, additionalLabels) => {
|
|
7771
|
+
this.emit(name, "gauge", value, additionalLabels);
|
|
7772
|
+
}
|
|
7773
|
+
};
|
|
7774
|
+
}
|
|
7775
|
+
/**
|
|
7776
|
+
* Create a histogram instrument. Call `.record(value)` to observe a measurement.
|
|
7777
|
+
*
|
|
7778
|
+
* @param name - Metric name (e.g. `mastra_request_duration_ms`).
|
|
7779
|
+
*/
|
|
7780
|
+
histogram(name) {
|
|
7781
|
+
return {
|
|
7782
|
+
record: (value, additionalLabels) => {
|
|
7783
|
+
this.emit(name, "histogram", value, additionalLabels);
|
|
7784
|
+
}
|
|
7785
|
+
};
|
|
7786
|
+
}
|
|
7787
|
+
/** Merge base + additional labels, apply cardinality filtering, and emit a MetricEvent. Non-finite values are silently dropped. */
|
|
7788
|
+
emit(name, metricType, value, additionalLabels) {
|
|
7789
|
+
if (!Number.isFinite(value)) return;
|
|
7790
|
+
const allLabels = {
|
|
7791
|
+
...this.config.labels,
|
|
7792
|
+
...additionalLabels
|
|
7793
|
+
};
|
|
7794
|
+
const filteredLabels = this.config.cardinalityFilter.filterLabels(allLabels);
|
|
7795
|
+
const exportedMetric = {
|
|
7796
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
7797
|
+
name,
|
|
7798
|
+
metricType,
|
|
7799
|
+
value,
|
|
7800
|
+
labels: filteredLabels
|
|
7801
|
+
};
|
|
7802
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
7803
|
+
this.config.observabilityBus.emit(event);
|
|
7804
|
+
}
|
|
7805
|
+
};
|
|
7806
|
+
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
7807
|
+
var CardinalityFilter = class {
|
|
7808
|
+
blockedLabels;
|
|
7809
|
+
blockUUIDs;
|
|
7810
|
+
/**
|
|
7811
|
+
* @param config - Optional configuration. When omitted, uses
|
|
7812
|
+
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
7813
|
+
*/
|
|
7814
|
+
constructor(config) {
|
|
7815
|
+
const blocked = config?.blockedLabels ?? [...DEFAULT_BLOCKED_LABELS];
|
|
7816
|
+
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
7817
|
+
this.blockUUIDs = config?.blockUUIDs ?? true;
|
|
7818
|
+
}
|
|
7819
|
+
/**
|
|
7820
|
+
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
7821
|
+
*
|
|
7822
|
+
* @param labels - Raw metric labels to filter.
|
|
7823
|
+
* @returns A new object containing only the allowed labels.
|
|
7824
|
+
*/
|
|
7825
|
+
filterLabels(labels) {
|
|
7826
|
+
const filtered = {};
|
|
7827
|
+
for (const [key, value] of Object.entries(labels)) {
|
|
7828
|
+
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
7829
|
+
continue;
|
|
7830
|
+
}
|
|
7831
|
+
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
7832
|
+
continue;
|
|
7833
|
+
}
|
|
7834
|
+
filtered[key] = value;
|
|
7835
|
+
}
|
|
7836
|
+
return filtered;
|
|
7837
|
+
}
|
|
7838
|
+
};
|
|
6921
7839
|
|
|
6922
7840
|
// src/usage.ts
|
|
6923
7841
|
function extractUsageMetrics(usage, providerMetadata) {
|
|
@@ -7940,6 +8858,15 @@ var NoOpSpan = class extends BaseSpan {
|
|
|
7940
8858
|
// src/instances/base.ts
|
|
7941
8859
|
var BaseObservabilityInstance = class extends MastraBase {
|
|
7942
8860
|
config;
|
|
8861
|
+
/**
|
|
8862
|
+
* Unified event bus for all observability signals.
|
|
8863
|
+
* Routes events to registered exporters based on event type.
|
|
8864
|
+
*/
|
|
8865
|
+
observabilityBus;
|
|
8866
|
+
/**
|
|
8867
|
+
* Cardinality filter for metrics label protection.
|
|
8868
|
+
*/
|
|
8869
|
+
cardinalityFilter;
|
|
7943
8870
|
constructor(config) {
|
|
7944
8871
|
super({ component: RegisteredLogger.OBSERVABILITY, name: config.serviceName });
|
|
7945
8872
|
this.config = {
|
|
@@ -7953,6 +8880,15 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
7953
8880
|
requestContextKeys: config.requestContextKeys ?? [],
|
|
7954
8881
|
serializationOptions: config.serializationOptions
|
|
7955
8882
|
};
|
|
8883
|
+
this.cardinalityFilter = new CardinalityFilter();
|
|
8884
|
+
this.observabilityBus = new ObservabilityBus();
|
|
8885
|
+
for (const exporter of this.exporters) {
|
|
8886
|
+
this.observabilityBus.registerExporter(exporter);
|
|
8887
|
+
}
|
|
8888
|
+
if (this.config.bridge) {
|
|
8889
|
+
this.observabilityBus.registerBridge(this.config.bridge);
|
|
8890
|
+
}
|
|
8891
|
+
this.observabilityBus.enableAutoExtractedMetrics(this.cardinalityFilter);
|
|
7956
8892
|
if (this.config.bridge?.init) {
|
|
7957
8893
|
this.config.bridge.init({ config: this.config });
|
|
7958
8894
|
}
|
|
@@ -8105,6 +9041,108 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
8105
9041
|
getLogger() {
|
|
8106
9042
|
return this.logger;
|
|
8107
9043
|
}
|
|
9044
|
+
/**
|
|
9045
|
+
* Get the ObservabilityBus for this instance.
|
|
9046
|
+
* The bus routes all observability events (tracing, logs, metrics, scores, feedback)
|
|
9047
|
+
* to registered exporters based on event type.
|
|
9048
|
+
*/
|
|
9049
|
+
getObservabilityBus() {
|
|
9050
|
+
return this.observabilityBus;
|
|
9051
|
+
}
|
|
9052
|
+
// ============================================================================
|
|
9053
|
+
// Context-factory bridge methods
|
|
9054
|
+
// ============================================================================
|
|
9055
|
+
/**
|
|
9056
|
+
* Extract entity context labels from a span's entity hierarchy by
|
|
9057
|
+
* walking the parent chain.
|
|
9058
|
+
*
|
|
9059
|
+
* Returns labels for: entity_type/name, parent_type/name, root_type/name.
|
|
9060
|
+
* Internal spans are skipped when resolving parent and root entities.
|
|
9061
|
+
*/
|
|
9062
|
+
extractEntityLabels(span) {
|
|
9063
|
+
const labels = {};
|
|
9064
|
+
if (span.entityType) labels.entity_type = span.entityType;
|
|
9065
|
+
if (span.entityName) labels.entity_name = span.entityName;
|
|
9066
|
+
let parentSpan = span.parent;
|
|
9067
|
+
while (parentSpan && parentSpan.isInternal) {
|
|
9068
|
+
parentSpan = parentSpan.parent;
|
|
9069
|
+
}
|
|
9070
|
+
if (parentSpan?.entityType && parentSpan.entityName) {
|
|
9071
|
+
labels.parent_type = parentSpan.entityType;
|
|
9072
|
+
labels.parent_name = parentSpan.entityName;
|
|
9073
|
+
let rootEntity = parentSpan;
|
|
9074
|
+
let current = parentSpan.parent;
|
|
9075
|
+
while (current) {
|
|
9076
|
+
if (!current.isInternal && current.entityType && current.entityName) {
|
|
9077
|
+
rootEntity = current;
|
|
9078
|
+
}
|
|
9079
|
+
current = current.parent;
|
|
9080
|
+
}
|
|
9081
|
+
if (rootEntity !== parentSpan) {
|
|
9082
|
+
labels.root_type = rootEntity.entityType;
|
|
9083
|
+
labels.root_name = rootEntity.entityName;
|
|
9084
|
+
}
|
|
9085
|
+
}
|
|
9086
|
+
return labels;
|
|
9087
|
+
}
|
|
9088
|
+
/**
|
|
9089
|
+
* Resolve tags for a span. Uses the span's own tags if present,
|
|
9090
|
+
* otherwise walks to the root span to inherit its tags.
|
|
9091
|
+
*/
|
|
9092
|
+
resolveSpanTags(span) {
|
|
9093
|
+
if (span.tags) return span.tags;
|
|
9094
|
+
let root = span;
|
|
9095
|
+
while (root.parent) {
|
|
9096
|
+
root = root.parent;
|
|
9097
|
+
}
|
|
9098
|
+
return root.tags;
|
|
9099
|
+
}
|
|
9100
|
+
/**
|
|
9101
|
+
* Get a LoggerContext correlated to a span.
|
|
9102
|
+
* Called by the context-factory in core (deriveLoggerContext) so that
|
|
9103
|
+
* `observabilityContext.loggerVNext` is a real logger instead of no-op.
|
|
9104
|
+
*/
|
|
9105
|
+
getLoggerContext(span) {
|
|
9106
|
+
const entityLabels = span ? this.extractEntityLabels(span) : void 0;
|
|
9107
|
+
const hasEntityLabels = entityLabels && Object.keys(entityLabels).length > 0;
|
|
9108
|
+
const metadata = hasEntityLabels || span?.metadata || this.config.serviceName ? {
|
|
9109
|
+
...hasEntityLabels ? entityLabels : void 0,
|
|
9110
|
+
...span?.metadata,
|
|
9111
|
+
...this.config.serviceName ? { serviceName: this.config.serviceName } : void 0
|
|
9112
|
+
} : void 0;
|
|
9113
|
+
return new LoggerContextImpl({
|
|
9114
|
+
traceId: span?.traceId,
|
|
9115
|
+
spanId: span?.id,
|
|
9116
|
+
tags: span ? this.resolveSpanTags(span) : void 0,
|
|
9117
|
+
metadata,
|
|
9118
|
+
observabilityBus: this.observabilityBus
|
|
9119
|
+
});
|
|
9120
|
+
}
|
|
9121
|
+
/**
|
|
9122
|
+
* Get a MetricsContext, optionally tagged from a span's entity info.
|
|
9123
|
+
* Called by the context-factory in core (deriveMetricsContext) so that
|
|
9124
|
+
* `observabilityContext.metrics` is a real metrics context instead of no-op.
|
|
9125
|
+
*/
|
|
9126
|
+
getMetricsContext(span) {
|
|
9127
|
+
const labels = span ? this.extractEntityLabels(span) : {};
|
|
9128
|
+
const attrs = span?.attributes;
|
|
9129
|
+
if (attrs?.model && typeof attrs.model === "string") labels.model = attrs.model;
|
|
9130
|
+
if (attrs?.provider && typeof attrs.provider === "string") labels.provider = attrs.provider;
|
|
9131
|
+
if (this.config.serviceName) labels.service_name = this.config.serviceName;
|
|
9132
|
+
return new MetricsContextImpl({
|
|
9133
|
+
labels: Object.keys(labels).length > 0 ? labels : void 0,
|
|
9134
|
+
observabilityBus: this.observabilityBus,
|
|
9135
|
+
cardinalityFilter: this.cardinalityFilter
|
|
9136
|
+
});
|
|
9137
|
+
}
|
|
9138
|
+
/**
|
|
9139
|
+
* Emit any observability event through the bus.
|
|
9140
|
+
* The bus routes the event to the appropriate handler on each registered exporter,
|
|
9141
|
+
* and for tracing events triggers auto-extracted metrics.
|
|
9142
|
+
*/
|
|
9143
|
+
emitObservabilityEvent(event) {
|
|
9144
|
+
this.observabilityBus.emit(event);
|
|
9145
|
+
}
|
|
8108
9146
|
// ============================================================================
|
|
8109
9147
|
// Span Lifecycle Management
|
|
8110
9148
|
// ============================================================================
|
|
@@ -8248,40 +9286,56 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
8248
9286
|
return processedSpan?.exportSpan(this.config.includeInternalSpans);
|
|
8249
9287
|
}
|
|
8250
9288
|
/**
|
|
8251
|
-
* Emit a span started event
|
|
9289
|
+
* Emit a span started event.
|
|
9290
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9291
|
+
* and auto-extracted metrics are generated.
|
|
8252
9292
|
*/
|
|
8253
9293
|
emitSpanStarted(span) {
|
|
8254
9294
|
const exportedSpan = this.getSpanForExport(span);
|
|
8255
9295
|
if (exportedSpan) {
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
});
|
|
9296
|
+
const event = { type: TracingEventType.SPAN_STARTED, exportedSpan };
|
|
9297
|
+
this.emitTracingEvent(event);
|
|
8259
9298
|
}
|
|
8260
9299
|
}
|
|
8261
9300
|
/**
|
|
8262
|
-
* Emit a span ended event (called automatically when spans end)
|
|
9301
|
+
* Emit a span ended event (called automatically when spans end).
|
|
9302
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9303
|
+
* and auto-extracted metrics are generated.
|
|
8263
9304
|
*/
|
|
8264
9305
|
emitSpanEnded(span) {
|
|
8265
9306
|
const exportedSpan = this.getSpanForExport(span);
|
|
8266
9307
|
if (exportedSpan) {
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
});
|
|
9308
|
+
const event = { type: TracingEventType.SPAN_ENDED, exportedSpan };
|
|
9309
|
+
this.emitTracingEvent(event);
|
|
8270
9310
|
}
|
|
8271
9311
|
}
|
|
8272
9312
|
/**
|
|
8273
|
-
* Emit a span updated event
|
|
9313
|
+
* Emit a span updated event.
|
|
9314
|
+
* Routes through the ObservabilityBus so exporters receive it via onTracingEvent
|
|
9315
|
+
* and auto-extracted metrics are generated.
|
|
8274
9316
|
*/
|
|
8275
9317
|
emitSpanUpdated(span) {
|
|
8276
9318
|
const exportedSpan = this.getSpanForExport(span);
|
|
8277
9319
|
if (exportedSpan) {
|
|
8278
|
-
|
|
8279
|
-
|
|
8280
|
-
});
|
|
9320
|
+
const event = { type: TracingEventType.SPAN_UPDATED, exportedSpan };
|
|
9321
|
+
this.emitTracingEvent(event);
|
|
8281
9322
|
}
|
|
8282
9323
|
}
|
|
8283
9324
|
/**
|
|
8284
|
-
*
|
|
9325
|
+
* Emit a tracing event through the bus.
|
|
9326
|
+
*
|
|
9327
|
+
* The bus routes the event to each registered exporter's and bridge's
|
|
9328
|
+
* onTracingEvent handler and triggers auto-extracted metrics (e.g.,
|
|
9329
|
+
* mastra_agent_runs_started, mastra_model_duration_ms).
|
|
9330
|
+
*/
|
|
9331
|
+
emitTracingEvent(event) {
|
|
9332
|
+
this.observabilityBus.emit(event);
|
|
9333
|
+
}
|
|
9334
|
+
/**
|
|
9335
|
+
* Export tracing event through all exporters and bridge.
|
|
9336
|
+
*
|
|
9337
|
+
* @deprecated Prefer emitTracingEvent() which routes through the bus.
|
|
9338
|
+
* Kept for backward compatibility with subclasses that may override it.
|
|
8285
9339
|
*/
|
|
8286
9340
|
async exportTracingEvent(event) {
|
|
8287
9341
|
const targets = [
|
|
@@ -8311,26 +9365,18 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
8311
9365
|
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
8312
9366
|
}
|
|
8313
9367
|
/**
|
|
8314
|
-
*
|
|
8315
|
-
*
|
|
9368
|
+
* Flush all observability data: awaits in-flight handler promises, then
|
|
9369
|
+
* drains exporter and bridge SDK-internal buffers.
|
|
9370
|
+
*
|
|
9371
|
+
* Delegates to ObservabilityBus.flush() which owns the two-phase logic.
|
|
8316
9372
|
*
|
|
8317
|
-
* This is
|
|
8318
|
-
*
|
|
8319
|
-
*
|
|
9373
|
+
* This is critical for durable execution engines (e.g., Inngest) where
|
|
9374
|
+
* the process may be interrupted after a step completes. Calling flush()
|
|
9375
|
+
* outside the durable step ensures all span data reaches external systems.
|
|
8320
9376
|
*/
|
|
8321
9377
|
async flush() {
|
|
8322
9378
|
this.logger.debug(`[Observability] Flush started [name=${this.name}]`);
|
|
8323
|
-
|
|
8324
|
-
if (this.config.bridge) {
|
|
8325
|
-
flushPromises.push(this.config.bridge.flush());
|
|
8326
|
-
}
|
|
8327
|
-
const results = await Promise.allSettled(flushPromises);
|
|
8328
|
-
results.forEach((result, index) => {
|
|
8329
|
-
if (result.status === "rejected") {
|
|
8330
|
-
const targetName = index < this.exporters.length ? this.exporters[index]?.name : "bridge";
|
|
8331
|
-
this.logger.error(`[Observability] Flush error [target=${targetName}]`, result.reason);
|
|
8332
|
-
}
|
|
8333
|
-
});
|
|
9379
|
+
await this.observabilityBus.flush();
|
|
8334
9380
|
this.logger.debug(`[Observability] Flush completed [name=${this.name}]`);
|
|
8335
9381
|
}
|
|
8336
9382
|
/**
|
|
@@ -8338,6 +9384,7 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
8338
9384
|
*/
|
|
8339
9385
|
async shutdown() {
|
|
8340
9386
|
this.logger.debug(`[Observability] Shutdown started [name=${this.name}]`);
|
|
9387
|
+
await this.observabilityBus.shutdown();
|
|
8341
9388
|
const shutdownPromises = [
|
|
8342
9389
|
...this.exporters.map((e) => e.shutdown()),
|
|
8343
9390
|
...this.spanOutputProcessors.map((p) => p.shutdown())
|
|
@@ -8345,7 +9392,14 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
8345
9392
|
if (this.config.bridge) {
|
|
8346
9393
|
shutdownPromises.push(this.config.bridge.shutdown());
|
|
8347
9394
|
}
|
|
8348
|
-
|
|
9395
|
+
if (shutdownPromises.length > 0) {
|
|
9396
|
+
const results = await Promise.allSettled(shutdownPromises);
|
|
9397
|
+
for (const result of results) {
|
|
9398
|
+
if (result.status === "rejected") {
|
|
9399
|
+
this.logger.error(`[Observability] Component shutdown failed [name=${this.name}]:`, result.reason);
|
|
9400
|
+
}
|
|
9401
|
+
}
|
|
9402
|
+
}
|
|
8349
9403
|
this.logger.info(`[Observability] Shutdown completed [name=${this.name}]`);
|
|
8350
9404
|
}
|
|
8351
9405
|
};
|
|
@@ -8732,6 +9786,6 @@ function buildTracingOptions(...updaters) {
|
|
|
8732
9786
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
8733
9787
|
}
|
|
8734
9788
|
|
|
8735
|
-
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, mergeSerializationOptions, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
9789
|
+
export { AutoExtractedMetrics, BaseExporter, BaseObservabilityEventBus, BaseObservabilityInstance, BaseSpan, CardinalityFilter, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, LoggerContextImpl, MetricsContextImpl, ModelSpanTracker, NoOpSpan, Observability, ObservabilityBus, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, mergeSerializationOptions, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, routeToHandler, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
8736
9790
|
//# sourceMappingURL=index.js.map
|
|
8737
9791
|
//# sourceMappingURL=index.js.map
|