@mastra/observability 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +9 -9
- package/dist/bus/observability-bus.d.ts +19 -10
- package/dist/bus/observability-bus.d.ts.map +1 -1
- package/dist/config.d.ts +8 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/context/metrics.d.ts +15 -24
- package/dist/context/metrics.d.ts.map +1 -1
- package/dist/default.d.ts +16 -3
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +39 -49
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/event-buffer.d.ts +63 -0
- package/dist/exporters/event-buffer.d.ts.map +1 -0
- package/dist/exporters/test.d.ts +5 -7
- package/dist/exporters/test.d.ts.map +1 -1
- package/dist/index.cjs +529 -655
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +529 -655
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +2 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/metrics/auto-extract.d.ts +15 -31
- package/dist/metrics/auto-extract.d.ts.map +1 -1
- package/package.json +7 -7
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
|
3
3
|
import { ConsoleLogger, LogLevel, RegisteredLogger } from '@mastra/core/logger';
|
|
4
4
|
import { TracingEventType, SpanType, DEFAULT_BLOCKED_LABELS, InternalSpans } from '@mastra/core/observability';
|
|
5
5
|
import { fetchWithRetry, getNestedValue, setNestedValue } from '@mastra/core/utils';
|
|
6
|
+
import { buildUpdateSpanRecord, buildFeedbackRecord, buildLogRecord, buildMetricRecord, buildScoreRecord, buildCreateSpanRecord } from '@mastra/core/storage';
|
|
6
7
|
import { TransformStream } from 'stream/web';
|
|
7
8
|
|
|
8
9
|
var __defProp = Object.defineProperty;
|
|
@@ -13811,7 +13812,8 @@ var observabilityInstanceConfigSchema = external_exports.object({
|
|
|
13811
13812
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
13812
13813
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
13813
13814
|
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
13814
|
-
serializationOptions: serializationOptionsSchema
|
|
13815
|
+
serializationOptions: serializationOptionsSchema,
|
|
13816
|
+
cardinality: external_exports.any().optional()
|
|
13815
13817
|
}).refine(
|
|
13816
13818
|
(data) => {
|
|
13817
13819
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -15260,44 +15262,197 @@ var ConsoleExporter = class extends BaseExporter {
|
|
|
15260
15262
|
this.logger.info("ConsoleExporter shutdown");
|
|
15261
15263
|
}
|
|
15262
15264
|
};
|
|
15263
|
-
|
|
15265
|
+
var EventBuffer = class {
|
|
15266
|
+
#preInit = [];
|
|
15267
|
+
#creates = [];
|
|
15268
|
+
#updates = [];
|
|
15269
|
+
#allCreatedSpans = /* @__PURE__ */ new Set();
|
|
15270
|
+
#firstEventTime;
|
|
15271
|
+
#storageStrategy;
|
|
15272
|
+
#maxRetries;
|
|
15273
|
+
constructor(args) {
|
|
15274
|
+
this.#maxRetries = args.maxRetries;
|
|
15275
|
+
}
|
|
15276
|
+
/** Initialize with a storage strategy and replay any pre-init events. */
|
|
15277
|
+
init(args) {
|
|
15278
|
+
if (!this.#storageStrategy) {
|
|
15279
|
+
this.#storageStrategy = args.strategy;
|
|
15280
|
+
for (const event of this.#preInit) {
|
|
15281
|
+
this.addEvent(event);
|
|
15282
|
+
}
|
|
15283
|
+
this.#preInit = [];
|
|
15284
|
+
}
|
|
15285
|
+
}
|
|
15286
|
+
/** Clear the create and update buffers and reset the event timer. */
|
|
15287
|
+
reset() {
|
|
15288
|
+
this.#creates = [];
|
|
15289
|
+
this.#updates = [];
|
|
15290
|
+
this.#firstEventTime = void 0;
|
|
15291
|
+
}
|
|
15292
|
+
setFirstEventTime() {
|
|
15293
|
+
if (!this.#firstEventTime) {
|
|
15294
|
+
this.#firstEventTime = /* @__PURE__ */ new Date();
|
|
15295
|
+
}
|
|
15296
|
+
}
|
|
15297
|
+
pushCreate(event) {
|
|
15298
|
+
this.setFirstEventTime();
|
|
15299
|
+
this.#creates.push({ ...event, retryCount: 0 });
|
|
15300
|
+
}
|
|
15301
|
+
pushUpdate(event) {
|
|
15302
|
+
this.setFirstEventTime();
|
|
15303
|
+
this.#updates.push({ ...event, retryCount: 0 });
|
|
15304
|
+
}
|
|
15305
|
+
/** Route an event to the create or update buffer based on its type and the storage strategy. */
|
|
15306
|
+
addEvent(event) {
|
|
15307
|
+
if (!this.#storageStrategy) {
|
|
15308
|
+
this.#preInit.push({ ...event, retryCount: 0 });
|
|
15309
|
+
return;
|
|
15310
|
+
}
|
|
15311
|
+
switch (event.type) {
|
|
15312
|
+
case TracingEventType.SPAN_STARTED:
|
|
15313
|
+
switch (this.#storageStrategy) {
|
|
15314
|
+
case "realtime":
|
|
15315
|
+
case "event-sourced":
|
|
15316
|
+
case "batch-with-updates":
|
|
15317
|
+
this.pushCreate(event);
|
|
15318
|
+
break;
|
|
15319
|
+
}
|
|
15320
|
+
break;
|
|
15321
|
+
case TracingEventType.SPAN_UPDATED:
|
|
15322
|
+
switch (this.#storageStrategy) {
|
|
15323
|
+
case "realtime":
|
|
15324
|
+
case "batch-with-updates":
|
|
15325
|
+
this.pushUpdate(event);
|
|
15326
|
+
break;
|
|
15327
|
+
}
|
|
15328
|
+
break;
|
|
15329
|
+
case TracingEventType.SPAN_ENDED:
|
|
15330
|
+
if (event.exportedSpan.isEvent) {
|
|
15331
|
+
this.pushCreate(event);
|
|
15332
|
+
} else {
|
|
15333
|
+
switch (this.#storageStrategy) {
|
|
15334
|
+
case "realtime":
|
|
15335
|
+
case "batch-with-updates":
|
|
15336
|
+
this.pushUpdate(event);
|
|
15337
|
+
break;
|
|
15338
|
+
default:
|
|
15339
|
+
this.pushCreate(event);
|
|
15340
|
+
break;
|
|
15341
|
+
}
|
|
15342
|
+
}
|
|
15343
|
+
break;
|
|
15344
|
+
default:
|
|
15345
|
+
this.pushCreate(event);
|
|
15346
|
+
break;
|
|
15347
|
+
}
|
|
15348
|
+
}
|
|
15349
|
+
/** Re-add failed create events to the buffer, dropping those that exceed max retries. */
|
|
15350
|
+
reAddCreates(events) {
|
|
15351
|
+
const retryable = [];
|
|
15352
|
+
for (const e of events) {
|
|
15353
|
+
if (++e.retryCount <= this.#maxRetries) {
|
|
15354
|
+
retryable.push(e);
|
|
15355
|
+
}
|
|
15356
|
+
}
|
|
15357
|
+
if (retryable.length > 0) {
|
|
15358
|
+
this.setFirstEventTime();
|
|
15359
|
+
this.#creates.push(...retryable);
|
|
15360
|
+
}
|
|
15361
|
+
}
|
|
15362
|
+
/** Re-add failed update events to the buffer, dropping those that exceed max retries. */
|
|
15363
|
+
reAddUpdates(events) {
|
|
15364
|
+
const retryable = [];
|
|
15365
|
+
for (const e of events) {
|
|
15366
|
+
if (++e.retryCount <= this.#maxRetries) {
|
|
15367
|
+
retryable.push(e);
|
|
15368
|
+
}
|
|
15369
|
+
}
|
|
15370
|
+
if (retryable.length > 0) {
|
|
15371
|
+
this.setFirstEventTime();
|
|
15372
|
+
this.#updates.push(...retryable);
|
|
15373
|
+
}
|
|
15374
|
+
}
|
|
15375
|
+
/** Snapshot of buffered create events. */
|
|
15376
|
+
get creates() {
|
|
15377
|
+
return [...this.#creates];
|
|
15378
|
+
}
|
|
15379
|
+
/** Snapshot of buffered update events. */
|
|
15380
|
+
get updates() {
|
|
15381
|
+
return [...this.#updates];
|
|
15382
|
+
}
|
|
15383
|
+
/** Total number of buffered events (creates + updates). */
|
|
15384
|
+
get totalSize() {
|
|
15385
|
+
return this.#creates.length + this.#updates.length;
|
|
15386
|
+
}
|
|
15387
|
+
/** Milliseconds since the first event was buffered in the current batch. */
|
|
15388
|
+
get elapsed() {
|
|
15389
|
+
if (!this.#firstEventTime) {
|
|
15390
|
+
return 0;
|
|
15391
|
+
}
|
|
15392
|
+
return Date.now() - this.#firstEventTime.getTime();
|
|
15393
|
+
}
|
|
15394
|
+
/**
|
|
15395
|
+
* Builds a unique span key for tracking
|
|
15396
|
+
*/
|
|
15397
|
+
buildSpanKey(span) {
|
|
15398
|
+
return `${span.traceId}:${span.spanId}`;
|
|
15399
|
+
}
|
|
15400
|
+
/** Track successfully created spans so updates can verify span existence before flushing. */
|
|
15401
|
+
addCreatedSpans(args) {
|
|
15402
|
+
if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") {
|
|
15403
|
+
return;
|
|
15404
|
+
}
|
|
15405
|
+
for (const createRecord of args.records) {
|
|
15406
|
+
if (!createRecord.isEvent) {
|
|
15407
|
+
this.#allCreatedSpans.add(this.buildSpanKey(createRecord));
|
|
15408
|
+
}
|
|
15409
|
+
}
|
|
15410
|
+
}
|
|
15411
|
+
/** Check whether a span's create record has already been flushed to storage. */
|
|
15412
|
+
spanExists(span) {
|
|
15413
|
+
return this.#allCreatedSpans?.has(this.buildSpanKey({ traceId: span.traceId, spanId: span.id }));
|
|
15414
|
+
}
|
|
15415
|
+
/** Remove completed spans from tracking after their SPAN_ENDED updates are flushed. */
|
|
15416
|
+
endFinishedSpans(args) {
|
|
15417
|
+
if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") {
|
|
15418
|
+
return;
|
|
15419
|
+
}
|
|
15420
|
+
args.records.forEach((r) => {
|
|
15421
|
+
this.#allCreatedSpans.delete(this.buildSpanKey(r));
|
|
15422
|
+
});
|
|
15423
|
+
}
|
|
15424
|
+
};
|
|
15425
|
+
|
|
15426
|
+
// src/exporters/default.ts
|
|
15427
|
+
function resolveTracingStorageStrategy(config2, observabilityStorage, storageName, logger) {
|
|
15428
|
+
const observabilityStrategy = observabilityStorage.observabilityStrategy;
|
|
15264
15429
|
if (config2.strategy && config2.strategy !== "auto") {
|
|
15265
|
-
|
|
15266
|
-
if (hints.supported.includes(config2.strategy)) {
|
|
15430
|
+
if (observabilityStrategy.supported.includes(config2.strategy)) {
|
|
15267
15431
|
return config2.strategy;
|
|
15268
15432
|
}
|
|
15269
15433
|
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
15270
15434
|
userStrategy: config2.strategy,
|
|
15271
15435
|
storageAdapter: storageName,
|
|
15272
|
-
supportedStrategies:
|
|
15273
|
-
fallbackStrategy:
|
|
15436
|
+
supportedStrategies: observabilityStrategy.supported,
|
|
15437
|
+
fallbackStrategy: observabilityStrategy.preferred
|
|
15274
15438
|
});
|
|
15275
15439
|
}
|
|
15276
|
-
return
|
|
15277
|
-
}
|
|
15278
|
-
function getStringOrNull(value) {
|
|
15279
|
-
return typeof value === "string" ? value : null;
|
|
15280
|
-
}
|
|
15281
|
-
function getObjectOrNull(value) {
|
|
15282
|
-
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
15440
|
+
return observabilityStrategy.preferred;
|
|
15283
15441
|
}
|
|
15284
15442
|
var DefaultExporter = class extends BaseExporter {
|
|
15285
15443
|
name = "mastra-default-observability-exporter";
|
|
15286
|
-
#storage;
|
|
15287
|
-
#observability;
|
|
15288
15444
|
#config;
|
|
15289
|
-
#resolvedStrategy;
|
|
15290
|
-
buffer;
|
|
15291
|
-
#flushTimer = null;
|
|
15292
15445
|
#isInitializing = false;
|
|
15293
15446
|
#initPromises = /* @__PURE__ */ new Set();
|
|
15294
|
-
|
|
15295
|
-
|
|
15447
|
+
#eventBuffer;
|
|
15448
|
+
#storage;
|
|
15449
|
+
#observabilityStorage;
|
|
15450
|
+
#resolvedStrategy;
|
|
15451
|
+
#flushTimer;
|
|
15452
|
+
// Signals whose storage methods threw "not implemented" — skip on future flushes
|
|
15453
|
+
#unsupportedSignals = /* @__PURE__ */ new Set();
|
|
15296
15454
|
constructor(config2 = {}) {
|
|
15297
15455
|
super(config2);
|
|
15298
|
-
if (config2 === void 0) {
|
|
15299
|
-
config2 = {};
|
|
15300
|
-
}
|
|
15301
15456
|
this.#config = {
|
|
15302
15457
|
...config2,
|
|
15303
15458
|
maxBatchSize: config2.maxBatchSize ?? 1e3,
|
|
@@ -15307,19 +15462,8 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15307
15462
|
retryDelayMs: config2.retryDelayMs ?? 500,
|
|
15308
15463
|
strategy: config2.strategy ?? "auto"
|
|
15309
15464
|
};
|
|
15310
|
-
this
|
|
15311
|
-
creates: [],
|
|
15312
|
-
updates: [],
|
|
15313
|
-
insertOnly: [],
|
|
15314
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
15315
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
15316
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
15317
|
-
outOfOrderCount: 0,
|
|
15318
|
-
totalSize: 0
|
|
15319
|
-
};
|
|
15320
|
-
this.#resolvedStrategy = "batch-with-updates";
|
|
15465
|
+
this.#eventBuffer = new EventBuffer({ maxRetries: this.#config.maxRetries ?? 4 });
|
|
15321
15466
|
}
|
|
15322
|
-
#strategyInitialized = false;
|
|
15323
15467
|
/**
|
|
15324
15468
|
* Initialize the exporter (called after all dependencies are ready)
|
|
15325
15469
|
*/
|
|
@@ -15331,14 +15475,31 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15331
15475
|
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
15332
15476
|
return;
|
|
15333
15477
|
}
|
|
15334
|
-
this.#
|
|
15335
|
-
if (!this.#
|
|
15478
|
+
this.#observabilityStorage = await this.#storage.getStore("observability");
|
|
15479
|
+
if (!this.#observabilityStorage) {
|
|
15336
15480
|
this.logger.warn(
|
|
15337
15481
|
"DefaultExporter disabled: Observability storage not available. Traces will not be persisted."
|
|
15338
15482
|
);
|
|
15339
15483
|
return;
|
|
15340
15484
|
}
|
|
15341
|
-
|
|
15485
|
+
if (!this.#resolvedStrategy) {
|
|
15486
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(
|
|
15487
|
+
this.#config,
|
|
15488
|
+
this.#observabilityStorage,
|
|
15489
|
+
this.#storage.constructor.name,
|
|
15490
|
+
this.logger
|
|
15491
|
+
);
|
|
15492
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
15493
|
+
strategy: this.#resolvedStrategy,
|
|
15494
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
15495
|
+
storageAdapter: this.#storage.constructor.name,
|
|
15496
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
15497
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
15498
|
+
});
|
|
15499
|
+
}
|
|
15500
|
+
if (this.#resolvedStrategy) {
|
|
15501
|
+
this.#eventBuffer.init({ strategy: this.#resolvedStrategy });
|
|
15502
|
+
}
|
|
15342
15503
|
} finally {
|
|
15343
15504
|
this.#isInitializing = false;
|
|
15344
15505
|
this.#initPromises.forEach((resolve) => {
|
|
@@ -15347,144 +15508,26 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15347
15508
|
this.#initPromises.clear();
|
|
15348
15509
|
}
|
|
15349
15510
|
}
|
|
15350
|
-
/**
|
|
15351
|
-
* Initialize the resolved strategy once observability store is available
|
|
15352
|
-
*/
|
|
15353
|
-
initializeStrategy(observability, storageName) {
|
|
15354
|
-
if (this.#strategyInitialized) return;
|
|
15355
|
-
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, observability, storageName, this.logger);
|
|
15356
|
-
this.#strategyInitialized = true;
|
|
15357
|
-
this.logger.debug("tracing storage exporter initialized", {
|
|
15358
|
-
strategy: this.#resolvedStrategy,
|
|
15359
|
-
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
15360
|
-
storageAdapter: storageName,
|
|
15361
|
-
maxBatchSize: this.#config.maxBatchSize,
|
|
15362
|
-
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
15363
|
-
});
|
|
15364
|
-
}
|
|
15365
|
-
/**
|
|
15366
|
-
* Builds a unique span key for tracking
|
|
15367
|
-
*/
|
|
15368
|
-
buildSpanKey(traceId, spanId) {
|
|
15369
|
-
return `${traceId}:${spanId}`;
|
|
15370
|
-
}
|
|
15371
|
-
/**
|
|
15372
|
-
* Gets the next sequence number for a span
|
|
15373
|
-
*/
|
|
15374
|
-
getNextSequence(spanKey) {
|
|
15375
|
-
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
15376
|
-
const next = current + 1;
|
|
15377
|
-
this.buffer.spanSequences.set(spanKey, next);
|
|
15378
|
-
return next;
|
|
15379
|
-
}
|
|
15380
|
-
/**
|
|
15381
|
-
* Handles out-of-order span updates by logging and skipping
|
|
15382
|
-
*/
|
|
15383
|
-
handleOutOfOrderUpdate(event) {
|
|
15384
|
-
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
15385
|
-
spanId: event.exportedSpan.id,
|
|
15386
|
-
traceId: event.exportedSpan.traceId,
|
|
15387
|
-
spanName: event.exportedSpan.name,
|
|
15388
|
-
eventType: event.type
|
|
15389
|
-
});
|
|
15390
|
-
}
|
|
15391
|
-
/**
|
|
15392
|
-
* Adds an event to the appropriate buffer based on strategy
|
|
15393
|
-
*/
|
|
15394
|
-
addToBuffer(event) {
|
|
15395
|
-
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
15396
|
-
if (this.buffer.totalSize === 0) {
|
|
15397
|
-
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
15398
|
-
}
|
|
15399
|
-
switch (event.type) {
|
|
15400
|
-
case TracingEventType.SPAN_STARTED:
|
|
15401
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15402
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15403
|
-
this.buffer.creates.push(createRecord);
|
|
15404
|
-
this.buffer.seenSpans.add(spanKey);
|
|
15405
|
-
this.allCreatedSpans.add(spanKey);
|
|
15406
|
-
}
|
|
15407
|
-
break;
|
|
15408
|
-
case TracingEventType.SPAN_UPDATED:
|
|
15409
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15410
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
15411
|
-
this.buffer.updates.push({
|
|
15412
|
-
traceId: event.exportedSpan.traceId,
|
|
15413
|
-
spanId: event.exportedSpan.id,
|
|
15414
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
15415
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
15416
|
-
});
|
|
15417
|
-
} else {
|
|
15418
|
-
this.handleOutOfOrderUpdate(event);
|
|
15419
|
-
this.buffer.outOfOrderCount++;
|
|
15420
|
-
}
|
|
15421
|
-
}
|
|
15422
|
-
break;
|
|
15423
|
-
case TracingEventType.SPAN_ENDED:
|
|
15424
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15425
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
15426
|
-
this.buffer.updates.push({
|
|
15427
|
-
traceId: event.exportedSpan.traceId,
|
|
15428
|
-
spanId: event.exportedSpan.id,
|
|
15429
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
15430
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
15431
|
-
});
|
|
15432
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15433
|
-
} else if (event.exportedSpan.isEvent) {
|
|
15434
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15435
|
-
this.buffer.creates.push(createRecord);
|
|
15436
|
-
this.buffer.seenSpans.add(spanKey);
|
|
15437
|
-
this.allCreatedSpans.add(spanKey);
|
|
15438
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15439
|
-
} else {
|
|
15440
|
-
this.handleOutOfOrderUpdate(event);
|
|
15441
|
-
this.buffer.outOfOrderCount++;
|
|
15442
|
-
}
|
|
15443
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
15444
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15445
|
-
this.buffer.insertOnly.push(createRecord);
|
|
15446
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15447
|
-
this.allCreatedSpans.add(spanKey);
|
|
15448
|
-
}
|
|
15449
|
-
break;
|
|
15450
|
-
}
|
|
15451
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
15452
|
-
}
|
|
15453
15511
|
/**
|
|
15454
15512
|
* Checks if buffer should be flushed based on size or time triggers
|
|
15455
15513
|
*/
|
|
15456
15514
|
shouldFlush() {
|
|
15457
|
-
if (this
|
|
15515
|
+
if (this.#resolvedStrategy === "realtime") {
|
|
15458
15516
|
return true;
|
|
15459
15517
|
}
|
|
15460
|
-
if (this.
|
|
15518
|
+
if (this.#eventBuffer.totalSize >= this.#config.maxBufferSize) {
|
|
15461
15519
|
return true;
|
|
15462
15520
|
}
|
|
15463
|
-
if (this.
|
|
15464
|
-
|
|
15465
|
-
|
|
15521
|
+
if (this.#eventBuffer.totalSize >= this.#config.maxBatchSize) {
|
|
15522
|
+
return true;
|
|
15523
|
+
}
|
|
15524
|
+
if (this.#eventBuffer.totalSize > 0) {
|
|
15525
|
+
if (this.#eventBuffer.elapsed >= this.#config.maxBatchWaitMs) {
|
|
15466
15526
|
return true;
|
|
15467
15527
|
}
|
|
15468
15528
|
}
|
|
15469
15529
|
return false;
|
|
15470
15530
|
}
|
|
15471
|
-
/**
|
|
15472
|
-
* Resets the buffer after successful flush
|
|
15473
|
-
*/
|
|
15474
|
-
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
15475
|
-
this.buffer.creates = [];
|
|
15476
|
-
this.buffer.updates = [];
|
|
15477
|
-
this.buffer.insertOnly = [];
|
|
15478
|
-
this.buffer.seenSpans.clear();
|
|
15479
|
-
this.buffer.spanSequences.clear();
|
|
15480
|
-
this.buffer.completedSpans.clear();
|
|
15481
|
-
this.buffer.outOfOrderCount = 0;
|
|
15482
|
-
this.buffer.firstEventTime = void 0;
|
|
15483
|
-
this.buffer.totalSize = 0;
|
|
15484
|
-
for (const spanKey of completedSpansToCleanup) {
|
|
15485
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15486
|
-
}
|
|
15487
|
-
}
|
|
15488
15531
|
/**
|
|
15489
15532
|
* Schedules a flush using setTimeout
|
|
15490
15533
|
*/
|
|
@@ -15501,276 +15544,185 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15501
15544
|
}, this.#config.maxBatchWaitMs);
|
|
15502
15545
|
}
|
|
15503
15546
|
/**
|
|
15504
|
-
*
|
|
15505
|
-
*
|
|
15547
|
+
* Checks flush triggers and schedules/triggers flush as needed.
|
|
15548
|
+
* Called after adding any event to the buffer.
|
|
15549
|
+
* Returns the flush promise when flushing so callers can await it.
|
|
15506
15550
|
*/
|
|
15507
|
-
|
|
15508
|
-
if (
|
|
15509
|
-
|
|
15510
|
-
}
|
|
15511
|
-
|
|
15512
|
-
return JSON.parse(
|
|
15513
|
-
JSON.stringify(span.attributes, (_key, value) => {
|
|
15514
|
-
if (value instanceof Date) {
|
|
15515
|
-
return value.toISOString();
|
|
15516
|
-
}
|
|
15517
|
-
if (typeof value === "object" && value !== null) {
|
|
15518
|
-
return value;
|
|
15519
|
-
}
|
|
15520
|
-
return value;
|
|
15521
|
-
})
|
|
15522
|
-
);
|
|
15523
|
-
} catch (error48) {
|
|
15524
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
15525
|
-
spanId: span.id,
|
|
15526
|
-
spanType: span.type,
|
|
15527
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15528
|
-
});
|
|
15529
|
-
return null;
|
|
15551
|
+
async handleBatchedFlush() {
|
|
15552
|
+
if (this.shouldFlush()) {
|
|
15553
|
+
await this.flushBuffer();
|
|
15554
|
+
} else if (this.#eventBuffer.totalSize === 1) {
|
|
15555
|
+
this.scheduleFlush();
|
|
15530
15556
|
}
|
|
15531
15557
|
}
|
|
15532
|
-
buildCreateRecord(span) {
|
|
15533
|
-
const metadata = span.metadata ?? {};
|
|
15534
|
-
return {
|
|
15535
|
-
traceId: span.traceId,
|
|
15536
|
-
spanId: span.id,
|
|
15537
|
-
parentSpanId: span.parentSpanId ?? null,
|
|
15538
|
-
name: span.name,
|
|
15539
|
-
// Entity identification - from span
|
|
15540
|
-
entityType: span.entityType ?? null,
|
|
15541
|
-
entityId: span.entityId ?? null,
|
|
15542
|
-
entityName: span.entityName ?? null,
|
|
15543
|
-
// Identity & Tenancy - extracted from metadata if present
|
|
15544
|
-
userId: getStringOrNull(metadata.userId),
|
|
15545
|
-
organizationId: getStringOrNull(metadata.organizationId),
|
|
15546
|
-
resourceId: getStringOrNull(metadata.resourceId),
|
|
15547
|
-
// Correlation IDs - extracted from metadata if present
|
|
15548
|
-
runId: getStringOrNull(metadata.runId),
|
|
15549
|
-
sessionId: getStringOrNull(metadata.sessionId),
|
|
15550
|
-
threadId: getStringOrNull(metadata.threadId),
|
|
15551
|
-
requestId: getStringOrNull(metadata.requestId),
|
|
15552
|
-
// Deployment context - extracted from metadata if present
|
|
15553
|
-
environment: getStringOrNull(metadata.environment),
|
|
15554
|
-
source: getStringOrNull(metadata.source),
|
|
15555
|
-
serviceName: getStringOrNull(metadata.serviceName),
|
|
15556
|
-
scope: getObjectOrNull(metadata.scope),
|
|
15557
|
-
// Span data
|
|
15558
|
-
spanType: span.type,
|
|
15559
|
-
attributes: this.serializeAttributes(span),
|
|
15560
|
-
metadata: span.metadata ?? null,
|
|
15561
|
-
// Keep all metadata including extracted fields
|
|
15562
|
-
tags: span.tags ?? null,
|
|
15563
|
-
links: null,
|
|
15564
|
-
input: span.input ?? null,
|
|
15565
|
-
output: span.output ?? null,
|
|
15566
|
-
error: span.errorInfo ?? null,
|
|
15567
|
-
requestContext: span.requestContext ?? null,
|
|
15568
|
-
isEvent: span.isEvent,
|
|
15569
|
-
// Timestamps
|
|
15570
|
-
startedAt: span.startTime,
|
|
15571
|
-
endedAt: span.endTime ?? null
|
|
15572
|
-
};
|
|
15573
|
-
}
|
|
15574
|
-
buildUpdateRecord(span) {
|
|
15575
|
-
return {
|
|
15576
|
-
name: span.name,
|
|
15577
|
-
scope: null,
|
|
15578
|
-
attributes: this.serializeAttributes(span),
|
|
15579
|
-
metadata: span.metadata ?? null,
|
|
15580
|
-
links: null,
|
|
15581
|
-
endedAt: span.endTime ?? null,
|
|
15582
|
-
input: span.input,
|
|
15583
|
-
output: span.output,
|
|
15584
|
-
error: span.errorInfo ?? null,
|
|
15585
|
-
requestContext: span.requestContext ?? null
|
|
15586
|
-
};
|
|
15587
|
-
}
|
|
15588
15558
|
/**
|
|
15589
|
-
*
|
|
15559
|
+
* Flush a batch of create events for a single signal type.
|
|
15560
|
+
* On "not implemented" errors, disables the signal for future flushes.
|
|
15561
|
+
* On other errors, re-adds events to the buffer for retry.
|
|
15590
15562
|
*/
|
|
15591
|
-
async
|
|
15592
|
-
|
|
15593
|
-
|
|
15594
|
-
|
|
15595
|
-
|
|
15596
|
-
|
|
15563
|
+
async flushCreates(signal, events, storageCall) {
|
|
15564
|
+
if (this.#unsupportedSignals.has(signal) || events.length === 0) return;
|
|
15565
|
+
try {
|
|
15566
|
+
await storageCall(events);
|
|
15567
|
+
} catch (error48) {
|
|
15568
|
+
if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15569
|
+
this.logger.warn(error48.message);
|
|
15570
|
+
this.#unsupportedSignals.add(signal);
|
|
15597
15571
|
} else {
|
|
15598
|
-
this.
|
|
15599
|
-
}
|
|
15600
|
-
} else {
|
|
15601
|
-
switch (event.type) {
|
|
15602
|
-
case TracingEventType.SPAN_STARTED:
|
|
15603
|
-
await observability.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
15604
|
-
this.allCreatedSpans.add(spanKey);
|
|
15605
|
-
break;
|
|
15606
|
-
case TracingEventType.SPAN_UPDATED:
|
|
15607
|
-
await observability.updateSpan({
|
|
15608
|
-
traceId: span.traceId,
|
|
15609
|
-
spanId: span.id,
|
|
15610
|
-
updates: this.buildUpdateRecord(span)
|
|
15611
|
-
});
|
|
15612
|
-
break;
|
|
15613
|
-
case TracingEventType.SPAN_ENDED:
|
|
15614
|
-
await observability.updateSpan({
|
|
15615
|
-
traceId: span.traceId,
|
|
15616
|
-
spanId: span.id,
|
|
15617
|
-
updates: this.buildUpdateRecord(span)
|
|
15618
|
-
});
|
|
15619
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15620
|
-
break;
|
|
15621
|
-
default:
|
|
15622
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
15572
|
+
this.#eventBuffer.reAddCreates(events);
|
|
15623
15573
|
}
|
|
15624
15574
|
}
|
|
15625
15575
|
}
|
|
15626
15576
|
/**
|
|
15627
|
-
*
|
|
15577
|
+
* Flush span update/end events, deferring any whose span hasn't been created yet.
|
|
15578
|
+
* When `isEnd` is true, successfully flushed spans are removed from tracking.
|
|
15628
15579
|
*/
|
|
15629
|
-
|
|
15630
|
-
this.
|
|
15631
|
-
|
|
15632
|
-
|
|
15633
|
-
|
|
15634
|
-
|
|
15580
|
+
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
15581
|
+
if (this.#unsupportedSignals.has("tracing") || events.length === 0) return;
|
|
15582
|
+
const partials = [];
|
|
15583
|
+
for (const event of events) {
|
|
15584
|
+
const span = event.exportedSpan;
|
|
15585
|
+
if (this.#eventBuffer.spanExists(span)) {
|
|
15586
|
+
partials.push({
|
|
15587
|
+
traceId: span.traceId,
|
|
15588
|
+
spanId: span.id,
|
|
15589
|
+
updates: buildUpdateSpanRecord(span)
|
|
15635
15590
|
});
|
|
15636
|
-
}
|
|
15637
|
-
|
|
15638
|
-
|
|
15591
|
+
} else {
|
|
15592
|
+
deferredUpdates.push(event);
|
|
15593
|
+
}
|
|
15639
15594
|
}
|
|
15640
|
-
|
|
15641
|
-
|
|
15642
|
-
|
|
15643
|
-
|
|
15644
|
-
|
|
15645
|
-
|
|
15646
|
-
|
|
15647
|
-
if (
|
|
15648
|
-
this.
|
|
15649
|
-
|
|
15650
|
-
|
|
15651
|
-
|
|
15652
|
-
|
|
15653
|
-
} else if (this.buffer.totalSize === 1) {
|
|
15654
|
-
this.scheduleFlush();
|
|
15595
|
+
if (partials.length === 0) return;
|
|
15596
|
+
try {
|
|
15597
|
+
await this.#observabilityStorage.batchUpdateSpans({ records: partials });
|
|
15598
|
+
if (isEnd) {
|
|
15599
|
+
this.#eventBuffer.endFinishedSpans({ records: partials });
|
|
15600
|
+
}
|
|
15601
|
+
} catch (error48) {
|
|
15602
|
+
if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15603
|
+
this.logger.warn(error48.message);
|
|
15604
|
+
this.#unsupportedSignals.add("tracing");
|
|
15605
|
+
} else {
|
|
15606
|
+
deferredUpdates.length = 0;
|
|
15607
|
+
this.#eventBuffer.reAddUpdates(events);
|
|
15655
15608
|
}
|
|
15656
15609
|
}
|
|
15657
15610
|
}
|
|
15658
15611
|
/**
|
|
15659
|
-
*
|
|
15660
|
-
|
|
15661
|
-
|
|
15662
|
-
|
|
15663
|
-
|
|
15664
|
-
|
|
15665
|
-
* Flushes the current buffer to storage with retry logic (internal implementation)
|
|
15612
|
+
* Flushes the current buffer to storage.
|
|
15613
|
+
*
|
|
15614
|
+
* Creates are flushed first, then their span keys are added to allCreatedSpans.
|
|
15615
|
+
* Updates are checked against allCreatedSpans — those whose span hasn't been
|
|
15616
|
+
* created yet are re-inserted into the live buffer for the next flush.
|
|
15617
|
+
* Completed spans (SPAN_ENDED) are cleaned up from allCreatedSpans after success.
|
|
15666
15618
|
*/
|
|
15667
15619
|
async flushBuffer() {
|
|
15668
|
-
if (!this.#
|
|
15669
|
-
this.logger.debug("Cannot flush
|
|
15620
|
+
if (!this.#observabilityStorage) {
|
|
15621
|
+
this.logger.debug("Cannot flush. Observability storage is not initialized");
|
|
15622
|
+
return;
|
|
15623
|
+
}
|
|
15624
|
+
if (!this.#resolvedStrategy) {
|
|
15625
|
+
this.logger.debug("Cannot flush. Observability strategy is not resolved");
|
|
15670
15626
|
return;
|
|
15671
15627
|
}
|
|
15672
15628
|
if (this.#flushTimer) {
|
|
15673
15629
|
clearTimeout(this.#flushTimer);
|
|
15674
|
-
this.#flushTimer =
|
|
15630
|
+
this.#flushTimer = void 0;
|
|
15675
15631
|
}
|
|
15676
|
-
if (this.
|
|
15632
|
+
if (this.#eventBuffer.totalSize === 0) {
|
|
15677
15633
|
return;
|
|
15678
15634
|
}
|
|
15679
15635
|
const startTime = Date.now();
|
|
15680
|
-
const
|
|
15681
|
-
const
|
|
15682
|
-
|
|
15683
|
-
|
|
15684
|
-
|
|
15685
|
-
|
|
15686
|
-
|
|
15687
|
-
|
|
15688
|
-
|
|
15689
|
-
|
|
15690
|
-
|
|
15691
|
-
|
|
15692
|
-
|
|
15693
|
-
|
|
15636
|
+
const batchSize = this.#eventBuffer.totalSize;
|
|
15637
|
+
const creates = this.#eventBuffer.creates;
|
|
15638
|
+
const updates = this.#eventBuffer.updates;
|
|
15639
|
+
this.#eventBuffer.reset();
|
|
15640
|
+
const createFeedbackEvents = [];
|
|
15641
|
+
const createLogEvents = [];
|
|
15642
|
+
const createMetricEvents = [];
|
|
15643
|
+
const createScoreEvents = [];
|
|
15644
|
+
const createSpanEvents = [];
|
|
15645
|
+
const updateSpanEvents = [];
|
|
15646
|
+
const endSpanEvents = [];
|
|
15647
|
+
for (const createEvent of creates) {
|
|
15648
|
+
switch (createEvent.type) {
|
|
15649
|
+
case "feedback":
|
|
15650
|
+
createFeedbackEvents.push(createEvent);
|
|
15651
|
+
break;
|
|
15652
|
+
case "log":
|
|
15653
|
+
createLogEvents.push(createEvent);
|
|
15654
|
+
break;
|
|
15655
|
+
case "metric":
|
|
15656
|
+
createMetricEvents.push(createEvent);
|
|
15657
|
+
break;
|
|
15658
|
+
case "score":
|
|
15659
|
+
createScoreEvents.push(createEvent);
|
|
15660
|
+
break;
|
|
15661
|
+
default:
|
|
15662
|
+
createSpanEvents.push(createEvent);
|
|
15663
|
+
break;
|
|
15664
|
+
}
|
|
15665
|
+
}
|
|
15666
|
+
for (const updateEvent of updates) {
|
|
15667
|
+
switch (updateEvent.type) {
|
|
15668
|
+
case TracingEventType.SPAN_UPDATED:
|
|
15669
|
+
updateSpanEvents.push(updateEvent);
|
|
15670
|
+
break;
|
|
15671
|
+
case TracingEventType.SPAN_ENDED:
|
|
15672
|
+
endSpanEvents.push(updateEvent);
|
|
15673
|
+
break;
|
|
15674
|
+
}
|
|
15675
|
+
}
|
|
15676
|
+
await Promise.all([
|
|
15677
|
+
this.flushCreates(
|
|
15678
|
+
"feedback",
|
|
15679
|
+
createFeedbackEvents,
|
|
15680
|
+
(events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => buildFeedbackRecord(f)) })
|
|
15681
|
+
),
|
|
15682
|
+
this.flushCreates(
|
|
15683
|
+
"logs",
|
|
15684
|
+
createLogEvents,
|
|
15685
|
+
(events) => this.#observabilityStorage.batchCreateLogs({ logs: events.map((l) => buildLogRecord(l)) })
|
|
15686
|
+
),
|
|
15687
|
+
this.flushCreates(
|
|
15688
|
+
"metrics",
|
|
15689
|
+
createMetricEvents,
|
|
15690
|
+
(events) => this.#observabilityStorage.batchCreateMetrics({ metrics: events.map((m) => buildMetricRecord(m)) })
|
|
15691
|
+
),
|
|
15692
|
+
this.flushCreates(
|
|
15693
|
+
"scores",
|
|
15694
|
+
createScoreEvents,
|
|
15695
|
+
(events) => this.#observabilityStorage.batchCreateScores({ scores: events.map((s) => buildScoreRecord(s)) })
|
|
15696
|
+
),
|
|
15697
|
+
this.flushCreates("tracing", createSpanEvents, async (events) => {
|
|
15698
|
+
const records = events.map((t) => buildCreateSpanRecord(t.exportedSpan));
|
|
15699
|
+
await this.#observabilityStorage.batchCreateSpans({ records });
|
|
15700
|
+
this.#eventBuffer.addCreatedSpans({ records });
|
|
15701
|
+
})
|
|
15702
|
+
]);
|
|
15703
|
+
const deferredUpdates = [];
|
|
15704
|
+
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
15705
|
+
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
15706
|
+
if (deferredUpdates.length > 0) {
|
|
15707
|
+
this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
15708
|
+
}
|
|
15694
15709
|
const elapsed = Date.now() - startTime;
|
|
15695
15710
|
this.logger.debug("Batch flushed", {
|
|
15696
15711
|
strategy: this.#resolvedStrategy,
|
|
15697
|
-
batchSize
|
|
15698
|
-
flushReason,
|
|
15712
|
+
batchSize,
|
|
15699
15713
|
durationMs: elapsed,
|
|
15700
|
-
|
|
15714
|
+
deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
|
|
15701
15715
|
});
|
|
15702
|
-
|
|
15703
|
-
/**
|
|
15704
|
-
* Attempts to flush with exponential backoff retry logic
|
|
15705
|
-
*/
|
|
15706
|
-
async flushWithRetries(observability, buffer, attempt) {
|
|
15707
|
-
try {
|
|
15708
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15709
|
-
if (buffer.creates.length > 0) {
|
|
15710
|
-
await observability.batchCreateSpans({ records: buffer.creates });
|
|
15711
|
-
}
|
|
15712
|
-
if (buffer.updates.length > 0) {
|
|
15713
|
-
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
15714
|
-
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
15715
|
-
this.buildSpanKey(b.traceId, b.spanId)
|
|
15716
|
-
);
|
|
15717
|
-
if (spanCompare !== 0) return spanCompare;
|
|
15718
|
-
return a.sequenceNumber - b.sequenceNumber;
|
|
15719
|
-
});
|
|
15720
|
-
await observability.batchUpdateSpans({ records: sortedUpdates });
|
|
15721
|
-
}
|
|
15722
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
15723
|
-
if (buffer.insertOnly.length > 0) {
|
|
15724
|
-
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
15725
|
-
}
|
|
15726
|
-
}
|
|
15727
|
-
for (const spanKey of buffer.completedSpans) {
|
|
15728
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15729
|
-
}
|
|
15730
|
-
} catch (error48) {
|
|
15731
|
-
if (attempt < this.#config.maxRetries) {
|
|
15732
|
-
const retryDelay = this.calculateRetryDelay(attempt);
|
|
15733
|
-
this.logger.warn("Batch flush failed, retrying", {
|
|
15734
|
-
attempt: attempt + 1,
|
|
15735
|
-
maxRetries: this.#config.maxRetries,
|
|
15736
|
-
nextRetryInMs: retryDelay,
|
|
15737
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15738
|
-
});
|
|
15739
|
-
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
15740
|
-
return this.flushWithRetries(observability, buffer, attempt + 1);
|
|
15741
|
-
} else {
|
|
15742
|
-
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
15743
|
-
finalAttempt: attempt + 1,
|
|
15744
|
-
maxRetries: this.#config.maxRetries,
|
|
15745
|
-
droppedBatchSize: buffer.totalSize,
|
|
15746
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15747
|
-
});
|
|
15748
|
-
for (const spanKey of buffer.completedSpans) {
|
|
15749
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15750
|
-
}
|
|
15751
|
-
}
|
|
15752
|
-
}
|
|
15716
|
+
return;
|
|
15753
15717
|
}
|
|
15754
15718
|
async _exportTracingEvent(event) {
|
|
15755
15719
|
await this.waitForInit();
|
|
15756
|
-
if (!this.#
|
|
15720
|
+
if (!this.#observabilityStorage) {
|
|
15757
15721
|
this.logger.debug("Cannot store traces. Observability storage is not initialized");
|
|
15758
15722
|
return;
|
|
15759
15723
|
}
|
|
15760
|
-
|
|
15761
|
-
|
|
15762
|
-
}
|
|
15763
|
-
switch (this.#resolvedStrategy) {
|
|
15764
|
-
case "realtime":
|
|
15765
|
-
await this.handleRealtimeEvent(event, this.#observability);
|
|
15766
|
-
break;
|
|
15767
|
-
case "batch-with-updates":
|
|
15768
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
15769
|
-
break;
|
|
15770
|
-
case "insert-only":
|
|
15771
|
-
this.handleInsertOnlyEvent(event);
|
|
15772
|
-
break;
|
|
15773
|
-
}
|
|
15724
|
+
this.#eventBuffer.addEvent(event);
|
|
15725
|
+
await this.handleBatchedFlush();
|
|
15774
15726
|
}
|
|
15775
15727
|
/**
|
|
15776
15728
|
* Resolves when an ongoing init call is finished
|
|
@@ -15783,15 +15735,51 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15783
15735
|
this.#initPromises.add(resolve);
|
|
15784
15736
|
});
|
|
15785
15737
|
}
|
|
15738
|
+
/**
|
|
15739
|
+
* Handle metric events — buffer for batch flush.
|
|
15740
|
+
*/
|
|
15741
|
+
async onMetricEvent(event) {
|
|
15742
|
+
await this.waitForInit();
|
|
15743
|
+
if (!this.#observabilityStorage) return;
|
|
15744
|
+
this.#eventBuffer.addEvent(event);
|
|
15745
|
+
await this.handleBatchedFlush();
|
|
15746
|
+
}
|
|
15747
|
+
/**
|
|
15748
|
+
* Handle log events — buffer for batch flush.
|
|
15749
|
+
*/
|
|
15750
|
+
async onLogEvent(event) {
|
|
15751
|
+
await this.waitForInit();
|
|
15752
|
+
if (!this.#observabilityStorage) return;
|
|
15753
|
+
this.#eventBuffer.addEvent(event);
|
|
15754
|
+
await this.handleBatchedFlush();
|
|
15755
|
+
}
|
|
15756
|
+
/**
|
|
15757
|
+
* Handle score events — buffer for batch flush.
|
|
15758
|
+
*/
|
|
15759
|
+
async onScoreEvent(event) {
|
|
15760
|
+
await this.waitForInit();
|
|
15761
|
+
if (!this.#observabilityStorage) return;
|
|
15762
|
+
this.#eventBuffer.addEvent(event);
|
|
15763
|
+
await this.handleBatchedFlush();
|
|
15764
|
+
}
|
|
15765
|
+
/**
|
|
15766
|
+
* Handle feedback events — buffer for batch flush.
|
|
15767
|
+
*/
|
|
15768
|
+
async onFeedbackEvent(event) {
|
|
15769
|
+
await this.waitForInit();
|
|
15770
|
+
if (!this.#observabilityStorage) return;
|
|
15771
|
+
this.#eventBuffer.addEvent(event);
|
|
15772
|
+
await this.handleBatchedFlush();
|
|
15773
|
+
}
|
|
15786
15774
|
/**
|
|
15787
15775
|
* Force flush any buffered spans without shutting down the exporter.
|
|
15788
15776
|
* This is useful in serverless environments where you need to ensure spans
|
|
15789
15777
|
* are exported before the runtime instance is terminated.
|
|
15790
15778
|
*/
|
|
15791
15779
|
async flush() {
|
|
15792
|
-
if (this.
|
|
15780
|
+
if (this.#eventBuffer.totalSize > 0) {
|
|
15793
15781
|
this.logger.debug("Flushing buffered events", {
|
|
15794
|
-
bufferedEvents: this.
|
|
15782
|
+
bufferedEvents: this.#eventBuffer.totalSize
|
|
15795
15783
|
});
|
|
15796
15784
|
await this.flushBuffer();
|
|
15797
15785
|
}
|
|
@@ -15799,7 +15787,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15799
15787
|
async shutdown() {
|
|
15800
15788
|
if (this.#flushTimer) {
|
|
15801
15789
|
clearTimeout(this.#flushTimer);
|
|
15802
|
-
this.#flushTimer =
|
|
15790
|
+
this.#flushTimer = void 0;
|
|
15803
15791
|
}
|
|
15804
15792
|
await this.flush();
|
|
15805
15793
|
this.logger.info("DefaultExporter shutdown complete");
|
|
@@ -15915,7 +15903,7 @@ var TestExporter = class extends BaseExporter {
|
|
|
15915
15903
|
if (this.#config.storeLogs) {
|
|
15916
15904
|
const metric = event.metric;
|
|
15917
15905
|
const labelsStr = Object.entries(metric.labels).map(([k, v]) => `${k}=${v}`).join(", ");
|
|
15918
|
-
const logMessage = `[TestExporter] metric
|
|
15906
|
+
const logMessage = `[TestExporter] metric: ${metric.name}=${metric.value}${labelsStr ? ` {${labelsStr}}` : ""}`;
|
|
15919
15907
|
this.#debugLogs.push(logMessage);
|
|
15920
15908
|
}
|
|
15921
15909
|
this.#metricEvents.push(event);
|
|
@@ -15927,7 +15915,7 @@ var TestExporter = class extends BaseExporter {
|
|
|
15927
15915
|
this.#trackEvent("score");
|
|
15928
15916
|
if (this.#config.storeLogs) {
|
|
15929
15917
|
const score = event.score;
|
|
15930
|
-
const logMessage = `[TestExporter] score: ${score.
|
|
15918
|
+
const logMessage = `[TestExporter] score: ${score.scorerId}=${score.score} (trace: ${score.traceId.slice(-8)}${score.spanId ? `, span: ${score.spanId.slice(-8)}` : ""})`;
|
|
15931
15919
|
this.#debugLogs.push(logMessage);
|
|
15932
15920
|
}
|
|
15933
15921
|
this.#scoreEvents.push(event);
|
|
@@ -16145,10 +16133,12 @@ var TestExporter = class extends BaseExporter {
|
|
|
16145
16133
|
return this.#metricEvents.filter((e) => e.metric.name === name).map((e) => e.metric);
|
|
16146
16134
|
}
|
|
16147
16135
|
/**
|
|
16148
|
-
*
|
|
16136
|
+
* @deprecated MetricType is no longer stored. Use getMetricsByName() instead.
|
|
16149
16137
|
*/
|
|
16150
|
-
getMetricsByType(
|
|
16151
|
-
|
|
16138
|
+
getMetricsByType(_metricType) {
|
|
16139
|
+
throw new Error(
|
|
16140
|
+
"getMetricsByType() has been removed: metricType is no longer stored. Use getMetricsByName(metricName) instead to filter metrics by name."
|
|
16141
|
+
);
|
|
16152
16142
|
}
|
|
16153
16143
|
// ============================================================================
|
|
16154
16144
|
// Score Query Methods
|
|
@@ -16166,10 +16156,10 @@ var TestExporter = class extends BaseExporter {
|
|
|
16166
16156
|
return this.#scoreEvents.map((e) => e.score);
|
|
16167
16157
|
}
|
|
16168
16158
|
/**
|
|
16169
|
-
* Get scores filtered by scorer
|
|
16159
|
+
* Get scores filtered by scorer id
|
|
16170
16160
|
*/
|
|
16171
|
-
getScoresByScorer(
|
|
16172
|
-
return this.#scoreEvents.filter((e) => e.score.
|
|
16161
|
+
getScoresByScorer(scorerId) {
|
|
16162
|
+
return this.#scoreEvents.filter((e) => e.score.scorerId === scorerId).map((e) => e.score);
|
|
16173
16163
|
}
|
|
16174
16164
|
/**
|
|
16175
16165
|
* Get scores for a specific trace
|
|
@@ -16231,17 +16221,14 @@ var TestExporter = class extends BaseExporter {
|
|
|
16231
16221
|
const level = event.log.level;
|
|
16232
16222
|
logsByLevel[level] = (logsByLevel[level] || 0) + 1;
|
|
16233
16223
|
}
|
|
16234
|
-
const metricsByType = {};
|
|
16235
16224
|
const metricsByName = {};
|
|
16236
16225
|
for (const event of this.#metricEvents) {
|
|
16237
|
-
const mType = event.metric.metricType;
|
|
16238
|
-
metricsByType[mType] = (metricsByType[mType] || 0) + 1;
|
|
16239
16226
|
const mName = event.metric.name;
|
|
16240
16227
|
metricsByName[mName] = (metricsByName[mName] || 0) + 1;
|
|
16241
16228
|
}
|
|
16242
16229
|
const scoresByScorer = {};
|
|
16243
16230
|
for (const event of this.#scoreEvents) {
|
|
16244
|
-
const scorer = event.score.
|
|
16231
|
+
const scorer = event.score.scorerId;
|
|
16245
16232
|
scoresByScorer[scorer] = (scoresByScorer[scorer] || 0) + 1;
|
|
16246
16233
|
}
|
|
16247
16234
|
const feedbackByType = {};
|
|
@@ -16266,7 +16253,6 @@ var TestExporter = class extends BaseExporter {
|
|
|
16266
16253
|
totalLogs: this.#logEvents.length,
|
|
16267
16254
|
logsByLevel,
|
|
16268
16255
|
totalMetrics: this.#metricEvents.length,
|
|
16269
|
-
metricsByType,
|
|
16270
16256
|
metricsByName,
|
|
16271
16257
|
totalScores: this.#scoreEvents.length,
|
|
16272
16258
|
scoresByScorer,
|
|
@@ -16998,90 +16984,33 @@ var BaseObservabilityEventBus = class _BaseObservabilityEventBus extends MastraB
|
|
|
16998
16984
|
}
|
|
16999
16985
|
};
|
|
17000
16986
|
var AutoExtractedMetrics = class {
|
|
17001
|
-
|
|
17002
|
-
* @param observabilityBus - Bus used to emit derived MetricEvents.
|
|
17003
|
-
* @param cardinalityFilter - Optional filter applied to metric labels before emission.
|
|
17004
|
-
*/
|
|
17005
|
-
constructor(observabilityBus, cardinalityFilter) {
|
|
16987
|
+
constructor(observabilityBus) {
|
|
17006
16988
|
this.observabilityBus = observabilityBus;
|
|
17007
|
-
this.cardinalityFilter = cardinalityFilter;
|
|
17008
16989
|
}
|
|
17009
16990
|
/**
|
|
17010
|
-
* Route a tracing event to the appropriate
|
|
17011
|
-
*
|
|
17012
|
-
* duration histogram, and (for model spans) token counters.
|
|
16991
|
+
* Route a tracing event to the appropriate handler.
|
|
16992
|
+
* SPAN_ENDED emits duration and token metrics (for model spans).
|
|
17013
16993
|
*/
|
|
17014
16994
|
processTracingEvent(event) {
|
|
17015
|
-
|
|
17016
|
-
|
|
17017
|
-
this.onSpanStarted(event.exportedSpan);
|
|
17018
|
-
break;
|
|
17019
|
-
case TracingEventType.SPAN_ENDED:
|
|
17020
|
-
this.onSpanEnded(event.exportedSpan);
|
|
17021
|
-
break;
|
|
17022
|
-
}
|
|
17023
|
-
}
|
|
17024
|
-
/** Emit a `mastra_scores_total` counter for a score event. */
|
|
17025
|
-
processScoreEvent(event) {
|
|
17026
|
-
const labels = {
|
|
17027
|
-
scorer: event.score.scorerName
|
|
17028
|
-
};
|
|
17029
|
-
if (event.score.metadata?.entityType) {
|
|
17030
|
-
labels.entity_type = String(event.score.metadata.entityType);
|
|
17031
|
-
}
|
|
17032
|
-
if (event.score.experimentId) {
|
|
17033
|
-
labels.experiment = event.score.experimentId;
|
|
17034
|
-
}
|
|
17035
|
-
this.emit("mastra_scores_total", "counter", 1, labels);
|
|
17036
|
-
}
|
|
17037
|
-
/** Emit a `mastra_feedback_total` counter for a feedback event. */
|
|
17038
|
-
processFeedbackEvent(event) {
|
|
17039
|
-
const labels = {
|
|
17040
|
-
feedback_type: event.feedback.feedbackType,
|
|
17041
|
-
source: event.feedback.source
|
|
17042
|
-
};
|
|
17043
|
-
if (event.feedback.metadata?.entityType) {
|
|
17044
|
-
labels.entity_type = String(event.feedback.metadata.entityType);
|
|
17045
|
-
}
|
|
17046
|
-
if (event.feedback.experimentId) {
|
|
17047
|
-
labels.experiment = event.feedback.experimentId;
|
|
17048
|
-
}
|
|
17049
|
-
this.emit("mastra_feedback_total", "counter", 1, labels);
|
|
17050
|
-
}
|
|
17051
|
-
/** Emit a started counter (e.g. `mastra_agent_runs_started`) for the span type. */
|
|
17052
|
-
onSpanStarted(span) {
|
|
17053
|
-
const labels = this.extractLabels(span);
|
|
17054
|
-
const metricName = this.getStartedMetricName(span);
|
|
17055
|
-
if (metricName) {
|
|
17056
|
-
this.emit(metricName, "counter", 1, labels);
|
|
16995
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
16996
|
+
this.onSpanEnded(event.exportedSpan);
|
|
17057
16997
|
}
|
|
17058
16998
|
}
|
|
17059
|
-
/** Emit
|
|
16999
|
+
/** Emit duration and token metrics when a span ends. */
|
|
17060
17000
|
onSpanEnded(span) {
|
|
17061
17001
|
const labels = this.extractLabels(span);
|
|
17062
|
-
const endedMetricName = this.getEndedMetricName(span);
|
|
17063
|
-
if (endedMetricName) {
|
|
17064
|
-
const endedLabels = { ...labels };
|
|
17065
|
-
if (span.errorInfo) {
|
|
17066
|
-
endedLabels.status = "error";
|
|
17067
|
-
} else {
|
|
17068
|
-
endedLabels.status = "ok";
|
|
17069
|
-
}
|
|
17070
|
-
this.emit(endedMetricName, "counter", 1, endedLabels);
|
|
17071
|
-
}
|
|
17072
17002
|
const durationMetricName = this.getDurationMetricName(span);
|
|
17073
17003
|
if (durationMetricName && span.startTime && span.endTime) {
|
|
17074
|
-
const durationMs =
|
|
17004
|
+
const durationMs = span.endTime.getTime() - span.startTime.getTime();
|
|
17075
17005
|
const durationLabels = { ...labels };
|
|
17076
|
-
|
|
17077
|
-
|
|
17078
|
-
} else {
|
|
17079
|
-
durationLabels.status = "ok";
|
|
17080
|
-
}
|
|
17081
|
-
this.emit(durationMetricName, "histogram", durationMs, durationLabels);
|
|
17006
|
+
durationLabels.status = span.errorInfo ? "error" : "ok";
|
|
17007
|
+
this.observabilityBus.emitMetric(durationMetricName, durationMs, durationLabels);
|
|
17082
17008
|
}
|
|
17083
17009
|
if (span.type === SpanType.MODEL_GENERATION) {
|
|
17084
|
-
|
|
17010
|
+
const attrs = span.attributes;
|
|
17011
|
+
if (attrs?.usage) {
|
|
17012
|
+
this.extractTokenMetrics(attrs.usage, labels);
|
|
17013
|
+
}
|
|
17085
17014
|
}
|
|
17086
17015
|
}
|
|
17087
17016
|
/** Build base metric labels from a span's entity and model attributes. */
|
|
@@ -17092,65 +17021,34 @@ var AutoExtractedMetrics = class {
|
|
|
17092
17021
|
if (entityName) labels.entity_name = entityName;
|
|
17093
17022
|
if (span.type === SpanType.MODEL_GENERATION) {
|
|
17094
17023
|
const attrs = span.attributes;
|
|
17095
|
-
if (attrs?.model) labels.model =
|
|
17096
|
-
if (attrs?.provider) labels.provider =
|
|
17024
|
+
if (attrs?.model) labels.model = attrs.model;
|
|
17025
|
+
if (attrs?.provider) labels.provider = attrs.provider;
|
|
17097
17026
|
}
|
|
17098
17027
|
return labels;
|
|
17099
17028
|
}
|
|
17100
|
-
/** Emit token usage
|
|
17101
|
-
extractTokenMetrics(
|
|
17102
|
-
const
|
|
17103
|
-
const
|
|
17104
|
-
|
|
17105
|
-
|
|
17106
|
-
|
|
17107
|
-
|
|
17108
|
-
|
|
17109
|
-
|
|
17110
|
-
|
|
17111
|
-
|
|
17112
|
-
|
|
17113
|
-
|
|
17114
|
-
|
|
17115
|
-
if (
|
|
17116
|
-
|
|
17117
|
-
|
|
17118
|
-
|
|
17119
|
-
|
|
17120
|
-
|
|
17121
|
-
|
|
17122
|
-
|
|
17123
|
-
/** Map a span type to its `*_started` counter metric name, or `null` for unsupported types. */
|
|
17124
|
-
getStartedMetricName(span) {
|
|
17125
|
-
switch (span.type) {
|
|
17126
|
-
case SpanType.AGENT_RUN:
|
|
17127
|
-
return "mastra_agent_runs_started";
|
|
17128
|
-
case SpanType.TOOL_CALL:
|
|
17129
|
-
return "mastra_tool_calls_started";
|
|
17130
|
-
case SpanType.WORKFLOW_RUN:
|
|
17131
|
-
return "mastra_workflow_runs_started";
|
|
17132
|
-
case SpanType.MODEL_GENERATION:
|
|
17133
|
-
return "mastra_model_requests_started";
|
|
17134
|
-
default:
|
|
17135
|
-
return null;
|
|
17136
|
-
}
|
|
17137
|
-
}
|
|
17138
|
-
/** Map a span type to its `*_ended` counter metric name, or `null` for unsupported types. */
|
|
17139
|
-
getEndedMetricName(span) {
|
|
17140
|
-
switch (span.type) {
|
|
17141
|
-
case SpanType.AGENT_RUN:
|
|
17142
|
-
return "mastra_agent_runs_ended";
|
|
17143
|
-
case SpanType.TOOL_CALL:
|
|
17144
|
-
return "mastra_tool_calls_ended";
|
|
17145
|
-
case SpanType.WORKFLOW_RUN:
|
|
17146
|
-
return "mastra_workflow_runs_ended";
|
|
17147
|
-
case SpanType.MODEL_GENERATION:
|
|
17148
|
-
return "mastra_model_requests_ended";
|
|
17149
|
-
default:
|
|
17150
|
-
return null;
|
|
17151
|
-
}
|
|
17152
|
-
}
|
|
17153
|
-
/** Map a span type to its `*_duration_ms` histogram metric name, or `null` for unsupported types. */
|
|
17029
|
+
/** Emit token usage metrics from UsageStats. */
|
|
17030
|
+
extractTokenMetrics(usage, labels) {
|
|
17031
|
+
const emit = (name, value) => this.observabilityBus.emitMetric(name, value, labels);
|
|
17032
|
+
const emitNonZero = (name, value) => {
|
|
17033
|
+
if (value > 0) emit(name, value);
|
|
17034
|
+
};
|
|
17035
|
+
emit("mastra_model_total_input_tokens", usage.inputTokens ?? 0);
|
|
17036
|
+
emit("mastra_model_total_output_tokens", usage.outputTokens ?? 0);
|
|
17037
|
+
if (usage.inputDetails) {
|
|
17038
|
+
emitNonZero("mastra_model_input_text_tokens", usage.inputDetails.text ?? 0);
|
|
17039
|
+
emitNonZero("mastra_model_input_cache_read_tokens", usage.inputDetails.cacheRead ?? 0);
|
|
17040
|
+
emitNonZero("mastra_model_input_cache_write_tokens", usage.inputDetails.cacheWrite ?? 0);
|
|
17041
|
+
emitNonZero("mastra_model_input_audio_tokens", usage.inputDetails.audio ?? 0);
|
|
17042
|
+
emitNonZero("mastra_model_input_image_tokens", usage.inputDetails.image ?? 0);
|
|
17043
|
+
}
|
|
17044
|
+
if (usage.outputDetails) {
|
|
17045
|
+
emitNonZero("mastra_model_output_text_tokens", usage.outputDetails.text ?? 0);
|
|
17046
|
+
emitNonZero("mastra_model_output_reasoning_tokens", usage.outputDetails.reasoning ?? 0);
|
|
17047
|
+
emitNonZero("mastra_model_output_audio_tokens", usage.outputDetails.audio ?? 0);
|
|
17048
|
+
emitNonZero("mastra_model_output_image_tokens", usage.outputDetails.image ?? 0);
|
|
17049
|
+
}
|
|
17050
|
+
}
|
|
17051
|
+
/** Map a span type to its `*_duration_ms` metric name, or `null` for unsupported types. */
|
|
17154
17052
|
getDurationMetricName(span) {
|
|
17155
17053
|
switch (span.type) {
|
|
17156
17054
|
case SpanType.AGENT_RUN:
|
|
@@ -17165,18 +17063,38 @@ var AutoExtractedMetrics = class {
|
|
|
17165
17063
|
return null;
|
|
17166
17064
|
}
|
|
17167
17065
|
}
|
|
17168
|
-
|
|
17169
|
-
|
|
17170
|
-
|
|
17171
|
-
|
|
17172
|
-
|
|
17173
|
-
|
|
17174
|
-
|
|
17175
|
-
|
|
17176
|
-
|
|
17177
|
-
|
|
17178
|
-
const
|
|
17179
|
-
this.
|
|
17066
|
+
};
|
|
17067
|
+
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17068
|
+
var CardinalityFilter = class {
|
|
17069
|
+
blockedLabels;
|
|
17070
|
+
blockUUIDs;
|
|
17071
|
+
/**
|
|
17072
|
+
* @param config - Optional configuration. When omitted, uses
|
|
17073
|
+
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
17074
|
+
*/
|
|
17075
|
+
constructor(config2) {
|
|
17076
|
+
const blocked = config2?.blockedLabels ?? [...DEFAULT_BLOCKED_LABELS];
|
|
17077
|
+
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
17078
|
+
this.blockUUIDs = config2?.blockUUIDs ?? true;
|
|
17079
|
+
}
|
|
17080
|
+
/**
|
|
17081
|
+
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
17082
|
+
*
|
|
17083
|
+
* @param labels - Raw metric labels to filter.
|
|
17084
|
+
* @returns A new object containing only the allowed labels.
|
|
17085
|
+
*/
|
|
17086
|
+
filterLabels(labels) {
|
|
17087
|
+
const filtered = {};
|
|
17088
|
+
for (const [key, value] of Object.entries(labels)) {
|
|
17089
|
+
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
17090
|
+
continue;
|
|
17091
|
+
}
|
|
17092
|
+
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
17093
|
+
continue;
|
|
17094
|
+
}
|
|
17095
|
+
filtered[key] = value;
|
|
17096
|
+
}
|
|
17097
|
+
return filtered;
|
|
17180
17098
|
}
|
|
17181
17099
|
};
|
|
17182
17100
|
function routeToHandler(handler, event, logger) {
|
|
@@ -17230,25 +17148,32 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17230
17148
|
exporters = [];
|
|
17231
17149
|
bridge;
|
|
17232
17150
|
autoExtractor;
|
|
17151
|
+
cardinalityFilter;
|
|
17233
17152
|
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
17234
17153
|
pendingHandlers = /* @__PURE__ */ new Set();
|
|
17235
|
-
constructor() {
|
|
17154
|
+
constructor(config2) {
|
|
17236
17155
|
super({ name: "ObservabilityBus" });
|
|
17156
|
+
this.cardinalityFilter = config2?.cardinalityFilter ?? new CardinalityFilter();
|
|
17157
|
+
if (config2?.autoExtractMetrics !== false) {
|
|
17158
|
+
this.autoExtractor = new AutoExtractedMetrics(this);
|
|
17159
|
+
}
|
|
17237
17160
|
}
|
|
17238
17161
|
/**
|
|
17239
|
-
*
|
|
17240
|
-
*
|
|
17241
|
-
*
|
|
17242
|
-
*
|
|
17243
|
-
* No-ops if auto-extraction is already enabled.
|
|
17244
|
-
*
|
|
17245
|
-
* @param cardinalityFilter - Optional filter applied to auto-extracted metric labels.
|
|
17162
|
+
* Emit a metric event with validation and cardinality filtering.
|
|
17163
|
+
* Non-finite or negative values are silently dropped.
|
|
17164
|
+
* This is the single entry point for all metric emission (auto-extracted and user-defined).
|
|
17246
17165
|
*/
|
|
17247
|
-
|
|
17248
|
-
if (
|
|
17249
|
-
|
|
17250
|
-
|
|
17251
|
-
|
|
17166
|
+
emitMetric(name, value, labels) {
|
|
17167
|
+
if (!Number.isFinite(value) || value < 0) return;
|
|
17168
|
+
const filteredLabels = this.cardinalityFilter.filterLabels(labels);
|
|
17169
|
+
const exportedMetric = {
|
|
17170
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
17171
|
+
name,
|
|
17172
|
+
value,
|
|
17173
|
+
labels: filteredLabels
|
|
17174
|
+
};
|
|
17175
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
17176
|
+
this.emit(event);
|
|
17252
17177
|
}
|
|
17253
17178
|
/**
|
|
17254
17179
|
* Register an exporter to receive routed events.
|
|
@@ -17327,15 +17252,9 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17327
17252
|
if (this.bridge) {
|
|
17328
17253
|
this.trackPromise(routeToHandler(this.bridge, event, this.logger));
|
|
17329
17254
|
}
|
|
17330
|
-
if (this.autoExtractor) {
|
|
17255
|
+
if (this.autoExtractor && isTracingEvent(event)) {
|
|
17331
17256
|
try {
|
|
17332
|
-
|
|
17333
|
-
this.autoExtractor.processTracingEvent(event);
|
|
17334
|
-
} else if (event.type === "score") {
|
|
17335
|
-
this.autoExtractor.processScoreEvent(event);
|
|
17336
|
-
} else if (event.type === "feedback") {
|
|
17337
|
-
this.autoExtractor.processFeedbackEvent(event);
|
|
17338
|
-
}
|
|
17257
|
+
this.autoExtractor.processTracingEvent(event);
|
|
17339
17258
|
} catch (err) {
|
|
17340
17259
|
this.logger.error("[ObservabilityBus] Auto-extraction error:", err);
|
|
17341
17260
|
}
|
|
@@ -17462,104 +17381,45 @@ var LoggerContextImpl = class {
|
|
|
17462
17381
|
|
|
17463
17382
|
// src/context/metrics.ts
|
|
17464
17383
|
var MetricsContextImpl = class {
|
|
17465
|
-
|
|
17384
|
+
baseLabels;
|
|
17385
|
+
observabilityBus;
|
|
17466
17386
|
/**
|
|
17467
17387
|
* Create a metrics context. Base labels are defensively copied so
|
|
17468
17388
|
* mutations after construction do not affect emitted metrics.
|
|
17469
17389
|
*/
|
|
17470
17390
|
constructor(config2) {
|
|
17471
|
-
this.
|
|
17472
|
-
|
|
17473
|
-
labels: config2.labels ? { ...config2.labels } : void 0
|
|
17474
|
-
};
|
|
17391
|
+
this.baseLabels = config2.labels ? { ...config2.labels } : {};
|
|
17392
|
+
this.observabilityBus = config2.observabilityBus;
|
|
17475
17393
|
}
|
|
17476
|
-
/**
|
|
17477
|
-
|
|
17478
|
-
|
|
17479
|
-
|
|
17480
|
-
|
|
17394
|
+
/** Emit a metric observation. */
|
|
17395
|
+
emit(name, value, labels) {
|
|
17396
|
+
const allLabels = { ...this.baseLabels, ...labels };
|
|
17397
|
+
this.observabilityBus.emitMetric(name, value, allLabels);
|
|
17398
|
+
}
|
|
17399
|
+
/** @deprecated Use `emit()` instead. */
|
|
17481
17400
|
counter(name) {
|
|
17482
17401
|
return {
|
|
17483
17402
|
add: (value, additionalLabels) => {
|
|
17484
|
-
this.emit(name,
|
|
17403
|
+
this.emit(name, value, additionalLabels);
|
|
17485
17404
|
}
|
|
17486
17405
|
};
|
|
17487
17406
|
}
|
|
17488
|
-
/**
|
|
17489
|
-
* Create a gauge instrument. Call `.set(value)` to record a point-in-time value.
|
|
17490
|
-
*
|
|
17491
|
-
* @param name - Metric name (e.g. `mastra_queue_depth`).
|
|
17492
|
-
*/
|
|
17407
|
+
/** @deprecated Use `emit()` instead. */
|
|
17493
17408
|
gauge(name) {
|
|
17494
17409
|
return {
|
|
17495
17410
|
set: (value, additionalLabels) => {
|
|
17496
|
-
this.emit(name,
|
|
17411
|
+
this.emit(name, value, additionalLabels);
|
|
17497
17412
|
}
|
|
17498
17413
|
};
|
|
17499
17414
|
}
|
|
17500
|
-
/**
|
|
17501
|
-
* Create a histogram instrument. Call `.record(value)` to observe a measurement.
|
|
17502
|
-
*
|
|
17503
|
-
* @param name - Metric name (e.g. `mastra_request_duration_ms`).
|
|
17504
|
-
*/
|
|
17415
|
+
/** @deprecated Use `emit()` instead. */
|
|
17505
17416
|
histogram(name) {
|
|
17506
17417
|
return {
|
|
17507
17418
|
record: (value, additionalLabels) => {
|
|
17508
|
-
this.emit(name,
|
|
17419
|
+
this.emit(name, value, additionalLabels);
|
|
17509
17420
|
}
|
|
17510
17421
|
};
|
|
17511
17422
|
}
|
|
17512
|
-
/** Merge base + additional labels, apply cardinality filtering, and emit a MetricEvent. Non-finite values are silently dropped. */
|
|
17513
|
-
emit(name, metricType, value, additionalLabels) {
|
|
17514
|
-
if (!Number.isFinite(value)) return;
|
|
17515
|
-
const allLabels = {
|
|
17516
|
-
...this.config.labels,
|
|
17517
|
-
...additionalLabels
|
|
17518
|
-
};
|
|
17519
|
-
const filteredLabels = this.config.cardinalityFilter.filterLabels(allLabels);
|
|
17520
|
-
const exportedMetric = {
|
|
17521
|
-
timestamp: /* @__PURE__ */ new Date(),
|
|
17522
|
-
name,
|
|
17523
|
-
metricType,
|
|
17524
|
-
value,
|
|
17525
|
-
labels: filteredLabels
|
|
17526
|
-
};
|
|
17527
|
-
const event = { type: "metric", metric: exportedMetric };
|
|
17528
|
-
this.config.observabilityBus.emit(event);
|
|
17529
|
-
}
|
|
17530
|
-
};
|
|
17531
|
-
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17532
|
-
var CardinalityFilter = class {
|
|
17533
|
-
blockedLabels;
|
|
17534
|
-
blockUUIDs;
|
|
17535
|
-
/**
|
|
17536
|
-
* @param config - Optional configuration. When omitted, uses
|
|
17537
|
-
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
17538
|
-
*/
|
|
17539
|
-
constructor(config2) {
|
|
17540
|
-
const blocked = config2?.blockedLabels ?? [...DEFAULT_BLOCKED_LABELS];
|
|
17541
|
-
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
17542
|
-
this.blockUUIDs = config2?.blockUUIDs ?? true;
|
|
17543
|
-
}
|
|
17544
|
-
/**
|
|
17545
|
-
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
17546
|
-
*
|
|
17547
|
-
* @param labels - Raw metric labels to filter.
|
|
17548
|
-
* @returns A new object containing only the allowed labels.
|
|
17549
|
-
*/
|
|
17550
|
-
filterLabels(labels) {
|
|
17551
|
-
const filtered = {};
|
|
17552
|
-
for (const [key, value] of Object.entries(labels)) {
|
|
17553
|
-
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
17554
|
-
continue;
|
|
17555
|
-
}
|
|
17556
|
-
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
17557
|
-
continue;
|
|
17558
|
-
}
|
|
17559
|
-
filtered[key] = value;
|
|
17560
|
-
}
|
|
17561
|
-
return filtered;
|
|
17562
|
-
}
|
|
17563
17423
|
};
|
|
17564
17424
|
|
|
17565
17425
|
// src/usage.ts
|
|
@@ -18614,15 +18474,16 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
18614
18474
|
requestContextKeys: config2.requestContextKeys ?? [],
|
|
18615
18475
|
serializationOptions: config2.serializationOptions
|
|
18616
18476
|
};
|
|
18617
|
-
this.cardinalityFilter = new CardinalityFilter();
|
|
18618
|
-
this.observabilityBus = new ObservabilityBus(
|
|
18477
|
+
this.cardinalityFilter = new CardinalityFilter(config2.cardinality);
|
|
18478
|
+
this.observabilityBus = new ObservabilityBus({
|
|
18479
|
+
cardinalityFilter: this.cardinalityFilter
|
|
18480
|
+
});
|
|
18619
18481
|
for (const exporter of this.exporters) {
|
|
18620
18482
|
this.observabilityBus.registerExporter(exporter);
|
|
18621
18483
|
}
|
|
18622
18484
|
if (this.config.bridge) {
|
|
18623
18485
|
this.observabilityBus.registerBridge(this.config.bridge);
|
|
18624
18486
|
}
|
|
18625
|
-
this.observabilityBus.enableAutoExtractedMetrics(this.cardinalityFilter);
|
|
18626
18487
|
if (this.config.bridge?.init) {
|
|
18627
18488
|
this.config.bridge.init({ config: this.config });
|
|
18628
18489
|
}
|
|
@@ -18866,8 +18727,7 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
18866
18727
|
if (this.config.serviceName) labels.service_name = this.config.serviceName;
|
|
18867
18728
|
return new MetricsContextImpl({
|
|
18868
18729
|
labels: Object.keys(labels).length > 0 ? labels : void 0,
|
|
18869
|
-
observabilityBus: this.observabilityBus
|
|
18870
|
-
cardinalityFilter: this.cardinalityFilter
|
|
18730
|
+
observabilityBus: this.observabilityBus
|
|
18871
18731
|
});
|
|
18872
18732
|
}
|
|
18873
18733
|
/**
|
|
@@ -19014,6 +18874,7 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
19014
18874
|
// ============================================================================
|
|
19015
18875
|
// Event-driven Export Methods
|
|
19016
18876
|
// ============================================================================
|
|
18877
|
+
/** Process a span through output processors and export it, returning undefined if filtered out. */
|
|
19017
18878
|
getSpanForExport(span) {
|
|
19018
18879
|
if (!span.isValid) return void 0;
|
|
19019
18880
|
if (span.isInternal && !this.config.includeInternalSpans) return void 0;
|
|
@@ -19061,7 +18922,7 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
19061
18922
|
*
|
|
19062
18923
|
* The bus routes the event to each registered exporter's and bridge's
|
|
19063
18924
|
* onTracingEvent handler and triggers auto-extracted metrics (e.g.,
|
|
19064
|
-
*
|
|
18925
|
+
* mastra_agent_duration_ms, mastra_model_duration_ms).
|
|
19065
18926
|
*/
|
|
19066
18927
|
emitTracingEvent(event) {
|
|
19067
18928
|
this.observabilityBus.emit(event);
|
|
@@ -19399,7 +19260,9 @@ var Observability = class extends MastraBase {
|
|
|
19399
19260
|
}
|
|
19400
19261
|
const validationResult = observabilityRegistryConfigSchema.safeParse(config2);
|
|
19401
19262
|
if (!validationResult.success) {
|
|
19402
|
-
const errorMessages = validationResult.error.issues.map(
|
|
19263
|
+
const errorMessages = validationResult.error.issues.map(
|
|
19264
|
+
(err) => `${err.path.join(".") || "config"}: ${err.message}`
|
|
19265
|
+
).join("; ");
|
|
19403
19266
|
throw new MastraError({
|
|
19404
19267
|
id: "OBSERVABILITY_INVALID_CONFIG",
|
|
19405
19268
|
text: `Invalid observability configuration: ${errorMessages}`,
|
|
@@ -19415,7 +19278,9 @@ var Observability = class extends MastraBase {
|
|
|
19415
19278
|
if (!isInstance(configValue)) {
|
|
19416
19279
|
const configValidation = observabilityConfigValueSchema.safeParse(configValue);
|
|
19417
19280
|
if (!configValidation.success) {
|
|
19418
|
-
const errorMessages = configValidation.error.issues.map(
|
|
19281
|
+
const errorMessages = configValidation.error.issues.map(
|
|
19282
|
+
(err) => `${err.path.join(".")}: ${err.message}`
|
|
19283
|
+
).join("; ");
|
|
19419
19284
|
throw new MastraError({
|
|
19420
19285
|
id: "OBSERVABILITY_INVALID_INSTANCE_CONFIG",
|
|
19421
19286
|
text: `Invalid configuration for observability instance '${name}': ${errorMessages}`,
|
|
@@ -19455,6 +19320,7 @@ var Observability = class extends MastraBase {
|
|
|
19455
19320
|
this.#registry.setSelector(config2.configSelector);
|
|
19456
19321
|
}
|
|
19457
19322
|
}
|
|
19323
|
+
/** Initialize all exporter instances with the Mastra context (storage, config, etc.). */
|
|
19458
19324
|
setMastraContext(options) {
|
|
19459
19325
|
const instances = this.listInstances();
|
|
19460
19326
|
const { mastra } = options;
|
|
@@ -19475,42 +19341,50 @@ var Observability = class extends MastraBase {
|
|
|
19475
19341
|
});
|
|
19476
19342
|
});
|
|
19477
19343
|
}
|
|
19344
|
+
/** Propagate a logger to this instance and all registered observability instances. */
|
|
19478
19345
|
setLogger(options) {
|
|
19479
19346
|
super.__setLogger(options.logger);
|
|
19480
19347
|
this.listInstances().forEach((instance) => {
|
|
19481
19348
|
instance.__setLogger(options.logger);
|
|
19482
19349
|
});
|
|
19483
19350
|
}
|
|
19351
|
+
/** Get the observability instance chosen by the config selector for the given options. */
|
|
19484
19352
|
getSelectedInstance(options) {
|
|
19485
19353
|
return this.#registry.getSelected(options);
|
|
19486
19354
|
}
|
|
19487
|
-
/**
|
|
19488
|
-
* Registry management methods
|
|
19489
|
-
*/
|
|
19355
|
+
/** Register a named observability instance, optionally marking it as default. */
|
|
19490
19356
|
registerInstance(name, instance, isDefault = false) {
|
|
19491
19357
|
this.#registry.register(name, instance, isDefault);
|
|
19492
19358
|
}
|
|
19359
|
+
/** Get a registered instance by name. */
|
|
19493
19360
|
getInstance(name) {
|
|
19494
19361
|
return this.#registry.get(name);
|
|
19495
19362
|
}
|
|
19363
|
+
/** Get the default observability instance. */
|
|
19496
19364
|
getDefaultInstance() {
|
|
19497
19365
|
return this.#registry.getDefault();
|
|
19498
19366
|
}
|
|
19367
|
+
/** List all registered observability instances. */
|
|
19499
19368
|
listInstances() {
|
|
19500
19369
|
return this.#registry.list();
|
|
19501
19370
|
}
|
|
19371
|
+
/** Unregister an instance by name. Returns true if it was found and removed. */
|
|
19502
19372
|
unregisterInstance(name) {
|
|
19503
19373
|
return this.#registry.unregister(name);
|
|
19504
19374
|
}
|
|
19375
|
+
/** Check whether an instance with the given name is registered. */
|
|
19505
19376
|
hasInstance(name) {
|
|
19506
19377
|
return !!this.#registry.get(name);
|
|
19507
19378
|
}
|
|
19379
|
+
/** Set the config selector used to choose an instance at runtime. */
|
|
19508
19380
|
setConfigSelector(selector) {
|
|
19509
19381
|
this.#registry.setSelector(selector);
|
|
19510
19382
|
}
|
|
19383
|
+
/** Remove all registered instances and reset the registry. */
|
|
19511
19384
|
clear() {
|
|
19512
19385
|
this.#registry.clear();
|
|
19513
19386
|
}
|
|
19387
|
+
/** Shut down all registered instances, flushing any pending data. */
|
|
19514
19388
|
async shutdown() {
|
|
19515
19389
|
await this.#registry.shutdown();
|
|
19516
19390
|
}
|