@mastra/observability 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +9 -9
- package/dist/bus/observability-bus.d.ts +19 -10
- package/dist/bus/observability-bus.d.ts.map +1 -1
- package/dist/config.d.ts +8 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/context/metrics.d.ts +15 -24
- package/dist/context/metrics.d.ts.map +1 -1
- package/dist/default.d.ts +16 -3
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +39 -49
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/event-buffer.d.ts +63 -0
- package/dist/exporters/event-buffer.d.ts.map +1 -0
- package/dist/exporters/test.d.ts +5 -7
- package/dist/exporters/test.d.ts.map +1 -1
- package/dist/index.cjs +529 -655
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +529 -655
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +2 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/metrics/auto-extract.d.ts +15 -31
- package/dist/metrics/auto-extract.d.ts.map +1 -1
- package/package.json +7 -7
package/dist/index.cjs
CHANGED
|
@@ -5,6 +5,7 @@ var error$1 = require('@mastra/core/error');
|
|
|
5
5
|
var logger = require('@mastra/core/logger');
|
|
6
6
|
var observability = require('@mastra/core/observability');
|
|
7
7
|
var utils = require('@mastra/core/utils');
|
|
8
|
+
var storage = require('@mastra/core/storage');
|
|
8
9
|
var web = require('stream/web');
|
|
9
10
|
|
|
10
11
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
@@ -13814,7 +13815,8 @@ var observabilityInstanceConfigSchema = external_exports.object({
|
|
|
13814
13815
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
13815
13816
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
13816
13817
|
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
13817
|
-
serializationOptions: serializationOptionsSchema
|
|
13818
|
+
serializationOptions: serializationOptionsSchema,
|
|
13819
|
+
cardinality: external_exports.any().optional()
|
|
13818
13820
|
}).refine(
|
|
13819
13821
|
(data) => {
|
|
13820
13822
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -15263,44 +15265,197 @@ var ConsoleExporter = class extends BaseExporter {
|
|
|
15263
15265
|
this.logger.info("ConsoleExporter shutdown");
|
|
15264
15266
|
}
|
|
15265
15267
|
};
|
|
15266
|
-
|
|
15268
|
+
var EventBuffer = class {
|
|
15269
|
+
#preInit = [];
|
|
15270
|
+
#creates = [];
|
|
15271
|
+
#updates = [];
|
|
15272
|
+
#allCreatedSpans = /* @__PURE__ */ new Set();
|
|
15273
|
+
#firstEventTime;
|
|
15274
|
+
#storageStrategy;
|
|
15275
|
+
#maxRetries;
|
|
15276
|
+
constructor(args) {
|
|
15277
|
+
this.#maxRetries = args.maxRetries;
|
|
15278
|
+
}
|
|
15279
|
+
/** Initialize with a storage strategy and replay any pre-init events. */
|
|
15280
|
+
init(args) {
|
|
15281
|
+
if (!this.#storageStrategy) {
|
|
15282
|
+
this.#storageStrategy = args.strategy;
|
|
15283
|
+
for (const event of this.#preInit) {
|
|
15284
|
+
this.addEvent(event);
|
|
15285
|
+
}
|
|
15286
|
+
this.#preInit = [];
|
|
15287
|
+
}
|
|
15288
|
+
}
|
|
15289
|
+
/** Clear the create and update buffers and reset the event timer. */
|
|
15290
|
+
reset() {
|
|
15291
|
+
this.#creates = [];
|
|
15292
|
+
this.#updates = [];
|
|
15293
|
+
this.#firstEventTime = void 0;
|
|
15294
|
+
}
|
|
15295
|
+
setFirstEventTime() {
|
|
15296
|
+
if (!this.#firstEventTime) {
|
|
15297
|
+
this.#firstEventTime = /* @__PURE__ */ new Date();
|
|
15298
|
+
}
|
|
15299
|
+
}
|
|
15300
|
+
pushCreate(event) {
|
|
15301
|
+
this.setFirstEventTime();
|
|
15302
|
+
this.#creates.push({ ...event, retryCount: 0 });
|
|
15303
|
+
}
|
|
15304
|
+
pushUpdate(event) {
|
|
15305
|
+
this.setFirstEventTime();
|
|
15306
|
+
this.#updates.push({ ...event, retryCount: 0 });
|
|
15307
|
+
}
|
|
15308
|
+
/** Route an event to the create or update buffer based on its type and the storage strategy. */
|
|
15309
|
+
addEvent(event) {
|
|
15310
|
+
if (!this.#storageStrategy) {
|
|
15311
|
+
this.#preInit.push({ ...event, retryCount: 0 });
|
|
15312
|
+
return;
|
|
15313
|
+
}
|
|
15314
|
+
switch (event.type) {
|
|
15315
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
15316
|
+
switch (this.#storageStrategy) {
|
|
15317
|
+
case "realtime":
|
|
15318
|
+
case "event-sourced":
|
|
15319
|
+
case "batch-with-updates":
|
|
15320
|
+
this.pushCreate(event);
|
|
15321
|
+
break;
|
|
15322
|
+
}
|
|
15323
|
+
break;
|
|
15324
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
15325
|
+
switch (this.#storageStrategy) {
|
|
15326
|
+
case "realtime":
|
|
15327
|
+
case "batch-with-updates":
|
|
15328
|
+
this.pushUpdate(event);
|
|
15329
|
+
break;
|
|
15330
|
+
}
|
|
15331
|
+
break;
|
|
15332
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
15333
|
+
if (event.exportedSpan.isEvent) {
|
|
15334
|
+
this.pushCreate(event);
|
|
15335
|
+
} else {
|
|
15336
|
+
switch (this.#storageStrategy) {
|
|
15337
|
+
case "realtime":
|
|
15338
|
+
case "batch-with-updates":
|
|
15339
|
+
this.pushUpdate(event);
|
|
15340
|
+
break;
|
|
15341
|
+
default:
|
|
15342
|
+
this.pushCreate(event);
|
|
15343
|
+
break;
|
|
15344
|
+
}
|
|
15345
|
+
}
|
|
15346
|
+
break;
|
|
15347
|
+
default:
|
|
15348
|
+
this.pushCreate(event);
|
|
15349
|
+
break;
|
|
15350
|
+
}
|
|
15351
|
+
}
|
|
15352
|
+
/** Re-add failed create events to the buffer, dropping those that exceed max retries. */
|
|
15353
|
+
reAddCreates(events) {
|
|
15354
|
+
const retryable = [];
|
|
15355
|
+
for (const e of events) {
|
|
15356
|
+
if (++e.retryCount <= this.#maxRetries) {
|
|
15357
|
+
retryable.push(e);
|
|
15358
|
+
}
|
|
15359
|
+
}
|
|
15360
|
+
if (retryable.length > 0) {
|
|
15361
|
+
this.setFirstEventTime();
|
|
15362
|
+
this.#creates.push(...retryable);
|
|
15363
|
+
}
|
|
15364
|
+
}
|
|
15365
|
+
/** Re-add failed update events to the buffer, dropping those that exceed max retries. */
|
|
15366
|
+
reAddUpdates(events) {
|
|
15367
|
+
const retryable = [];
|
|
15368
|
+
for (const e of events) {
|
|
15369
|
+
if (++e.retryCount <= this.#maxRetries) {
|
|
15370
|
+
retryable.push(e);
|
|
15371
|
+
}
|
|
15372
|
+
}
|
|
15373
|
+
if (retryable.length > 0) {
|
|
15374
|
+
this.setFirstEventTime();
|
|
15375
|
+
this.#updates.push(...retryable);
|
|
15376
|
+
}
|
|
15377
|
+
}
|
|
15378
|
+
/** Snapshot of buffered create events. */
|
|
15379
|
+
get creates() {
|
|
15380
|
+
return [...this.#creates];
|
|
15381
|
+
}
|
|
15382
|
+
/** Snapshot of buffered update events. */
|
|
15383
|
+
get updates() {
|
|
15384
|
+
return [...this.#updates];
|
|
15385
|
+
}
|
|
15386
|
+
/** Total number of buffered events (creates + updates). */
|
|
15387
|
+
get totalSize() {
|
|
15388
|
+
return this.#creates.length + this.#updates.length;
|
|
15389
|
+
}
|
|
15390
|
+
/** Milliseconds since the first event was buffered in the current batch. */
|
|
15391
|
+
get elapsed() {
|
|
15392
|
+
if (!this.#firstEventTime) {
|
|
15393
|
+
return 0;
|
|
15394
|
+
}
|
|
15395
|
+
return Date.now() - this.#firstEventTime.getTime();
|
|
15396
|
+
}
|
|
15397
|
+
/**
|
|
15398
|
+
* Builds a unique span key for tracking
|
|
15399
|
+
*/
|
|
15400
|
+
buildSpanKey(span) {
|
|
15401
|
+
return `${span.traceId}:${span.spanId}`;
|
|
15402
|
+
}
|
|
15403
|
+
/** Track successfully created spans so updates can verify span existence before flushing. */
|
|
15404
|
+
addCreatedSpans(args) {
|
|
15405
|
+
if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") {
|
|
15406
|
+
return;
|
|
15407
|
+
}
|
|
15408
|
+
for (const createRecord of args.records) {
|
|
15409
|
+
if (!createRecord.isEvent) {
|
|
15410
|
+
this.#allCreatedSpans.add(this.buildSpanKey(createRecord));
|
|
15411
|
+
}
|
|
15412
|
+
}
|
|
15413
|
+
}
|
|
15414
|
+
/** Check whether a span's create record has already been flushed to storage. */
|
|
15415
|
+
spanExists(span) {
|
|
15416
|
+
return this.#allCreatedSpans?.has(this.buildSpanKey({ traceId: span.traceId, spanId: span.id }));
|
|
15417
|
+
}
|
|
15418
|
+
/** Remove completed spans from tracking after their SPAN_ENDED updates are flushed. */
|
|
15419
|
+
endFinishedSpans(args) {
|
|
15420
|
+
if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") {
|
|
15421
|
+
return;
|
|
15422
|
+
}
|
|
15423
|
+
args.records.forEach((r) => {
|
|
15424
|
+
this.#allCreatedSpans.delete(this.buildSpanKey(r));
|
|
15425
|
+
});
|
|
15426
|
+
}
|
|
15427
|
+
};
|
|
15428
|
+
|
|
15429
|
+
// src/exporters/default.ts
|
|
15430
|
+
function resolveTracingStorageStrategy(config2, observabilityStorage, storageName, logger) {
|
|
15431
|
+
const observabilityStrategy = observabilityStorage.observabilityStrategy;
|
|
15267
15432
|
if (config2.strategy && config2.strategy !== "auto") {
|
|
15268
|
-
|
|
15269
|
-
if (hints.supported.includes(config2.strategy)) {
|
|
15433
|
+
if (observabilityStrategy.supported.includes(config2.strategy)) {
|
|
15270
15434
|
return config2.strategy;
|
|
15271
15435
|
}
|
|
15272
15436
|
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
15273
15437
|
userStrategy: config2.strategy,
|
|
15274
15438
|
storageAdapter: storageName,
|
|
15275
|
-
supportedStrategies:
|
|
15276
|
-
fallbackStrategy:
|
|
15439
|
+
supportedStrategies: observabilityStrategy.supported,
|
|
15440
|
+
fallbackStrategy: observabilityStrategy.preferred
|
|
15277
15441
|
});
|
|
15278
15442
|
}
|
|
15279
|
-
return
|
|
15280
|
-
}
|
|
15281
|
-
function getStringOrNull(value) {
|
|
15282
|
-
return typeof value === "string" ? value : null;
|
|
15283
|
-
}
|
|
15284
|
-
function getObjectOrNull(value) {
|
|
15285
|
-
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
15443
|
+
return observabilityStrategy.preferred;
|
|
15286
15444
|
}
|
|
15287
15445
|
var DefaultExporter = class extends BaseExporter {
|
|
15288
15446
|
name = "mastra-default-observability-exporter";
|
|
15289
|
-
#storage;
|
|
15290
|
-
#observability;
|
|
15291
15447
|
#config;
|
|
15292
|
-
#resolvedStrategy;
|
|
15293
|
-
buffer;
|
|
15294
|
-
#flushTimer = null;
|
|
15295
15448
|
#isInitializing = false;
|
|
15296
15449
|
#initPromises = /* @__PURE__ */ new Set();
|
|
15297
|
-
|
|
15298
|
-
|
|
15450
|
+
#eventBuffer;
|
|
15451
|
+
#storage;
|
|
15452
|
+
#observabilityStorage;
|
|
15453
|
+
#resolvedStrategy;
|
|
15454
|
+
#flushTimer;
|
|
15455
|
+
// Signals whose storage methods threw "not implemented" — skip on future flushes
|
|
15456
|
+
#unsupportedSignals = /* @__PURE__ */ new Set();
|
|
15299
15457
|
constructor(config2 = {}) {
|
|
15300
15458
|
super(config2);
|
|
15301
|
-
if (config2 === void 0) {
|
|
15302
|
-
config2 = {};
|
|
15303
|
-
}
|
|
15304
15459
|
this.#config = {
|
|
15305
15460
|
...config2,
|
|
15306
15461
|
maxBatchSize: config2.maxBatchSize ?? 1e3,
|
|
@@ -15310,19 +15465,8 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15310
15465
|
retryDelayMs: config2.retryDelayMs ?? 500,
|
|
15311
15466
|
strategy: config2.strategy ?? "auto"
|
|
15312
15467
|
};
|
|
15313
|
-
this
|
|
15314
|
-
creates: [],
|
|
15315
|
-
updates: [],
|
|
15316
|
-
insertOnly: [],
|
|
15317
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
15318
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
15319
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
15320
|
-
outOfOrderCount: 0,
|
|
15321
|
-
totalSize: 0
|
|
15322
|
-
};
|
|
15323
|
-
this.#resolvedStrategy = "batch-with-updates";
|
|
15468
|
+
this.#eventBuffer = new EventBuffer({ maxRetries: this.#config.maxRetries ?? 4 });
|
|
15324
15469
|
}
|
|
15325
|
-
#strategyInitialized = false;
|
|
15326
15470
|
/**
|
|
15327
15471
|
* Initialize the exporter (called after all dependencies are ready)
|
|
15328
15472
|
*/
|
|
@@ -15334,14 +15478,31 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15334
15478
|
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
15335
15479
|
return;
|
|
15336
15480
|
}
|
|
15337
|
-
this.#
|
|
15338
|
-
if (!this.#
|
|
15481
|
+
this.#observabilityStorage = await this.#storage.getStore("observability");
|
|
15482
|
+
if (!this.#observabilityStorage) {
|
|
15339
15483
|
this.logger.warn(
|
|
15340
15484
|
"DefaultExporter disabled: Observability storage not available. Traces will not be persisted."
|
|
15341
15485
|
);
|
|
15342
15486
|
return;
|
|
15343
15487
|
}
|
|
15344
|
-
|
|
15488
|
+
if (!this.#resolvedStrategy) {
|
|
15489
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(
|
|
15490
|
+
this.#config,
|
|
15491
|
+
this.#observabilityStorage,
|
|
15492
|
+
this.#storage.constructor.name,
|
|
15493
|
+
this.logger
|
|
15494
|
+
);
|
|
15495
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
15496
|
+
strategy: this.#resolvedStrategy,
|
|
15497
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
15498
|
+
storageAdapter: this.#storage.constructor.name,
|
|
15499
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
15500
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
15501
|
+
});
|
|
15502
|
+
}
|
|
15503
|
+
if (this.#resolvedStrategy) {
|
|
15504
|
+
this.#eventBuffer.init({ strategy: this.#resolvedStrategy });
|
|
15505
|
+
}
|
|
15345
15506
|
} finally {
|
|
15346
15507
|
this.#isInitializing = false;
|
|
15347
15508
|
this.#initPromises.forEach((resolve) => {
|
|
@@ -15350,144 +15511,26 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15350
15511
|
this.#initPromises.clear();
|
|
15351
15512
|
}
|
|
15352
15513
|
}
|
|
15353
|
-
/**
|
|
15354
|
-
* Initialize the resolved strategy once observability store is available
|
|
15355
|
-
*/
|
|
15356
|
-
initializeStrategy(observability, storageName) {
|
|
15357
|
-
if (this.#strategyInitialized) return;
|
|
15358
|
-
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, observability, storageName, this.logger);
|
|
15359
|
-
this.#strategyInitialized = true;
|
|
15360
|
-
this.logger.debug("tracing storage exporter initialized", {
|
|
15361
|
-
strategy: this.#resolvedStrategy,
|
|
15362
|
-
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
15363
|
-
storageAdapter: storageName,
|
|
15364
|
-
maxBatchSize: this.#config.maxBatchSize,
|
|
15365
|
-
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
15366
|
-
});
|
|
15367
|
-
}
|
|
15368
|
-
/**
|
|
15369
|
-
* Builds a unique span key for tracking
|
|
15370
|
-
*/
|
|
15371
|
-
buildSpanKey(traceId, spanId) {
|
|
15372
|
-
return `${traceId}:${spanId}`;
|
|
15373
|
-
}
|
|
15374
|
-
/**
|
|
15375
|
-
* Gets the next sequence number for a span
|
|
15376
|
-
*/
|
|
15377
|
-
getNextSequence(spanKey) {
|
|
15378
|
-
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
15379
|
-
const next = current + 1;
|
|
15380
|
-
this.buffer.spanSequences.set(spanKey, next);
|
|
15381
|
-
return next;
|
|
15382
|
-
}
|
|
15383
|
-
/**
|
|
15384
|
-
* Handles out-of-order span updates by logging and skipping
|
|
15385
|
-
*/
|
|
15386
|
-
handleOutOfOrderUpdate(event) {
|
|
15387
|
-
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
15388
|
-
spanId: event.exportedSpan.id,
|
|
15389
|
-
traceId: event.exportedSpan.traceId,
|
|
15390
|
-
spanName: event.exportedSpan.name,
|
|
15391
|
-
eventType: event.type
|
|
15392
|
-
});
|
|
15393
|
-
}
|
|
15394
|
-
/**
|
|
15395
|
-
* Adds an event to the appropriate buffer based on strategy
|
|
15396
|
-
*/
|
|
15397
|
-
addToBuffer(event) {
|
|
15398
|
-
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
15399
|
-
if (this.buffer.totalSize === 0) {
|
|
15400
|
-
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
15401
|
-
}
|
|
15402
|
-
switch (event.type) {
|
|
15403
|
-
case observability.TracingEventType.SPAN_STARTED:
|
|
15404
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15405
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15406
|
-
this.buffer.creates.push(createRecord);
|
|
15407
|
-
this.buffer.seenSpans.add(spanKey);
|
|
15408
|
-
this.allCreatedSpans.add(spanKey);
|
|
15409
|
-
}
|
|
15410
|
-
break;
|
|
15411
|
-
case observability.TracingEventType.SPAN_UPDATED:
|
|
15412
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15413
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
15414
|
-
this.buffer.updates.push({
|
|
15415
|
-
traceId: event.exportedSpan.traceId,
|
|
15416
|
-
spanId: event.exportedSpan.id,
|
|
15417
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
15418
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
15419
|
-
});
|
|
15420
|
-
} else {
|
|
15421
|
-
this.handleOutOfOrderUpdate(event);
|
|
15422
|
-
this.buffer.outOfOrderCount++;
|
|
15423
|
-
}
|
|
15424
|
-
}
|
|
15425
|
-
break;
|
|
15426
|
-
case observability.TracingEventType.SPAN_ENDED:
|
|
15427
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15428
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
15429
|
-
this.buffer.updates.push({
|
|
15430
|
-
traceId: event.exportedSpan.traceId,
|
|
15431
|
-
spanId: event.exportedSpan.id,
|
|
15432
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
15433
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
15434
|
-
});
|
|
15435
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15436
|
-
} else if (event.exportedSpan.isEvent) {
|
|
15437
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15438
|
-
this.buffer.creates.push(createRecord);
|
|
15439
|
-
this.buffer.seenSpans.add(spanKey);
|
|
15440
|
-
this.allCreatedSpans.add(spanKey);
|
|
15441
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15442
|
-
} else {
|
|
15443
|
-
this.handleOutOfOrderUpdate(event);
|
|
15444
|
-
this.buffer.outOfOrderCount++;
|
|
15445
|
-
}
|
|
15446
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
15447
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
15448
|
-
this.buffer.insertOnly.push(createRecord);
|
|
15449
|
-
this.buffer.completedSpans.add(spanKey);
|
|
15450
|
-
this.allCreatedSpans.add(spanKey);
|
|
15451
|
-
}
|
|
15452
|
-
break;
|
|
15453
|
-
}
|
|
15454
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
15455
|
-
}
|
|
15456
15514
|
/**
|
|
15457
15515
|
* Checks if buffer should be flushed based on size or time triggers
|
|
15458
15516
|
*/
|
|
15459
15517
|
shouldFlush() {
|
|
15460
|
-
if (this
|
|
15518
|
+
if (this.#resolvedStrategy === "realtime") {
|
|
15461
15519
|
return true;
|
|
15462
15520
|
}
|
|
15463
|
-
if (this.
|
|
15521
|
+
if (this.#eventBuffer.totalSize >= this.#config.maxBufferSize) {
|
|
15464
15522
|
return true;
|
|
15465
15523
|
}
|
|
15466
|
-
if (this.
|
|
15467
|
-
|
|
15468
|
-
|
|
15524
|
+
if (this.#eventBuffer.totalSize >= this.#config.maxBatchSize) {
|
|
15525
|
+
return true;
|
|
15526
|
+
}
|
|
15527
|
+
if (this.#eventBuffer.totalSize > 0) {
|
|
15528
|
+
if (this.#eventBuffer.elapsed >= this.#config.maxBatchWaitMs) {
|
|
15469
15529
|
return true;
|
|
15470
15530
|
}
|
|
15471
15531
|
}
|
|
15472
15532
|
return false;
|
|
15473
15533
|
}
|
|
15474
|
-
/**
|
|
15475
|
-
* Resets the buffer after successful flush
|
|
15476
|
-
*/
|
|
15477
|
-
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
15478
|
-
this.buffer.creates = [];
|
|
15479
|
-
this.buffer.updates = [];
|
|
15480
|
-
this.buffer.insertOnly = [];
|
|
15481
|
-
this.buffer.seenSpans.clear();
|
|
15482
|
-
this.buffer.spanSequences.clear();
|
|
15483
|
-
this.buffer.completedSpans.clear();
|
|
15484
|
-
this.buffer.outOfOrderCount = 0;
|
|
15485
|
-
this.buffer.firstEventTime = void 0;
|
|
15486
|
-
this.buffer.totalSize = 0;
|
|
15487
|
-
for (const spanKey of completedSpansToCleanup) {
|
|
15488
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15489
|
-
}
|
|
15490
|
-
}
|
|
15491
15534
|
/**
|
|
15492
15535
|
* Schedules a flush using setTimeout
|
|
15493
15536
|
*/
|
|
@@ -15504,276 +15547,185 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15504
15547
|
}, this.#config.maxBatchWaitMs);
|
|
15505
15548
|
}
|
|
15506
15549
|
/**
|
|
15507
|
-
*
|
|
15508
|
-
*
|
|
15550
|
+
* Checks flush triggers and schedules/triggers flush as needed.
|
|
15551
|
+
* Called after adding any event to the buffer.
|
|
15552
|
+
* Returns the flush promise when flushing so callers can await it.
|
|
15509
15553
|
*/
|
|
15510
|
-
|
|
15511
|
-
if (
|
|
15512
|
-
|
|
15513
|
-
}
|
|
15514
|
-
|
|
15515
|
-
return JSON.parse(
|
|
15516
|
-
JSON.stringify(span.attributes, (_key, value) => {
|
|
15517
|
-
if (value instanceof Date) {
|
|
15518
|
-
return value.toISOString();
|
|
15519
|
-
}
|
|
15520
|
-
if (typeof value === "object" && value !== null) {
|
|
15521
|
-
return value;
|
|
15522
|
-
}
|
|
15523
|
-
return value;
|
|
15524
|
-
})
|
|
15525
|
-
);
|
|
15526
|
-
} catch (error48) {
|
|
15527
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
15528
|
-
spanId: span.id,
|
|
15529
|
-
spanType: span.type,
|
|
15530
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15531
|
-
});
|
|
15532
|
-
return null;
|
|
15554
|
+
async handleBatchedFlush() {
|
|
15555
|
+
if (this.shouldFlush()) {
|
|
15556
|
+
await this.flushBuffer();
|
|
15557
|
+
} else if (this.#eventBuffer.totalSize === 1) {
|
|
15558
|
+
this.scheduleFlush();
|
|
15533
15559
|
}
|
|
15534
15560
|
}
|
|
15535
|
-
buildCreateRecord(span) {
|
|
15536
|
-
const metadata = span.metadata ?? {};
|
|
15537
|
-
return {
|
|
15538
|
-
traceId: span.traceId,
|
|
15539
|
-
spanId: span.id,
|
|
15540
|
-
parentSpanId: span.parentSpanId ?? null,
|
|
15541
|
-
name: span.name,
|
|
15542
|
-
// Entity identification - from span
|
|
15543
|
-
entityType: span.entityType ?? null,
|
|
15544
|
-
entityId: span.entityId ?? null,
|
|
15545
|
-
entityName: span.entityName ?? null,
|
|
15546
|
-
// Identity & Tenancy - extracted from metadata if present
|
|
15547
|
-
userId: getStringOrNull(metadata.userId),
|
|
15548
|
-
organizationId: getStringOrNull(metadata.organizationId),
|
|
15549
|
-
resourceId: getStringOrNull(metadata.resourceId),
|
|
15550
|
-
// Correlation IDs - extracted from metadata if present
|
|
15551
|
-
runId: getStringOrNull(metadata.runId),
|
|
15552
|
-
sessionId: getStringOrNull(metadata.sessionId),
|
|
15553
|
-
threadId: getStringOrNull(metadata.threadId),
|
|
15554
|
-
requestId: getStringOrNull(metadata.requestId),
|
|
15555
|
-
// Deployment context - extracted from metadata if present
|
|
15556
|
-
environment: getStringOrNull(metadata.environment),
|
|
15557
|
-
source: getStringOrNull(metadata.source),
|
|
15558
|
-
serviceName: getStringOrNull(metadata.serviceName),
|
|
15559
|
-
scope: getObjectOrNull(metadata.scope),
|
|
15560
|
-
// Span data
|
|
15561
|
-
spanType: span.type,
|
|
15562
|
-
attributes: this.serializeAttributes(span),
|
|
15563
|
-
metadata: span.metadata ?? null,
|
|
15564
|
-
// Keep all metadata including extracted fields
|
|
15565
|
-
tags: span.tags ?? null,
|
|
15566
|
-
links: null,
|
|
15567
|
-
input: span.input ?? null,
|
|
15568
|
-
output: span.output ?? null,
|
|
15569
|
-
error: span.errorInfo ?? null,
|
|
15570
|
-
requestContext: span.requestContext ?? null,
|
|
15571
|
-
isEvent: span.isEvent,
|
|
15572
|
-
// Timestamps
|
|
15573
|
-
startedAt: span.startTime,
|
|
15574
|
-
endedAt: span.endTime ?? null
|
|
15575
|
-
};
|
|
15576
|
-
}
|
|
15577
|
-
buildUpdateRecord(span) {
|
|
15578
|
-
return {
|
|
15579
|
-
name: span.name,
|
|
15580
|
-
scope: null,
|
|
15581
|
-
attributes: this.serializeAttributes(span),
|
|
15582
|
-
metadata: span.metadata ?? null,
|
|
15583
|
-
links: null,
|
|
15584
|
-
endedAt: span.endTime ?? null,
|
|
15585
|
-
input: span.input,
|
|
15586
|
-
output: span.output,
|
|
15587
|
-
error: span.errorInfo ?? null,
|
|
15588
|
-
requestContext: span.requestContext ?? null
|
|
15589
|
-
};
|
|
15590
|
-
}
|
|
15591
15561
|
/**
|
|
15592
|
-
*
|
|
15562
|
+
* Flush a batch of create events for a single signal type.
|
|
15563
|
+
* On "not implemented" errors, disables the signal for future flushes.
|
|
15564
|
+
* On other errors, re-adds events to the buffer for retry.
|
|
15593
15565
|
*/
|
|
15594
|
-
async
|
|
15595
|
-
|
|
15596
|
-
|
|
15597
|
-
|
|
15598
|
-
|
|
15599
|
-
|
|
15566
|
+
async flushCreates(signal, events, storageCall) {
|
|
15567
|
+
if (this.#unsupportedSignals.has(signal) || events.length === 0) return;
|
|
15568
|
+
try {
|
|
15569
|
+
await storageCall(events);
|
|
15570
|
+
} catch (error48) {
|
|
15571
|
+
if (error48 instanceof error$1.MastraError && error48.domain === error$1.ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15572
|
+
this.logger.warn(error48.message);
|
|
15573
|
+
this.#unsupportedSignals.add(signal);
|
|
15600
15574
|
} else {
|
|
15601
|
-
this.
|
|
15602
|
-
}
|
|
15603
|
-
} else {
|
|
15604
|
-
switch (event.type) {
|
|
15605
|
-
case observability.TracingEventType.SPAN_STARTED:
|
|
15606
|
-
await observability$1.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
15607
|
-
this.allCreatedSpans.add(spanKey);
|
|
15608
|
-
break;
|
|
15609
|
-
case observability.TracingEventType.SPAN_UPDATED:
|
|
15610
|
-
await observability$1.updateSpan({
|
|
15611
|
-
traceId: span.traceId,
|
|
15612
|
-
spanId: span.id,
|
|
15613
|
-
updates: this.buildUpdateRecord(span)
|
|
15614
|
-
});
|
|
15615
|
-
break;
|
|
15616
|
-
case observability.TracingEventType.SPAN_ENDED:
|
|
15617
|
-
await observability$1.updateSpan({
|
|
15618
|
-
traceId: span.traceId,
|
|
15619
|
-
spanId: span.id,
|
|
15620
|
-
updates: this.buildUpdateRecord(span)
|
|
15621
|
-
});
|
|
15622
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15623
|
-
break;
|
|
15624
|
-
default:
|
|
15625
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
15575
|
+
this.#eventBuffer.reAddCreates(events);
|
|
15626
15576
|
}
|
|
15627
15577
|
}
|
|
15628
15578
|
}
|
|
15629
15579
|
/**
|
|
15630
|
-
*
|
|
15580
|
+
* Flush span update/end events, deferring any whose span hasn't been created yet.
|
|
15581
|
+
* When `isEnd` is true, successfully flushed spans are removed from tracking.
|
|
15631
15582
|
*/
|
|
15632
|
-
|
|
15633
|
-
this.
|
|
15634
|
-
|
|
15635
|
-
|
|
15636
|
-
|
|
15637
|
-
|
|
15583
|
+
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
15584
|
+
if (this.#unsupportedSignals.has("tracing") || events.length === 0) return;
|
|
15585
|
+
const partials = [];
|
|
15586
|
+
for (const event of events) {
|
|
15587
|
+
const span = event.exportedSpan;
|
|
15588
|
+
if (this.#eventBuffer.spanExists(span)) {
|
|
15589
|
+
partials.push({
|
|
15590
|
+
traceId: span.traceId,
|
|
15591
|
+
spanId: span.id,
|
|
15592
|
+
updates: storage.buildUpdateSpanRecord(span)
|
|
15638
15593
|
});
|
|
15639
|
-
}
|
|
15640
|
-
|
|
15641
|
-
|
|
15594
|
+
} else {
|
|
15595
|
+
deferredUpdates.push(event);
|
|
15596
|
+
}
|
|
15642
15597
|
}
|
|
15643
|
-
|
|
15644
|
-
|
|
15645
|
-
|
|
15646
|
-
|
|
15647
|
-
|
|
15648
|
-
|
|
15649
|
-
|
|
15650
|
-
if (
|
|
15651
|
-
this.
|
|
15652
|
-
|
|
15653
|
-
|
|
15654
|
-
|
|
15655
|
-
|
|
15656
|
-
} else if (this.buffer.totalSize === 1) {
|
|
15657
|
-
this.scheduleFlush();
|
|
15598
|
+
if (partials.length === 0) return;
|
|
15599
|
+
try {
|
|
15600
|
+
await this.#observabilityStorage.batchUpdateSpans({ records: partials });
|
|
15601
|
+
if (isEnd) {
|
|
15602
|
+
this.#eventBuffer.endFinishedSpans({ records: partials });
|
|
15603
|
+
}
|
|
15604
|
+
} catch (error48) {
|
|
15605
|
+
if (error48 instanceof error$1.MastraError && error48.domain === error$1.ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15606
|
+
this.logger.warn(error48.message);
|
|
15607
|
+
this.#unsupportedSignals.add("tracing");
|
|
15608
|
+
} else {
|
|
15609
|
+
deferredUpdates.length = 0;
|
|
15610
|
+
this.#eventBuffer.reAddUpdates(events);
|
|
15658
15611
|
}
|
|
15659
15612
|
}
|
|
15660
15613
|
}
|
|
15661
15614
|
/**
|
|
15662
|
-
*
|
|
15663
|
-
|
|
15664
|
-
|
|
15665
|
-
|
|
15666
|
-
|
|
15667
|
-
|
|
15668
|
-
* Flushes the current buffer to storage with retry logic (internal implementation)
|
|
15615
|
+
* Flushes the current buffer to storage.
|
|
15616
|
+
*
|
|
15617
|
+
* Creates are flushed first, then their span keys are added to allCreatedSpans.
|
|
15618
|
+
* Updates are checked against allCreatedSpans — those whose span hasn't been
|
|
15619
|
+
* created yet are re-inserted into the live buffer for the next flush.
|
|
15620
|
+
* Completed spans (SPAN_ENDED) are cleaned up from allCreatedSpans after success.
|
|
15669
15621
|
*/
|
|
15670
15622
|
async flushBuffer() {
|
|
15671
|
-
if (!this.#
|
|
15672
|
-
this.logger.debug("Cannot flush
|
|
15623
|
+
if (!this.#observabilityStorage) {
|
|
15624
|
+
this.logger.debug("Cannot flush. Observability storage is not initialized");
|
|
15625
|
+
return;
|
|
15626
|
+
}
|
|
15627
|
+
if (!this.#resolvedStrategy) {
|
|
15628
|
+
this.logger.debug("Cannot flush. Observability strategy is not resolved");
|
|
15673
15629
|
return;
|
|
15674
15630
|
}
|
|
15675
15631
|
if (this.#flushTimer) {
|
|
15676
15632
|
clearTimeout(this.#flushTimer);
|
|
15677
|
-
this.#flushTimer =
|
|
15633
|
+
this.#flushTimer = void 0;
|
|
15678
15634
|
}
|
|
15679
|
-
if (this.
|
|
15635
|
+
if (this.#eventBuffer.totalSize === 0) {
|
|
15680
15636
|
return;
|
|
15681
15637
|
}
|
|
15682
15638
|
const startTime = Date.now();
|
|
15683
|
-
const
|
|
15684
|
-
const
|
|
15685
|
-
|
|
15686
|
-
|
|
15687
|
-
|
|
15688
|
-
|
|
15689
|
-
|
|
15690
|
-
|
|
15691
|
-
|
|
15692
|
-
|
|
15693
|
-
|
|
15694
|
-
|
|
15695
|
-
|
|
15696
|
-
|
|
15639
|
+
const batchSize = this.#eventBuffer.totalSize;
|
|
15640
|
+
const creates = this.#eventBuffer.creates;
|
|
15641
|
+
const updates = this.#eventBuffer.updates;
|
|
15642
|
+
this.#eventBuffer.reset();
|
|
15643
|
+
const createFeedbackEvents = [];
|
|
15644
|
+
const createLogEvents = [];
|
|
15645
|
+
const createMetricEvents = [];
|
|
15646
|
+
const createScoreEvents = [];
|
|
15647
|
+
const createSpanEvents = [];
|
|
15648
|
+
const updateSpanEvents = [];
|
|
15649
|
+
const endSpanEvents = [];
|
|
15650
|
+
for (const createEvent of creates) {
|
|
15651
|
+
switch (createEvent.type) {
|
|
15652
|
+
case "feedback":
|
|
15653
|
+
createFeedbackEvents.push(createEvent);
|
|
15654
|
+
break;
|
|
15655
|
+
case "log":
|
|
15656
|
+
createLogEvents.push(createEvent);
|
|
15657
|
+
break;
|
|
15658
|
+
case "metric":
|
|
15659
|
+
createMetricEvents.push(createEvent);
|
|
15660
|
+
break;
|
|
15661
|
+
case "score":
|
|
15662
|
+
createScoreEvents.push(createEvent);
|
|
15663
|
+
break;
|
|
15664
|
+
default:
|
|
15665
|
+
createSpanEvents.push(createEvent);
|
|
15666
|
+
break;
|
|
15667
|
+
}
|
|
15668
|
+
}
|
|
15669
|
+
for (const updateEvent of updates) {
|
|
15670
|
+
switch (updateEvent.type) {
|
|
15671
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
15672
|
+
updateSpanEvents.push(updateEvent);
|
|
15673
|
+
break;
|
|
15674
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
15675
|
+
endSpanEvents.push(updateEvent);
|
|
15676
|
+
break;
|
|
15677
|
+
}
|
|
15678
|
+
}
|
|
15679
|
+
await Promise.all([
|
|
15680
|
+
this.flushCreates(
|
|
15681
|
+
"feedback",
|
|
15682
|
+
createFeedbackEvents,
|
|
15683
|
+
(events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => storage.buildFeedbackRecord(f)) })
|
|
15684
|
+
),
|
|
15685
|
+
this.flushCreates(
|
|
15686
|
+
"logs",
|
|
15687
|
+
createLogEvents,
|
|
15688
|
+
(events) => this.#observabilityStorage.batchCreateLogs({ logs: events.map((l) => storage.buildLogRecord(l)) })
|
|
15689
|
+
),
|
|
15690
|
+
this.flushCreates(
|
|
15691
|
+
"metrics",
|
|
15692
|
+
createMetricEvents,
|
|
15693
|
+
(events) => this.#observabilityStorage.batchCreateMetrics({ metrics: events.map((m) => storage.buildMetricRecord(m)) })
|
|
15694
|
+
),
|
|
15695
|
+
this.flushCreates(
|
|
15696
|
+
"scores",
|
|
15697
|
+
createScoreEvents,
|
|
15698
|
+
(events) => this.#observabilityStorage.batchCreateScores({ scores: events.map((s) => storage.buildScoreRecord(s)) })
|
|
15699
|
+
),
|
|
15700
|
+
this.flushCreates("tracing", createSpanEvents, async (events) => {
|
|
15701
|
+
const records = events.map((t) => storage.buildCreateSpanRecord(t.exportedSpan));
|
|
15702
|
+
await this.#observabilityStorage.batchCreateSpans({ records });
|
|
15703
|
+
this.#eventBuffer.addCreatedSpans({ records });
|
|
15704
|
+
})
|
|
15705
|
+
]);
|
|
15706
|
+
const deferredUpdates = [];
|
|
15707
|
+
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
15708
|
+
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
15709
|
+
if (deferredUpdates.length > 0) {
|
|
15710
|
+
this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
15711
|
+
}
|
|
15697
15712
|
const elapsed = Date.now() - startTime;
|
|
15698
15713
|
this.logger.debug("Batch flushed", {
|
|
15699
15714
|
strategy: this.#resolvedStrategy,
|
|
15700
|
-
batchSize
|
|
15701
|
-
flushReason,
|
|
15715
|
+
batchSize,
|
|
15702
15716
|
durationMs: elapsed,
|
|
15703
|
-
|
|
15717
|
+
deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
|
|
15704
15718
|
});
|
|
15705
|
-
|
|
15706
|
-
/**
|
|
15707
|
-
* Attempts to flush with exponential backoff retry logic
|
|
15708
|
-
*/
|
|
15709
|
-
async flushWithRetries(observability, buffer, attempt) {
|
|
15710
|
-
try {
|
|
15711
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
15712
|
-
if (buffer.creates.length > 0) {
|
|
15713
|
-
await observability.batchCreateSpans({ records: buffer.creates });
|
|
15714
|
-
}
|
|
15715
|
-
if (buffer.updates.length > 0) {
|
|
15716
|
-
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
15717
|
-
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
15718
|
-
this.buildSpanKey(b.traceId, b.spanId)
|
|
15719
|
-
);
|
|
15720
|
-
if (spanCompare !== 0) return spanCompare;
|
|
15721
|
-
return a.sequenceNumber - b.sequenceNumber;
|
|
15722
|
-
});
|
|
15723
|
-
await observability.batchUpdateSpans({ records: sortedUpdates });
|
|
15724
|
-
}
|
|
15725
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
15726
|
-
if (buffer.insertOnly.length > 0) {
|
|
15727
|
-
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
15728
|
-
}
|
|
15729
|
-
}
|
|
15730
|
-
for (const spanKey of buffer.completedSpans) {
|
|
15731
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15732
|
-
}
|
|
15733
|
-
} catch (error48) {
|
|
15734
|
-
if (attempt < this.#config.maxRetries) {
|
|
15735
|
-
const retryDelay = this.calculateRetryDelay(attempt);
|
|
15736
|
-
this.logger.warn("Batch flush failed, retrying", {
|
|
15737
|
-
attempt: attempt + 1,
|
|
15738
|
-
maxRetries: this.#config.maxRetries,
|
|
15739
|
-
nextRetryInMs: retryDelay,
|
|
15740
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15741
|
-
});
|
|
15742
|
-
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
15743
|
-
return this.flushWithRetries(observability, buffer, attempt + 1);
|
|
15744
|
-
} else {
|
|
15745
|
-
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
15746
|
-
finalAttempt: attempt + 1,
|
|
15747
|
-
maxRetries: this.#config.maxRetries,
|
|
15748
|
-
droppedBatchSize: buffer.totalSize,
|
|
15749
|
-
error: error48 instanceof Error ? error48.message : String(error48)
|
|
15750
|
-
});
|
|
15751
|
-
for (const spanKey of buffer.completedSpans) {
|
|
15752
|
-
this.allCreatedSpans.delete(spanKey);
|
|
15753
|
-
}
|
|
15754
|
-
}
|
|
15755
|
-
}
|
|
15719
|
+
return;
|
|
15756
15720
|
}
|
|
15757
15721
|
async _exportTracingEvent(event) {
|
|
15758
15722
|
await this.waitForInit();
|
|
15759
|
-
if (!this.#
|
|
15723
|
+
if (!this.#observabilityStorage) {
|
|
15760
15724
|
this.logger.debug("Cannot store traces. Observability storage is not initialized");
|
|
15761
15725
|
return;
|
|
15762
15726
|
}
|
|
15763
|
-
|
|
15764
|
-
|
|
15765
|
-
}
|
|
15766
|
-
switch (this.#resolvedStrategy) {
|
|
15767
|
-
case "realtime":
|
|
15768
|
-
await this.handleRealtimeEvent(event, this.#observability);
|
|
15769
|
-
break;
|
|
15770
|
-
case "batch-with-updates":
|
|
15771
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
15772
|
-
break;
|
|
15773
|
-
case "insert-only":
|
|
15774
|
-
this.handleInsertOnlyEvent(event);
|
|
15775
|
-
break;
|
|
15776
|
-
}
|
|
15727
|
+
this.#eventBuffer.addEvent(event);
|
|
15728
|
+
await this.handleBatchedFlush();
|
|
15777
15729
|
}
|
|
15778
15730
|
/**
|
|
15779
15731
|
* Resolves when an ongoing init call is finished
|
|
@@ -15786,15 +15738,51 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15786
15738
|
this.#initPromises.add(resolve);
|
|
15787
15739
|
});
|
|
15788
15740
|
}
|
|
15741
|
+
/**
|
|
15742
|
+
* Handle metric events — buffer for batch flush.
|
|
15743
|
+
*/
|
|
15744
|
+
async onMetricEvent(event) {
|
|
15745
|
+
await this.waitForInit();
|
|
15746
|
+
if (!this.#observabilityStorage) return;
|
|
15747
|
+
this.#eventBuffer.addEvent(event);
|
|
15748
|
+
await this.handleBatchedFlush();
|
|
15749
|
+
}
|
|
15750
|
+
/**
|
|
15751
|
+
* Handle log events — buffer for batch flush.
|
|
15752
|
+
*/
|
|
15753
|
+
async onLogEvent(event) {
|
|
15754
|
+
await this.waitForInit();
|
|
15755
|
+
if (!this.#observabilityStorage) return;
|
|
15756
|
+
this.#eventBuffer.addEvent(event);
|
|
15757
|
+
await this.handleBatchedFlush();
|
|
15758
|
+
}
|
|
15759
|
+
/**
|
|
15760
|
+
* Handle score events — buffer for batch flush.
|
|
15761
|
+
*/
|
|
15762
|
+
async onScoreEvent(event) {
|
|
15763
|
+
await this.waitForInit();
|
|
15764
|
+
if (!this.#observabilityStorage) return;
|
|
15765
|
+
this.#eventBuffer.addEvent(event);
|
|
15766
|
+
await this.handleBatchedFlush();
|
|
15767
|
+
}
|
|
15768
|
+
/**
|
|
15769
|
+
* Handle feedback events — buffer for batch flush.
|
|
15770
|
+
*/
|
|
15771
|
+
async onFeedbackEvent(event) {
|
|
15772
|
+
await this.waitForInit();
|
|
15773
|
+
if (!this.#observabilityStorage) return;
|
|
15774
|
+
this.#eventBuffer.addEvent(event);
|
|
15775
|
+
await this.handleBatchedFlush();
|
|
15776
|
+
}
|
|
15789
15777
|
/**
|
|
15790
15778
|
* Force flush any buffered spans without shutting down the exporter.
|
|
15791
15779
|
* This is useful in serverless environments where you need to ensure spans
|
|
15792
15780
|
* are exported before the runtime instance is terminated.
|
|
15793
15781
|
*/
|
|
15794
15782
|
async flush() {
|
|
15795
|
-
if (this.
|
|
15783
|
+
if (this.#eventBuffer.totalSize > 0) {
|
|
15796
15784
|
this.logger.debug("Flushing buffered events", {
|
|
15797
|
-
bufferedEvents: this.
|
|
15785
|
+
bufferedEvents: this.#eventBuffer.totalSize
|
|
15798
15786
|
});
|
|
15799
15787
|
await this.flushBuffer();
|
|
15800
15788
|
}
|
|
@@ -15802,7 +15790,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15802
15790
|
async shutdown() {
|
|
15803
15791
|
if (this.#flushTimer) {
|
|
15804
15792
|
clearTimeout(this.#flushTimer);
|
|
15805
|
-
this.#flushTimer =
|
|
15793
|
+
this.#flushTimer = void 0;
|
|
15806
15794
|
}
|
|
15807
15795
|
await this.flush();
|
|
15808
15796
|
this.logger.info("DefaultExporter shutdown complete");
|
|
@@ -15918,7 +15906,7 @@ var TestExporter = class extends BaseExporter {
|
|
|
15918
15906
|
if (this.#config.storeLogs) {
|
|
15919
15907
|
const metric = event.metric;
|
|
15920
15908
|
const labelsStr = Object.entries(metric.labels).map(([k, v]) => `${k}=${v}`).join(", ");
|
|
15921
|
-
const logMessage = `[TestExporter] metric
|
|
15909
|
+
const logMessage = `[TestExporter] metric: ${metric.name}=${metric.value}${labelsStr ? ` {${labelsStr}}` : ""}`;
|
|
15922
15910
|
this.#debugLogs.push(logMessage);
|
|
15923
15911
|
}
|
|
15924
15912
|
this.#metricEvents.push(event);
|
|
@@ -15930,7 +15918,7 @@ var TestExporter = class extends BaseExporter {
|
|
|
15930
15918
|
this.#trackEvent("score");
|
|
15931
15919
|
if (this.#config.storeLogs) {
|
|
15932
15920
|
const score = event.score;
|
|
15933
|
-
const logMessage = `[TestExporter] score: ${score.
|
|
15921
|
+
const logMessage = `[TestExporter] score: ${score.scorerId}=${score.score} (trace: ${score.traceId.slice(-8)}${score.spanId ? `, span: ${score.spanId.slice(-8)}` : ""})`;
|
|
15934
15922
|
this.#debugLogs.push(logMessage);
|
|
15935
15923
|
}
|
|
15936
15924
|
this.#scoreEvents.push(event);
|
|
@@ -16148,10 +16136,12 @@ var TestExporter = class extends BaseExporter {
|
|
|
16148
16136
|
return this.#metricEvents.filter((e) => e.metric.name === name).map((e) => e.metric);
|
|
16149
16137
|
}
|
|
16150
16138
|
/**
|
|
16151
|
-
*
|
|
16139
|
+
* @deprecated MetricType is no longer stored. Use getMetricsByName() instead.
|
|
16152
16140
|
*/
|
|
16153
|
-
getMetricsByType(
|
|
16154
|
-
|
|
16141
|
+
getMetricsByType(_metricType) {
|
|
16142
|
+
throw new Error(
|
|
16143
|
+
"getMetricsByType() has been removed: metricType is no longer stored. Use getMetricsByName(metricName) instead to filter metrics by name."
|
|
16144
|
+
);
|
|
16155
16145
|
}
|
|
16156
16146
|
// ============================================================================
|
|
16157
16147
|
// Score Query Methods
|
|
@@ -16169,10 +16159,10 @@ var TestExporter = class extends BaseExporter {
|
|
|
16169
16159
|
return this.#scoreEvents.map((e) => e.score);
|
|
16170
16160
|
}
|
|
16171
16161
|
/**
|
|
16172
|
-
* Get scores filtered by scorer
|
|
16162
|
+
* Get scores filtered by scorer id
|
|
16173
16163
|
*/
|
|
16174
|
-
getScoresByScorer(
|
|
16175
|
-
return this.#scoreEvents.filter((e) => e.score.
|
|
16164
|
+
getScoresByScorer(scorerId) {
|
|
16165
|
+
return this.#scoreEvents.filter((e) => e.score.scorerId === scorerId).map((e) => e.score);
|
|
16176
16166
|
}
|
|
16177
16167
|
/**
|
|
16178
16168
|
* Get scores for a specific trace
|
|
@@ -16234,17 +16224,14 @@ var TestExporter = class extends BaseExporter {
|
|
|
16234
16224
|
const level = event.log.level;
|
|
16235
16225
|
logsByLevel[level] = (logsByLevel[level] || 0) + 1;
|
|
16236
16226
|
}
|
|
16237
|
-
const metricsByType = {};
|
|
16238
16227
|
const metricsByName = {};
|
|
16239
16228
|
for (const event of this.#metricEvents) {
|
|
16240
|
-
const mType = event.metric.metricType;
|
|
16241
|
-
metricsByType[mType] = (metricsByType[mType] || 0) + 1;
|
|
16242
16229
|
const mName = event.metric.name;
|
|
16243
16230
|
metricsByName[mName] = (metricsByName[mName] || 0) + 1;
|
|
16244
16231
|
}
|
|
16245
16232
|
const scoresByScorer = {};
|
|
16246
16233
|
for (const event of this.#scoreEvents) {
|
|
16247
|
-
const scorer = event.score.
|
|
16234
|
+
const scorer = event.score.scorerId;
|
|
16248
16235
|
scoresByScorer[scorer] = (scoresByScorer[scorer] || 0) + 1;
|
|
16249
16236
|
}
|
|
16250
16237
|
const feedbackByType = {};
|
|
@@ -16269,7 +16256,6 @@ var TestExporter = class extends BaseExporter {
|
|
|
16269
16256
|
totalLogs: this.#logEvents.length,
|
|
16270
16257
|
logsByLevel,
|
|
16271
16258
|
totalMetrics: this.#metricEvents.length,
|
|
16272
|
-
metricsByType,
|
|
16273
16259
|
metricsByName,
|
|
16274
16260
|
totalScores: this.#scoreEvents.length,
|
|
16275
16261
|
scoresByScorer,
|
|
@@ -17001,90 +16987,33 @@ var BaseObservabilityEventBus = class _BaseObservabilityEventBus extends base.Ma
|
|
|
17001
16987
|
}
|
|
17002
16988
|
};
|
|
17003
16989
|
var AutoExtractedMetrics = class {
|
|
17004
|
-
|
|
17005
|
-
* @param observabilityBus - Bus used to emit derived MetricEvents.
|
|
17006
|
-
* @param cardinalityFilter - Optional filter applied to metric labels before emission.
|
|
17007
|
-
*/
|
|
17008
|
-
constructor(observabilityBus, cardinalityFilter) {
|
|
16990
|
+
constructor(observabilityBus) {
|
|
17009
16991
|
this.observabilityBus = observabilityBus;
|
|
17010
|
-
this.cardinalityFilter = cardinalityFilter;
|
|
17011
16992
|
}
|
|
17012
16993
|
/**
|
|
17013
|
-
* Route a tracing event to the appropriate
|
|
17014
|
-
*
|
|
17015
|
-
* duration histogram, and (for model spans) token counters.
|
|
16994
|
+
* Route a tracing event to the appropriate handler.
|
|
16995
|
+
* SPAN_ENDED emits duration and token metrics (for model spans).
|
|
17016
16996
|
*/
|
|
17017
16997
|
processTracingEvent(event) {
|
|
17018
|
-
|
|
17019
|
-
|
|
17020
|
-
this.onSpanStarted(event.exportedSpan);
|
|
17021
|
-
break;
|
|
17022
|
-
case observability.TracingEventType.SPAN_ENDED:
|
|
17023
|
-
this.onSpanEnded(event.exportedSpan);
|
|
17024
|
-
break;
|
|
17025
|
-
}
|
|
17026
|
-
}
|
|
17027
|
-
/** Emit a `mastra_scores_total` counter for a score event. */
|
|
17028
|
-
processScoreEvent(event) {
|
|
17029
|
-
const labels = {
|
|
17030
|
-
scorer: event.score.scorerName
|
|
17031
|
-
};
|
|
17032
|
-
if (event.score.metadata?.entityType) {
|
|
17033
|
-
labels.entity_type = String(event.score.metadata.entityType);
|
|
17034
|
-
}
|
|
17035
|
-
if (event.score.experimentId) {
|
|
17036
|
-
labels.experiment = event.score.experimentId;
|
|
17037
|
-
}
|
|
17038
|
-
this.emit("mastra_scores_total", "counter", 1, labels);
|
|
17039
|
-
}
|
|
17040
|
-
/** Emit a `mastra_feedback_total` counter for a feedback event. */
|
|
17041
|
-
processFeedbackEvent(event) {
|
|
17042
|
-
const labels = {
|
|
17043
|
-
feedback_type: event.feedback.feedbackType,
|
|
17044
|
-
source: event.feedback.source
|
|
17045
|
-
};
|
|
17046
|
-
if (event.feedback.metadata?.entityType) {
|
|
17047
|
-
labels.entity_type = String(event.feedback.metadata.entityType);
|
|
17048
|
-
}
|
|
17049
|
-
if (event.feedback.experimentId) {
|
|
17050
|
-
labels.experiment = event.feedback.experimentId;
|
|
17051
|
-
}
|
|
17052
|
-
this.emit("mastra_feedback_total", "counter", 1, labels);
|
|
17053
|
-
}
|
|
17054
|
-
/** Emit a started counter (e.g. `mastra_agent_runs_started`) for the span type. */
|
|
17055
|
-
onSpanStarted(span) {
|
|
17056
|
-
const labels = this.extractLabels(span);
|
|
17057
|
-
const metricName = this.getStartedMetricName(span);
|
|
17058
|
-
if (metricName) {
|
|
17059
|
-
this.emit(metricName, "counter", 1, labels);
|
|
16998
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
16999
|
+
this.onSpanEnded(event.exportedSpan);
|
|
17060
17000
|
}
|
|
17061
17001
|
}
|
|
17062
|
-
/** Emit
|
|
17002
|
+
/** Emit duration and token metrics when a span ends. */
|
|
17063
17003
|
onSpanEnded(span) {
|
|
17064
17004
|
const labels = this.extractLabels(span);
|
|
17065
|
-
const endedMetricName = this.getEndedMetricName(span);
|
|
17066
|
-
if (endedMetricName) {
|
|
17067
|
-
const endedLabels = { ...labels };
|
|
17068
|
-
if (span.errorInfo) {
|
|
17069
|
-
endedLabels.status = "error";
|
|
17070
|
-
} else {
|
|
17071
|
-
endedLabels.status = "ok";
|
|
17072
|
-
}
|
|
17073
|
-
this.emit(endedMetricName, "counter", 1, endedLabels);
|
|
17074
|
-
}
|
|
17075
17005
|
const durationMetricName = this.getDurationMetricName(span);
|
|
17076
17006
|
if (durationMetricName && span.startTime && span.endTime) {
|
|
17077
|
-
const durationMs =
|
|
17007
|
+
const durationMs = span.endTime.getTime() - span.startTime.getTime();
|
|
17078
17008
|
const durationLabels = { ...labels };
|
|
17079
|
-
|
|
17080
|
-
|
|
17081
|
-
} else {
|
|
17082
|
-
durationLabels.status = "ok";
|
|
17083
|
-
}
|
|
17084
|
-
this.emit(durationMetricName, "histogram", durationMs, durationLabels);
|
|
17009
|
+
durationLabels.status = span.errorInfo ? "error" : "ok";
|
|
17010
|
+
this.observabilityBus.emitMetric(durationMetricName, durationMs, durationLabels);
|
|
17085
17011
|
}
|
|
17086
17012
|
if (span.type === observability.SpanType.MODEL_GENERATION) {
|
|
17087
|
-
|
|
17013
|
+
const attrs = span.attributes;
|
|
17014
|
+
if (attrs?.usage) {
|
|
17015
|
+
this.extractTokenMetrics(attrs.usage, labels);
|
|
17016
|
+
}
|
|
17088
17017
|
}
|
|
17089
17018
|
}
|
|
17090
17019
|
/** Build base metric labels from a span's entity and model attributes. */
|
|
@@ -17095,65 +17024,34 @@ var AutoExtractedMetrics = class {
|
|
|
17095
17024
|
if (entityName) labels.entity_name = entityName;
|
|
17096
17025
|
if (span.type === observability.SpanType.MODEL_GENERATION) {
|
|
17097
17026
|
const attrs = span.attributes;
|
|
17098
|
-
if (attrs?.model) labels.model =
|
|
17099
|
-
if (attrs?.provider) labels.provider =
|
|
17027
|
+
if (attrs?.model) labels.model = attrs.model;
|
|
17028
|
+
if (attrs?.provider) labels.provider = attrs.provider;
|
|
17100
17029
|
}
|
|
17101
17030
|
return labels;
|
|
17102
17031
|
}
|
|
17103
|
-
/** Emit token usage
|
|
17104
|
-
extractTokenMetrics(
|
|
17105
|
-
const
|
|
17106
|
-
const
|
|
17107
|
-
|
|
17108
|
-
|
|
17109
|
-
|
|
17110
|
-
|
|
17111
|
-
|
|
17112
|
-
|
|
17113
|
-
|
|
17114
|
-
|
|
17115
|
-
|
|
17116
|
-
|
|
17117
|
-
|
|
17118
|
-
if (
|
|
17119
|
-
|
|
17120
|
-
|
|
17121
|
-
|
|
17122
|
-
|
|
17123
|
-
|
|
17124
|
-
|
|
17125
|
-
|
|
17126
|
-
/** Map a span type to its `*_started` counter metric name, or `null` for unsupported types. */
|
|
17127
|
-
getStartedMetricName(span) {
|
|
17128
|
-
switch (span.type) {
|
|
17129
|
-
case observability.SpanType.AGENT_RUN:
|
|
17130
|
-
return "mastra_agent_runs_started";
|
|
17131
|
-
case observability.SpanType.TOOL_CALL:
|
|
17132
|
-
return "mastra_tool_calls_started";
|
|
17133
|
-
case observability.SpanType.WORKFLOW_RUN:
|
|
17134
|
-
return "mastra_workflow_runs_started";
|
|
17135
|
-
case observability.SpanType.MODEL_GENERATION:
|
|
17136
|
-
return "mastra_model_requests_started";
|
|
17137
|
-
default:
|
|
17138
|
-
return null;
|
|
17139
|
-
}
|
|
17140
|
-
}
|
|
17141
|
-
/** Map a span type to its `*_ended` counter metric name, or `null` for unsupported types. */
|
|
17142
|
-
getEndedMetricName(span) {
|
|
17143
|
-
switch (span.type) {
|
|
17144
|
-
case observability.SpanType.AGENT_RUN:
|
|
17145
|
-
return "mastra_agent_runs_ended";
|
|
17146
|
-
case observability.SpanType.TOOL_CALL:
|
|
17147
|
-
return "mastra_tool_calls_ended";
|
|
17148
|
-
case observability.SpanType.WORKFLOW_RUN:
|
|
17149
|
-
return "mastra_workflow_runs_ended";
|
|
17150
|
-
case observability.SpanType.MODEL_GENERATION:
|
|
17151
|
-
return "mastra_model_requests_ended";
|
|
17152
|
-
default:
|
|
17153
|
-
return null;
|
|
17154
|
-
}
|
|
17155
|
-
}
|
|
17156
|
-
/** Map a span type to its `*_duration_ms` histogram metric name, or `null` for unsupported types. */
|
|
17032
|
+
/** Emit token usage metrics from UsageStats. */
|
|
17033
|
+
extractTokenMetrics(usage, labels) {
|
|
17034
|
+
const emit = (name, value) => this.observabilityBus.emitMetric(name, value, labels);
|
|
17035
|
+
const emitNonZero = (name, value) => {
|
|
17036
|
+
if (value > 0) emit(name, value);
|
|
17037
|
+
};
|
|
17038
|
+
emit("mastra_model_total_input_tokens", usage.inputTokens ?? 0);
|
|
17039
|
+
emit("mastra_model_total_output_tokens", usage.outputTokens ?? 0);
|
|
17040
|
+
if (usage.inputDetails) {
|
|
17041
|
+
emitNonZero("mastra_model_input_text_tokens", usage.inputDetails.text ?? 0);
|
|
17042
|
+
emitNonZero("mastra_model_input_cache_read_tokens", usage.inputDetails.cacheRead ?? 0);
|
|
17043
|
+
emitNonZero("mastra_model_input_cache_write_tokens", usage.inputDetails.cacheWrite ?? 0);
|
|
17044
|
+
emitNonZero("mastra_model_input_audio_tokens", usage.inputDetails.audio ?? 0);
|
|
17045
|
+
emitNonZero("mastra_model_input_image_tokens", usage.inputDetails.image ?? 0);
|
|
17046
|
+
}
|
|
17047
|
+
if (usage.outputDetails) {
|
|
17048
|
+
emitNonZero("mastra_model_output_text_tokens", usage.outputDetails.text ?? 0);
|
|
17049
|
+
emitNonZero("mastra_model_output_reasoning_tokens", usage.outputDetails.reasoning ?? 0);
|
|
17050
|
+
emitNonZero("mastra_model_output_audio_tokens", usage.outputDetails.audio ?? 0);
|
|
17051
|
+
emitNonZero("mastra_model_output_image_tokens", usage.outputDetails.image ?? 0);
|
|
17052
|
+
}
|
|
17053
|
+
}
|
|
17054
|
+
/** Map a span type to its `*_duration_ms` metric name, or `null` for unsupported types. */
|
|
17157
17055
|
getDurationMetricName(span) {
|
|
17158
17056
|
switch (span.type) {
|
|
17159
17057
|
case observability.SpanType.AGENT_RUN:
|
|
@@ -17168,18 +17066,38 @@ var AutoExtractedMetrics = class {
|
|
|
17168
17066
|
return null;
|
|
17169
17067
|
}
|
|
17170
17068
|
}
|
|
17171
|
-
|
|
17172
|
-
|
|
17173
|
-
|
|
17174
|
-
|
|
17175
|
-
|
|
17176
|
-
|
|
17177
|
-
|
|
17178
|
-
|
|
17179
|
-
|
|
17180
|
-
|
|
17181
|
-
const
|
|
17182
|
-
this.
|
|
17069
|
+
};
|
|
17070
|
+
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17071
|
+
var CardinalityFilter = class {
|
|
17072
|
+
blockedLabels;
|
|
17073
|
+
blockUUIDs;
|
|
17074
|
+
/**
|
|
17075
|
+
* @param config - Optional configuration. When omitted, uses
|
|
17076
|
+
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
17077
|
+
*/
|
|
17078
|
+
constructor(config2) {
|
|
17079
|
+
const blocked = config2?.blockedLabels ?? [...observability.DEFAULT_BLOCKED_LABELS];
|
|
17080
|
+
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
17081
|
+
this.blockUUIDs = config2?.blockUUIDs ?? true;
|
|
17082
|
+
}
|
|
17083
|
+
/**
|
|
17084
|
+
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
17085
|
+
*
|
|
17086
|
+
* @param labels - Raw metric labels to filter.
|
|
17087
|
+
* @returns A new object containing only the allowed labels.
|
|
17088
|
+
*/
|
|
17089
|
+
filterLabels(labels) {
|
|
17090
|
+
const filtered = {};
|
|
17091
|
+
for (const [key, value] of Object.entries(labels)) {
|
|
17092
|
+
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
17093
|
+
continue;
|
|
17094
|
+
}
|
|
17095
|
+
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
17096
|
+
continue;
|
|
17097
|
+
}
|
|
17098
|
+
filtered[key] = value;
|
|
17099
|
+
}
|
|
17100
|
+
return filtered;
|
|
17183
17101
|
}
|
|
17184
17102
|
};
|
|
17185
17103
|
function routeToHandler(handler, event, logger) {
|
|
@@ -17233,25 +17151,32 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17233
17151
|
exporters = [];
|
|
17234
17152
|
bridge;
|
|
17235
17153
|
autoExtractor;
|
|
17154
|
+
cardinalityFilter;
|
|
17236
17155
|
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
17237
17156
|
pendingHandlers = /* @__PURE__ */ new Set();
|
|
17238
|
-
constructor() {
|
|
17157
|
+
constructor(config2) {
|
|
17239
17158
|
super({ name: "ObservabilityBus" });
|
|
17159
|
+
this.cardinalityFilter = config2?.cardinalityFilter ?? new CardinalityFilter();
|
|
17160
|
+
if (config2?.autoExtractMetrics !== false) {
|
|
17161
|
+
this.autoExtractor = new AutoExtractedMetrics(this);
|
|
17162
|
+
}
|
|
17240
17163
|
}
|
|
17241
17164
|
/**
|
|
17242
|
-
*
|
|
17243
|
-
*
|
|
17244
|
-
*
|
|
17245
|
-
*
|
|
17246
|
-
* No-ops if auto-extraction is already enabled.
|
|
17247
|
-
*
|
|
17248
|
-
* @param cardinalityFilter - Optional filter applied to auto-extracted metric labels.
|
|
17165
|
+
* Emit a metric event with validation and cardinality filtering.
|
|
17166
|
+
* Non-finite or negative values are silently dropped.
|
|
17167
|
+
* This is the single entry point for all metric emission (auto-extracted and user-defined).
|
|
17249
17168
|
*/
|
|
17250
|
-
|
|
17251
|
-
if (
|
|
17252
|
-
|
|
17253
|
-
|
|
17254
|
-
|
|
17169
|
+
emitMetric(name, value, labels) {
|
|
17170
|
+
if (!Number.isFinite(value) || value < 0) return;
|
|
17171
|
+
const filteredLabels = this.cardinalityFilter.filterLabels(labels);
|
|
17172
|
+
const exportedMetric = {
|
|
17173
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
17174
|
+
name,
|
|
17175
|
+
value,
|
|
17176
|
+
labels: filteredLabels
|
|
17177
|
+
};
|
|
17178
|
+
const event = { type: "metric", metric: exportedMetric };
|
|
17179
|
+
this.emit(event);
|
|
17255
17180
|
}
|
|
17256
17181
|
/**
|
|
17257
17182
|
* Register an exporter to receive routed events.
|
|
@@ -17330,15 +17255,9 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17330
17255
|
if (this.bridge) {
|
|
17331
17256
|
this.trackPromise(routeToHandler(this.bridge, event, this.logger));
|
|
17332
17257
|
}
|
|
17333
|
-
if (this.autoExtractor) {
|
|
17258
|
+
if (this.autoExtractor && isTracingEvent(event)) {
|
|
17334
17259
|
try {
|
|
17335
|
-
|
|
17336
|
-
this.autoExtractor.processTracingEvent(event);
|
|
17337
|
-
} else if (event.type === "score") {
|
|
17338
|
-
this.autoExtractor.processScoreEvent(event);
|
|
17339
|
-
} else if (event.type === "feedback") {
|
|
17340
|
-
this.autoExtractor.processFeedbackEvent(event);
|
|
17341
|
-
}
|
|
17260
|
+
this.autoExtractor.processTracingEvent(event);
|
|
17342
17261
|
} catch (err) {
|
|
17343
17262
|
this.logger.error("[ObservabilityBus] Auto-extraction error:", err);
|
|
17344
17263
|
}
|
|
@@ -17465,104 +17384,45 @@ var LoggerContextImpl = class {
|
|
|
17465
17384
|
|
|
17466
17385
|
// src/context/metrics.ts
|
|
17467
17386
|
var MetricsContextImpl = class {
|
|
17468
|
-
|
|
17387
|
+
baseLabels;
|
|
17388
|
+
observabilityBus;
|
|
17469
17389
|
/**
|
|
17470
17390
|
* Create a metrics context. Base labels are defensively copied so
|
|
17471
17391
|
* mutations after construction do not affect emitted metrics.
|
|
17472
17392
|
*/
|
|
17473
17393
|
constructor(config2) {
|
|
17474
|
-
this.
|
|
17475
|
-
|
|
17476
|
-
labels: config2.labels ? { ...config2.labels } : void 0
|
|
17477
|
-
};
|
|
17394
|
+
this.baseLabels = config2.labels ? { ...config2.labels } : {};
|
|
17395
|
+
this.observabilityBus = config2.observabilityBus;
|
|
17478
17396
|
}
|
|
17479
|
-
/**
|
|
17480
|
-
|
|
17481
|
-
|
|
17482
|
-
|
|
17483
|
-
|
|
17397
|
+
/** Emit a metric observation. */
|
|
17398
|
+
emit(name, value, labels) {
|
|
17399
|
+
const allLabels = { ...this.baseLabels, ...labels };
|
|
17400
|
+
this.observabilityBus.emitMetric(name, value, allLabels);
|
|
17401
|
+
}
|
|
17402
|
+
/** @deprecated Use `emit()` instead. */
|
|
17484
17403
|
counter(name) {
|
|
17485
17404
|
return {
|
|
17486
17405
|
add: (value, additionalLabels) => {
|
|
17487
|
-
this.emit(name,
|
|
17406
|
+
this.emit(name, value, additionalLabels);
|
|
17488
17407
|
}
|
|
17489
17408
|
};
|
|
17490
17409
|
}
|
|
17491
|
-
/**
|
|
17492
|
-
* Create a gauge instrument. Call `.set(value)` to record a point-in-time value.
|
|
17493
|
-
*
|
|
17494
|
-
* @param name - Metric name (e.g. `mastra_queue_depth`).
|
|
17495
|
-
*/
|
|
17410
|
+
/** @deprecated Use `emit()` instead. */
|
|
17496
17411
|
gauge(name) {
|
|
17497
17412
|
return {
|
|
17498
17413
|
set: (value, additionalLabels) => {
|
|
17499
|
-
this.emit(name,
|
|
17414
|
+
this.emit(name, value, additionalLabels);
|
|
17500
17415
|
}
|
|
17501
17416
|
};
|
|
17502
17417
|
}
|
|
17503
|
-
/**
|
|
17504
|
-
* Create a histogram instrument. Call `.record(value)` to observe a measurement.
|
|
17505
|
-
*
|
|
17506
|
-
* @param name - Metric name (e.g. `mastra_request_duration_ms`).
|
|
17507
|
-
*/
|
|
17418
|
+
/** @deprecated Use `emit()` instead. */
|
|
17508
17419
|
histogram(name) {
|
|
17509
17420
|
return {
|
|
17510
17421
|
record: (value, additionalLabels) => {
|
|
17511
|
-
this.emit(name,
|
|
17422
|
+
this.emit(name, value, additionalLabels);
|
|
17512
17423
|
}
|
|
17513
17424
|
};
|
|
17514
17425
|
}
|
|
17515
|
-
/** Merge base + additional labels, apply cardinality filtering, and emit a MetricEvent. Non-finite values are silently dropped. */
|
|
17516
|
-
emit(name, metricType, value, additionalLabels) {
|
|
17517
|
-
if (!Number.isFinite(value)) return;
|
|
17518
|
-
const allLabels = {
|
|
17519
|
-
...this.config.labels,
|
|
17520
|
-
...additionalLabels
|
|
17521
|
-
};
|
|
17522
|
-
const filteredLabels = this.config.cardinalityFilter.filterLabels(allLabels);
|
|
17523
|
-
const exportedMetric = {
|
|
17524
|
-
timestamp: /* @__PURE__ */ new Date(),
|
|
17525
|
-
name,
|
|
17526
|
-
metricType,
|
|
17527
|
-
value,
|
|
17528
|
-
labels: filteredLabels
|
|
17529
|
-
};
|
|
17530
|
-
const event = { type: "metric", metric: exportedMetric };
|
|
17531
|
-
this.config.observabilityBus.emit(event);
|
|
17532
|
-
}
|
|
17533
|
-
};
|
|
17534
|
-
var UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17535
|
-
var CardinalityFilter = class {
|
|
17536
|
-
blockedLabels;
|
|
17537
|
-
blockUUIDs;
|
|
17538
|
-
/**
|
|
17539
|
-
* @param config - Optional configuration. When omitted, uses
|
|
17540
|
-
* {@link DEFAULT_BLOCKED_LABELS} and blocks UUID-valued labels.
|
|
17541
|
-
*/
|
|
17542
|
-
constructor(config2) {
|
|
17543
|
-
const blocked = config2?.blockedLabels ?? [...observability.DEFAULT_BLOCKED_LABELS];
|
|
17544
|
-
this.blockedLabels = new Set(blocked.map((l) => l.toLowerCase()));
|
|
17545
|
-
this.blockUUIDs = config2?.blockUUIDs ?? true;
|
|
17546
|
-
}
|
|
17547
|
-
/**
|
|
17548
|
-
* Return a copy of `labels` with blocked keys and UUID values removed.
|
|
17549
|
-
*
|
|
17550
|
-
* @param labels - Raw metric labels to filter.
|
|
17551
|
-
* @returns A new object containing only the allowed labels.
|
|
17552
|
-
*/
|
|
17553
|
-
filterLabels(labels) {
|
|
17554
|
-
const filtered = {};
|
|
17555
|
-
for (const [key, value] of Object.entries(labels)) {
|
|
17556
|
-
if (this.blockedLabels.has(key.toLowerCase())) {
|
|
17557
|
-
continue;
|
|
17558
|
-
}
|
|
17559
|
-
if (this.blockUUIDs && UUID_REGEX.test(value)) {
|
|
17560
|
-
continue;
|
|
17561
|
-
}
|
|
17562
|
-
filtered[key] = value;
|
|
17563
|
-
}
|
|
17564
|
-
return filtered;
|
|
17565
|
-
}
|
|
17566
17426
|
};
|
|
17567
17427
|
|
|
17568
17428
|
// src/usage.ts
|
|
@@ -18617,15 +18477,16 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
18617
18477
|
requestContextKeys: config2.requestContextKeys ?? [],
|
|
18618
18478
|
serializationOptions: config2.serializationOptions
|
|
18619
18479
|
};
|
|
18620
|
-
this.cardinalityFilter = new CardinalityFilter();
|
|
18621
|
-
this.observabilityBus = new ObservabilityBus(
|
|
18480
|
+
this.cardinalityFilter = new CardinalityFilter(config2.cardinality);
|
|
18481
|
+
this.observabilityBus = new ObservabilityBus({
|
|
18482
|
+
cardinalityFilter: this.cardinalityFilter
|
|
18483
|
+
});
|
|
18622
18484
|
for (const exporter of this.exporters) {
|
|
18623
18485
|
this.observabilityBus.registerExporter(exporter);
|
|
18624
18486
|
}
|
|
18625
18487
|
if (this.config.bridge) {
|
|
18626
18488
|
this.observabilityBus.registerBridge(this.config.bridge);
|
|
18627
18489
|
}
|
|
18628
|
-
this.observabilityBus.enableAutoExtractedMetrics(this.cardinalityFilter);
|
|
18629
18490
|
if (this.config.bridge?.init) {
|
|
18630
18491
|
this.config.bridge.init({ config: this.config });
|
|
18631
18492
|
}
|
|
@@ -18869,8 +18730,7 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
18869
18730
|
if (this.config.serviceName) labels.service_name = this.config.serviceName;
|
|
18870
18731
|
return new MetricsContextImpl({
|
|
18871
18732
|
labels: Object.keys(labels).length > 0 ? labels : void 0,
|
|
18872
|
-
observabilityBus: this.observabilityBus
|
|
18873
|
-
cardinalityFilter: this.cardinalityFilter
|
|
18733
|
+
observabilityBus: this.observabilityBus
|
|
18874
18734
|
});
|
|
18875
18735
|
}
|
|
18876
18736
|
/**
|
|
@@ -19017,6 +18877,7 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
19017
18877
|
// ============================================================================
|
|
19018
18878
|
// Event-driven Export Methods
|
|
19019
18879
|
// ============================================================================
|
|
18880
|
+
/** Process a span through output processors and export it, returning undefined if filtered out. */
|
|
19020
18881
|
getSpanForExport(span) {
|
|
19021
18882
|
if (!span.isValid) return void 0;
|
|
19022
18883
|
if (span.isInternal && !this.config.includeInternalSpans) return void 0;
|
|
@@ -19064,7 +18925,7 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
19064
18925
|
*
|
|
19065
18926
|
* The bus routes the event to each registered exporter's and bridge's
|
|
19066
18927
|
* onTracingEvent handler and triggers auto-extracted metrics (e.g.,
|
|
19067
|
-
*
|
|
18928
|
+
* mastra_agent_duration_ms, mastra_model_duration_ms).
|
|
19068
18929
|
*/
|
|
19069
18930
|
emitTracingEvent(event) {
|
|
19070
18931
|
this.observabilityBus.emit(event);
|
|
@@ -19402,7 +19263,9 @@ var Observability = class extends base.MastraBase {
|
|
|
19402
19263
|
}
|
|
19403
19264
|
const validationResult = observabilityRegistryConfigSchema.safeParse(config2);
|
|
19404
19265
|
if (!validationResult.success) {
|
|
19405
|
-
const errorMessages = validationResult.error.issues.map(
|
|
19266
|
+
const errorMessages = validationResult.error.issues.map(
|
|
19267
|
+
(err) => `${err.path.join(".") || "config"}: ${err.message}`
|
|
19268
|
+
).join("; ");
|
|
19406
19269
|
throw new error$1.MastraError({
|
|
19407
19270
|
id: "OBSERVABILITY_INVALID_CONFIG",
|
|
19408
19271
|
text: `Invalid observability configuration: ${errorMessages}`,
|
|
@@ -19418,7 +19281,9 @@ var Observability = class extends base.MastraBase {
|
|
|
19418
19281
|
if (!isInstance(configValue)) {
|
|
19419
19282
|
const configValidation = observabilityConfigValueSchema.safeParse(configValue);
|
|
19420
19283
|
if (!configValidation.success) {
|
|
19421
|
-
const errorMessages = configValidation.error.issues.map(
|
|
19284
|
+
const errorMessages = configValidation.error.issues.map(
|
|
19285
|
+
(err) => `${err.path.join(".")}: ${err.message}`
|
|
19286
|
+
).join("; ");
|
|
19422
19287
|
throw new error$1.MastraError({
|
|
19423
19288
|
id: "OBSERVABILITY_INVALID_INSTANCE_CONFIG",
|
|
19424
19289
|
text: `Invalid configuration for observability instance '${name}': ${errorMessages}`,
|
|
@@ -19458,6 +19323,7 @@ var Observability = class extends base.MastraBase {
|
|
|
19458
19323
|
this.#registry.setSelector(config2.configSelector);
|
|
19459
19324
|
}
|
|
19460
19325
|
}
|
|
19326
|
+
/** Initialize all exporter instances with the Mastra context (storage, config, etc.). */
|
|
19461
19327
|
setMastraContext(options) {
|
|
19462
19328
|
const instances = this.listInstances();
|
|
19463
19329
|
const { mastra } = options;
|
|
@@ -19478,42 +19344,50 @@ var Observability = class extends base.MastraBase {
|
|
|
19478
19344
|
});
|
|
19479
19345
|
});
|
|
19480
19346
|
}
|
|
19347
|
+
/** Propagate a logger to this instance and all registered observability instances. */
|
|
19481
19348
|
setLogger(options) {
|
|
19482
19349
|
super.__setLogger(options.logger);
|
|
19483
19350
|
this.listInstances().forEach((instance) => {
|
|
19484
19351
|
instance.__setLogger(options.logger);
|
|
19485
19352
|
});
|
|
19486
19353
|
}
|
|
19354
|
+
/** Get the observability instance chosen by the config selector for the given options. */
|
|
19487
19355
|
getSelectedInstance(options) {
|
|
19488
19356
|
return this.#registry.getSelected(options);
|
|
19489
19357
|
}
|
|
19490
|
-
/**
|
|
19491
|
-
* Registry management methods
|
|
19492
|
-
*/
|
|
19358
|
+
/** Register a named observability instance, optionally marking it as default. */
|
|
19493
19359
|
registerInstance(name, instance, isDefault = false) {
|
|
19494
19360
|
this.#registry.register(name, instance, isDefault);
|
|
19495
19361
|
}
|
|
19362
|
+
/** Get a registered instance by name. */
|
|
19496
19363
|
getInstance(name) {
|
|
19497
19364
|
return this.#registry.get(name);
|
|
19498
19365
|
}
|
|
19366
|
+
/** Get the default observability instance. */
|
|
19499
19367
|
getDefaultInstance() {
|
|
19500
19368
|
return this.#registry.getDefault();
|
|
19501
19369
|
}
|
|
19370
|
+
/** List all registered observability instances. */
|
|
19502
19371
|
listInstances() {
|
|
19503
19372
|
return this.#registry.list();
|
|
19504
19373
|
}
|
|
19374
|
+
/** Unregister an instance by name. Returns true if it was found and removed. */
|
|
19505
19375
|
unregisterInstance(name) {
|
|
19506
19376
|
return this.#registry.unregister(name);
|
|
19507
19377
|
}
|
|
19378
|
+
/** Check whether an instance with the given name is registered. */
|
|
19508
19379
|
hasInstance(name) {
|
|
19509
19380
|
return !!this.#registry.get(name);
|
|
19510
19381
|
}
|
|
19382
|
+
/** Set the config selector used to choose an instance at runtime. */
|
|
19511
19383
|
setConfigSelector(selector) {
|
|
19512
19384
|
this.#registry.setSelector(selector);
|
|
19513
19385
|
}
|
|
19386
|
+
/** Remove all registered instances and reset the registry. */
|
|
19514
19387
|
clear() {
|
|
19515
19388
|
this.#registry.clear();
|
|
19516
19389
|
}
|
|
19390
|
+
/** Shut down all registered instances, flushing any pending data. */
|
|
19517
19391
|
async shutdown() {
|
|
19518
19392
|
await this.#registry.shutdown();
|
|
19519
19393
|
}
|