@mastra/observability 1.0.0-beta.9 → 1.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +697 -0
- package/README.md +34 -79
- package/dist/config.d.ts +110 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/base.d.ts +55 -5
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/cloud.d.ts +8 -2
- package/dist/exporters/cloud.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +8 -2
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +4 -2
- package/dist/exporters/index.d.ts.map +1 -1
- package/dist/exporters/json.d.ts +386 -0
- package/dist/exporters/json.d.ts.map +1 -0
- package/dist/exporters/span-formatters.d.ts +35 -0
- package/dist/exporters/span-formatters.d.ts.map +1 -0
- package/dist/exporters/tracking.d.ts +470 -0
- package/dist/exporters/tracking.d.ts.map +1 -0
- package/dist/index.cjs +2840 -812
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2833 -812
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +33 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/model-tracing.d.ts +6 -0
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/spans/base.d.ts +5 -0
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts.map +1 -1
- package/dist/spans/serialization.d.ts +30 -0
- package/dist/spans/serialization.d.ts.map +1 -1
- package/package.json +11 -11
package/dist/index.cjs
CHANGED
|
@@ -5,8 +5,12 @@ var error = require('@mastra/core/error');
|
|
|
5
5
|
var logger = require('@mastra/core/logger');
|
|
6
6
|
var observability = require('@mastra/core/observability');
|
|
7
7
|
var utils = require('@mastra/core/utils');
|
|
8
|
+
var promises = require('fs/promises');
|
|
9
|
+
var path = require('path');
|
|
10
|
+
var url = require('url');
|
|
8
11
|
var web = require('stream/web');
|
|
9
12
|
|
|
13
|
+
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
10
14
|
var __defProp = Object.defineProperty;
|
|
11
15
|
var __export = (target, all) => {
|
|
12
16
|
for (var name in all)
|
|
@@ -3839,7 +3843,7 @@ ZodNaN.create = (params) => {
|
|
|
3839
3843
|
...processCreateParams(params)
|
|
3840
3844
|
});
|
|
3841
3845
|
};
|
|
3842
|
-
var BRAND = Symbol("zod_brand");
|
|
3846
|
+
var BRAND = /* @__PURE__ */ Symbol("zod_brand");
|
|
3843
3847
|
var ZodBranded = class extends ZodType {
|
|
3844
3848
|
_parse(input) {
|
|
3845
3849
|
const { ctx } = this._processInputParams(input);
|
|
@@ -4076,6 +4080,12 @@ var samplingStrategySchema = external_exports.discriminatedUnion("type", [
|
|
|
4076
4080
|
sampler: external_exports.function().args(external_exports.any().optional()).returns(external_exports.boolean())
|
|
4077
4081
|
})
|
|
4078
4082
|
]);
|
|
4083
|
+
var serializationOptionsSchema = external_exports.object({
|
|
4084
|
+
maxStringLength: external_exports.number().int().positive().optional(),
|
|
4085
|
+
maxDepth: external_exports.number().int().positive().optional(),
|
|
4086
|
+
maxArrayLength: external_exports.number().int().positive().optional(),
|
|
4087
|
+
maxObjectKeys: external_exports.number().int().positive().optional()
|
|
4088
|
+
}).optional();
|
|
4079
4089
|
var observabilityInstanceConfigSchema = external_exports.object({
|
|
4080
4090
|
name: external_exports.string().min(1, "Name is required"),
|
|
4081
4091
|
serviceName: external_exports.string().min(1, "Service name is required"),
|
|
@@ -4084,7 +4094,8 @@ var observabilityInstanceConfigSchema = external_exports.object({
|
|
|
4084
4094
|
bridge: external_exports.any().optional(),
|
|
4085
4095
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4086
4096
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4087
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4097
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4098
|
+
serializationOptions: serializationOptionsSchema
|
|
4088
4099
|
}).refine(
|
|
4089
4100
|
(data) => {
|
|
4090
4101
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4102,7 +4113,8 @@ var observabilityConfigValueSchema = external_exports.object({
|
|
|
4102
4113
|
bridge: external_exports.any().optional(),
|
|
4103
4114
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4104
4115
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4105
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4116
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4117
|
+
serializationOptions: serializationOptionsSchema
|
|
4106
4118
|
}).refine(
|
|
4107
4119
|
(data) => {
|
|
4108
4120
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4155,12 +4167,19 @@ var observabilityRegistryConfigSchema = external_exports.object({
|
|
|
4155
4167
|
var BaseExporter = class {
|
|
4156
4168
|
/** Mastra logger instance */
|
|
4157
4169
|
logger;
|
|
4170
|
+
/** Base configuration (accessible by subclasses) */
|
|
4171
|
+
baseConfig;
|
|
4158
4172
|
/** Whether this exporter is disabled */
|
|
4159
|
-
|
|
4173
|
+
#disabled = false;
|
|
4174
|
+
/** Public getter for disabled state */
|
|
4175
|
+
get isDisabled() {
|
|
4176
|
+
return this.#disabled;
|
|
4177
|
+
}
|
|
4160
4178
|
/**
|
|
4161
4179
|
* Initialize the base exporter with logger
|
|
4162
4180
|
*/
|
|
4163
4181
|
constructor(config = {}) {
|
|
4182
|
+
this.baseConfig = config;
|
|
4164
4183
|
const logLevel = this.resolveLogLevel(config.logLevel);
|
|
4165
4184
|
this.logger = config.logger ?? new logger.ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
4166
4185
|
}
|
|
@@ -4195,20 +4214,62 @@ var BaseExporter = class {
|
|
|
4195
4214
|
* @param reason - Reason why the exporter is disabled
|
|
4196
4215
|
*/
|
|
4197
4216
|
setDisabled(reason) {
|
|
4198
|
-
this
|
|
4217
|
+
this.#disabled = true;
|
|
4199
4218
|
this.logger.warn(`${this.name} disabled: ${reason}`);
|
|
4200
4219
|
}
|
|
4220
|
+
/**
|
|
4221
|
+
* Apply the customSpanFormatter if configured.
|
|
4222
|
+
* This is called automatically by exportTracingEvent before _exportTracingEvent.
|
|
4223
|
+
*
|
|
4224
|
+
* Supports both synchronous and asynchronous formatters. If the formatter
|
|
4225
|
+
* returns a Promise, it will be awaited.
|
|
4226
|
+
*
|
|
4227
|
+
* @param event - The incoming tracing event
|
|
4228
|
+
* @returns The (possibly modified) event to process
|
|
4229
|
+
*/
|
|
4230
|
+
async applySpanFormatter(event) {
|
|
4231
|
+
if (this.baseConfig.customSpanFormatter) {
|
|
4232
|
+
try {
|
|
4233
|
+
const formattedSpan = await this.baseConfig.customSpanFormatter(event.exportedSpan);
|
|
4234
|
+
return {
|
|
4235
|
+
...event,
|
|
4236
|
+
exportedSpan: formattedSpan
|
|
4237
|
+
};
|
|
4238
|
+
} catch (error) {
|
|
4239
|
+
this.logger.error(`${this.name}: Error in customSpanFormatter`, {
|
|
4240
|
+
error,
|
|
4241
|
+
spanId: event.exportedSpan.id,
|
|
4242
|
+
traceId: event.exportedSpan.traceId
|
|
4243
|
+
});
|
|
4244
|
+
}
|
|
4245
|
+
}
|
|
4246
|
+
return event;
|
|
4247
|
+
}
|
|
4201
4248
|
/**
|
|
4202
4249
|
* Export a tracing event
|
|
4203
4250
|
*
|
|
4204
|
-
* This method checks if the exporter is disabled
|
|
4205
|
-
*
|
|
4251
|
+
* This method checks if the exporter is disabled, applies the customSpanFormatter,
|
|
4252
|
+
* then calls _exportTracingEvent.
|
|
4253
|
+
* Subclasses should implement _exportTracingEvent instead of overriding this method.
|
|
4206
4254
|
*/
|
|
4207
4255
|
async exportTracingEvent(event) {
|
|
4208
4256
|
if (this.isDisabled) {
|
|
4209
4257
|
return;
|
|
4210
4258
|
}
|
|
4211
|
-
await this.
|
|
4259
|
+
const processedEvent = await this.applySpanFormatter(event);
|
|
4260
|
+
await this._exportTracingEvent(processedEvent);
|
|
4261
|
+
}
|
|
4262
|
+
/**
|
|
4263
|
+
* Force flush any buffered/queued spans without shutting down the exporter.
|
|
4264
|
+
*
|
|
4265
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
4266
|
+
* are exported before the runtime instance is terminated, while keeping
|
|
4267
|
+
* the exporter active for future requests.
|
|
4268
|
+
*
|
|
4269
|
+
* Default implementation is a no-op. Override to add flush logic.
|
|
4270
|
+
*/
|
|
4271
|
+
async flush() {
|
|
4272
|
+
this.logger.debug(`${this.name} flush called (no-op in base class)`);
|
|
4212
4273
|
}
|
|
4213
4274
|
/**
|
|
4214
4275
|
* Shutdown the exporter and clean up resources
|
|
@@ -4219,811 +4280,2609 @@ var BaseExporter = class {
|
|
|
4219
4280
|
this.logger.info(`${this.name} shutdown complete`);
|
|
4220
4281
|
}
|
|
4221
4282
|
};
|
|
4222
|
-
var
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4283
|
+
var TraceData = class {
|
|
4284
|
+
/** The vendor-specific root/trace object */
|
|
4285
|
+
#rootSpan;
|
|
4286
|
+
/** The span ID of the root span */
|
|
4287
|
+
#rootSpanId;
|
|
4288
|
+
/** Whether a span with isRootSpan=true has been successfully processed */
|
|
4289
|
+
#rootSpanProcessed;
|
|
4290
|
+
/** Maps eventId to vendor-specific event objects */
|
|
4291
|
+
#events;
|
|
4292
|
+
/** Maps spanId to vendor-specific span objects */
|
|
4293
|
+
#spans;
|
|
4294
|
+
/** Maps spanId to parentSpanId, representing the span hierarchy */
|
|
4295
|
+
#tree;
|
|
4296
|
+
/** Set of span IDs that have started but not yet ended */
|
|
4297
|
+
#activeSpanIds;
|
|
4298
|
+
/** Maps spanId to vendor-specific metadata */
|
|
4299
|
+
#metadata;
|
|
4300
|
+
/** Arbitrary key-value storage for per-trace data */
|
|
4301
|
+
#extraData;
|
|
4302
|
+
/** Events waiting for the root span to be processed */
|
|
4303
|
+
#waitingForRoot;
|
|
4304
|
+
/** Events waiting for specific parent spans, keyed by parentSpanId */
|
|
4305
|
+
#waitingForParent;
|
|
4306
|
+
/** When this trace data was created, used for cap enforcement */
|
|
4307
|
+
createdAt;
|
|
4308
|
+
constructor() {
|
|
4309
|
+
this.#events = /* @__PURE__ */ new Map();
|
|
4310
|
+
this.#spans = /* @__PURE__ */ new Map();
|
|
4311
|
+
this.#activeSpanIds = /* @__PURE__ */ new Set();
|
|
4312
|
+
this.#tree = /* @__PURE__ */ new Map();
|
|
4313
|
+
this.#metadata = /* @__PURE__ */ new Map();
|
|
4314
|
+
this.#extraData = /* @__PURE__ */ new Map();
|
|
4315
|
+
this.#rootSpanProcessed = false;
|
|
4316
|
+
this.#waitingForRoot = [];
|
|
4317
|
+
this.#waitingForParent = /* @__PURE__ */ new Map();
|
|
4318
|
+
this.createdAt = /* @__PURE__ */ new Date();
|
|
4249
4319
|
}
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
this.flush().catch((error) => {
|
|
4257
|
-
this.logger.error("Batch flush failed", {
|
|
4258
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4259
|
-
});
|
|
4260
|
-
});
|
|
4261
|
-
} else if (this.buffer.totalSize === 1) {
|
|
4262
|
-
this.scheduleFlush();
|
|
4263
|
-
}
|
|
4320
|
+
/**
|
|
4321
|
+
* Check if this trace has a root span registered.
|
|
4322
|
+
* @returns True if addRoot() has been called
|
|
4323
|
+
*/
|
|
4324
|
+
hasRoot() {
|
|
4325
|
+
return !!this.#rootSpanId;
|
|
4264
4326
|
}
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
this.
|
|
4327
|
+
/**
|
|
4328
|
+
* Register the root span for this trace.
|
|
4329
|
+
* @param args.rootId - The span ID of the root span
|
|
4330
|
+
* @param args.rootData - The vendor-specific root object
|
|
4331
|
+
*/
|
|
4332
|
+
addRoot(args) {
|
|
4333
|
+
this.#rootSpanId = args.rootId;
|
|
4334
|
+
this.#rootSpan = args.rootData;
|
|
4335
|
+
this.#rootSpanProcessed = true;
|
|
4272
4336
|
}
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
spanType: span.type,
|
|
4280
|
-
attributes: span.attributes ?? null,
|
|
4281
|
-
metadata: span.metadata ?? null,
|
|
4282
|
-
startedAt: span.startTime,
|
|
4283
|
-
endedAt: span.endTime ?? null,
|
|
4284
|
-
input: span.input ?? null,
|
|
4285
|
-
output: span.output ?? null,
|
|
4286
|
-
error: span.errorInfo,
|
|
4287
|
-
isEvent: span.isEvent,
|
|
4288
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
4289
|
-
updatedAt: null
|
|
4290
|
-
};
|
|
4291
|
-
return spanRecord;
|
|
4337
|
+
/**
|
|
4338
|
+
* Get the vendor-specific root object.
|
|
4339
|
+
* @returns The root object, or undefined if not yet set
|
|
4340
|
+
*/
|
|
4341
|
+
getRoot() {
|
|
4342
|
+
return this.#rootSpan;
|
|
4292
4343
|
}
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
return true;
|
|
4301
|
-
}
|
|
4302
|
-
}
|
|
4303
|
-
return false;
|
|
4344
|
+
/**
|
|
4345
|
+
* Check if a span with isRootSpan=true has been successfully processed.
|
|
4346
|
+
* Set via addRoot() or markRootSpanProcessed().
|
|
4347
|
+
* @returns True if the root span has been processed
|
|
4348
|
+
*/
|
|
4349
|
+
isRootProcessed() {
|
|
4350
|
+
return this.#rootSpanProcessed;
|
|
4304
4351
|
}
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
{
|
|
4313
|
-
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
4314
|
-
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4315
|
-
category: error.ErrorCategory.USER
|
|
4316
|
-
},
|
|
4317
|
-
error$1
|
|
4318
|
-
);
|
|
4319
|
-
this.logger.trackException(mastraError);
|
|
4320
|
-
this.logger.error("Scheduled flush failed", mastraError);
|
|
4321
|
-
});
|
|
4322
|
-
}, this.config.maxBatchWaitMs);
|
|
4352
|
+
/**
|
|
4353
|
+
* Mark that the root span has been processed.
|
|
4354
|
+
* Used by exporters with skipBuildRootTask=true where root goes through _buildSpan
|
|
4355
|
+
* instead of _buildRoot.
|
|
4356
|
+
*/
|
|
4357
|
+
markRootSpanProcessed() {
|
|
4358
|
+
this.#rootSpanProcessed = true;
|
|
4323
4359
|
}
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
}
|
|
4332
|
-
const startTime = Date.now();
|
|
4333
|
-
const spansCopy = [...this.buffer.spans];
|
|
4334
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
4335
|
-
this.resetBuffer();
|
|
4336
|
-
try {
|
|
4337
|
-
await this.batchUpload(spansCopy);
|
|
4338
|
-
const elapsed = Date.now() - startTime;
|
|
4339
|
-
this.logger.debug("Batch flushed successfully", {
|
|
4340
|
-
batchSize: spansCopy.length,
|
|
4341
|
-
flushReason,
|
|
4342
|
-
durationMs: elapsed
|
|
4343
|
-
});
|
|
4344
|
-
} catch (error$1) {
|
|
4345
|
-
const mastraError = new error.MastraError(
|
|
4346
|
-
{
|
|
4347
|
-
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
4348
|
-
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4349
|
-
category: error.ErrorCategory.USER,
|
|
4350
|
-
details: {
|
|
4351
|
-
droppedBatchSize: spansCopy.length
|
|
4352
|
-
}
|
|
4353
|
-
},
|
|
4354
|
-
error$1
|
|
4355
|
-
);
|
|
4356
|
-
this.logger.trackException(mastraError);
|
|
4357
|
-
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
4358
|
-
}
|
|
4360
|
+
/**
|
|
4361
|
+
* Store an arbitrary value in per-trace storage.
|
|
4362
|
+
* @param key - Storage key
|
|
4363
|
+
* @param value - Value to store
|
|
4364
|
+
*/
|
|
4365
|
+
setExtraValue(key, value) {
|
|
4366
|
+
this.#extraData.set(key, value);
|
|
4359
4367
|
}
|
|
4360
4368
|
/**
|
|
4361
|
-
*
|
|
4369
|
+
* Check if a key exists in per-trace storage.
|
|
4370
|
+
* @param key - Storage key
|
|
4371
|
+
* @returns True if the key exists
|
|
4362
4372
|
*/
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
Authorization: `Bearer ${this.config.accessToken}`,
|
|
4366
|
-
"Content-Type": "application/json"
|
|
4367
|
-
};
|
|
4368
|
-
const options = {
|
|
4369
|
-
method: "POST",
|
|
4370
|
-
headers,
|
|
4371
|
-
body: JSON.stringify({ spans })
|
|
4372
|
-
};
|
|
4373
|
-
await utils.fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
4373
|
+
hasExtraValue(key) {
|
|
4374
|
+
return this.#extraData.has(key);
|
|
4374
4375
|
}
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4376
|
+
/**
|
|
4377
|
+
* Get a value from per-trace storage.
|
|
4378
|
+
* @param key - Storage key
|
|
4379
|
+
* @returns The stored value, or undefined if not found
|
|
4380
|
+
*/
|
|
4381
|
+
getExtraValue(key) {
|
|
4382
|
+
return this.#extraData.get(key);
|
|
4379
4383
|
}
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4384
|
+
// ============================================================================
|
|
4385
|
+
// Early Queue Methods
|
|
4386
|
+
// ============================================================================
|
|
4387
|
+
/**
|
|
4388
|
+
* Add an event to the waiting queue.
|
|
4389
|
+
* @param args.event - The tracing event to queue
|
|
4390
|
+
* @param args.waitingFor - 'root' or a specific parentSpanId
|
|
4391
|
+
* @param args.attempts - Optional: preserve attempts count when re-queuing
|
|
4392
|
+
* @param args.queuedAt - Optional: preserve original queue time when re-queuing
|
|
4393
|
+
*/
|
|
4394
|
+
addToWaitingQueue(args) {
|
|
4395
|
+
const queuedEvent = {
|
|
4396
|
+
event: args.event,
|
|
4397
|
+
waitingFor: args.waitingFor,
|
|
4398
|
+
attempts: args.attempts ?? 0,
|
|
4399
|
+
queuedAt: args.queuedAt ?? /* @__PURE__ */ new Date()
|
|
4400
|
+
};
|
|
4401
|
+
if (args.waitingFor === "root") {
|
|
4402
|
+
this.#waitingForRoot.push(queuedEvent);
|
|
4403
|
+
} else {
|
|
4404
|
+
const queue = this.#waitingForParent.get(args.waitingFor) ?? [];
|
|
4405
|
+
queue.push(queuedEvent);
|
|
4406
|
+
this.#waitingForParent.set(args.waitingFor, queue);
|
|
4383
4407
|
}
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4408
|
+
}
|
|
4409
|
+
/**
|
|
4410
|
+
* Get all events waiting for the root span.
|
|
4411
|
+
* Returns a copy of the internal array.
|
|
4412
|
+
*/
|
|
4413
|
+
getEventsWaitingForRoot() {
|
|
4414
|
+
return [...this.#waitingForRoot];
|
|
4415
|
+
}
|
|
4416
|
+
/**
|
|
4417
|
+
* Get all events waiting for a specific parent span.
|
|
4418
|
+
* Returns a copy of the internal array.
|
|
4419
|
+
*/
|
|
4420
|
+
getEventsWaitingFor(args) {
|
|
4421
|
+
return [...this.#waitingForParent.get(args.spanId) ?? []];
|
|
4422
|
+
}
|
|
4423
|
+
/**
|
|
4424
|
+
* Clear the waiting-for-root queue.
|
|
4425
|
+
*/
|
|
4426
|
+
clearWaitingForRoot() {
|
|
4427
|
+
this.#waitingForRoot = [];
|
|
4428
|
+
}
|
|
4429
|
+
/**
|
|
4430
|
+
* Clear the waiting queue for a specific parent span.
|
|
4431
|
+
*/
|
|
4432
|
+
clearWaitingFor(args) {
|
|
4433
|
+
this.#waitingForParent.delete(args.spanId);
|
|
4434
|
+
}
|
|
4435
|
+
/**
|
|
4436
|
+
* Get total count of events in all waiting queues.
|
|
4437
|
+
*/
|
|
4438
|
+
waitingQueueSize() {
|
|
4439
|
+
let count = this.#waitingForRoot.length;
|
|
4440
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4441
|
+
count += queue.length;
|
|
4387
4442
|
}
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
category: error.ErrorCategory.USER,
|
|
4400
|
-
details: {
|
|
4401
|
-
remainingEvents: this.buffer.totalSize
|
|
4402
|
-
}
|
|
4403
|
-
},
|
|
4404
|
-
error$1
|
|
4405
|
-
);
|
|
4406
|
-
this.logger.trackException(mastraError);
|
|
4407
|
-
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
4408
|
-
}
|
|
4443
|
+
return count;
|
|
4444
|
+
}
|
|
4445
|
+
/**
|
|
4446
|
+
* Get all queued events across all waiting queues.
|
|
4447
|
+
* Used for cleanup and logging orphaned events.
|
|
4448
|
+
* @returns Array of all queued events
|
|
4449
|
+
*/
|
|
4450
|
+
getAllQueuedEvents() {
|
|
4451
|
+
const all = [...this.#waitingForRoot];
|
|
4452
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4453
|
+
all.push(...queue);
|
|
4409
4454
|
}
|
|
4410
|
-
|
|
4455
|
+
return all;
|
|
4411
4456
|
}
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4457
|
+
// ============================================================================
|
|
4458
|
+
// Span Tree Methods
|
|
4459
|
+
// ============================================================================
|
|
4460
|
+
/**
|
|
4461
|
+
* Record the parent-child relationship for a span.
|
|
4462
|
+
* @param args.spanId - The child span ID
|
|
4463
|
+
* @param args.parentSpanId - The parent span ID, or undefined for root spans
|
|
4464
|
+
*/
|
|
4465
|
+
addBranch(args) {
|
|
4466
|
+
this.#tree.set(args.spanId, args.parentSpanId);
|
|
4417
4467
|
}
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4468
|
+
/**
|
|
4469
|
+
* Get the parent span ID for a given span.
|
|
4470
|
+
* @param args.spanId - The span ID to look up
|
|
4471
|
+
* @returns The parent span ID, or undefined if root or not found
|
|
4472
|
+
*/
|
|
4473
|
+
getParentId(args) {
|
|
4474
|
+
return this.#tree.get(args.spanId);
|
|
4475
|
+
}
|
|
4476
|
+
// ============================================================================
|
|
4477
|
+
// Span Management Methods
|
|
4478
|
+
// ============================================================================
|
|
4479
|
+
/**
|
|
4480
|
+
* Register a span and mark it as active.
|
|
4481
|
+
* @param args.spanId - The span ID
|
|
4482
|
+
* @param args.spanData - The vendor-specific span object
|
|
4483
|
+
*/
|
|
4484
|
+
addSpan(args) {
|
|
4485
|
+
this.#spans.set(args.spanId, args.spanData);
|
|
4486
|
+
this.#activeSpanIds.add(args.spanId);
|
|
4487
|
+
}
|
|
4488
|
+
/**
|
|
4489
|
+
* Check if a span exists (regardless of active state).
|
|
4490
|
+
* @param args.spanId - The span ID to check
|
|
4491
|
+
* @returns True if the span exists
|
|
4492
|
+
*/
|
|
4493
|
+
hasSpan(args) {
|
|
4494
|
+
return this.#spans.has(args.spanId);
|
|
4495
|
+
}
|
|
4496
|
+
/**
|
|
4497
|
+
* Get a span by ID.
|
|
4498
|
+
* @param args.spanId - The span ID to look up
|
|
4499
|
+
* @returns The vendor-specific span object, or undefined if not found
|
|
4500
|
+
*/
|
|
4501
|
+
getSpan(args) {
|
|
4502
|
+
return this.#spans.get(args.spanId);
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4505
|
+
* Mark a span as ended (no longer active).
|
|
4506
|
+
* @param args.spanId - The span ID to mark as ended
|
|
4507
|
+
*/
|
|
4508
|
+
endSpan(args) {
|
|
4509
|
+
this.#activeSpanIds.delete(args.spanId);
|
|
4510
|
+
}
|
|
4511
|
+
/**
|
|
4512
|
+
* Check if a span is currently active (started but not ended).
|
|
4513
|
+
* @param args.spanId - The span ID to check
|
|
4514
|
+
* @returns True if the span is active
|
|
4515
|
+
*/
|
|
4516
|
+
isActiveSpan(args) {
|
|
4517
|
+
return this.#activeSpanIds.has(args.spanId);
|
|
4518
|
+
}
|
|
4519
|
+
/**
|
|
4520
|
+
* Get the count of currently active spans.
|
|
4521
|
+
* @returns Number of active spans
|
|
4522
|
+
*/
|
|
4523
|
+
activeSpanCount() {
|
|
4524
|
+
return this.#activeSpanIds.size;
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Get all active span IDs.
|
|
4528
|
+
* @returns Array of active span IDs
|
|
4529
|
+
*/
|
|
4530
|
+
get activeSpanIds() {
|
|
4531
|
+
return [...this.#activeSpanIds];
|
|
4532
|
+
}
|
|
4533
|
+
// ============================================================================
|
|
4534
|
+
// Event Management Methods
|
|
4535
|
+
// ============================================================================
|
|
4536
|
+
/**
|
|
4537
|
+
* Register an event.
|
|
4538
|
+
* @param args.eventId - The event ID
|
|
4539
|
+
* @param args.eventData - The vendor-specific event object
|
|
4540
|
+
*/
|
|
4541
|
+
addEvent(args) {
|
|
4542
|
+
this.#events.set(args.eventId, args.eventData);
|
|
4543
|
+
}
|
|
4544
|
+
// ============================================================================
|
|
4545
|
+
// Metadata Methods
|
|
4546
|
+
// ============================================================================
|
|
4547
|
+
/**
|
|
4548
|
+
* Store vendor-specific metadata for a span.
|
|
4549
|
+
* Note: This overwrites any existing metadata for the span.
|
|
4550
|
+
* @param args.spanId - The span ID
|
|
4551
|
+
* @param args.metadata - The vendor-specific metadata
|
|
4552
|
+
*/
|
|
4553
|
+
addMetadata(args) {
|
|
4554
|
+
this.#metadata.set(args.spanId, args.metadata);
|
|
4555
|
+
}
|
|
4556
|
+
/**
|
|
4557
|
+
* Get vendor-specific metadata for a span.
|
|
4558
|
+
* @param args.spanId - The span ID
|
|
4559
|
+
* @returns The metadata, or undefined if not found
|
|
4560
|
+
*/
|
|
4561
|
+
getMetadata(args) {
|
|
4562
|
+
return this.#metadata.get(args.spanId);
|
|
4563
|
+
}
|
|
4564
|
+
// ============================================================================
|
|
4565
|
+
// Parent Lookup Methods
|
|
4566
|
+
// ============================================================================
|
|
4567
|
+
/**
|
|
4568
|
+
* Get the parent span or event for a given span.
|
|
4569
|
+
* Looks up in both spans and events maps.
|
|
4570
|
+
* @param args.span - The span to find the parent for
|
|
4571
|
+
* @returns The parent span/event object, or undefined if root or not found
|
|
4572
|
+
*/
|
|
4573
|
+
getParent(args) {
|
|
4574
|
+
const parentId = args.span.parentSpanId;
|
|
4575
|
+
if (parentId) {
|
|
4576
|
+
if (this.#spans.has(parentId)) {
|
|
4577
|
+
return this.#spans.get(parentId);
|
|
4578
|
+
}
|
|
4579
|
+
if (this.#events.has(parentId)) {
|
|
4580
|
+
return this.#events.get(parentId);
|
|
4581
|
+
}
|
|
4582
|
+
}
|
|
4583
|
+
return void 0;
|
|
4584
|
+
}
|
|
4585
|
+
/**
|
|
4586
|
+
* Get the parent span/event or fall back to the root object.
|
|
4587
|
+
* Useful for vendors that attach child spans to either parent spans or the trace root.
|
|
4588
|
+
* @param args.span - The span to find the parent for
|
|
4589
|
+
* @returns The parent span/event, the root object, or undefined
|
|
4590
|
+
*/
|
|
4591
|
+
getParentOrRoot(args) {
|
|
4592
|
+
return this.getParent(args) ?? this.getRoot();
|
|
4593
|
+
}
|
|
4594
|
+
};
|
|
4595
|
+
var DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS = 5;
|
|
4596
|
+
var DEFAULT_EARLY_QUEUE_TTL_MS = 3e4;
|
|
4597
|
+
var DEFAULT_TRACE_CLEANUP_DELAY_MS = 3e4;
|
|
4598
|
+
var DEFAULT_MAX_PENDING_CLEANUP_TRACES = 100;
|
|
4599
|
+
var DEFAULT_MAX_TOTAL_TRACES = 500;
|
|
4600
|
+
var TrackingExporter = class extends BaseExporter {
|
|
4601
|
+
/** Map of traceId to per-trace data container */
|
|
4602
|
+
#traceMap = /* @__PURE__ */ new Map();
|
|
4603
|
+
/** Flag to prevent processing during shutdown */
|
|
4604
|
+
#shutdownStarted = false;
|
|
4605
|
+
/** Flag to prevent concurrent hard cap enforcement */
|
|
4606
|
+
#hardCapEnforcementInProgress = false;
|
|
4607
|
+
/** Map of traceId to scheduled cleanup timeout */
|
|
4608
|
+
#pendingCleanups = /* @__PURE__ */ new Map();
|
|
4609
|
+
// Note: #traceMap maintains insertion order (JS Map spec), so we use
|
|
4610
|
+
// #traceMap.keys() to iterate traces oldest-first for cap enforcement.
|
|
4611
|
+
/** Subclass configuration with resolved values */
|
|
4612
|
+
config;
|
|
4613
|
+
/** Maximum attempts to process a queued event before dropping */
|
|
4614
|
+
#earlyQueueMaxAttempts;
|
|
4615
|
+
/** TTL in milliseconds for queued events */
|
|
4616
|
+
#earlyQueueTTLMs;
|
|
4617
|
+
/** Delay before cleaning up completed traces */
|
|
4618
|
+
#traceCleanupDelayMs;
|
|
4619
|
+
/** Soft cap on traces awaiting cleanup */
|
|
4620
|
+
#maxPendingCleanupTraces;
|
|
4621
|
+
/** Hard cap on total traces (will abort active spans if exceeded) */
|
|
4622
|
+
#maxTotalTraces;
|
|
4623
|
+
constructor(config) {
|
|
4624
|
+
super(config);
|
|
4625
|
+
this.config = config;
|
|
4626
|
+
this.#earlyQueueMaxAttempts = config.earlyQueueMaxAttempts ?? DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS;
|
|
4627
|
+
this.#earlyQueueTTLMs = config.earlyQueueTTLMs ?? DEFAULT_EARLY_QUEUE_TTL_MS;
|
|
4628
|
+
this.#traceCleanupDelayMs = config.traceCleanupDelayMs ?? DEFAULT_TRACE_CLEANUP_DELAY_MS;
|
|
4629
|
+
this.#maxPendingCleanupTraces = config.maxPendingCleanupTraces ?? DEFAULT_MAX_PENDING_CLEANUP_TRACES;
|
|
4630
|
+
this.#maxTotalTraces = config.maxTotalTraces ?? DEFAULT_MAX_TOTAL_TRACES;
|
|
4631
|
+
}
|
|
4632
|
+
// ============================================================================
|
|
4633
|
+
// Early Queue Processing
|
|
4634
|
+
// ============================================================================
|
|
4635
|
+
/**
|
|
4636
|
+
* Schedule async processing of events waiting for root span.
|
|
4637
|
+
* Called after root span is successfully processed.
|
|
4638
|
+
*/
|
|
4639
|
+
#scheduleProcessWaitingForRoot(traceId) {
|
|
4640
|
+
setImmediate(() => {
|
|
4641
|
+
this.#processWaitingForRoot(traceId).catch((error) => {
|
|
4642
|
+
this.logger.error(`${this.name}: Error processing waiting-for-root queue`, { error, traceId });
|
|
4643
|
+
});
|
|
4644
|
+
});
|
|
4645
|
+
}
|
|
4646
|
+
/**
|
|
4647
|
+
* Schedule async processing of events waiting for a specific parent span.
|
|
4648
|
+
* Called after a span/event is successfully created.
|
|
4649
|
+
*/
|
|
4650
|
+
#scheduleProcessWaitingFor(traceId, spanId) {
|
|
4651
|
+
setImmediate(() => {
|
|
4652
|
+
this.#processWaitingFor(traceId, spanId).catch((error) => {
|
|
4653
|
+
this.logger.error(`${this.name}: Error processing waiting queue`, { error, traceId, spanId });
|
|
4654
|
+
});
|
|
4655
|
+
});
|
|
4656
|
+
}
|
|
4657
|
+
/**
|
|
4658
|
+
* Process all events waiting for root span.
|
|
4659
|
+
*/
|
|
4660
|
+
async #processWaitingForRoot(traceId) {
|
|
4661
|
+
if (this.#shutdownStarted) return;
|
|
4662
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4663
|
+
if (!traceData) return;
|
|
4664
|
+
const queue = traceData.getEventsWaitingForRoot();
|
|
4665
|
+
if (queue.length === 0) return;
|
|
4666
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for root`, { traceId });
|
|
4667
|
+
const toKeep = [];
|
|
4668
|
+
const now = Date.now();
|
|
4669
|
+
for (const queuedEvent of queue) {
|
|
4670
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4671
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4672
|
+
traceId,
|
|
4673
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4674
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4675
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4676
|
+
attempts: queuedEvent.attempts
|
|
4677
|
+
});
|
|
4678
|
+
continue;
|
|
4679
|
+
}
|
|
4680
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4681
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4682
|
+
traceId,
|
|
4683
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4684
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4685
|
+
attempts: queuedEvent.attempts
|
|
4686
|
+
});
|
|
4687
|
+
continue;
|
|
4688
|
+
}
|
|
4689
|
+
queuedEvent.attempts++;
|
|
4690
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4691
|
+
if (!processed) {
|
|
4692
|
+
const parentId = queuedEvent.event.exportedSpan.parentSpanId;
|
|
4693
|
+
if (parentId && traceData.isRootProcessed()) {
|
|
4694
|
+
traceData.addToWaitingQueue({
|
|
4695
|
+
event: queuedEvent.event,
|
|
4696
|
+
waitingFor: parentId,
|
|
4697
|
+
attempts: queuedEvent.attempts,
|
|
4698
|
+
queuedAt: queuedEvent.queuedAt
|
|
4699
|
+
});
|
|
4700
|
+
} else {
|
|
4701
|
+
toKeep.push(queuedEvent);
|
|
4702
|
+
}
|
|
4703
|
+
}
|
|
4704
|
+
}
|
|
4705
|
+
traceData.clearWaitingForRoot();
|
|
4706
|
+
for (const event of toKeep) {
|
|
4707
|
+
traceData.addToWaitingQueue({
|
|
4708
|
+
event: event.event,
|
|
4709
|
+
waitingFor: "root",
|
|
4710
|
+
attempts: event.attempts,
|
|
4711
|
+
queuedAt: event.queuedAt
|
|
4712
|
+
});
|
|
4713
|
+
}
|
|
4714
|
+
}
|
|
4715
|
+
/**
|
|
4716
|
+
* Process events waiting for a specific parent span.
|
|
4717
|
+
*/
|
|
4718
|
+
async #processWaitingFor(traceId, spanId) {
|
|
4719
|
+
if (this.#shutdownStarted) return;
|
|
4720
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4721
|
+
if (!traceData) return;
|
|
4722
|
+
const queue = traceData.getEventsWaitingFor({ spanId });
|
|
4723
|
+
if (queue.length === 0) return;
|
|
4724
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for span`, { traceId, spanId });
|
|
4725
|
+
const toKeep = [];
|
|
4726
|
+
const now = Date.now();
|
|
4727
|
+
for (const queuedEvent of queue) {
|
|
4728
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4729
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4730
|
+
traceId,
|
|
4731
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4732
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4733
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4734
|
+
attempts: queuedEvent.attempts
|
|
4735
|
+
});
|
|
4736
|
+
continue;
|
|
4737
|
+
}
|
|
4738
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4739
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4740
|
+
traceId,
|
|
4741
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4742
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4743
|
+
attempts: queuedEvent.attempts
|
|
4744
|
+
});
|
|
4745
|
+
continue;
|
|
4746
|
+
}
|
|
4747
|
+
queuedEvent.attempts++;
|
|
4748
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4749
|
+
if (!processed) {
|
|
4750
|
+
toKeep.push(queuedEvent);
|
|
4751
|
+
}
|
|
4752
|
+
}
|
|
4753
|
+
traceData.clearWaitingFor({ spanId });
|
|
4754
|
+
for (const event of toKeep) {
|
|
4755
|
+
traceData.addToWaitingQueue({
|
|
4756
|
+
event: event.event,
|
|
4757
|
+
waitingFor: spanId,
|
|
4758
|
+
attempts: event.attempts,
|
|
4759
|
+
queuedAt: event.queuedAt
|
|
4760
|
+
});
|
|
4761
|
+
}
|
|
4762
|
+
}
|
|
4763
|
+
/**
|
|
4764
|
+
* Try to process a queued event.
|
|
4765
|
+
* Returns true if successfully processed, false if still waiting for dependencies.
|
|
4766
|
+
*/
|
|
4767
|
+
async #tryProcessQueuedEvent(queuedEvent, traceData) {
|
|
4768
|
+
const { event } = queuedEvent;
|
|
4769
|
+
const { exportedSpan } = event;
|
|
4770
|
+
const method = this.getMethod(event);
|
|
4771
|
+
try {
|
|
4772
|
+
switch (method) {
|
|
4773
|
+
case "handleEventSpan": {
|
|
4774
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4775
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
4776
|
+
if (eventData) {
|
|
4777
|
+
if (!this.skipCachingEventSpans) {
|
|
4778
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
4779
|
+
}
|
|
4780
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4781
|
+
return true;
|
|
4782
|
+
}
|
|
4783
|
+
return false;
|
|
4784
|
+
}
|
|
4785
|
+
case "handleSpanStart": {
|
|
4786
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4787
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
4788
|
+
if (spanData) {
|
|
4789
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
4790
|
+
if (exportedSpan.isRootSpan) {
|
|
4791
|
+
traceData.markRootSpanProcessed();
|
|
4792
|
+
}
|
|
4793
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4794
|
+
return true;
|
|
4795
|
+
}
|
|
4796
|
+
return false;
|
|
4797
|
+
}
|
|
4798
|
+
case "handleSpanUpdate": {
|
|
4799
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
4800
|
+
return true;
|
|
4801
|
+
}
|
|
4802
|
+
case "handleSpanEnd": {
|
|
4803
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
4804
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
4805
|
+
if (traceData.activeSpanCount() === 0) {
|
|
4806
|
+
this.#scheduleCleanup(exportedSpan.traceId);
|
|
4807
|
+
}
|
|
4808
|
+
return true;
|
|
4809
|
+
}
|
|
4810
|
+
default:
|
|
4811
|
+
return false;
|
|
4812
|
+
}
|
|
4813
|
+
} catch (error) {
|
|
4814
|
+
this.logger.error(`${this.name}: Error processing queued event`, { error, event, method });
|
|
4815
|
+
return false;
|
|
4816
|
+
}
|
|
4817
|
+
}
|
|
4818
|
+
// ============================================================================
|
|
4819
|
+
// Delayed Cleanup
|
|
4820
|
+
// ============================================================================
|
|
4821
|
+
/**
|
|
4822
|
+
* Schedule cleanup of trace data after a delay.
|
|
4823
|
+
* Allows late-arriving data to still be processed.
|
|
4824
|
+
*/
|
|
4825
|
+
#scheduleCleanup(traceId) {
|
|
4826
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4827
|
+
this.logger.debug(`${this.name}: Scheduling cleanup in ${this.#traceCleanupDelayMs}ms`, { traceId });
|
|
4828
|
+
const timeout = setTimeout(() => {
|
|
4829
|
+
this.#pendingCleanups.delete(traceId);
|
|
4830
|
+
this.#performCleanup(traceId);
|
|
4831
|
+
}, this.#traceCleanupDelayMs);
|
|
4832
|
+
this.#pendingCleanups.set(traceId, timeout);
|
|
4833
|
+
this.#enforcePendingCleanupCap();
|
|
4834
|
+
}
|
|
4835
|
+
/**
|
|
4836
|
+
* Cancel a scheduled cleanup for a trace.
|
|
4837
|
+
*/
|
|
4838
|
+
#cancelScheduledCleanup(traceId) {
|
|
4839
|
+
const existingTimeout = this.#pendingCleanups.get(traceId);
|
|
4840
|
+
if (existingTimeout) {
|
|
4841
|
+
clearTimeout(existingTimeout);
|
|
4842
|
+
this.#pendingCleanups.delete(traceId);
|
|
4843
|
+
this.logger.debug(`${this.name}: Cancelled scheduled cleanup`, { traceId });
|
|
4844
|
+
}
|
|
4845
|
+
}
|
|
4846
|
+
/**
|
|
4847
|
+
* Perform the actual cleanup of trace data.
|
|
4848
|
+
*/
|
|
4849
|
+
#performCleanup(traceId) {
|
|
4850
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4851
|
+
if (!traceData) return;
|
|
4852
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
4853
|
+
if (orphanedEvents.length > 0) {
|
|
4854
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on cleanup`, {
|
|
4855
|
+
traceId,
|
|
4856
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
4857
|
+
spanId: e.event.exportedSpan.id,
|
|
4858
|
+
waitingFor: e.waitingFor,
|
|
4859
|
+
attempts: e.attempts,
|
|
4860
|
+
queuedAt: e.queuedAt
|
|
4861
|
+
}))
|
|
4862
|
+
});
|
|
4863
|
+
}
|
|
4864
|
+
this.#traceMap.delete(traceId);
|
|
4865
|
+
this.logger.debug(`${this.name}: Cleaned up trace data`, { traceId });
|
|
4866
|
+
}
|
|
4867
|
+
// ============================================================================
|
|
4868
|
+
// Cap Enforcement
|
|
4869
|
+
// ============================================================================
|
|
4870
|
+
/**
|
|
4871
|
+
* Enforce soft cap on pending cleanup traces.
|
|
4872
|
+
* Only removes traces with activeSpanCount == 0.
|
|
4873
|
+
*/
|
|
4874
|
+
#enforcePendingCleanupCap() {
|
|
4875
|
+
if (this.#pendingCleanups.size <= this.#maxPendingCleanupTraces) {
|
|
4876
|
+
return;
|
|
4877
|
+
}
|
|
4878
|
+
const toRemove = this.#pendingCleanups.size - this.#maxPendingCleanupTraces;
|
|
4879
|
+
this.logger.warn(`${this.name}: Pending cleanup cap exceeded, force-cleaning ${toRemove} traces`, {
|
|
4880
|
+
pendingCount: this.#pendingCleanups.size,
|
|
4881
|
+
cap: this.#maxPendingCleanupTraces
|
|
4882
|
+
});
|
|
4883
|
+
let removed = 0;
|
|
4884
|
+
for (const traceId of this.#traceMap.keys()) {
|
|
4885
|
+
if (removed >= toRemove) break;
|
|
4886
|
+
if (this.#pendingCleanups.has(traceId)) {
|
|
4887
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4888
|
+
this.#performCleanup(traceId);
|
|
4889
|
+
removed++;
|
|
4890
|
+
}
|
|
4891
|
+
}
|
|
4892
|
+
}
|
|
4893
|
+
/**
|
|
4894
|
+
* Enforce hard cap on total traces.
|
|
4895
|
+
* Will kill even active traces if necessary.
|
|
4896
|
+
* Uses a flag to prevent concurrent executions when called fire-and-forget.
|
|
4897
|
+
*/
|
|
4898
|
+
async #enforceHardCap() {
|
|
4899
|
+
if (this.#traceMap.size <= this.#maxTotalTraces || this.#hardCapEnforcementInProgress) {
|
|
4900
|
+
return;
|
|
4901
|
+
}
|
|
4902
|
+
this.#hardCapEnforcementInProgress = true;
|
|
4903
|
+
try {
|
|
4904
|
+
if (this.#traceMap.size <= this.#maxTotalTraces) {
|
|
4905
|
+
return;
|
|
4906
|
+
}
|
|
4907
|
+
const toRemove = this.#traceMap.size - this.#maxTotalTraces;
|
|
4908
|
+
this.logger.warn(`${this.name}: Total trace cap exceeded, killing ${toRemove} oldest traces`, {
|
|
4909
|
+
traceCount: this.#traceMap.size,
|
|
4910
|
+
cap: this.#maxTotalTraces
|
|
4911
|
+
});
|
|
4912
|
+
const reason = {
|
|
4913
|
+
id: "TRACE_CAP_EXCEEDED",
|
|
4914
|
+
message: "Trace killed due to memory cap enforcement.",
|
|
4915
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
4916
|
+
category: "SYSTEM"
|
|
4917
|
+
};
|
|
4918
|
+
let removed = 0;
|
|
4919
|
+
for (const traceId of [...this.#traceMap.keys()]) {
|
|
4920
|
+
if (removed >= toRemove) break;
|
|
4921
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4922
|
+
if (traceData) {
|
|
4923
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
4924
|
+
const span = traceData.getSpan({ spanId });
|
|
4925
|
+
if (span) {
|
|
4926
|
+
await this._abortSpan({ span, traceData, reason });
|
|
4927
|
+
}
|
|
4928
|
+
}
|
|
4929
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4930
|
+
this.#performCleanup(traceId);
|
|
4931
|
+
removed++;
|
|
4932
|
+
}
|
|
4933
|
+
}
|
|
4934
|
+
} finally {
|
|
4935
|
+
this.#hardCapEnforcementInProgress = false;
|
|
4936
|
+
}
|
|
4937
|
+
}
|
|
4938
|
+
// ============================================================================
|
|
4939
|
+
// Lifecycle Hooks (Override in subclass)
|
|
4940
|
+
// ============================================================================
|
|
4941
|
+
/**
|
|
4942
|
+
* Hook called before processing each tracing event.
|
|
4943
|
+
* Override to transform or enrich the event before processing.
|
|
4944
|
+
*
|
|
4945
|
+
* Note: The customSpanFormatter is applied at the BaseExporter level before this hook.
|
|
4946
|
+
* Subclasses can override this to add additional pre-processing logic.
|
|
4947
|
+
*
|
|
4948
|
+
* @param event - The incoming tracing event
|
|
4949
|
+
* @returns The (possibly modified) event to process
|
|
4950
|
+
*/
|
|
4951
|
+
async _preExportTracingEvent(event) {
|
|
4952
|
+
return event;
|
|
4953
|
+
}
|
|
4954
|
+
/**
|
|
4955
|
+
* Hook called after processing each tracing event.
|
|
4956
|
+
* Override to perform post-processing actions like flushing.
|
|
4957
|
+
*/
|
|
4958
|
+
async _postExportTracingEvent() {
|
|
4959
|
+
}
|
|
4960
|
+
// ============================================================================
|
|
4961
|
+
// Behavior Flags (Override in subclass as needed)
|
|
4962
|
+
// ============================================================================
|
|
4963
|
+
/**
|
|
4964
|
+
* If true, skip calling _buildRoot and let root spans go through _buildSpan.
|
|
4965
|
+
* Use when the vendor doesn't have a separate trace/root concept.
|
|
4966
|
+
* @default false
|
|
4967
|
+
*/
|
|
4968
|
+
skipBuildRootTask = false;
|
|
4969
|
+
/**
|
|
4970
|
+
* If true, skip processing span_updated events entirely.
|
|
4971
|
+
* Use when the vendor doesn't support incremental span updates.
|
|
4972
|
+
* @default false
|
|
4973
|
+
*/
|
|
4974
|
+
skipSpanUpdateEvents = false;
|
|
4975
|
+
/**
|
|
4976
|
+
* If true, don't cache event spans in TraceData.
|
|
4977
|
+
* Use when events can't be parents of other spans.
|
|
4978
|
+
* @default false
|
|
4979
|
+
*/
|
|
4980
|
+
skipCachingEventSpans = false;
|
|
4981
|
+
getMethod(event) {
|
|
4982
|
+
if (event.exportedSpan.isEvent) {
|
|
4983
|
+
return "handleEventSpan";
|
|
4984
|
+
}
|
|
4985
|
+
const eventType = event.type;
|
|
4986
|
+
switch (eventType) {
|
|
4987
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
4988
|
+
return "handleSpanStart";
|
|
4989
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
4990
|
+
return "handleSpanUpdate";
|
|
4991
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
4992
|
+
return "handleSpanEnd";
|
|
4993
|
+
default: {
|
|
4994
|
+
const _exhaustiveCheck = eventType;
|
|
4995
|
+
throw new Error(`Unhandled event type: ${_exhaustiveCheck}`);
|
|
4996
|
+
}
|
|
4997
|
+
}
|
|
4998
|
+
}
|
|
4999
|
+
async _exportTracingEvent(event) {
|
|
5000
|
+
if (this.#shutdownStarted) {
|
|
5001
|
+
return;
|
|
5002
|
+
}
|
|
5003
|
+
const method = this.getMethod(event);
|
|
5004
|
+
if (method == "handleSpanUpdate" && this.skipSpanUpdateEvents) {
|
|
5005
|
+
return;
|
|
5006
|
+
}
|
|
5007
|
+
const traceId = event.exportedSpan.traceId;
|
|
5008
|
+
const traceData = this.getTraceData({ traceId, method });
|
|
5009
|
+
const { exportedSpan } = await this._preExportTracingEvent(event);
|
|
5010
|
+
if (!this.skipBuildRootTask && !traceData.hasRoot()) {
|
|
5011
|
+
if (exportedSpan.isRootSpan) {
|
|
5012
|
+
this.logger.debug(`${this.name}: Building root`, {
|
|
5013
|
+
traceId: exportedSpan.traceId,
|
|
5014
|
+
spanId: exportedSpan.id
|
|
5015
|
+
});
|
|
5016
|
+
const rootData = await this._buildRoot({ span: exportedSpan, traceData });
|
|
5017
|
+
if (rootData) {
|
|
5018
|
+
this.logger.debug(`${this.name}: Adding root`, {
|
|
5019
|
+
traceId: exportedSpan.traceId,
|
|
5020
|
+
spanId: exportedSpan.id
|
|
5021
|
+
});
|
|
5022
|
+
traceData.addRoot({ rootId: exportedSpan.id, rootData });
|
|
5023
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5024
|
+
}
|
|
5025
|
+
} else {
|
|
5026
|
+
this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
|
|
5027
|
+
traceId: exportedSpan.traceId,
|
|
5028
|
+
spanId: exportedSpan.id
|
|
5029
|
+
});
|
|
5030
|
+
traceData.addToWaitingQueue({ event, waitingFor: "root" });
|
|
5031
|
+
return;
|
|
5032
|
+
}
|
|
5033
|
+
}
|
|
5034
|
+
if (exportedSpan.metadata && this.name in exportedSpan.metadata) {
|
|
5035
|
+
const metadata = exportedSpan.metadata[this.name];
|
|
5036
|
+
this.logger.debug(`${this.name}: Found provider metadata in span`, {
|
|
5037
|
+
traceId: exportedSpan.traceId,
|
|
5038
|
+
spanId: exportedSpan.id,
|
|
5039
|
+
metadata
|
|
5040
|
+
});
|
|
5041
|
+
traceData.addMetadata({ spanId: exportedSpan.id, metadata });
|
|
5042
|
+
}
|
|
5043
|
+
try {
|
|
5044
|
+
switch (method) {
|
|
5045
|
+
case "handleEventSpan": {
|
|
5046
|
+
this.logger.debug(`${this.name}: handling event`, {
|
|
5047
|
+
traceId: exportedSpan.traceId,
|
|
5048
|
+
spanId: exportedSpan.id
|
|
5049
|
+
});
|
|
5050
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5051
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
5052
|
+
if (eventData) {
|
|
5053
|
+
if (!this.skipCachingEventSpans) {
|
|
5054
|
+
this.logger.debug(`${this.name}: adding event to traceData`, {
|
|
5055
|
+
traceId: exportedSpan.traceId,
|
|
5056
|
+
spanId: exportedSpan.id
|
|
5057
|
+
});
|
|
5058
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
5059
|
+
}
|
|
5060
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5061
|
+
} else {
|
|
5062
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5063
|
+
this.logger.debug(`${this.name}: adding event to waiting queue`, {
|
|
5064
|
+
traceId: exportedSpan.traceId,
|
|
5065
|
+
spanId: exportedSpan.id,
|
|
5066
|
+
waitingFor: parentId ?? "root"
|
|
5067
|
+
});
|
|
5068
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5069
|
+
}
|
|
5070
|
+
break;
|
|
5071
|
+
}
|
|
5072
|
+
case "handleSpanStart": {
|
|
5073
|
+
this.logger.debug(`${this.name}: handling span start`, {
|
|
5074
|
+
traceId: exportedSpan.traceId,
|
|
5075
|
+
spanId: exportedSpan.id
|
|
5076
|
+
});
|
|
5077
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5078
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
5079
|
+
if (spanData) {
|
|
5080
|
+
this.logger.debug(`${this.name}: adding span to traceData`, {
|
|
5081
|
+
traceId: exportedSpan.traceId,
|
|
5082
|
+
spanId: exportedSpan.id
|
|
5083
|
+
});
|
|
5084
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
5085
|
+
if (exportedSpan.isRootSpan) {
|
|
5086
|
+
traceData.markRootSpanProcessed();
|
|
5087
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5088
|
+
}
|
|
5089
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5090
|
+
} else {
|
|
5091
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5092
|
+
this.logger.debug(`${this.name}: adding span to waiting queue`, {
|
|
5093
|
+
traceId: exportedSpan.traceId,
|
|
5094
|
+
waitingFor: parentId ?? "root"
|
|
5095
|
+
});
|
|
5096
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5097
|
+
}
|
|
5098
|
+
break;
|
|
5099
|
+
}
|
|
5100
|
+
case "handleSpanUpdate":
|
|
5101
|
+
this.logger.debug(`${this.name}: handling span update`, {
|
|
5102
|
+
traceId: exportedSpan.traceId,
|
|
5103
|
+
spanId: exportedSpan.id
|
|
5104
|
+
});
|
|
5105
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
5106
|
+
break;
|
|
5107
|
+
case "handleSpanEnd":
|
|
5108
|
+
this.logger.debug(`${this.name}: handling span end`, {
|
|
5109
|
+
traceId: exportedSpan.traceId,
|
|
5110
|
+
spanId: exportedSpan.id
|
|
5111
|
+
});
|
|
5112
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
5113
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
5114
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5115
|
+
this.#scheduleCleanup(traceId);
|
|
5116
|
+
}
|
|
5117
|
+
break;
|
|
5118
|
+
}
|
|
5119
|
+
} catch (error) {
|
|
5120
|
+
this.logger.error(`${this.name}: exporter error`, { error, event, method });
|
|
5121
|
+
}
|
|
5122
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5123
|
+
this.#scheduleCleanup(traceId);
|
|
5124
|
+
}
|
|
5125
|
+
await this._postExportTracingEvent();
|
|
5126
|
+
}
|
|
5127
|
+
// ============================================================================
|
|
5128
|
+
// Protected Helpers
|
|
5129
|
+
// ============================================================================
|
|
5130
|
+
/**
|
|
5131
|
+
* Get or create the TraceData container for a trace.
|
|
5132
|
+
* Also cancels any pending cleanup since new data has arrived.
|
|
5133
|
+
*
|
|
5134
|
+
* @param args.traceId - The trace ID
|
|
5135
|
+
* @param args.method - The calling method name (for logging)
|
|
5136
|
+
* @returns The TraceData container for this trace
|
|
5137
|
+
*/
|
|
5138
|
+
getTraceData(args) {
|
|
5139
|
+
const { traceId, method } = args;
|
|
5140
|
+
this.#cancelScheduledCleanup(traceId);
|
|
5141
|
+
if (!this.#traceMap.has(traceId)) {
|
|
5142
|
+
this.#traceMap.set(traceId, new TraceData());
|
|
5143
|
+
this.logger.debug(`${this.name}: Created new trace data cache`, {
|
|
5144
|
+
traceId,
|
|
5145
|
+
method
|
|
5146
|
+
});
|
|
5147
|
+
this.#enforceHardCap().catch((error) => {
|
|
5148
|
+
this.logger.error(`${this.name}: Error enforcing hard cap`, { error });
|
|
5149
|
+
});
|
|
5150
|
+
}
|
|
5151
|
+
return this.#traceMap.get(traceId);
|
|
5152
|
+
}
|
|
5153
|
+
/**
|
|
5154
|
+
* Get the current number of traces being tracked.
|
|
5155
|
+
* @returns The trace count
|
|
5156
|
+
*/
|
|
5157
|
+
traceMapSize() {
|
|
5158
|
+
return this.#traceMap.size;
|
|
5159
|
+
}
|
|
5160
|
+
// ============================================================================
|
|
5161
|
+
// Flush and Shutdown Hooks (Override in subclass as needed)
|
|
5162
|
+
// ============================================================================
|
|
5163
|
+
/**
|
|
5164
|
+
* Hook called by flush() to perform vendor-specific flush logic.
|
|
5165
|
+
* Override to send buffered data to the vendor's API.
|
|
5166
|
+
*
|
|
5167
|
+
* Unlike _postShutdown(), this method should NOT release resources,
|
|
5168
|
+
* as the exporter will continue to be used after flushing.
|
|
5169
|
+
*/
|
|
5170
|
+
async _flush() {
|
|
5171
|
+
}
|
|
5172
|
+
/**
|
|
5173
|
+
* Force flush any buffered data without shutting down the exporter.
|
|
5174
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5175
|
+
* are exported before the runtime instance is terminated.
|
|
5176
|
+
*
|
|
5177
|
+
* Subclasses should override _flush() to implement vendor-specific flush logic.
|
|
5178
|
+
*/
|
|
5179
|
+
async flush() {
|
|
5180
|
+
if (this.isDisabled) {
|
|
5181
|
+
return;
|
|
5182
|
+
}
|
|
5183
|
+
this.logger.debug(`${this.name}: Flushing`);
|
|
5184
|
+
await this._flush();
|
|
5185
|
+
}
|
|
5186
|
+
/**
|
|
5187
|
+
* Hook called at the start of shutdown, before cancelling timers and aborting spans.
|
|
5188
|
+
* Override to perform vendor-specific pre-shutdown tasks.
|
|
5189
|
+
*/
|
|
5190
|
+
async _preShutdown() {
|
|
5191
|
+
}
|
|
5192
|
+
/**
|
|
5193
|
+
* Hook called at the end of shutdown, after all spans are aborted.
|
|
5194
|
+
* Override to perform vendor-specific cleanup (e.g., flushing).
|
|
5195
|
+
*/
|
|
5196
|
+
async _postShutdown() {
|
|
5197
|
+
}
|
|
5198
|
+
/**
|
|
5199
|
+
* Gracefully shut down the exporter.
|
|
5200
|
+
* Cancels all pending cleanup timers, aborts all active spans, and clears state.
|
|
5201
|
+
*/
|
|
5202
|
+
async shutdown() {
|
|
5203
|
+
if (this.isDisabled) {
|
|
5204
|
+
return;
|
|
5205
|
+
}
|
|
5206
|
+
this.#shutdownStarted = true;
|
|
5207
|
+
await this._preShutdown();
|
|
5208
|
+
for (const [traceId, timeout] of this.#pendingCleanups) {
|
|
5209
|
+
clearTimeout(timeout);
|
|
5210
|
+
this.logger.debug(`${this.name}: Cancelled pending cleanup on shutdown`, { traceId });
|
|
5211
|
+
}
|
|
5212
|
+
this.#pendingCleanups.clear();
|
|
5213
|
+
const reason = {
|
|
5214
|
+
id: "SHUTDOWN",
|
|
5215
|
+
message: "Observability is shutting down.",
|
|
5216
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
5217
|
+
category: "SYSTEM"
|
|
5218
|
+
};
|
|
5219
|
+
for (const [traceId, traceData] of this.#traceMap) {
|
|
5220
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
5221
|
+
if (orphanedEvents.length > 0) {
|
|
5222
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on shutdown`, {
|
|
5223
|
+
traceId,
|
|
5224
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
5225
|
+
spanId: e.event.exportedSpan.id,
|
|
5226
|
+
waitingFor: e.waitingFor,
|
|
5227
|
+
attempts: e.attempts
|
|
5228
|
+
}))
|
|
5229
|
+
});
|
|
5230
|
+
}
|
|
5231
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
5232
|
+
const span = traceData.getSpan({ spanId });
|
|
5233
|
+
if (span) {
|
|
5234
|
+
await this._abortSpan({ span, traceData, reason });
|
|
5235
|
+
}
|
|
5236
|
+
}
|
|
5237
|
+
}
|
|
5238
|
+
this.#traceMap.clear();
|
|
5239
|
+
await this._postShutdown();
|
|
5240
|
+
await super.shutdown();
|
|
5241
|
+
}
|
|
5242
|
+
};
|
|
5243
|
+
|
|
5244
|
+
// src/exporters/span-formatters.ts
|
|
5245
|
+
function chainFormatters(formatters) {
|
|
5246
|
+
return async (span) => {
|
|
5247
|
+
let currentSpan = span;
|
|
5248
|
+
for (const formatter of formatters) {
|
|
5249
|
+
currentSpan = await formatter(currentSpan);
|
|
5250
|
+
}
|
|
5251
|
+
return currentSpan;
|
|
5252
|
+
};
|
|
5253
|
+
}
|
|
5254
|
+
var CloudExporter = class extends BaseExporter {
|
|
5255
|
+
name = "mastra-cloud-observability-exporter";
|
|
5256
|
+
cloudConfig;
|
|
5257
|
+
buffer;
|
|
5258
|
+
flushTimer = null;
|
|
5259
|
+
constructor(config = {}) {
|
|
5260
|
+
super(config);
|
|
5261
|
+
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
5262
|
+
if (!accessToken) {
|
|
5263
|
+
this.setDisabled("MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.");
|
|
5264
|
+
}
|
|
5265
|
+
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
5266
|
+
this.cloudConfig = {
|
|
5267
|
+
logger: this.logger,
|
|
5268
|
+
logLevel: config.logLevel ?? logger.LogLevel.INFO,
|
|
5269
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5270
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5271
|
+
maxRetries: config.maxRetries ?? 3,
|
|
5272
|
+
accessToken: accessToken || "",
|
|
5273
|
+
endpoint
|
|
5274
|
+
};
|
|
5275
|
+
this.buffer = {
|
|
5276
|
+
spans: [],
|
|
5277
|
+
totalSize: 0
|
|
5278
|
+
};
|
|
5279
|
+
}
|
|
5280
|
+
async _exportTracingEvent(event) {
|
|
5281
|
+
if (event.type !== observability.TracingEventType.SPAN_ENDED) {
|
|
5282
|
+
return;
|
|
5283
|
+
}
|
|
5284
|
+
this.addToBuffer(event);
|
|
5285
|
+
if (this.shouldFlush()) {
|
|
5286
|
+
this.flush().catch((error) => {
|
|
5287
|
+
this.logger.error("Batch flush failed", {
|
|
5288
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5289
|
+
});
|
|
5290
|
+
});
|
|
5291
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5292
|
+
this.scheduleFlush();
|
|
5293
|
+
}
|
|
5294
|
+
}
|
|
5295
|
+
addToBuffer(event) {
|
|
5296
|
+
if (this.buffer.totalSize === 0) {
|
|
5297
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5298
|
+
}
|
|
5299
|
+
const spanRecord = this.formatSpan(event.exportedSpan);
|
|
5300
|
+
this.buffer.spans.push(spanRecord);
|
|
5301
|
+
this.buffer.totalSize++;
|
|
5302
|
+
}
|
|
5303
|
+
formatSpan(span) {
|
|
5304
|
+
const spanRecord = {
|
|
5305
|
+
traceId: span.traceId,
|
|
5306
|
+
spanId: span.id,
|
|
5307
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5308
|
+
name: span.name,
|
|
5309
|
+
spanType: span.type,
|
|
5310
|
+
attributes: span.attributes ?? null,
|
|
5311
|
+
metadata: span.metadata ?? null,
|
|
5312
|
+
startedAt: span.startTime,
|
|
5313
|
+
endedAt: span.endTime ?? null,
|
|
5314
|
+
input: span.input ?? null,
|
|
5315
|
+
output: span.output ?? null,
|
|
5316
|
+
error: span.errorInfo,
|
|
5317
|
+
isEvent: span.isEvent,
|
|
5318
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
5319
|
+
updatedAt: null
|
|
5320
|
+
};
|
|
5321
|
+
return spanRecord;
|
|
5322
|
+
}
|
|
5323
|
+
shouldFlush() {
|
|
5324
|
+
if (this.buffer.totalSize >= this.cloudConfig.maxBatchSize) {
|
|
5325
|
+
return true;
|
|
5326
|
+
}
|
|
5327
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5328
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5329
|
+
if (elapsed >= this.cloudConfig.maxBatchWaitMs) {
|
|
5330
|
+
return true;
|
|
5331
|
+
}
|
|
5332
|
+
}
|
|
5333
|
+
return false;
|
|
5334
|
+
}
|
|
5335
|
+
scheduleFlush() {
|
|
5336
|
+
if (this.flushTimer) {
|
|
5337
|
+
clearTimeout(this.flushTimer);
|
|
5338
|
+
}
|
|
5339
|
+
this.flushTimer = setTimeout(() => {
|
|
5340
|
+
this.flush().catch((error$1) => {
|
|
5341
|
+
const mastraError = new error.MastraError(
|
|
5342
|
+
{
|
|
5343
|
+
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
5344
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5345
|
+
category: error.ErrorCategory.USER
|
|
5346
|
+
},
|
|
5347
|
+
error$1
|
|
5348
|
+
);
|
|
5349
|
+
this.logger.trackException(mastraError);
|
|
5350
|
+
this.logger.error("Scheduled flush failed", mastraError);
|
|
5351
|
+
});
|
|
5352
|
+
}, this.cloudConfig.maxBatchWaitMs);
|
|
5353
|
+
}
|
|
5354
|
+
async flushBuffer() {
|
|
5355
|
+
if (this.flushTimer) {
|
|
5356
|
+
clearTimeout(this.flushTimer);
|
|
5357
|
+
this.flushTimer = null;
|
|
5358
|
+
}
|
|
5359
|
+
if (this.buffer.totalSize === 0) {
|
|
5360
|
+
return;
|
|
5361
|
+
}
|
|
5362
|
+
const startTime = Date.now();
|
|
5363
|
+
const spansCopy = [...this.buffer.spans];
|
|
5364
|
+
const flushReason = this.buffer.totalSize >= this.cloudConfig.maxBatchSize ? "size" : "time";
|
|
5365
|
+
this.resetBuffer();
|
|
5366
|
+
try {
|
|
5367
|
+
await this.batchUpload(spansCopy);
|
|
5368
|
+
const elapsed = Date.now() - startTime;
|
|
5369
|
+
this.logger.debug("Batch flushed successfully", {
|
|
5370
|
+
batchSize: spansCopy.length,
|
|
5371
|
+
flushReason,
|
|
5372
|
+
durationMs: elapsed
|
|
5373
|
+
});
|
|
5374
|
+
} catch (error$1) {
|
|
5375
|
+
const mastraError = new error.MastraError(
|
|
5376
|
+
{
|
|
5377
|
+
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
5378
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5379
|
+
category: error.ErrorCategory.USER,
|
|
5380
|
+
details: {
|
|
5381
|
+
droppedBatchSize: spansCopy.length
|
|
5382
|
+
}
|
|
5383
|
+
},
|
|
5384
|
+
error$1
|
|
5385
|
+
);
|
|
5386
|
+
this.logger.trackException(mastraError);
|
|
5387
|
+
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
5388
|
+
}
|
|
5389
|
+
}
|
|
5390
|
+
/**
|
|
5391
|
+
* Uploads spans to cloud API using fetchWithRetry for all retry logic
|
|
5392
|
+
*/
|
|
5393
|
+
async batchUpload(spans) {
|
|
5394
|
+
const headers = {
|
|
5395
|
+
Authorization: `Bearer ${this.cloudConfig.accessToken}`,
|
|
5396
|
+
"Content-Type": "application/json"
|
|
5397
|
+
};
|
|
5398
|
+
const options = {
|
|
5399
|
+
method: "POST",
|
|
5400
|
+
headers,
|
|
5401
|
+
body: JSON.stringify({ spans })
|
|
5402
|
+
};
|
|
5403
|
+
await utils.fetchWithRetry(this.cloudConfig.endpoint, options, this.cloudConfig.maxRetries);
|
|
5404
|
+
}
|
|
5405
|
+
resetBuffer() {
|
|
5406
|
+
this.buffer.spans = [];
|
|
5407
|
+
this.buffer.firstEventTime = void 0;
|
|
5408
|
+
this.buffer.totalSize = 0;
|
|
5409
|
+
}
|
|
5410
|
+
/**
|
|
5411
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
5412
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5413
|
+
* are exported before the runtime instance is terminated.
|
|
5414
|
+
*/
|
|
5415
|
+
async flush() {
|
|
5416
|
+
if (this.isDisabled) {
|
|
5417
|
+
return;
|
|
5418
|
+
}
|
|
5419
|
+
if (this.buffer.totalSize > 0) {
|
|
5420
|
+
this.logger.debug("Flushing buffered events", {
|
|
5421
|
+
bufferedEvents: this.buffer.totalSize
|
|
5422
|
+
});
|
|
5423
|
+
await this.flushBuffer();
|
|
5424
|
+
}
|
|
5425
|
+
}
|
|
5426
|
+
async shutdown() {
|
|
5427
|
+
if (this.isDisabled) {
|
|
5428
|
+
return;
|
|
5429
|
+
}
|
|
5430
|
+
if (this.flushTimer) {
|
|
5431
|
+
clearTimeout(this.flushTimer);
|
|
5432
|
+
this.flushTimer = null;
|
|
5433
|
+
}
|
|
5434
|
+
try {
|
|
5435
|
+
await this.flush();
|
|
5436
|
+
} catch (error$1) {
|
|
5437
|
+
const mastraError = new error.MastraError(
|
|
5438
|
+
{
|
|
5439
|
+
id: `CLOUD_EXPORTER_FAILED_TO_FLUSH_REMAINING_EVENTS_DURING_SHUTDOWN`,
|
|
5440
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5441
|
+
category: error.ErrorCategory.USER,
|
|
5442
|
+
details: {
|
|
5443
|
+
remainingEvents: this.buffer.totalSize
|
|
5444
|
+
}
|
|
5445
|
+
},
|
|
5446
|
+
error$1
|
|
5447
|
+
);
|
|
5448
|
+
this.logger.trackException(mastraError);
|
|
5449
|
+
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
5450
|
+
}
|
|
5451
|
+
this.logger.info("CloudExporter shutdown complete");
|
|
5452
|
+
}
|
|
5453
|
+
};
|
|
5454
|
+
var ConsoleExporter = class extends BaseExporter {
|
|
5455
|
+
name = "tracing-console-exporter";
|
|
5456
|
+
constructor(config = {}) {
|
|
5457
|
+
super(config);
|
|
5458
|
+
}
|
|
5459
|
+
async _exportTracingEvent(event) {
|
|
5460
|
+
const span = event.exportedSpan;
|
|
5461
|
+
const formatAttributes = (attributes) => {
|
|
5462
|
+
try {
|
|
5463
|
+
return JSON.stringify(attributes, null, 2);
|
|
5464
|
+
} catch (error) {
|
|
5465
|
+
const errMsg = error instanceof Error ? error.message : "Unknown formatting error";
|
|
5466
|
+
return `[Unable to serialize attributes: ${errMsg}]`;
|
|
5467
|
+
}
|
|
5468
|
+
};
|
|
5469
|
+
const formatDuration = (startTime, endTime) => {
|
|
5470
|
+
if (!endTime) return "N/A";
|
|
5471
|
+
const duration = endTime.getTime() - startTime.getTime();
|
|
5472
|
+
return `${duration}ms`;
|
|
5473
|
+
};
|
|
5474
|
+
switch (event.type) {
|
|
5475
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
5476
|
+
this.logger.info(`\u{1F680} SPAN_STARTED`);
|
|
5477
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5478
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5479
|
+
this.logger.info(` ID: ${span.id}`);
|
|
5480
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5481
|
+
if (span.input !== void 0) {
|
|
5482
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5483
|
+
}
|
|
5484
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5485
|
+
this.logger.info("\u2500".repeat(80));
|
|
5486
|
+
break;
|
|
5487
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
5488
|
+
const duration = formatDuration(span.startTime, span.endTime);
|
|
5489
|
+
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
5490
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5491
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5492
|
+
this.logger.info(` ID: ${span.id}`);
|
|
5493
|
+
this.logger.info(` Duration: ${duration}`);
|
|
5494
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5495
|
+
if (span.input !== void 0) {
|
|
5496
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5497
|
+
}
|
|
5498
|
+
if (span.output !== void 0) {
|
|
5499
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5500
|
+
}
|
|
5501
|
+
if (span.errorInfo) {
|
|
5502
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5503
|
+
}
|
|
5504
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5505
|
+
this.logger.info("\u2500".repeat(80));
|
|
5506
|
+
break;
|
|
5507
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
5508
|
+
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
4449
5509
|
this.logger.info(` Type: ${span.type}`);
|
|
4450
5510
|
this.logger.info(` Name: ${span.name}`);
|
|
4451
5511
|
this.logger.info(` ID: ${span.id}`);
|
|
4452
|
-
this.logger.info(` Duration: ${duration}`);
|
|
4453
5512
|
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
4454
5513
|
if (span.input !== void 0) {
|
|
4455
5514
|
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
4456
5515
|
}
|
|
4457
|
-
if (span.output !== void 0) {
|
|
4458
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5516
|
+
if (span.output !== void 0) {
|
|
5517
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5518
|
+
}
|
|
5519
|
+
if (span.errorInfo) {
|
|
5520
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5521
|
+
}
|
|
5522
|
+
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
5523
|
+
this.logger.info("\u2500".repeat(80));
|
|
5524
|
+
break;
|
|
5525
|
+
default:
|
|
5526
|
+
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
5527
|
+
}
|
|
5528
|
+
}
|
|
5529
|
+
async shutdown() {
|
|
5530
|
+
this.logger.info("ConsoleExporter shutdown");
|
|
5531
|
+
}
|
|
5532
|
+
};
|
|
5533
|
+
function resolveTracingStorageStrategy(config, observability, storageName, logger) {
|
|
5534
|
+
if (config.strategy && config.strategy !== "auto") {
|
|
5535
|
+
const hints = observability.tracingStrategy;
|
|
5536
|
+
if (hints.supported.includes(config.strategy)) {
|
|
5537
|
+
return config.strategy;
|
|
5538
|
+
}
|
|
5539
|
+
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
5540
|
+
userStrategy: config.strategy,
|
|
5541
|
+
storageAdapter: storageName,
|
|
5542
|
+
supportedStrategies: hints.supported,
|
|
5543
|
+
fallbackStrategy: hints.preferred
|
|
5544
|
+
});
|
|
5545
|
+
}
|
|
5546
|
+
return observability.tracingStrategy.preferred;
|
|
5547
|
+
}
|
|
5548
|
+
function getStringOrNull(value) {
|
|
5549
|
+
return typeof value === "string" ? value : null;
|
|
5550
|
+
}
|
|
5551
|
+
function getObjectOrNull(value) {
|
|
5552
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
5553
|
+
}
|
|
5554
|
+
var DefaultExporter = class extends BaseExporter {
|
|
5555
|
+
name = "mastra-default-observability-exporter";
|
|
5556
|
+
#storage;
|
|
5557
|
+
#observability;
|
|
5558
|
+
#config;
|
|
5559
|
+
#resolvedStrategy;
|
|
5560
|
+
buffer;
|
|
5561
|
+
#flushTimer = null;
|
|
5562
|
+
// Track all spans that have been created, persists across flushes
|
|
5563
|
+
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
5564
|
+
constructor(config = {}) {
|
|
5565
|
+
super(config);
|
|
5566
|
+
if (config === void 0) {
|
|
5567
|
+
config = {};
|
|
5568
|
+
}
|
|
5569
|
+
this.#config = {
|
|
5570
|
+
...config,
|
|
5571
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5572
|
+
maxBufferSize: config.maxBufferSize ?? 1e4,
|
|
5573
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5574
|
+
maxRetries: config.maxRetries ?? 4,
|
|
5575
|
+
retryDelayMs: config.retryDelayMs ?? 500,
|
|
5576
|
+
strategy: config.strategy ?? "auto"
|
|
5577
|
+
};
|
|
5578
|
+
this.buffer = {
|
|
5579
|
+
creates: [],
|
|
5580
|
+
updates: [],
|
|
5581
|
+
insertOnly: [],
|
|
5582
|
+
seenSpans: /* @__PURE__ */ new Set(),
|
|
5583
|
+
spanSequences: /* @__PURE__ */ new Map(),
|
|
5584
|
+
completedSpans: /* @__PURE__ */ new Set(),
|
|
5585
|
+
outOfOrderCount: 0,
|
|
5586
|
+
totalSize: 0
|
|
5587
|
+
};
|
|
5588
|
+
this.#resolvedStrategy = "batch-with-updates";
|
|
5589
|
+
}
|
|
5590
|
+
#strategyInitialized = false;
|
|
5591
|
+
/**
|
|
5592
|
+
* Initialize the exporter (called after all dependencies are ready)
|
|
5593
|
+
*/
|
|
5594
|
+
async init(options) {
|
|
5595
|
+
this.#storage = options.mastra?.getStorage();
|
|
5596
|
+
if (!this.#storage) {
|
|
5597
|
+
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
5598
|
+
return;
|
|
5599
|
+
}
|
|
5600
|
+
this.#observability = await this.#storage.getStore("observability");
|
|
5601
|
+
if (!this.#observability) {
|
|
5602
|
+
this.logger.warn("DefaultExporter disabled: Observability storage not available. Traces will not be persisted.");
|
|
5603
|
+
return;
|
|
5604
|
+
}
|
|
5605
|
+
this.initializeStrategy(this.#observability, this.#storage.constructor.name);
|
|
5606
|
+
}
|
|
5607
|
+
/**
|
|
5608
|
+
* Initialize the resolved strategy once observability store is available
|
|
5609
|
+
*/
|
|
5610
|
+
initializeStrategy(observability, storageName) {
|
|
5611
|
+
if (this.#strategyInitialized) return;
|
|
5612
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, observability, storageName, this.logger);
|
|
5613
|
+
this.#strategyInitialized = true;
|
|
5614
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
5615
|
+
strategy: this.#resolvedStrategy,
|
|
5616
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
5617
|
+
storageAdapter: storageName,
|
|
5618
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
5619
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
5620
|
+
});
|
|
5621
|
+
}
|
|
5622
|
+
/**
|
|
5623
|
+
* Builds a unique span key for tracking
|
|
5624
|
+
*/
|
|
5625
|
+
buildSpanKey(traceId, spanId) {
|
|
5626
|
+
return `${traceId}:${spanId}`;
|
|
5627
|
+
}
|
|
5628
|
+
/**
|
|
5629
|
+
* Gets the next sequence number for a span
|
|
5630
|
+
*/
|
|
5631
|
+
getNextSequence(spanKey) {
|
|
5632
|
+
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
5633
|
+
const next = current + 1;
|
|
5634
|
+
this.buffer.spanSequences.set(spanKey, next);
|
|
5635
|
+
return next;
|
|
5636
|
+
}
|
|
5637
|
+
/**
|
|
5638
|
+
* Handles out-of-order span updates by logging and skipping
|
|
5639
|
+
*/
|
|
5640
|
+
handleOutOfOrderUpdate(event) {
|
|
5641
|
+
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
5642
|
+
spanId: event.exportedSpan.id,
|
|
5643
|
+
traceId: event.exportedSpan.traceId,
|
|
5644
|
+
spanName: event.exportedSpan.name,
|
|
5645
|
+
eventType: event.type
|
|
5646
|
+
});
|
|
5647
|
+
}
|
|
5648
|
+
/**
|
|
5649
|
+
* Adds an event to the appropriate buffer based on strategy
|
|
5650
|
+
*/
|
|
5651
|
+
addToBuffer(event) {
|
|
5652
|
+
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
5653
|
+
if (this.buffer.totalSize === 0) {
|
|
5654
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5655
|
+
}
|
|
5656
|
+
switch (event.type) {
|
|
5657
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
5658
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5659
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5660
|
+
this.buffer.creates.push(createRecord);
|
|
5661
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5662
|
+
this.allCreatedSpans.add(spanKey);
|
|
5663
|
+
}
|
|
5664
|
+
break;
|
|
5665
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
5666
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5667
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5668
|
+
this.buffer.updates.push({
|
|
5669
|
+
traceId: event.exportedSpan.traceId,
|
|
5670
|
+
spanId: event.exportedSpan.id,
|
|
5671
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5672
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5673
|
+
});
|
|
5674
|
+
} else {
|
|
5675
|
+
this.handleOutOfOrderUpdate(event);
|
|
5676
|
+
this.buffer.outOfOrderCount++;
|
|
5677
|
+
}
|
|
5678
|
+
}
|
|
5679
|
+
break;
|
|
5680
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
5681
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5682
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5683
|
+
this.buffer.updates.push({
|
|
5684
|
+
traceId: event.exportedSpan.traceId,
|
|
5685
|
+
spanId: event.exportedSpan.id,
|
|
5686
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5687
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5688
|
+
});
|
|
5689
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5690
|
+
} else if (event.exportedSpan.isEvent) {
|
|
5691
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5692
|
+
this.buffer.creates.push(createRecord);
|
|
5693
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5694
|
+
this.allCreatedSpans.add(spanKey);
|
|
5695
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5696
|
+
} else {
|
|
5697
|
+
this.handleOutOfOrderUpdate(event);
|
|
5698
|
+
this.buffer.outOfOrderCount++;
|
|
5699
|
+
}
|
|
5700
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5701
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5702
|
+
this.buffer.insertOnly.push(createRecord);
|
|
5703
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5704
|
+
this.allCreatedSpans.add(spanKey);
|
|
5705
|
+
}
|
|
5706
|
+
break;
|
|
5707
|
+
}
|
|
5708
|
+
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
5709
|
+
}
|
|
5710
|
+
/**
|
|
5711
|
+
* Checks if buffer should be flushed based on size or time triggers
|
|
5712
|
+
*/
|
|
5713
|
+
shouldFlush() {
|
|
5714
|
+
if (this.buffer.totalSize >= this.#config.maxBufferSize) {
|
|
5715
|
+
return true;
|
|
5716
|
+
}
|
|
5717
|
+
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
5718
|
+
return true;
|
|
5719
|
+
}
|
|
5720
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5721
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5722
|
+
if (elapsed >= this.#config.maxBatchWaitMs) {
|
|
5723
|
+
return true;
|
|
5724
|
+
}
|
|
5725
|
+
}
|
|
5726
|
+
return false;
|
|
5727
|
+
}
|
|
5728
|
+
/**
|
|
5729
|
+
* Resets the buffer after successful flush
|
|
5730
|
+
*/
|
|
5731
|
+
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
5732
|
+
this.buffer.creates = [];
|
|
5733
|
+
this.buffer.updates = [];
|
|
5734
|
+
this.buffer.insertOnly = [];
|
|
5735
|
+
this.buffer.seenSpans.clear();
|
|
5736
|
+
this.buffer.spanSequences.clear();
|
|
5737
|
+
this.buffer.completedSpans.clear();
|
|
5738
|
+
this.buffer.outOfOrderCount = 0;
|
|
5739
|
+
this.buffer.firstEventTime = void 0;
|
|
5740
|
+
this.buffer.totalSize = 0;
|
|
5741
|
+
for (const spanKey of completedSpansToCleanup) {
|
|
5742
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5743
|
+
}
|
|
5744
|
+
}
|
|
5745
|
+
/**
|
|
5746
|
+
* Schedules a flush using setTimeout
|
|
5747
|
+
*/
|
|
5748
|
+
scheduleFlush() {
|
|
5749
|
+
if (this.#flushTimer) {
|
|
5750
|
+
clearTimeout(this.#flushTimer);
|
|
5751
|
+
}
|
|
5752
|
+
this.#flushTimer = setTimeout(() => {
|
|
5753
|
+
this.flushBuffer().catch((error) => {
|
|
5754
|
+
this.logger.error("Scheduled flush failed", {
|
|
5755
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5756
|
+
});
|
|
5757
|
+
});
|
|
5758
|
+
}, this.#config.maxBatchWaitMs);
|
|
5759
|
+
}
|
|
5760
|
+
/**
|
|
5761
|
+
* Serializes span attributes to storage record format
|
|
5762
|
+
* Handles all Span types and their specific attributes
|
|
5763
|
+
*/
|
|
5764
|
+
serializeAttributes(span) {
|
|
5765
|
+
if (!span.attributes) {
|
|
5766
|
+
return null;
|
|
5767
|
+
}
|
|
5768
|
+
try {
|
|
5769
|
+
return JSON.parse(
|
|
5770
|
+
JSON.stringify(span.attributes, (_key, value) => {
|
|
5771
|
+
if (value instanceof Date) {
|
|
5772
|
+
return value.toISOString();
|
|
5773
|
+
}
|
|
5774
|
+
if (typeof value === "object" && value !== null) {
|
|
5775
|
+
return value;
|
|
5776
|
+
}
|
|
5777
|
+
return value;
|
|
5778
|
+
})
|
|
5779
|
+
);
|
|
5780
|
+
} catch (error) {
|
|
5781
|
+
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
5782
|
+
spanId: span.id,
|
|
5783
|
+
spanType: span.type,
|
|
5784
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5785
|
+
});
|
|
5786
|
+
return null;
|
|
5787
|
+
}
|
|
5788
|
+
}
|
|
5789
|
+
buildCreateRecord(span) {
|
|
5790
|
+
const metadata = span.metadata ?? {};
|
|
5791
|
+
return {
|
|
5792
|
+
traceId: span.traceId,
|
|
5793
|
+
spanId: span.id,
|
|
5794
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5795
|
+
name: span.name,
|
|
5796
|
+
// Entity identification - from span
|
|
5797
|
+
entityType: span.entityType ?? null,
|
|
5798
|
+
entityId: span.entityId ?? null,
|
|
5799
|
+
entityName: span.entityName ?? null,
|
|
5800
|
+
// Identity & Tenancy - extracted from metadata if present
|
|
5801
|
+
userId: getStringOrNull(metadata.userId),
|
|
5802
|
+
organizationId: getStringOrNull(metadata.organizationId),
|
|
5803
|
+
resourceId: getStringOrNull(metadata.resourceId),
|
|
5804
|
+
// Correlation IDs - extracted from metadata if present
|
|
5805
|
+
runId: getStringOrNull(metadata.runId),
|
|
5806
|
+
sessionId: getStringOrNull(metadata.sessionId),
|
|
5807
|
+
threadId: getStringOrNull(metadata.threadId),
|
|
5808
|
+
requestId: getStringOrNull(metadata.requestId),
|
|
5809
|
+
// Deployment context - extracted from metadata if present
|
|
5810
|
+
environment: getStringOrNull(metadata.environment),
|
|
5811
|
+
source: getStringOrNull(metadata.source),
|
|
5812
|
+
serviceName: getStringOrNull(metadata.serviceName),
|
|
5813
|
+
scope: getObjectOrNull(metadata.scope),
|
|
5814
|
+
// Span data
|
|
5815
|
+
spanType: span.type,
|
|
5816
|
+
attributes: this.serializeAttributes(span),
|
|
5817
|
+
metadata: span.metadata ?? null,
|
|
5818
|
+
// Keep all metadata including extracted fields
|
|
5819
|
+
tags: span.tags ?? null,
|
|
5820
|
+
links: null,
|
|
5821
|
+
input: span.input ?? null,
|
|
5822
|
+
output: span.output ?? null,
|
|
5823
|
+
error: span.errorInfo ?? null,
|
|
5824
|
+
isEvent: span.isEvent,
|
|
5825
|
+
// Timestamps
|
|
5826
|
+
startedAt: span.startTime,
|
|
5827
|
+
endedAt: span.endTime ?? null
|
|
5828
|
+
};
|
|
5829
|
+
}
|
|
5830
|
+
buildUpdateRecord(span) {
|
|
5831
|
+
return {
|
|
5832
|
+
name: span.name,
|
|
5833
|
+
scope: null,
|
|
5834
|
+
attributes: this.serializeAttributes(span),
|
|
5835
|
+
metadata: span.metadata ?? null,
|
|
5836
|
+
links: null,
|
|
5837
|
+
endedAt: span.endTime ?? null,
|
|
5838
|
+
input: span.input,
|
|
5839
|
+
output: span.output,
|
|
5840
|
+
error: span.errorInfo ?? null
|
|
5841
|
+
};
|
|
5842
|
+
}
|
|
5843
|
+
/**
|
|
5844
|
+
* Handles realtime strategy - processes each event immediately
|
|
5845
|
+
*/
|
|
5846
|
+
async handleRealtimeEvent(event, observability$1) {
|
|
5847
|
+
const span = event.exportedSpan;
|
|
5848
|
+
const spanKey = this.buildSpanKey(span.traceId, span.id);
|
|
5849
|
+
if (span.isEvent) {
|
|
5850
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
5851
|
+
await observability$1.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5852
|
+
} else {
|
|
5853
|
+
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
5854
|
+
}
|
|
5855
|
+
} else {
|
|
5856
|
+
switch (event.type) {
|
|
5857
|
+
case observability.TracingEventType.SPAN_STARTED:
|
|
5858
|
+
await observability$1.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5859
|
+
this.allCreatedSpans.add(spanKey);
|
|
5860
|
+
break;
|
|
5861
|
+
case observability.TracingEventType.SPAN_UPDATED:
|
|
5862
|
+
await observability$1.updateSpan({
|
|
5863
|
+
traceId: span.traceId,
|
|
5864
|
+
spanId: span.id,
|
|
5865
|
+
updates: this.buildUpdateRecord(span)
|
|
5866
|
+
});
|
|
5867
|
+
break;
|
|
5868
|
+
case observability.TracingEventType.SPAN_ENDED:
|
|
5869
|
+
await observability$1.updateSpan({
|
|
5870
|
+
traceId: span.traceId,
|
|
5871
|
+
spanId: span.id,
|
|
5872
|
+
updates: this.buildUpdateRecord(span)
|
|
5873
|
+
});
|
|
5874
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5875
|
+
break;
|
|
5876
|
+
default:
|
|
5877
|
+
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
5878
|
+
}
|
|
5879
|
+
}
|
|
5880
|
+
}
|
|
5881
|
+
/**
|
|
5882
|
+
* Handles batch-with-updates strategy - buffers events and processes in batches
|
|
5883
|
+
*/
|
|
5884
|
+
handleBatchWithUpdatesEvent(event) {
|
|
5885
|
+
this.addToBuffer(event);
|
|
5886
|
+
if (this.shouldFlush()) {
|
|
5887
|
+
this.flushBuffer().catch((error) => {
|
|
5888
|
+
this.logger.error("Batch flush failed", {
|
|
5889
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5890
|
+
});
|
|
5891
|
+
});
|
|
5892
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5893
|
+
this.scheduleFlush();
|
|
5894
|
+
}
|
|
5895
|
+
}
|
|
5896
|
+
/**
|
|
5897
|
+
* Handles insert-only strategy - only processes SPAN_ENDED events in batches
|
|
5898
|
+
*/
|
|
5899
|
+
handleInsertOnlyEvent(event) {
|
|
5900
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
5901
|
+
this.addToBuffer(event);
|
|
5902
|
+
if (this.shouldFlush()) {
|
|
5903
|
+
this.flushBuffer().catch((error) => {
|
|
5904
|
+
this.logger.error("Batch flush failed", {
|
|
5905
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5906
|
+
});
|
|
5907
|
+
});
|
|
5908
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5909
|
+
this.scheduleFlush();
|
|
5910
|
+
}
|
|
5911
|
+
}
|
|
5912
|
+
}
|
|
5913
|
+
/**
|
|
5914
|
+
* Calculates retry delay using exponential backoff
|
|
5915
|
+
*/
|
|
5916
|
+
calculateRetryDelay(attempt) {
|
|
5917
|
+
return this.#config.retryDelayMs * Math.pow(2, attempt);
|
|
5918
|
+
}
|
|
5919
|
+
/**
|
|
5920
|
+
* Flushes the current buffer to storage with retry logic (internal implementation)
|
|
5921
|
+
*/
|
|
5922
|
+
async flushBuffer() {
|
|
5923
|
+
if (!this.#observability) {
|
|
5924
|
+
this.logger.debug("Cannot flush traces. Observability storage is not initialized");
|
|
5925
|
+
return;
|
|
5926
|
+
}
|
|
5927
|
+
if (this.#flushTimer) {
|
|
5928
|
+
clearTimeout(this.#flushTimer);
|
|
5929
|
+
this.#flushTimer = null;
|
|
5930
|
+
}
|
|
5931
|
+
if (this.buffer.totalSize === 0) {
|
|
5932
|
+
return;
|
|
5933
|
+
}
|
|
5934
|
+
const startTime = Date.now();
|
|
5935
|
+
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
5936
|
+
const bufferCopy = {
|
|
5937
|
+
creates: [...this.buffer.creates],
|
|
5938
|
+
updates: [...this.buffer.updates],
|
|
5939
|
+
insertOnly: [...this.buffer.insertOnly],
|
|
5940
|
+
seenSpans: new Set(this.buffer.seenSpans),
|
|
5941
|
+
spanSequences: new Map(this.buffer.spanSequences),
|
|
5942
|
+
completedSpans: new Set(this.buffer.completedSpans),
|
|
5943
|
+
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
5944
|
+
firstEventTime: this.buffer.firstEventTime,
|
|
5945
|
+
totalSize: this.buffer.totalSize
|
|
5946
|
+
};
|
|
5947
|
+
this.resetBuffer();
|
|
5948
|
+
await this.flushWithRetries(this.#observability, bufferCopy, 0);
|
|
5949
|
+
const elapsed = Date.now() - startTime;
|
|
5950
|
+
this.logger.debug("Batch flushed", {
|
|
5951
|
+
strategy: this.#resolvedStrategy,
|
|
5952
|
+
batchSize: bufferCopy.totalSize,
|
|
5953
|
+
flushReason,
|
|
5954
|
+
durationMs: elapsed,
|
|
5955
|
+
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
5956
|
+
});
|
|
5957
|
+
}
|
|
5958
|
+
/**
|
|
5959
|
+
* Attempts to flush with exponential backoff retry logic
|
|
5960
|
+
*/
|
|
5961
|
+
async flushWithRetries(observability, buffer, attempt) {
|
|
5962
|
+
try {
|
|
5963
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5964
|
+
if (buffer.creates.length > 0) {
|
|
5965
|
+
await observability.batchCreateSpans({ records: buffer.creates });
|
|
5966
|
+
}
|
|
5967
|
+
if (buffer.updates.length > 0) {
|
|
5968
|
+
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
5969
|
+
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
5970
|
+
this.buildSpanKey(b.traceId, b.spanId)
|
|
5971
|
+
);
|
|
5972
|
+
if (spanCompare !== 0) return spanCompare;
|
|
5973
|
+
return a.sequenceNumber - b.sequenceNumber;
|
|
5974
|
+
});
|
|
5975
|
+
await observability.batchUpdateSpans({ records: sortedUpdates });
|
|
4459
5976
|
}
|
|
4460
|
-
|
|
4461
|
-
|
|
5977
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5978
|
+
if (buffer.insertOnly.length > 0) {
|
|
5979
|
+
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
4462
5980
|
}
|
|
4463
|
-
|
|
4464
|
-
|
|
5981
|
+
}
|
|
5982
|
+
for (const spanKey of buffer.completedSpans) {
|
|
5983
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5984
|
+
}
|
|
5985
|
+
} catch (error) {
|
|
5986
|
+
if (attempt < this.#config.maxRetries) {
|
|
5987
|
+
const retryDelay = this.calculateRetryDelay(attempt);
|
|
5988
|
+
this.logger.warn("Batch flush failed, retrying", {
|
|
5989
|
+
attempt: attempt + 1,
|
|
5990
|
+
maxRetries: this.#config.maxRetries,
|
|
5991
|
+
nextRetryInMs: retryDelay,
|
|
5992
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5993
|
+
});
|
|
5994
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
5995
|
+
return this.flushWithRetries(observability, buffer, attempt + 1);
|
|
5996
|
+
} else {
|
|
5997
|
+
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
5998
|
+
finalAttempt: attempt + 1,
|
|
5999
|
+
maxRetries: this.#config.maxRetries,
|
|
6000
|
+
droppedBatchSize: buffer.totalSize,
|
|
6001
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6002
|
+
});
|
|
6003
|
+
for (const spanKey of buffer.completedSpans) {
|
|
6004
|
+
this.allCreatedSpans.delete(spanKey);
|
|
6005
|
+
}
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
}
|
|
6009
|
+
async _exportTracingEvent(event) {
|
|
6010
|
+
if (!this.#observability) {
|
|
6011
|
+
this.logger.debug("Cannot store traces. Observability storage is not initialized");
|
|
6012
|
+
return;
|
|
6013
|
+
}
|
|
6014
|
+
if (!this.#strategyInitialized) {
|
|
6015
|
+
this.initializeStrategy(this.#observability, this.#storage?.constructor.name ?? "Unknown");
|
|
6016
|
+
}
|
|
6017
|
+
switch (this.#resolvedStrategy) {
|
|
6018
|
+
case "realtime":
|
|
6019
|
+
await this.handleRealtimeEvent(event, this.#observability);
|
|
6020
|
+
break;
|
|
6021
|
+
case "batch-with-updates":
|
|
6022
|
+
this.handleBatchWithUpdatesEvent(event);
|
|
4465
6023
|
break;
|
|
4466
|
-
case
|
|
4467
|
-
this.
|
|
4468
|
-
this.logger.info(` Type: ${span.type}`);
|
|
4469
|
-
this.logger.info(` Name: ${span.name}`);
|
|
4470
|
-
this.logger.info(` ID: ${span.id}`);
|
|
4471
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
4472
|
-
if (span.input !== void 0) {
|
|
4473
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
4474
|
-
}
|
|
4475
|
-
if (span.output !== void 0) {
|
|
4476
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
4477
|
-
}
|
|
4478
|
-
if (span.errorInfo) {
|
|
4479
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
4480
|
-
}
|
|
4481
|
-
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
4482
|
-
this.logger.info("\u2500".repeat(80));
|
|
6024
|
+
case "insert-only":
|
|
6025
|
+
this.handleInsertOnlyEvent(event);
|
|
4483
6026
|
break;
|
|
4484
|
-
|
|
4485
|
-
|
|
6027
|
+
}
|
|
6028
|
+
}
|
|
6029
|
+
/**
|
|
6030
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
6031
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
6032
|
+
* are exported before the runtime instance is terminated.
|
|
6033
|
+
*/
|
|
6034
|
+
async flush() {
|
|
6035
|
+
if (this.buffer.totalSize > 0) {
|
|
6036
|
+
this.logger.debug("Flushing buffered events", {
|
|
6037
|
+
bufferedEvents: this.buffer.totalSize
|
|
6038
|
+
});
|
|
6039
|
+
await this.flushBuffer();
|
|
4486
6040
|
}
|
|
4487
6041
|
}
|
|
4488
6042
|
async shutdown() {
|
|
4489
|
-
this
|
|
6043
|
+
if (this.#flushTimer) {
|
|
6044
|
+
clearTimeout(this.#flushTimer);
|
|
6045
|
+
this.#flushTimer = null;
|
|
6046
|
+
}
|
|
6047
|
+
await this.flush();
|
|
6048
|
+
this.logger.info("DefaultExporter shutdown complete");
|
|
4490
6049
|
}
|
|
4491
6050
|
};
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
userStrategy: config.strategy,
|
|
4500
|
-
storageAdapter: storageName,
|
|
4501
|
-
supportedStrategies: hints.supported,
|
|
4502
|
-
fallbackStrategy: hints.preferred
|
|
4503
|
-
});
|
|
6051
|
+
|
|
6052
|
+
// src/exporters/test.ts
|
|
6053
|
+
var TestExporter = class extends BaseExporter {
|
|
6054
|
+
name = "tracing-test-exporter";
|
|
6055
|
+
#events = [];
|
|
6056
|
+
constructor(config = {}) {
|
|
6057
|
+
super(config);
|
|
4504
6058
|
}
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
6059
|
+
async _exportTracingEvent(event) {
|
|
6060
|
+
this.#events.push(event);
|
|
6061
|
+
}
|
|
6062
|
+
clearEvents() {
|
|
6063
|
+
this.#events = [];
|
|
6064
|
+
}
|
|
6065
|
+
get events() {
|
|
6066
|
+
return this.#events;
|
|
6067
|
+
}
|
|
6068
|
+
async shutdown() {
|
|
6069
|
+
this.logger.info("TestExporter shutdown");
|
|
6070
|
+
}
|
|
6071
|
+
};
|
|
6072
|
+
var __filename$1 = url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
6073
|
+
var __dirname$1 = path.dirname(__filename$1);
|
|
6074
|
+
var SNAPSHOTS_DIR = path.join(__dirname$1, "..", "__snapshots__");
|
|
6075
|
+
var JsonExporter = class extends BaseExporter {
|
|
6076
|
+
name = "json-exporter";
|
|
6077
|
+
/** All collected events */
|
|
6078
|
+
#events = [];
|
|
6079
|
+
/** Per-span state tracking */
|
|
6080
|
+
#spanStates = /* @__PURE__ */ new Map();
|
|
6081
|
+
/** Logs for debugging */
|
|
6082
|
+
#logs = [];
|
|
6083
|
+
/** Configuration */
|
|
4517
6084
|
#config;
|
|
4518
|
-
#resolvedStrategy;
|
|
4519
|
-
buffer;
|
|
4520
|
-
#flushTimer = null;
|
|
4521
|
-
// Track all spans that have been created, persists across flushes
|
|
4522
|
-
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
4523
6085
|
constructor(config = {}) {
|
|
4524
6086
|
super(config);
|
|
4525
|
-
if (config === void 0) {
|
|
4526
|
-
config = {};
|
|
4527
|
-
}
|
|
4528
6087
|
this.#config = {
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
maxRetries: config.maxRetries ?? 4,
|
|
4534
|
-
retryDelayMs: config.retryDelayMs ?? 500,
|
|
4535
|
-
strategy: config.strategy ?? "auto"
|
|
6088
|
+
validateLifecycle: true,
|
|
6089
|
+
storeLogs: true,
|
|
6090
|
+
jsonIndent: 2,
|
|
6091
|
+
...config
|
|
4536
6092
|
};
|
|
4537
|
-
this.buffer = {
|
|
4538
|
-
creates: [],
|
|
4539
|
-
updates: [],
|
|
4540
|
-
insertOnly: [],
|
|
4541
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
4542
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
4543
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
4544
|
-
outOfOrderCount: 0,
|
|
4545
|
-
totalSize: 0
|
|
4546
|
-
};
|
|
4547
|
-
this.#resolvedStrategy = "batch-with-updates";
|
|
4548
6093
|
}
|
|
4549
|
-
#strategyInitialized = false;
|
|
4550
6094
|
/**
|
|
4551
|
-
*
|
|
6095
|
+
* Process incoming tracing events with lifecycle tracking
|
|
4552
6096
|
*/
|
|
4553
|
-
async
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
6097
|
+
async _exportTracingEvent(event) {
|
|
6098
|
+
const span = event.exportedSpan;
|
|
6099
|
+
const spanId = span.id;
|
|
6100
|
+
const logMessage = `[JsonExporter] ${event.type}: ${span.type} "${span.name}" (entity: ${span.entityName ?? span.entityId ?? "unknown"}, trace: ${span.traceId.slice(-8)}, span: ${spanId.slice(-8)})`;
|
|
6101
|
+
if (this.#config.storeLogs) {
|
|
6102
|
+
this.#logs.push(logMessage);
|
|
6103
|
+
}
|
|
6104
|
+
const state = this.#spanStates.get(spanId) || {
|
|
6105
|
+
hasStart: false,
|
|
6106
|
+
hasEnd: false,
|
|
6107
|
+
hasUpdate: false,
|
|
6108
|
+
events: []
|
|
6109
|
+
};
|
|
6110
|
+
if (this.#config.validateLifecycle) {
|
|
6111
|
+
this.#validateLifecycle(event, state, spanId);
|
|
4558
6112
|
}
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
6113
|
+
if (event.type === observability.TracingEventType.SPAN_STARTED) {
|
|
6114
|
+
state.hasStart = true;
|
|
6115
|
+
} else if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
6116
|
+
state.hasEnd = true;
|
|
6117
|
+
if (span.isEvent) {
|
|
6118
|
+
state.isEventSpan = true;
|
|
6119
|
+
}
|
|
6120
|
+
} else if (event.type === observability.TracingEventType.SPAN_UPDATED) {
|
|
6121
|
+
state.hasUpdate = true;
|
|
4563
6122
|
}
|
|
4564
|
-
|
|
6123
|
+
state.events.push(event);
|
|
6124
|
+
this.#spanStates.set(spanId, state);
|
|
6125
|
+
this.#events.push(event);
|
|
4565
6126
|
}
|
|
4566
6127
|
/**
|
|
4567
|
-
*
|
|
6128
|
+
* Validate span lifecycle rules
|
|
4568
6129
|
*/
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
6130
|
+
#validateLifecycle(event, state, spanId) {
|
|
6131
|
+
const span = event.exportedSpan;
|
|
6132
|
+
if (event.type === observability.TracingEventType.SPAN_STARTED) {
|
|
6133
|
+
if (state.hasStart) {
|
|
6134
|
+
this.logger.warn(`Span ${spanId} (${span.type} "${span.name}") started twice`);
|
|
6135
|
+
}
|
|
6136
|
+
} else if (event.type === observability.TracingEventType.SPAN_ENDED) {
|
|
6137
|
+
if (span.isEvent) {
|
|
6138
|
+
if (state.hasStart) {
|
|
6139
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_STARTED`);
|
|
6140
|
+
}
|
|
6141
|
+
if (state.hasUpdate) {
|
|
6142
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_UPDATED`);
|
|
6143
|
+
}
|
|
6144
|
+
} else {
|
|
6145
|
+
if (!state.hasStart) {
|
|
6146
|
+
this.logger.warn(`Normal span ${spanId} (${span.type} "${span.name}") ended without starting`);
|
|
6147
|
+
}
|
|
6148
|
+
}
|
|
6149
|
+
}
|
|
4580
6150
|
}
|
|
6151
|
+
// ============================================================================
|
|
6152
|
+
// Query Methods
|
|
6153
|
+
// ============================================================================
|
|
4581
6154
|
/**
|
|
4582
|
-
*
|
|
6155
|
+
* Get all collected events
|
|
4583
6156
|
*/
|
|
4584
|
-
|
|
4585
|
-
return
|
|
6157
|
+
get events() {
|
|
6158
|
+
return [...this.#events];
|
|
4586
6159
|
}
|
|
4587
6160
|
/**
|
|
4588
|
-
*
|
|
6161
|
+
* Get completed spans by SpanType (e.g., 'agent_run', 'tool_call')
|
|
6162
|
+
*
|
|
6163
|
+
* @param type - The SpanType to filter by
|
|
6164
|
+
* @returns Array of completed exported spans of the specified type
|
|
4589
6165
|
*/
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
6166
|
+
getSpansByType(type) {
|
|
6167
|
+
return Array.from(this.#spanStates.values()).filter((state) => {
|
|
6168
|
+
if (!state.hasEnd) return false;
|
|
6169
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6170
|
+
return endEvent?.exportedSpan.type === type;
|
|
6171
|
+
}).map((state) => {
|
|
6172
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6173
|
+
return endEvent?.exportedSpan;
|
|
6174
|
+
}).filter((span) => span !== void 0);
|
|
4595
6175
|
}
|
|
4596
6176
|
/**
|
|
4597
|
-
*
|
|
6177
|
+
* Get events by TracingEventType (SPAN_STARTED, SPAN_UPDATED, SPAN_ENDED)
|
|
6178
|
+
*
|
|
6179
|
+
* @param type - The TracingEventType to filter by
|
|
6180
|
+
* @returns Array of events of the specified type
|
|
4598
6181
|
*/
|
|
4599
|
-
|
|
4600
|
-
this.
|
|
4601
|
-
spanId: event.exportedSpan.id,
|
|
4602
|
-
traceId: event.exportedSpan.traceId,
|
|
4603
|
-
spanName: event.exportedSpan.name,
|
|
4604
|
-
eventType: event.type
|
|
4605
|
-
});
|
|
6182
|
+
getByEventType(type) {
|
|
6183
|
+
return this.#events.filter((e) => e.type === type);
|
|
4606
6184
|
}
|
|
4607
6185
|
/**
|
|
4608
|
-
*
|
|
6186
|
+
* Get all events and spans for a specific trace
|
|
6187
|
+
*
|
|
6188
|
+
* @param traceId - The trace ID to filter by
|
|
6189
|
+
* @returns Object containing events and final spans for the trace
|
|
4609
6190
|
*/
|
|
4610
|
-
|
|
4611
|
-
const
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
}
|
|
4615
|
-
switch (event.type) {
|
|
4616
|
-
case observability.TracingEventType.SPAN_STARTED:
|
|
4617
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4618
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4619
|
-
this.buffer.creates.push(createRecord);
|
|
4620
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4621
|
-
this.allCreatedSpans.add(spanKey);
|
|
4622
|
-
}
|
|
4623
|
-
break;
|
|
4624
|
-
case observability.TracingEventType.SPAN_UPDATED:
|
|
4625
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4626
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4627
|
-
this.buffer.updates.push({
|
|
4628
|
-
traceId: event.exportedSpan.traceId,
|
|
4629
|
-
spanId: event.exportedSpan.id,
|
|
4630
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4631
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4632
|
-
});
|
|
4633
|
-
} else {
|
|
4634
|
-
this.handleOutOfOrderUpdate(event);
|
|
4635
|
-
this.buffer.outOfOrderCount++;
|
|
4636
|
-
}
|
|
4637
|
-
}
|
|
4638
|
-
break;
|
|
4639
|
-
case observability.TracingEventType.SPAN_ENDED:
|
|
4640
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4641
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4642
|
-
this.buffer.updates.push({
|
|
4643
|
-
traceId: event.exportedSpan.traceId,
|
|
4644
|
-
spanId: event.exportedSpan.id,
|
|
4645
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4646
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4647
|
-
});
|
|
4648
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4649
|
-
} else if (event.exportedSpan.isEvent) {
|
|
4650
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4651
|
-
this.buffer.creates.push(createRecord);
|
|
4652
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4653
|
-
this.allCreatedSpans.add(spanKey);
|
|
4654
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4655
|
-
} else {
|
|
4656
|
-
this.handleOutOfOrderUpdate(event);
|
|
4657
|
-
this.buffer.outOfOrderCount++;
|
|
4658
|
-
}
|
|
4659
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
4660
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4661
|
-
this.buffer.insertOnly.push(createRecord);
|
|
4662
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4663
|
-
this.allCreatedSpans.add(spanKey);
|
|
4664
|
-
}
|
|
4665
|
-
break;
|
|
4666
|
-
}
|
|
4667
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
6191
|
+
getByTraceId(traceId) {
|
|
6192
|
+
const events = this.#events.filter((e) => e.exportedSpan.traceId === traceId);
|
|
6193
|
+
const spans = this.#getUniqueSpansFromEvents(events);
|
|
6194
|
+
return { events, spans };
|
|
4668
6195
|
}
|
|
4669
6196
|
/**
|
|
4670
|
-
*
|
|
6197
|
+
* Get all events for a specific span
|
|
6198
|
+
*
|
|
6199
|
+
* @param spanId - The span ID to filter by
|
|
6200
|
+
* @returns Object containing events and final span state
|
|
4671
6201
|
*/
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
4677
|
-
return true;
|
|
6202
|
+
getBySpanId(spanId) {
|
|
6203
|
+
const state = this.#spanStates.get(spanId);
|
|
6204
|
+
if (!state) {
|
|
6205
|
+
return { events: [], span: void 0, state: void 0 };
|
|
4678
6206
|
}
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
6207
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6208
|
+
const span = endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6209
|
+
return { events: state.events, span, state };
|
|
6210
|
+
}
|
|
6211
|
+
/**
|
|
6212
|
+
* Get all unique spans (returns the final state of each span)
|
|
6213
|
+
*/
|
|
6214
|
+
getAllSpans() {
|
|
6215
|
+
return Array.from(this.#spanStates.values()).map((state) => {
|
|
6216
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6217
|
+
return endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6218
|
+
}).filter((span) => span !== void 0);
|
|
6219
|
+
}
|
|
6220
|
+
/**
|
|
6221
|
+
* Get only completed spans (those that have received SPAN_ENDED)
|
|
6222
|
+
*/
|
|
6223
|
+
getCompletedSpans() {
|
|
6224
|
+
return Array.from(this.#spanStates.values()).filter((state) => state.hasEnd).map((state) => {
|
|
6225
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6226
|
+
return endEvent.exportedSpan;
|
|
6227
|
+
});
|
|
6228
|
+
}
|
|
6229
|
+
/**
|
|
6230
|
+
* Get root spans only (spans with no parent)
|
|
6231
|
+
*/
|
|
6232
|
+
getRootSpans() {
|
|
6233
|
+
return this.getAllSpans().filter((span) => span.isRootSpan);
|
|
6234
|
+
}
|
|
6235
|
+
/**
|
|
6236
|
+
* Get incomplete spans (started but not yet ended)
|
|
6237
|
+
*/
|
|
6238
|
+
getIncompleteSpans() {
|
|
6239
|
+
return Array.from(this.#spanStates.entries()).filter(([_, state]) => !state.hasEnd).map(([spanId, state]) => ({
|
|
6240
|
+
spanId,
|
|
6241
|
+
span: state.events[0]?.exportedSpan,
|
|
6242
|
+
state: {
|
|
6243
|
+
hasStart: state.hasStart,
|
|
6244
|
+
hasUpdate: state.hasUpdate,
|
|
6245
|
+
hasEnd: state.hasEnd
|
|
4683
6246
|
}
|
|
4684
|
-
}
|
|
4685
|
-
return false;
|
|
6247
|
+
}));
|
|
4686
6248
|
}
|
|
4687
6249
|
/**
|
|
4688
|
-
*
|
|
6250
|
+
* Get unique trace IDs from all collected spans
|
|
4689
6251
|
*/
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
this
|
|
4693
|
-
|
|
4694
|
-
this.buffer.seenSpans.clear();
|
|
4695
|
-
this.buffer.spanSequences.clear();
|
|
4696
|
-
this.buffer.completedSpans.clear();
|
|
4697
|
-
this.buffer.outOfOrderCount = 0;
|
|
4698
|
-
this.buffer.firstEventTime = void 0;
|
|
4699
|
-
this.buffer.totalSize = 0;
|
|
4700
|
-
for (const spanKey of completedSpansToCleanup) {
|
|
4701
|
-
this.allCreatedSpans.delete(spanKey);
|
|
6252
|
+
getTraceIds() {
|
|
6253
|
+
const traceIds = /* @__PURE__ */ new Set();
|
|
6254
|
+
for (const event of this.#events) {
|
|
6255
|
+
traceIds.add(event.exportedSpan.traceId);
|
|
4702
6256
|
}
|
|
6257
|
+
return Array.from(traceIds);
|
|
4703
6258
|
}
|
|
6259
|
+
// ============================================================================
|
|
6260
|
+
// Statistics
|
|
6261
|
+
// ============================================================================
|
|
4704
6262
|
/**
|
|
4705
|
-
*
|
|
6263
|
+
* Get comprehensive statistics about collected spans
|
|
4706
6264
|
*/
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
6265
|
+
getStatistics() {
|
|
6266
|
+
const bySpanType = {};
|
|
6267
|
+
let completedSpans = 0;
|
|
6268
|
+
let incompleteSpans = 0;
|
|
6269
|
+
for (const state of this.#spanStates.values()) {
|
|
6270
|
+
if (state.hasEnd) {
|
|
6271
|
+
completedSpans++;
|
|
6272
|
+
const endEvent = state.events.find((e) => e.type === observability.TracingEventType.SPAN_ENDED);
|
|
6273
|
+
const spanType = endEvent?.exportedSpan.type;
|
|
6274
|
+
if (spanType) {
|
|
6275
|
+
bySpanType[spanType] = (bySpanType[spanType] || 0) + 1;
|
|
6276
|
+
}
|
|
6277
|
+
} else {
|
|
6278
|
+
incompleteSpans++;
|
|
6279
|
+
}
|
|
4710
6280
|
}
|
|
4711
|
-
|
|
4712
|
-
this.
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
6281
|
+
return {
|
|
6282
|
+
totalEvents: this.#events.length,
|
|
6283
|
+
totalSpans: this.#spanStates.size,
|
|
6284
|
+
totalTraces: this.getTraceIds().length,
|
|
6285
|
+
completedSpans,
|
|
6286
|
+
incompleteSpans,
|
|
6287
|
+
byEventType: {
|
|
6288
|
+
started: this.#events.filter((e) => e.type === observability.TracingEventType.SPAN_STARTED).length,
|
|
6289
|
+
updated: this.#events.filter((e) => e.type === observability.TracingEventType.SPAN_UPDATED).length,
|
|
6290
|
+
ended: this.#events.filter((e) => e.type === observability.TracingEventType.SPAN_ENDED).length
|
|
6291
|
+
},
|
|
6292
|
+
bySpanType
|
|
6293
|
+
};
|
|
4718
6294
|
}
|
|
6295
|
+
// ============================================================================
|
|
6296
|
+
// JSON Output
|
|
6297
|
+
// ============================================================================
|
|
4719
6298
|
/**
|
|
4720
|
-
*
|
|
4721
|
-
*
|
|
6299
|
+
* Serialize all collected data to JSON string
|
|
6300
|
+
*
|
|
6301
|
+
* @param options - Serialization options
|
|
6302
|
+
* @returns JSON string of all collected data
|
|
4722
6303
|
*/
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
6304
|
+
toJSON(options) {
|
|
6305
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6306
|
+
const includeEvents = options?.includeEvents ?? true;
|
|
6307
|
+
const includeStats = options?.includeStats ?? true;
|
|
6308
|
+
const data = {
|
|
6309
|
+
spans: this.getAllSpans()
|
|
6310
|
+
};
|
|
6311
|
+
if (includeEvents) {
|
|
6312
|
+
data.events = this.#events;
|
|
4726
6313
|
}
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
JSON.stringify(span.attributes, (_key, value) => {
|
|
4730
|
-
if (value instanceof Date) {
|
|
4731
|
-
return value.toISOString();
|
|
4732
|
-
}
|
|
4733
|
-
if (typeof value === "object" && value !== null) {
|
|
4734
|
-
return value;
|
|
4735
|
-
}
|
|
4736
|
-
return value;
|
|
4737
|
-
})
|
|
4738
|
-
);
|
|
4739
|
-
} catch (error) {
|
|
4740
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
4741
|
-
spanId: span.id,
|
|
4742
|
-
spanType: span.type,
|
|
4743
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4744
|
-
});
|
|
4745
|
-
return null;
|
|
6314
|
+
if (includeStats) {
|
|
6315
|
+
data.statistics = this.getStatistics();
|
|
4746
6316
|
}
|
|
6317
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4747
6318
|
}
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
serviceName: getStringOrNull(metadata.serviceName),
|
|
4772
|
-
scope: getObjectOrNull(metadata.scope),
|
|
4773
|
-
// Span data
|
|
4774
|
-
spanType: span.type,
|
|
4775
|
-
attributes: this.serializeAttributes(span),
|
|
4776
|
-
metadata: span.metadata ?? null,
|
|
4777
|
-
// Keep all metadata including extracted fields
|
|
4778
|
-
tags: span.tags ?? null,
|
|
4779
|
-
links: null,
|
|
4780
|
-
input: span.input ?? null,
|
|
4781
|
-
output: span.output ?? null,
|
|
4782
|
-
error: span.errorInfo ?? null,
|
|
4783
|
-
isEvent: span.isEvent,
|
|
4784
|
-
// Timestamps
|
|
4785
|
-
startedAt: span.startTime,
|
|
4786
|
-
endedAt: span.endTime ?? null
|
|
6319
|
+
/**
|
|
6320
|
+
* Build a tree structure from spans, nesting children under their parents
|
|
6321
|
+
*
|
|
6322
|
+
* @returns Array of root span tree nodes (spans with no parent)
|
|
6323
|
+
*/
|
|
6324
|
+
buildSpanTree() {
|
|
6325
|
+
const spans = this.getAllSpans();
|
|
6326
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
6327
|
+
const roots = [];
|
|
6328
|
+
for (const span of spans) {
|
|
6329
|
+
nodeMap.set(span.id, { span, children: [] });
|
|
6330
|
+
}
|
|
6331
|
+
for (const span of spans) {
|
|
6332
|
+
const node = nodeMap.get(span.id);
|
|
6333
|
+
if (span.parentSpanId && nodeMap.has(span.parentSpanId)) {
|
|
6334
|
+
nodeMap.get(span.parentSpanId).children.push(node);
|
|
6335
|
+
} else {
|
|
6336
|
+
roots.push(node);
|
|
6337
|
+
}
|
|
6338
|
+
}
|
|
6339
|
+
const sortChildren = (node) => {
|
|
6340
|
+
node.children.sort((a, b) => new Date(a.span.startTime).getTime() - new Date(b.span.startTime).getTime());
|
|
6341
|
+
node.children.forEach(sortChildren);
|
|
4787
6342
|
};
|
|
6343
|
+
roots.forEach(sortChildren);
|
|
6344
|
+
return roots;
|
|
4788
6345
|
}
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
6346
|
+
/**
|
|
6347
|
+
* Serialize spans as a tree structure to JSON string
|
|
6348
|
+
*
|
|
6349
|
+
* @param options - Serialization options
|
|
6350
|
+
* @returns JSON string with spans nested in tree format
|
|
6351
|
+
*/
|
|
6352
|
+
toTreeJSON(options) {
|
|
6353
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6354
|
+
const includeStats = options?.includeStats ?? true;
|
|
6355
|
+
const data = {
|
|
6356
|
+
tree: this.buildSpanTree()
|
|
4800
6357
|
};
|
|
6358
|
+
if (includeStats) {
|
|
6359
|
+
data.statistics = this.getStatistics();
|
|
6360
|
+
}
|
|
6361
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4801
6362
|
}
|
|
4802
6363
|
/**
|
|
4803
|
-
*
|
|
6364
|
+
* Build a normalized tree structure suitable for snapshot testing.
|
|
6365
|
+
*
|
|
6366
|
+
* Normalizations applied:
|
|
6367
|
+
* - Span IDs replaced with stable placeholders (<span-1>, <span-2>, etc.)
|
|
6368
|
+
* - Trace IDs replaced with stable placeholders (<trace-1>, <trace-2>, etc.)
|
|
6369
|
+
* - parentSpanId replaced with normalized parent ID
|
|
6370
|
+
* - Timestamps replaced with durationMs (or null if not ended)
|
|
6371
|
+
* - Empty children arrays are omitted
|
|
6372
|
+
*
|
|
6373
|
+
* @returns Array of normalized root tree nodes
|
|
4804
6374
|
*/
|
|
4805
|
-
|
|
4806
|
-
const
|
|
4807
|
-
const
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
6375
|
+
buildNormalizedTree() {
|
|
6376
|
+
const tree = this.buildSpanTree();
|
|
6377
|
+
const spanIdMap = /* @__PURE__ */ new Map();
|
|
6378
|
+
const traceIdMap = /* @__PURE__ */ new Map();
|
|
6379
|
+
const uuidMapsByKey = /* @__PURE__ */ new Map();
|
|
6380
|
+
const uuidCountersByKey = /* @__PURE__ */ new Map();
|
|
6381
|
+
let spanIdCounter = 1;
|
|
6382
|
+
let traceIdCounter = 1;
|
|
6383
|
+
const uuidRegex2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6384
|
+
const hexId32Regex = /^[0-9a-f]{32}$/i;
|
|
6385
|
+
const prefixedUuidRegex = /^([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
|
|
6386
|
+
const embeddedPrefixedUuidRegex = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
|
|
6387
|
+
const normalizeUuid = (uuid, key) => {
|
|
6388
|
+
if (!uuidMapsByKey.has(key)) {
|
|
6389
|
+
uuidMapsByKey.set(key, /* @__PURE__ */ new Map());
|
|
6390
|
+
uuidCountersByKey.set(key, 1);
|
|
4813
6391
|
}
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
6392
|
+
const keyMap = uuidMapsByKey.get(key);
|
|
6393
|
+
if (!keyMap.has(uuid)) {
|
|
6394
|
+
const counter = uuidCountersByKey.get(key);
|
|
6395
|
+
keyMap.set(uuid, `<${key}-${counter}>`);
|
|
6396
|
+
uuidCountersByKey.set(key, counter + 1);
|
|
6397
|
+
}
|
|
6398
|
+
return keyMap.get(uuid);
|
|
6399
|
+
};
|
|
6400
|
+
const normalizeValue = (value, key) => {
|
|
6401
|
+
if (value instanceof Date) {
|
|
6402
|
+
return "<date>";
|
|
6403
|
+
}
|
|
6404
|
+
if (typeof value === "string") {
|
|
6405
|
+
if (key === "traceId" && (uuidRegex2.test(value) || hexId32Regex.test(value))) {
|
|
6406
|
+
if (!traceIdMap.has(value)) {
|
|
6407
|
+
traceIdMap.set(value, `<trace-${traceIdCounter++}>`);
|
|
6408
|
+
}
|
|
6409
|
+
return traceIdMap.get(value);
|
|
6410
|
+
}
|
|
6411
|
+
if (uuidRegex2.test(value)) {
|
|
6412
|
+
return normalizeUuid(value, key ?? "uuid");
|
|
6413
|
+
}
|
|
6414
|
+
const prefixMatch = prefixedUuidRegex.exec(value);
|
|
6415
|
+
if (prefixMatch && prefixMatch[1] && prefixMatch[2]) {
|
|
6416
|
+
const prefix = prefixMatch[1];
|
|
6417
|
+
const uuid = prefixMatch[2];
|
|
6418
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6419
|
+
}
|
|
6420
|
+
if (embeddedPrefixedUuidRegex.test(value)) {
|
|
6421
|
+
embeddedPrefixedUuidRegex.lastIndex = 0;
|
|
6422
|
+
return value.replace(embeddedPrefixedUuidRegex, (_match, prefix, uuid) => {
|
|
6423
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
4832
6424
|
});
|
|
4833
|
-
|
|
4834
|
-
break;
|
|
4835
|
-
default:
|
|
4836
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
6425
|
+
}
|
|
4837
6426
|
}
|
|
4838
|
-
|
|
6427
|
+
if (Array.isArray(value)) {
|
|
6428
|
+
return value.map((v) => normalizeValue(v, key));
|
|
6429
|
+
}
|
|
6430
|
+
if (value && typeof value === "object") {
|
|
6431
|
+
const normalized = {};
|
|
6432
|
+
for (const [k, v] of Object.entries(value)) {
|
|
6433
|
+
normalized[k] = normalizeValue(v, k);
|
|
6434
|
+
}
|
|
6435
|
+
return normalized;
|
|
6436
|
+
}
|
|
6437
|
+
return value;
|
|
6438
|
+
};
|
|
6439
|
+
const assignIds = (nodes) => {
|
|
6440
|
+
for (const node of nodes) {
|
|
6441
|
+
spanIdMap.set(node.span.id, `<span-${spanIdCounter++}>`);
|
|
6442
|
+
if (!traceIdMap.has(node.span.traceId)) {
|
|
6443
|
+
traceIdMap.set(node.span.traceId, `<trace-${traceIdCounter++}>`);
|
|
6444
|
+
}
|
|
6445
|
+
assignIds(node.children);
|
|
6446
|
+
}
|
|
6447
|
+
};
|
|
6448
|
+
assignIds(tree);
|
|
6449
|
+
const normalizeNode = (node) => {
|
|
6450
|
+
const span = node.span;
|
|
6451
|
+
const completed = span.endTime !== void 0 && span.endTime !== null;
|
|
6452
|
+
const normalizedSpan = {
|
|
6453
|
+
id: spanIdMap.get(span.id),
|
|
6454
|
+
traceId: traceIdMap.get(span.traceId),
|
|
6455
|
+
name: normalizeValue(span.name, "name"),
|
|
6456
|
+
type: span.type,
|
|
6457
|
+
completed,
|
|
6458
|
+
isEvent: span.isEvent,
|
|
6459
|
+
isRootSpan: span.isRootSpan
|
|
6460
|
+
};
|
|
6461
|
+
if (span.parentSpanId && spanIdMap.has(span.parentSpanId)) {
|
|
6462
|
+
normalizedSpan.parentId = spanIdMap.get(span.parentSpanId);
|
|
6463
|
+
}
|
|
6464
|
+
if (span.entityType) {
|
|
6465
|
+
normalizedSpan.entityType = span.entityType;
|
|
6466
|
+
}
|
|
6467
|
+
if (span.entityId) {
|
|
6468
|
+
normalizedSpan.entityId = normalizeValue(span.entityId, "entityId");
|
|
6469
|
+
}
|
|
6470
|
+
if (span.attributes && Object.keys(span.attributes).length > 0) {
|
|
6471
|
+
normalizedSpan.attributes = normalizeValue(span.attributes);
|
|
6472
|
+
}
|
|
6473
|
+
if (span.metadata && Object.keys(span.metadata).length > 0) {
|
|
6474
|
+
normalizedSpan.metadata = normalizeValue(span.metadata);
|
|
6475
|
+
}
|
|
6476
|
+
if (span.input !== void 0) {
|
|
6477
|
+
normalizedSpan.input = normalizeValue(span.input);
|
|
6478
|
+
}
|
|
6479
|
+
if (span.output !== void 0) {
|
|
6480
|
+
normalizedSpan.output = normalizeValue(span.output);
|
|
6481
|
+
}
|
|
6482
|
+
if (span.errorInfo) {
|
|
6483
|
+
normalizedSpan.errorInfo = span.errorInfo;
|
|
6484
|
+
}
|
|
6485
|
+
if (span.tags && span.tags.length > 0) {
|
|
6486
|
+
normalizedSpan.tags = span.tags;
|
|
6487
|
+
}
|
|
6488
|
+
const result = { span: normalizedSpan };
|
|
6489
|
+
if (node.children.length > 0) {
|
|
6490
|
+
result.children = node.children.map(normalizeNode);
|
|
6491
|
+
}
|
|
6492
|
+
return result;
|
|
6493
|
+
};
|
|
6494
|
+
return tree.map(normalizeNode);
|
|
4839
6495
|
}
|
|
4840
6496
|
/**
|
|
4841
|
-
*
|
|
6497
|
+
* Generate an ASCII tree structure graph for debugging.
|
|
6498
|
+
* Shows span type and name in a hierarchical format.
|
|
6499
|
+
*
|
|
6500
|
+
* @param nodes - Normalized tree nodes (defaults to current normalized tree)
|
|
6501
|
+
* @returns Array of strings representing the tree structure
|
|
6502
|
+
*
|
|
6503
|
+
* @example
|
|
6504
|
+
* ```
|
|
6505
|
+
* agent_run: "agent run: 'test-agent'"
|
|
6506
|
+
* ├── processor_run: "input processor: validator"
|
|
6507
|
+
* │ └── agent_run: "agent run: 'validator-agent'"
|
|
6508
|
+
* └── model_generation: "llm: 'mock-model-id'"
|
|
6509
|
+
* ```
|
|
4842
6510
|
*/
|
|
4843
|
-
|
|
4844
|
-
this.
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
6511
|
+
generateStructureGraph(nodes) {
|
|
6512
|
+
const tree = nodes ?? this.buildNormalizedTree();
|
|
6513
|
+
const lines = [];
|
|
6514
|
+
const buildLines = (node, prefix, isLast, isRoot) => {
|
|
6515
|
+
const connector = isRoot ? "" : isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ";
|
|
6516
|
+
const line = `${prefix}${connector}${node.span.type}: "${node.span.name}"`;
|
|
6517
|
+
lines.push(line);
|
|
6518
|
+
const children = node.children ?? [];
|
|
6519
|
+
const childPrefix = isRoot ? "" : prefix + (isLast ? " " : "\u2502 ");
|
|
6520
|
+
children.forEach((child, index) => {
|
|
6521
|
+
const childIsLast = index === children.length - 1;
|
|
6522
|
+
buildLines(child, childPrefix, childIsLast, false);
|
|
4850
6523
|
});
|
|
4851
|
-
}
|
|
4852
|
-
|
|
4853
|
-
|
|
6524
|
+
};
|
|
6525
|
+
tree.forEach((rootNode, index) => {
|
|
6526
|
+
if (index > 0) {
|
|
6527
|
+
lines.push("");
|
|
6528
|
+
}
|
|
6529
|
+
buildLines(rootNode, "", true, true);
|
|
6530
|
+
});
|
|
6531
|
+
return lines;
|
|
4854
6532
|
}
|
|
4855
6533
|
/**
|
|
4856
|
-
*
|
|
6534
|
+
* Serialize spans as a normalized tree structure for snapshot testing.
|
|
6535
|
+
* Includes a __structure__ field with an ASCII tree graph for readability.
|
|
6536
|
+
*
|
|
6537
|
+
* @param options - Serialization options
|
|
6538
|
+
* @returns JSON string with normalized spans in tree format
|
|
4857
6539
|
*/
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
4867
|
-
}
|
|
4868
|
-
|
|
4869
|
-
}
|
|
6540
|
+
toNormalizedTreeJSON(options) {
|
|
6541
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6542
|
+
const includeStructure = options?.includeStructure ?? true;
|
|
6543
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6544
|
+
if (includeStructure) {
|
|
6545
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6546
|
+
const data = {
|
|
6547
|
+
__structure__: structureGraph,
|
|
6548
|
+
spans: normalizedTree
|
|
6549
|
+
};
|
|
6550
|
+
return JSON.stringify(data, null, indent);
|
|
4870
6551
|
}
|
|
6552
|
+
return JSON.stringify(normalizedTree, null, indent);
|
|
4871
6553
|
}
|
|
4872
6554
|
/**
|
|
4873
|
-
*
|
|
6555
|
+
* Write collected data to a JSON file
|
|
6556
|
+
*
|
|
6557
|
+
* @param filePath - Path to write the JSON file
|
|
6558
|
+
* @param options - Serialization options
|
|
4874
6559
|
*/
|
|
4875
|
-
|
|
4876
|
-
|
|
6560
|
+
async writeToFile(filePath, options) {
|
|
6561
|
+
const format = options?.format ?? "flat";
|
|
6562
|
+
let json;
|
|
6563
|
+
if (format === "normalized") {
|
|
6564
|
+
json = this.toNormalizedTreeJSON({ indent: options?.indent });
|
|
6565
|
+
} else if (format === "tree") {
|
|
6566
|
+
json = this.toTreeJSON({ indent: options?.indent, includeStats: options?.includeStats });
|
|
6567
|
+
} else {
|
|
6568
|
+
json = this.toJSON(options);
|
|
6569
|
+
}
|
|
6570
|
+
await promises.writeFile(filePath, json, "utf-8");
|
|
6571
|
+
this.logger.info(`JsonExporter: wrote ${this.#events.length} events to ${filePath}`);
|
|
4877
6572
|
}
|
|
4878
6573
|
/**
|
|
4879
|
-
*
|
|
6574
|
+
* Assert that the current normalized tree matches a snapshot file.
|
|
6575
|
+
* Throws an error with a diff if they don't match.
|
|
6576
|
+
*
|
|
6577
|
+
* The snapshot format includes:
|
|
6578
|
+
* - `__structure__`: ASCII tree graph (compared first for quick validation)
|
|
6579
|
+
* - `spans`: The normalized span tree (detailed comparison)
|
|
6580
|
+
*
|
|
6581
|
+
* Supports special markers in the snapshot:
|
|
6582
|
+
* - `{"__or__": ["value1", "value2"]}` - matches if actual equals any listed value
|
|
6583
|
+
* - `{"__any__": "string"}` - matches any string value
|
|
6584
|
+
* - `{"__any__": "number"}` - matches any number value
|
|
6585
|
+
* - `{"__any__": "boolean"}` - matches any boolean value
|
|
6586
|
+
* - `{"__any__": "object"}` - matches any object value
|
|
6587
|
+
* - `{"__any__": "array"}` - matches any array value
|
|
6588
|
+
* - `{"__any__": true}` - matches any non-null/undefined value
|
|
6589
|
+
*
|
|
6590
|
+
* Environment variables:
|
|
6591
|
+
* Use `{ updateSnapshot: true }` option to update the snapshot instead of comparing
|
|
6592
|
+
*
|
|
6593
|
+
* @param snapshotName - Name of the snapshot file (resolved relative to __snapshots__ directory)
|
|
6594
|
+
* @param options - Options for snapshot comparison
|
|
6595
|
+
* @param options.updateSnapshot - If true, update the snapshot file instead of comparing
|
|
6596
|
+
* @throws Error if the snapshot doesn't match (and updateSnapshot is false)
|
|
4880
6597
|
*/
|
|
4881
|
-
async
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
6598
|
+
async assertMatchesSnapshot(snapshotName, options) {
|
|
6599
|
+
const snapshotPath = path.join(SNAPSHOTS_DIR, snapshotName);
|
|
6600
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6601
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6602
|
+
const currentData = {
|
|
6603
|
+
__structure__: structureGraph,
|
|
6604
|
+
spans: normalizedTree
|
|
6605
|
+
};
|
|
6606
|
+
const currentJson = JSON.stringify(currentData, null, this.#config.jsonIndent);
|
|
6607
|
+
const shouldUpdate = options?.updateSnapshot;
|
|
6608
|
+
if (shouldUpdate) {
|
|
6609
|
+
await promises.writeFile(snapshotPath, currentJson, "utf-8");
|
|
6610
|
+
this.logger.info(`JsonExporter: updated snapshot ${snapshotPath}`);
|
|
4891
6611
|
return;
|
|
4892
6612
|
}
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
|
|
4902
|
-
|
|
4903
|
-
|
|
4904
|
-
|
|
4905
|
-
}
|
|
4906
|
-
|
|
4907
|
-
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
6613
|
+
let snapshotData;
|
|
6614
|
+
try {
|
|
6615
|
+
const snapshotContent = await promises.readFile(snapshotPath, "utf-8");
|
|
6616
|
+
snapshotData = JSON.parse(snapshotContent);
|
|
6617
|
+
} catch {
|
|
6618
|
+
throw new Error(`Snapshot file not found: ${snapshotPath}
|
|
6619
|
+
Run with { updateSnapshot: true } to create it.`);
|
|
6620
|
+
}
|
|
6621
|
+
let expectedSpans;
|
|
6622
|
+
let expectedStructure;
|
|
6623
|
+
if (Array.isArray(snapshotData)) {
|
|
6624
|
+
expectedSpans = snapshotData;
|
|
6625
|
+
} else if (snapshotData && typeof snapshotData === "object" && "spans" in snapshotData) {
|
|
6626
|
+
expectedSpans = snapshotData.spans;
|
|
6627
|
+
expectedStructure = snapshotData.__structure__;
|
|
6628
|
+
} else {
|
|
6629
|
+
throw new Error(
|
|
6630
|
+
`Invalid snapshot format in ${snapshotPath}.
|
|
6631
|
+
Expected an array or object with 'spans' property.`
|
|
6632
|
+
);
|
|
6633
|
+
}
|
|
6634
|
+
if (expectedStructure) {
|
|
6635
|
+
const structureMismatches = this.#compareStructure(structureGraph, expectedStructure);
|
|
6636
|
+
if (structureMismatches.length > 0) {
|
|
6637
|
+
throw new Error(
|
|
6638
|
+
`Structure mismatch in snapshot:
|
|
6639
|
+
|
|
6640
|
+
Expected:
|
|
6641
|
+
${expectedStructure.join("\n")}
|
|
6642
|
+
|
|
6643
|
+
Actual:
|
|
6644
|
+
${structureGraph.join("\n")}
|
|
6645
|
+
|
|
6646
|
+
Differences:
|
|
6647
|
+
${structureMismatches.join("\n")}
|
|
6648
|
+
|
|
6649
|
+
Snapshot: ${snapshotPath}
|
|
6650
|
+
Run with { updateSnapshot: true } to update.`
|
|
6651
|
+
);
|
|
6652
|
+
}
|
|
6653
|
+
}
|
|
6654
|
+
const mismatches = [];
|
|
6655
|
+
this.#deepCompareWithMarkers(normalizedTree, expectedSpans, "$.spans", mismatches);
|
|
6656
|
+
if (mismatches.length > 0) {
|
|
6657
|
+
const mismatchDetails = mismatches.map(
|
|
6658
|
+
(m, i) => `${i + 1}. ${m.path}
|
|
6659
|
+
Expected: ${JSON.stringify(m.expected)}
|
|
6660
|
+
Actual: ${JSON.stringify(m.actual)}`
|
|
6661
|
+
).join("\n\n");
|
|
6662
|
+
throw new Error(
|
|
6663
|
+
`Snapshot has ${mismatches.length} mismatch${mismatches.length > 1 ? "es" : ""}:
|
|
6664
|
+
|
|
6665
|
+
${mismatchDetails}
|
|
6666
|
+
|
|
6667
|
+
Snapshot: ${snapshotPath}
|
|
6668
|
+
Run with { updateSnapshot: true } to update.`
|
|
6669
|
+
);
|
|
6670
|
+
}
|
|
4916
6671
|
}
|
|
4917
6672
|
/**
|
|
4918
|
-
*
|
|
6673
|
+
* Compare two structure graphs and return differences
|
|
4919
6674
|
*/
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
if (buffer.insertOnly.length > 0) {
|
|
4938
|
-
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
6675
|
+
#compareStructure(actual, expected) {
|
|
6676
|
+
const diffs = [];
|
|
6677
|
+
const maxLen = Math.max(actual.length, expected.length);
|
|
6678
|
+
for (let i = 0; i < maxLen; i++) {
|
|
6679
|
+
const actualLine = actual[i];
|
|
6680
|
+
const expectedLine = expected[i];
|
|
6681
|
+
if (actualLine !== expectedLine) {
|
|
6682
|
+
if (actualLine === void 0) {
|
|
6683
|
+
diffs.push(`Line ${i + 1}: Missing in actual`);
|
|
6684
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6685
|
+
} else if (expectedLine === void 0) {
|
|
6686
|
+
diffs.push(`Line ${i + 1}: Extra in actual`);
|
|
6687
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
6688
|
+
} else {
|
|
6689
|
+
diffs.push(`Line ${i + 1}:`);
|
|
6690
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6691
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
4939
6692
|
}
|
|
4940
6693
|
}
|
|
4941
|
-
|
|
4942
|
-
|
|
6694
|
+
}
|
|
6695
|
+
return diffs;
|
|
6696
|
+
}
|
|
6697
|
+
/**
|
|
6698
|
+
* Deep compare two values, supporting special markers like __or__ and __any__.
|
|
6699
|
+
* Collects all mismatches into the provided array.
|
|
6700
|
+
*/
|
|
6701
|
+
#deepCompareWithMarkers(actual, expected, path, mismatches) {
|
|
6702
|
+
if (this.#isOrMarker(expected)) {
|
|
6703
|
+
const allowedValues = expected.__or__;
|
|
6704
|
+
const matches = allowedValues.some((allowed) => {
|
|
6705
|
+
const tempMismatches = [];
|
|
6706
|
+
this.#deepCompareWithMarkers(actual, allowed, path, tempMismatches);
|
|
6707
|
+
return tempMismatches.length === 0;
|
|
6708
|
+
});
|
|
6709
|
+
if (!matches) {
|
|
6710
|
+
mismatches.push({ path, expected: { __or__: allowedValues }, actual });
|
|
4943
6711
|
}
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
6712
|
+
return;
|
|
6713
|
+
}
|
|
6714
|
+
if (this.#isAnyMarker(expected)) {
|
|
6715
|
+
const typeConstraint = expected.__any__;
|
|
6716
|
+
if (actual === null || actual === void 0) {
|
|
6717
|
+
mismatches.push({ path, expected: { __any__: typeConstraint }, actual });
|
|
6718
|
+
return;
|
|
6719
|
+
}
|
|
6720
|
+
if (typeConstraint === true) {
|
|
6721
|
+
return;
|
|
6722
|
+
}
|
|
6723
|
+
const actualType = Array.isArray(actual) ? "array" : typeof actual;
|
|
6724
|
+
if (actualType !== typeConstraint) {
|
|
6725
|
+
mismatches.push({
|
|
6726
|
+
path,
|
|
6727
|
+
expected: { __any__: typeConstraint },
|
|
6728
|
+
actual: `(${actualType}) ${JSON.stringify(actual).slice(0, 50)}...`
|
|
4961
6729
|
});
|
|
4962
|
-
for (const spanKey of buffer.completedSpans) {
|
|
4963
|
-
this.allCreatedSpans.delete(spanKey);
|
|
4964
|
-
}
|
|
4965
6730
|
}
|
|
6731
|
+
return;
|
|
4966
6732
|
}
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
6733
|
+
if (Array.isArray(expected)) {
|
|
6734
|
+
if (!Array.isArray(actual)) {
|
|
6735
|
+
mismatches.push({ path, expected, actual });
|
|
6736
|
+
return;
|
|
6737
|
+
}
|
|
6738
|
+
if (actual.length !== expected.length) {
|
|
6739
|
+
mismatches.push({ path: `${path}.length`, expected: expected.length, actual: actual.length });
|
|
6740
|
+
return;
|
|
6741
|
+
}
|
|
6742
|
+
for (let i = 0; i < expected.length; i++) {
|
|
6743
|
+
this.#deepCompareWithMarkers(actual[i], expected[i], `${path}[${i}]`, mismatches);
|
|
6744
|
+
}
|
|
4971
6745
|
return;
|
|
4972
6746
|
}
|
|
4973
|
-
if (
|
|
4974
|
-
|
|
6747
|
+
if (expected !== null && typeof expected === "object") {
|
|
6748
|
+
if (actual === null || typeof actual !== "object" || Array.isArray(actual)) {
|
|
6749
|
+
mismatches.push({ path, expected, actual });
|
|
6750
|
+
return;
|
|
6751
|
+
}
|
|
6752
|
+
const expectedObj = expected;
|
|
6753
|
+
const actualObj = actual;
|
|
6754
|
+
for (const key of Object.keys(expectedObj)) {
|
|
6755
|
+
if (this.#isMetadataKey(key)) {
|
|
6756
|
+
continue;
|
|
6757
|
+
}
|
|
6758
|
+
if (!(key in actualObj)) {
|
|
6759
|
+
if (expectedObj[key] !== void 0) {
|
|
6760
|
+
mismatches.push({ path: `${path}.${key}`, expected: expectedObj[key], actual: void 0 });
|
|
6761
|
+
}
|
|
6762
|
+
continue;
|
|
6763
|
+
}
|
|
6764
|
+
this.#deepCompareWithMarkers(actualObj[key], expectedObj[key], `${path}.${key}`, mismatches);
|
|
6765
|
+
}
|
|
6766
|
+
for (const key of Object.keys(actualObj)) {
|
|
6767
|
+
if (this.#isMetadataKey(key)) {
|
|
6768
|
+
continue;
|
|
6769
|
+
}
|
|
6770
|
+
if (!(key in expectedObj)) {
|
|
6771
|
+
if (actualObj[key] !== void 0) {
|
|
6772
|
+
mismatches.push({ path: `${path}.${key}`, expected: void 0, actual: actualObj[key] });
|
|
6773
|
+
}
|
|
6774
|
+
}
|
|
6775
|
+
}
|
|
6776
|
+
return;
|
|
4975
6777
|
}
|
|
4976
|
-
|
|
4977
|
-
|
|
4978
|
-
await this.handleRealtimeEvent(event, this.#observability);
|
|
4979
|
-
break;
|
|
4980
|
-
case "batch-with-updates":
|
|
4981
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
4982
|
-
break;
|
|
4983
|
-
case "insert-only":
|
|
4984
|
-
this.handleInsertOnlyEvent(event);
|
|
4985
|
-
break;
|
|
6778
|
+
if (actual !== expected) {
|
|
6779
|
+
mismatches.push({ path, expected, actual });
|
|
4986
6780
|
}
|
|
4987
6781
|
}
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
6782
|
+
/**
|
|
6783
|
+
* Check if a value is an __or__ marker object
|
|
6784
|
+
*/
|
|
6785
|
+
#isOrMarker(value) {
|
|
6786
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) && "__or__" in value && Array.isArray(value.__or__);
|
|
6787
|
+
}
|
|
6788
|
+
/**
|
|
6789
|
+
* Check if a value is an __any__ marker object
|
|
6790
|
+
*/
|
|
6791
|
+
#isAnyMarker(value) {
|
|
6792
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
6793
|
+
return false;
|
|
4992
6794
|
}
|
|
4993
|
-
if (
|
|
4994
|
-
|
|
4995
|
-
remainingEvents: this.buffer.totalSize
|
|
4996
|
-
});
|
|
4997
|
-
try {
|
|
4998
|
-
await this.flush();
|
|
4999
|
-
} catch (error) {
|
|
5000
|
-
this.logger.error("Failed to flush remaining events during shutdown", {
|
|
5001
|
-
error: error instanceof Error ? error.message : String(error)
|
|
5002
|
-
});
|
|
5003
|
-
}
|
|
6795
|
+
if (!("__any__" in value)) {
|
|
6796
|
+
return false;
|
|
5004
6797
|
}
|
|
5005
|
-
|
|
6798
|
+
const constraint = value.__any__;
|
|
6799
|
+
return constraint === true || ["string", "number", "boolean", "object", "array"].includes(constraint);
|
|
5006
6800
|
}
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
|
|
5012
|
-
#events = [];
|
|
5013
|
-
constructor(config = {}) {
|
|
5014
|
-
super(config);
|
|
6801
|
+
/**
|
|
6802
|
+
* Check if a key should be skipped during comparison (metadata keys like __structure__)
|
|
6803
|
+
*/
|
|
6804
|
+
#isMetadataKey(key) {
|
|
6805
|
+
return key.startsWith("__") && key.endsWith("__");
|
|
5015
6806
|
}
|
|
5016
|
-
|
|
5017
|
-
|
|
6807
|
+
/**
|
|
6808
|
+
* Custom JSON replacer to handle Date objects and other special types
|
|
6809
|
+
*/
|
|
6810
|
+
#jsonReplacer = (_key, value) => {
|
|
6811
|
+
if (value instanceof Date) {
|
|
6812
|
+
return value.toISOString();
|
|
6813
|
+
}
|
|
6814
|
+
return value;
|
|
6815
|
+
};
|
|
6816
|
+
// ============================================================================
|
|
6817
|
+
// Debugging Helpers
|
|
6818
|
+
// ============================================================================
|
|
6819
|
+
/**
|
|
6820
|
+
* Get all stored logs
|
|
6821
|
+
*/
|
|
6822
|
+
getLogs() {
|
|
6823
|
+
return [...this.#logs];
|
|
6824
|
+
}
|
|
6825
|
+
/**
|
|
6826
|
+
* Dump logs to console for debugging (uses console.error for visibility in test output)
|
|
6827
|
+
*/
|
|
6828
|
+
dumpLogs() {
|
|
6829
|
+
console.error("\n=== JsonExporter Logs ===");
|
|
6830
|
+
this.#logs.forEach((log) => console.error(log));
|
|
6831
|
+
console.error("=== End Logs ===\n");
|
|
6832
|
+
}
|
|
6833
|
+
/**
|
|
6834
|
+
* Validate final state - useful for test assertions
|
|
6835
|
+
*
|
|
6836
|
+
* @returns Object with validation results
|
|
6837
|
+
*/
|
|
6838
|
+
validateFinalState() {
|
|
6839
|
+
const traceIds = this.getTraceIds();
|
|
6840
|
+
const incompleteSpans = this.getIncompleteSpans();
|
|
6841
|
+
const singleTraceId = traceIds.length === 1;
|
|
6842
|
+
const allSpansComplete = incompleteSpans.length === 0;
|
|
6843
|
+
return {
|
|
6844
|
+
valid: singleTraceId && allSpansComplete,
|
|
6845
|
+
singleTraceId,
|
|
6846
|
+
allSpansComplete,
|
|
6847
|
+
traceIds,
|
|
6848
|
+
incompleteSpans
|
|
6849
|
+
};
|
|
5018
6850
|
}
|
|
6851
|
+
// ============================================================================
|
|
6852
|
+
// Reset & Lifecycle
|
|
6853
|
+
// ============================================================================
|
|
6854
|
+
/**
|
|
6855
|
+
* Clear all collected events and state
|
|
6856
|
+
*/
|
|
5019
6857
|
clearEvents() {
|
|
5020
6858
|
this.#events = [];
|
|
6859
|
+
this.#spanStates.clear();
|
|
6860
|
+
this.#logs = [];
|
|
5021
6861
|
}
|
|
5022
|
-
|
|
5023
|
-
|
|
6862
|
+
/**
|
|
6863
|
+
* Alias for clearEvents (compatibility with TestExporter)
|
|
6864
|
+
*/
|
|
6865
|
+
reset() {
|
|
6866
|
+
this.clearEvents();
|
|
5024
6867
|
}
|
|
5025
6868
|
async shutdown() {
|
|
5026
|
-
this.logger.info("
|
|
6869
|
+
this.logger.info("JsonExporter shutdown");
|
|
6870
|
+
}
|
|
6871
|
+
// ============================================================================
|
|
6872
|
+
// Private Helpers
|
|
6873
|
+
// ============================================================================
|
|
6874
|
+
/**
|
|
6875
|
+
* Extract unique spans from a list of events
|
|
6876
|
+
*/
|
|
6877
|
+
#getUniqueSpansFromEvents(events) {
|
|
6878
|
+
const spanMap = /* @__PURE__ */ new Map();
|
|
6879
|
+
for (const event of events) {
|
|
6880
|
+
const span = event.exportedSpan;
|
|
6881
|
+
if (event.type === observability.TracingEventType.SPAN_ENDED || !spanMap.has(span.id)) {
|
|
6882
|
+
spanMap.set(span.id, span);
|
|
6883
|
+
}
|
|
6884
|
+
}
|
|
6885
|
+
return Array.from(spanMap.values());
|
|
5027
6886
|
}
|
|
5028
6887
|
};
|
|
5029
6888
|
|
|
@@ -5082,14 +6941,11 @@ var ModelSpanTracker = class {
|
|
|
5082
6941
|
#modelSpan;
|
|
5083
6942
|
#currentStepSpan;
|
|
5084
6943
|
#currentChunkSpan;
|
|
6944
|
+
#currentChunkType;
|
|
5085
6945
|
#accumulator = {};
|
|
5086
6946
|
#stepIndex = 0;
|
|
5087
6947
|
#chunkSequence = 0;
|
|
5088
6948
|
#completionStartTime;
|
|
5089
|
-
/** Tracks tool output accumulators by toolCallId for consolidating sub-agent streams */
|
|
5090
|
-
#toolOutputAccumulators = /* @__PURE__ */ new Map();
|
|
5091
|
-
/** Tracks toolCallIds that had streaming output (to skip redundant tool-result spans) */
|
|
5092
|
-
#streamedToolCallIds = /* @__PURE__ */ new Set();
|
|
5093
6949
|
constructor(modelSpan) {
|
|
5094
6950
|
this.#modelSpan = modelSpan;
|
|
5095
6951
|
}
|
|
@@ -5176,6 +7032,7 @@ var ModelSpanTracker = class {
|
|
|
5176
7032
|
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
5177
7033
|
*/
|
|
5178
7034
|
#endStepSpan(payload) {
|
|
7035
|
+
this.#endChunkSpan();
|
|
5179
7036
|
if (!this.#currentStepSpan) return;
|
|
5180
7037
|
const output = payload.output;
|
|
5181
7038
|
const { usage: rawUsage, ...otherOutput } = output;
|
|
@@ -5183,8 +7040,10 @@ var ModelSpanTracker = class {
|
|
|
5183
7040
|
const metadata = payload.metadata;
|
|
5184
7041
|
const usage = extractUsageMetrics(rawUsage, metadata?.providerMetadata);
|
|
5185
7042
|
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
5186
|
-
if (cleanMetadata
|
|
5187
|
-
|
|
7043
|
+
if (cleanMetadata) {
|
|
7044
|
+
for (const key of ["request", "id", "timestamp", "modelId", "modelVersion", "modelProvider"]) {
|
|
7045
|
+
delete cleanMetadata[key];
|
|
7046
|
+
}
|
|
5188
7047
|
}
|
|
5189
7048
|
this.#currentStepSpan.end({
|
|
5190
7049
|
output: otherOutput,
|
|
@@ -5205,6 +7064,7 @@ var ModelSpanTracker = class {
|
|
|
5205
7064
|
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
5206
7065
|
*/
|
|
5207
7066
|
#startChunkSpan(chunkType, initialData) {
|
|
7067
|
+
this.#endChunkSpan();
|
|
5208
7068
|
if (!this.#currentStepSpan) {
|
|
5209
7069
|
this.startStep();
|
|
5210
7070
|
}
|
|
@@ -5216,6 +7076,7 @@ var ModelSpanTracker = class {
|
|
|
5216
7076
|
sequenceNumber: this.#chunkSequence
|
|
5217
7077
|
}
|
|
5218
7078
|
});
|
|
7079
|
+
this.#currentChunkType = chunkType;
|
|
5219
7080
|
this.#accumulator = initialData || {};
|
|
5220
7081
|
}
|
|
5221
7082
|
/**
|
|
@@ -5238,13 +7099,14 @@ var ModelSpanTracker = class {
|
|
|
5238
7099
|
output: output !== void 0 ? output : this.#accumulator
|
|
5239
7100
|
});
|
|
5240
7101
|
this.#currentChunkSpan = void 0;
|
|
7102
|
+
this.#currentChunkType = void 0;
|
|
5241
7103
|
this.#accumulator = {};
|
|
5242
7104
|
this.#chunkSequence++;
|
|
5243
7105
|
}
|
|
5244
7106
|
/**
|
|
5245
7107
|
* Create an event span (for single chunks like tool-call)
|
|
5246
7108
|
*/
|
|
5247
|
-
#createEventSpan(chunkType, output) {
|
|
7109
|
+
#createEventSpan(chunkType, output, options) {
|
|
5248
7110
|
if (!this.#currentStepSpan) {
|
|
5249
7111
|
this.startStep();
|
|
5250
7112
|
}
|
|
@@ -5253,8 +7115,10 @@ var ModelSpanTracker = class {
|
|
|
5253
7115
|
type: observability.SpanType.MODEL_CHUNK,
|
|
5254
7116
|
attributes: {
|
|
5255
7117
|
chunkType,
|
|
5256
|
-
sequenceNumber: this.#chunkSequence
|
|
7118
|
+
sequenceNumber: this.#chunkSequence,
|
|
7119
|
+
...options?.attributes
|
|
5257
7120
|
},
|
|
7121
|
+
metadata: options?.metadata,
|
|
5258
7122
|
output
|
|
5259
7123
|
});
|
|
5260
7124
|
if (span) {
|
|
@@ -5282,6 +7146,9 @@ var ModelSpanTracker = class {
|
|
|
5282
7146
|
this.#startChunkSpan("text");
|
|
5283
7147
|
break;
|
|
5284
7148
|
case "text-delta":
|
|
7149
|
+
if (this.#currentChunkType !== "text") {
|
|
7150
|
+
this.#startChunkSpan("text");
|
|
7151
|
+
}
|
|
5285
7152
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5286
7153
|
break;
|
|
5287
7154
|
case "text-end": {
|
|
@@ -5299,6 +7166,9 @@ var ModelSpanTracker = class {
|
|
|
5299
7166
|
this.#startChunkSpan("reasoning");
|
|
5300
7167
|
break;
|
|
5301
7168
|
case "reasoning-delta":
|
|
7169
|
+
if (this.#currentChunkType !== "reasoning") {
|
|
7170
|
+
this.#startChunkSpan("reasoning");
|
|
7171
|
+
}
|
|
5302
7172
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5303
7173
|
break;
|
|
5304
7174
|
case "reasoning-end": {
|
|
@@ -5345,7 +7215,7 @@ var ModelSpanTracker = class {
|
|
|
5345
7215
|
#handleObjectChunk(chunk) {
|
|
5346
7216
|
switch (chunk.type) {
|
|
5347
7217
|
case "object":
|
|
5348
|
-
if (
|
|
7218
|
+
if (this.#currentChunkType !== "object") {
|
|
5349
7219
|
this.#startChunkSpan("object");
|
|
5350
7220
|
}
|
|
5351
7221
|
break;
|
|
@@ -5355,75 +7225,27 @@ var ModelSpanTracker = class {
|
|
|
5355
7225
|
}
|
|
5356
7226
|
}
|
|
5357
7227
|
/**
|
|
5358
|
-
* Handle tool-
|
|
5359
|
-
*
|
|
7228
|
+
* Handle tool-call-approval chunks.
|
|
7229
|
+
* Creates a span for approval requests so they can be seen in traces for debugging.
|
|
5360
7230
|
*/
|
|
5361
|
-
#
|
|
5362
|
-
if (chunk.type !== "tool-
|
|
7231
|
+
#handleToolApprovalChunk(chunk) {
|
|
7232
|
+
if (chunk.type !== "tool-call-approval") return;
|
|
5363
7233
|
const payload = chunk.payload;
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
if (!acc) {
|
|
5367
|
-
if (!this.#currentStepSpan) {
|
|
5368
|
-
this.startStep();
|
|
5369
|
-
}
|
|
5370
|
-
acc = {
|
|
5371
|
-
toolName: toolName || "unknown",
|
|
5372
|
-
toolCallId,
|
|
5373
|
-
text: "",
|
|
5374
|
-
reasoning: "",
|
|
5375
|
-
sequenceNumber: this.#chunkSequence++,
|
|
5376
|
-
// Name the span 'tool-result' for consistency (tool-call → tool-result)
|
|
5377
|
-
span: this.#currentStepSpan?.createChildSpan({
|
|
5378
|
-
name: `chunk: 'tool-result'`,
|
|
5379
|
-
type: observability.SpanType.MODEL_CHUNK,
|
|
5380
|
-
attributes: {
|
|
5381
|
-
chunkType: "tool-result",
|
|
5382
|
-
sequenceNumber: this.#chunkSequence - 1
|
|
5383
|
-
}
|
|
5384
|
-
})
|
|
5385
|
-
};
|
|
5386
|
-
this.#toolOutputAccumulators.set(toolCallId, acc);
|
|
5387
|
-
}
|
|
5388
|
-
if (output && typeof output === "object" && "type" in output) {
|
|
5389
|
-
const innerType = output.type;
|
|
5390
|
-
switch (innerType) {
|
|
5391
|
-
case "text-delta":
|
|
5392
|
-
if (output.payload?.text) {
|
|
5393
|
-
acc.text += output.payload.text;
|
|
5394
|
-
}
|
|
5395
|
-
break;
|
|
5396
|
-
case "reasoning-delta":
|
|
5397
|
-
if (output.payload?.text) {
|
|
5398
|
-
acc.reasoning += output.payload.text;
|
|
5399
|
-
}
|
|
5400
|
-
break;
|
|
5401
|
-
case "finish":
|
|
5402
|
-
case "workflow-finish":
|
|
5403
|
-
this.#endToolOutputSpan(toolCallId);
|
|
5404
|
-
break;
|
|
5405
|
-
}
|
|
5406
|
-
}
|
|
5407
|
-
}
|
|
5408
|
-
/**
|
|
5409
|
-
* End a tool output span and clean up the accumulator
|
|
5410
|
-
*/
|
|
5411
|
-
#endToolOutputSpan(toolCallId) {
|
|
5412
|
-
const acc = this.#toolOutputAccumulators.get(toolCallId);
|
|
5413
|
-
if (!acc) return;
|
|
5414
|
-
const output = {
|
|
5415
|
-
toolCallId: acc.toolCallId,
|
|
5416
|
-
toolName: acc.toolName
|
|
5417
|
-
};
|
|
5418
|
-
if (acc.text) {
|
|
5419
|
-
output.text = acc.text;
|
|
7234
|
+
if (!this.#currentStepSpan) {
|
|
7235
|
+
this.startStep();
|
|
5420
7236
|
}
|
|
5421
|
-
|
|
5422
|
-
|
|
7237
|
+
const span = this.#currentStepSpan?.createEventSpan({
|
|
7238
|
+
name: `chunk: 'tool-call-approval'`,
|
|
7239
|
+
type: observability.SpanType.MODEL_CHUNK,
|
|
7240
|
+
attributes: {
|
|
7241
|
+
chunkType: "tool-call-approval",
|
|
7242
|
+
sequenceNumber: this.#chunkSequence
|
|
7243
|
+
},
|
|
7244
|
+
output: payload
|
|
7245
|
+
});
|
|
7246
|
+
if (span) {
|
|
7247
|
+
this.#chunkSequence++;
|
|
5423
7248
|
}
|
|
5424
|
-
acc.span?.end({ output });
|
|
5425
|
-
this.#toolOutputAccumulators.delete(toolCallId);
|
|
5426
|
-
this.#streamedToolCallIds.add(toolCallId);
|
|
5427
7249
|
}
|
|
5428
7250
|
/**
|
|
5429
7251
|
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
@@ -5474,36 +7296,60 @@ var ModelSpanTracker = class {
|
|
|
5474
7296
|
case "step-finish":
|
|
5475
7297
|
this.#endStepSpan(chunk.payload);
|
|
5476
7298
|
break;
|
|
7299
|
+
// Infrastructure chunks - skip creating spans for these
|
|
7300
|
+
// They are either redundant, metadata-only, or error/control flow
|
|
5477
7301
|
case "raw":
|
|
5478
|
-
//
|
|
7302
|
+
// Redundant raw data
|
|
5479
7303
|
case "start":
|
|
7304
|
+
// Stream start marker
|
|
5480
7305
|
case "finish":
|
|
7306
|
+
// Stream finish marker (step-finish already captures this)
|
|
7307
|
+
case "response-metadata":
|
|
7308
|
+
// Response metadata (not semantic content)
|
|
7309
|
+
case "source":
|
|
7310
|
+
// Source references (metadata)
|
|
7311
|
+
case "file":
|
|
7312
|
+
// Binary file data (too large/not semantic)
|
|
7313
|
+
case "error":
|
|
7314
|
+
// Error handling
|
|
7315
|
+
case "abort":
|
|
7316
|
+
// Abort signal
|
|
7317
|
+
case "tripwire":
|
|
7318
|
+
// Processor rejection
|
|
7319
|
+
case "watch":
|
|
7320
|
+
// Internal watch event
|
|
7321
|
+
case "tool-error":
|
|
7322
|
+
// Tool error handling
|
|
7323
|
+
case "tool-call-suspended":
|
|
7324
|
+
// Suspension (not content)
|
|
7325
|
+
case "reasoning-signature":
|
|
7326
|
+
// Signature metadata
|
|
7327
|
+
case "redacted-reasoning":
|
|
7328
|
+
// Redacted content metadata
|
|
7329
|
+
case "step-output":
|
|
7330
|
+
break;
|
|
7331
|
+
case "tool-call-approval":
|
|
7332
|
+
this.#handleToolApprovalChunk(chunk);
|
|
5481
7333
|
break;
|
|
5482
7334
|
case "tool-output":
|
|
5483
|
-
this.#handleToolOutputChunk(chunk);
|
|
5484
7335
|
break;
|
|
5485
7336
|
case "tool-result": {
|
|
5486
|
-
const
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
|
|
5492
|
-
|
|
5493
|
-
|
|
5494
|
-
|
|
5495
|
-
|
|
5496
|
-
|
|
5497
|
-
|
|
5498
|
-
if (
|
|
5499
|
-
|
|
5500
|
-
|
|
5501
|
-
|
|
5502
|
-
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
5503
|
-
delete outputPayload.data;
|
|
5504
|
-
}
|
|
5505
|
-
}
|
|
5506
|
-
this.#createEventSpan(chunk.type, outputPayload);
|
|
7337
|
+
const {
|
|
7338
|
+
// Metadata - tool call context (unique to tool-result chunks)
|
|
7339
|
+
toolCallId,
|
|
7340
|
+
toolName,
|
|
7341
|
+
isError,
|
|
7342
|
+
dynamic,
|
|
7343
|
+
providerExecuted,
|
|
7344
|
+
providerMetadata,
|
|
7345
|
+
// Output - the actual result
|
|
7346
|
+
result} = chunk.payload || {};
|
|
7347
|
+
const metadata = { toolCallId, toolName };
|
|
7348
|
+
if (isError !== void 0) metadata.isError = isError;
|
|
7349
|
+
if (dynamic !== void 0) metadata.dynamic = dynamic;
|
|
7350
|
+
if (providerExecuted !== void 0) metadata.providerExecuted = providerExecuted;
|
|
7351
|
+
if (providerMetadata !== void 0) metadata.providerMetadata = providerMetadata;
|
|
7352
|
+
this.#createEventSpan(chunk.type, result, { metadata });
|
|
5507
7353
|
break;
|
|
5508
7354
|
}
|
|
5509
7355
|
}
|
|
@@ -5519,7 +7365,11 @@ var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
|
5519
7365
|
"experimental_providerMetadata",
|
|
5520
7366
|
"providerMetadata",
|
|
5521
7367
|
"steps",
|
|
5522
|
-
"tracingContext"
|
|
7368
|
+
"tracingContext",
|
|
7369
|
+
"execute",
|
|
7370
|
+
// Tool execute functions
|
|
7371
|
+
"validate"
|
|
7372
|
+
// Schema validate functions
|
|
5523
7373
|
]);
|
|
5524
7374
|
var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
5525
7375
|
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
@@ -5528,12 +7378,68 @@ var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
|
5528
7378
|
maxArrayLength: 50,
|
|
5529
7379
|
maxObjectKeys: 50
|
|
5530
7380
|
});
|
|
7381
|
+
function mergeSerializationOptions(userOptions) {
|
|
7382
|
+
if (!userOptions) {
|
|
7383
|
+
return DEFAULT_DEEP_CLEAN_OPTIONS;
|
|
7384
|
+
}
|
|
7385
|
+
return {
|
|
7386
|
+
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
7387
|
+
maxDepth: userOptions.maxDepth ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxDepth,
|
|
7388
|
+
maxStringLength: userOptions.maxStringLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxStringLength,
|
|
7389
|
+
maxArrayLength: userOptions.maxArrayLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxArrayLength,
|
|
7390
|
+
maxObjectKeys: userOptions.maxObjectKeys ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxObjectKeys
|
|
7391
|
+
};
|
|
7392
|
+
}
|
|
5531
7393
|
function truncateString(s, maxChars) {
|
|
5532
7394
|
if (s.length <= maxChars) {
|
|
5533
7395
|
return s;
|
|
5534
7396
|
}
|
|
5535
7397
|
return s.slice(0, maxChars) + "\u2026[truncated]";
|
|
5536
7398
|
}
|
|
7399
|
+
function isJsonSchema(val) {
|
|
7400
|
+
if (typeof val !== "object" || val === null) return false;
|
|
7401
|
+
if (val.$schema && typeof val.$schema === "string" && val.$schema.includes("json-schema")) {
|
|
7402
|
+
return true;
|
|
7403
|
+
}
|
|
7404
|
+
if (val.type === "object" && val.properties && typeof val.properties === "object") {
|
|
7405
|
+
return true;
|
|
7406
|
+
}
|
|
7407
|
+
return false;
|
|
7408
|
+
}
|
|
7409
|
+
function compressJsonSchema(schema, depth = 0) {
|
|
7410
|
+
if (depth > 3) {
|
|
7411
|
+
return schema.type || "object";
|
|
7412
|
+
}
|
|
7413
|
+
if (schema.type !== "object" || !schema.properties) {
|
|
7414
|
+
return schema.type || schema;
|
|
7415
|
+
}
|
|
7416
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
7417
|
+
const compressed = {};
|
|
7418
|
+
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
7419
|
+
const prop = propSchema;
|
|
7420
|
+
let value = prop.type || "unknown";
|
|
7421
|
+
if (prop.type === "object" && prop.properties) {
|
|
7422
|
+
value = compressJsonSchema(prop, depth + 1);
|
|
7423
|
+
if (required.has(key)) {
|
|
7424
|
+
compressed[key + " (required)"] = value;
|
|
7425
|
+
continue;
|
|
7426
|
+
}
|
|
7427
|
+
} else if (prop.type === "array" && prop.items) {
|
|
7428
|
+
if (prop.items.type === "object" && prop.items.properties) {
|
|
7429
|
+
value = [compressJsonSchema(prop.items, depth + 1)];
|
|
7430
|
+
} else {
|
|
7431
|
+
value = `${prop.items.type || "any"}[]`;
|
|
7432
|
+
}
|
|
7433
|
+
} else if (prop.enum) {
|
|
7434
|
+
value = prop.enum.map((v) => JSON.stringify(v)).join(" | ");
|
|
7435
|
+
}
|
|
7436
|
+
if (required.has(key) && typeof value === "string") {
|
|
7437
|
+
value += " (required)";
|
|
7438
|
+
}
|
|
7439
|
+
compressed[key] = value;
|
|
7440
|
+
}
|
|
7441
|
+
return compressed;
|
|
7442
|
+
}
|
|
5537
7443
|
function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
5538
7444
|
const { keysToStrip, maxDepth, maxStringLength, maxArrayLength, maxObjectKeys } = options;
|
|
5539
7445
|
const seen = /* @__PURE__ */ new WeakSet();
|
|
@@ -5593,6 +7499,15 @@ function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
|
5593
7499
|
if (val instanceof ArrayBuffer) {
|
|
5594
7500
|
return `[ArrayBuffer byteLength=${val.byteLength}]`;
|
|
5595
7501
|
}
|
|
7502
|
+
if (typeof val.serializeForSpan === "function") {
|
|
7503
|
+
try {
|
|
7504
|
+
return helper(val.serializeForSpan(), depth);
|
|
7505
|
+
} catch {
|
|
7506
|
+
}
|
|
7507
|
+
}
|
|
7508
|
+
if (isJsonSchema(val)) {
|
|
7509
|
+
return helper(compressJsonSchema(val), depth);
|
|
7510
|
+
}
|
|
5596
7511
|
const cleaned = {};
|
|
5597
7512
|
const entries = Object.entries(val);
|
|
5598
7513
|
let keyCount = 0;
|
|
@@ -5684,25 +7599,30 @@ var BaseSpan = class {
|
|
|
5684
7599
|
entityName;
|
|
5685
7600
|
/** Parent span ID (for root spans that are children of external spans) */
|
|
5686
7601
|
parentSpanId;
|
|
7602
|
+
/** Deep clean options for serialization */
|
|
7603
|
+
deepCleanOptions;
|
|
5687
7604
|
constructor(options, observabilityInstance) {
|
|
7605
|
+
const serializationOptions = observabilityInstance.getConfig().serializationOptions;
|
|
7606
|
+
this.deepCleanOptions = mergeSerializationOptions(serializationOptions);
|
|
5688
7607
|
this.name = options.name;
|
|
5689
7608
|
this.type = options.type;
|
|
5690
|
-
this.attributes = deepClean(options.attributes) || {};
|
|
5691
|
-
this.metadata = deepClean(options.metadata);
|
|
7609
|
+
this.attributes = deepClean(options.attributes, this.deepCleanOptions) || {};
|
|
7610
|
+
this.metadata = deepClean(options.metadata, this.deepCleanOptions);
|
|
5692
7611
|
this.parent = options.parent;
|
|
5693
|
-
this.startTime = /* @__PURE__ */ new Date();
|
|
7612
|
+
this.startTime = options.startTime ?? /* @__PURE__ */ new Date();
|
|
5694
7613
|
this.observabilityInstance = observabilityInstance;
|
|
5695
7614
|
this.isEvent = options.isEvent ?? false;
|
|
5696
7615
|
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
5697
7616
|
this.traceState = options.traceState;
|
|
5698
7617
|
this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
|
|
5699
|
-
|
|
5700
|
-
this.
|
|
5701
|
-
this.
|
|
7618
|
+
const entityParent = this.getParentSpan(false);
|
|
7619
|
+
this.entityType = options.entityType ?? entityParent?.entityType;
|
|
7620
|
+
this.entityId = options.entityId ?? entityParent?.entityId;
|
|
7621
|
+
this.entityName = options.entityName ?? entityParent?.entityName;
|
|
5702
7622
|
if (this.isEvent) {
|
|
5703
|
-
this.output = deepClean(options.output);
|
|
7623
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5704
7624
|
} else {
|
|
5705
|
-
this.input = deepClean(options.input);
|
|
7625
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5706
7626
|
}
|
|
5707
7627
|
}
|
|
5708
7628
|
createChildSpan(options) {
|
|
@@ -5725,14 +7645,25 @@ var BaseSpan = class {
|
|
|
5725
7645
|
get isRootSpan() {
|
|
5726
7646
|
return !this.parent;
|
|
5727
7647
|
}
|
|
7648
|
+
/** Get the closest parent span, optionally skipping internal spans */
|
|
7649
|
+
getParentSpan(includeInternalSpans) {
|
|
7650
|
+
if (!this.parent) {
|
|
7651
|
+
return void 0;
|
|
7652
|
+
}
|
|
7653
|
+
if (includeInternalSpans) return this.parent;
|
|
7654
|
+
if (this.parent.isInternal) return this.parent.getParentSpan(includeInternalSpans);
|
|
7655
|
+
return this.parent;
|
|
7656
|
+
}
|
|
5728
7657
|
/** Get the closest parent spanId that isn't an internal span */
|
|
5729
7658
|
getParentSpanId(includeInternalSpans) {
|
|
5730
7659
|
if (!this.parent) {
|
|
5731
7660
|
return this.parentSpanId;
|
|
5732
7661
|
}
|
|
5733
|
-
|
|
5734
|
-
if (
|
|
5735
|
-
|
|
7662
|
+
const parentSpan = this.getParentSpan(includeInternalSpans);
|
|
7663
|
+
if (parentSpan) {
|
|
7664
|
+
return parentSpan.id;
|
|
7665
|
+
}
|
|
7666
|
+
return this.parent.getParentSpanId(includeInternalSpans);
|
|
5736
7667
|
}
|
|
5737
7668
|
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
5738
7669
|
findParent(spanType) {
|
|
@@ -5747,6 +7678,8 @@ var BaseSpan = class {
|
|
|
5747
7678
|
}
|
|
5748
7679
|
/** Returns a lightweight span ready for export */
|
|
5749
7680
|
exportSpan(includeInternalSpans) {
|
|
7681
|
+
const hideInput = this.traceState?.hideInput ?? false;
|
|
7682
|
+
const hideOutput = this.traceState?.hideOutput ?? false;
|
|
5750
7683
|
return {
|
|
5751
7684
|
id: this.id,
|
|
5752
7685
|
traceId: this.traceId,
|
|
@@ -5759,8 +7692,8 @@ var BaseSpan = class {
|
|
|
5759
7692
|
metadata: this.metadata,
|
|
5760
7693
|
startTime: this.startTime,
|
|
5761
7694
|
endTime: this.endTime,
|
|
5762
|
-
input: this.input,
|
|
5763
|
-
output: this.output,
|
|
7695
|
+
input: hideInput ? void 0 : this.input,
|
|
7696
|
+
output: hideOutput ? void 0 : this.output,
|
|
5764
7697
|
errorInfo: this.errorInfo,
|
|
5765
7698
|
isEvent: this.isEvent,
|
|
5766
7699
|
isRootSpan: this.isRootSpan,
|
|
@@ -5800,6 +7733,14 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5800
7733
|
traceId;
|
|
5801
7734
|
constructor(options, observabilityInstance) {
|
|
5802
7735
|
super(options, observabilityInstance);
|
|
7736
|
+
if (options.spanId && options.traceId) {
|
|
7737
|
+
this.id = options.spanId;
|
|
7738
|
+
this.traceId = options.traceId;
|
|
7739
|
+
if (options.parentSpanId) {
|
|
7740
|
+
this.parentSpanId = options.parentSpanId;
|
|
7741
|
+
}
|
|
7742
|
+
return;
|
|
7743
|
+
}
|
|
5803
7744
|
const bridge = observabilityInstance.getBridge();
|
|
5804
7745
|
if (bridge && !this.isInternal) {
|
|
5805
7746
|
const bridgeIds = bridge.createSpan(options);
|
|
@@ -5834,13 +7775,13 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5834
7775
|
}
|
|
5835
7776
|
this.endTime = /* @__PURE__ */ new Date();
|
|
5836
7777
|
if (options?.output !== void 0) {
|
|
5837
|
-
this.output = deepClean(options.output);
|
|
7778
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5838
7779
|
}
|
|
5839
7780
|
if (options?.attributes) {
|
|
5840
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7781
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5841
7782
|
}
|
|
5842
7783
|
if (options?.metadata) {
|
|
5843
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7784
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5844
7785
|
}
|
|
5845
7786
|
}
|
|
5846
7787
|
error(options) {
|
|
@@ -5858,10 +7799,10 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5858
7799
|
message: error$1.message
|
|
5859
7800
|
};
|
|
5860
7801
|
if (attributes) {
|
|
5861
|
-
this.attributes = { ...this.attributes, ...deepClean(attributes) };
|
|
7802
|
+
this.attributes = { ...this.attributes, ...deepClean(attributes, this.deepCleanOptions) };
|
|
5862
7803
|
}
|
|
5863
7804
|
if (metadata) {
|
|
5864
|
-
this.metadata = { ...this.metadata, ...deepClean(metadata) };
|
|
7805
|
+
this.metadata = { ...this.metadata, ...deepClean(metadata, this.deepCleanOptions) };
|
|
5865
7806
|
}
|
|
5866
7807
|
if (endSpan) {
|
|
5867
7808
|
this.end();
|
|
@@ -5874,16 +7815,16 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5874
7815
|
return;
|
|
5875
7816
|
}
|
|
5876
7817
|
if (options.input !== void 0) {
|
|
5877
|
-
this.input = deepClean(options.input);
|
|
7818
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5878
7819
|
}
|
|
5879
7820
|
if (options.output !== void 0) {
|
|
5880
|
-
this.output = deepClean(options.output);
|
|
7821
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5881
7822
|
}
|
|
5882
7823
|
if (options.attributes) {
|
|
5883
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7824
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5884
7825
|
}
|
|
5885
7826
|
if (options.metadata) {
|
|
5886
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7827
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5887
7828
|
}
|
|
5888
7829
|
}
|
|
5889
7830
|
get isValid() {
|
|
@@ -5974,7 +7915,8 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
5974
7915
|
spanOutputProcessors: config.spanOutputProcessors ?? [],
|
|
5975
7916
|
bridge: config.bridge ?? void 0,
|
|
5976
7917
|
includeInternalSpans: config.includeInternalSpans ?? false,
|
|
5977
|
-
requestContextKeys: config.requestContextKeys ?? []
|
|
7918
|
+
requestContextKeys: config.requestContextKeys ?? [],
|
|
7919
|
+
serializationOptions: config.serializationOptions
|
|
5978
7920
|
};
|
|
5979
7921
|
if (this.config.bridge?.init) {
|
|
5980
7922
|
this.config.bridge.init({ config: this.config });
|
|
@@ -6012,11 +7954,26 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
6012
7954
|
// ============================================================================
|
|
6013
7955
|
/**
|
|
6014
7956
|
* Start a new span of a specific SpanType
|
|
7957
|
+
*
|
|
7958
|
+
* Sampling Decision:
|
|
7959
|
+
* - For root spans (no parent): Perform sampling check using the configured strategy
|
|
7960
|
+
* - For child spans: Inherit the sampling decision from the parent
|
|
7961
|
+
* - If parent is a NoOpSpan (not sampled), child is also a NoOpSpan
|
|
7962
|
+
* - If parent is a valid span (sampled), child is also sampled
|
|
7963
|
+
*
|
|
7964
|
+
* This ensures trace-level sampling: either all spans in a trace are sampled or none are.
|
|
7965
|
+
* See: https://github.com/mastra-ai/mastra/issues/11504
|
|
6015
7966
|
*/
|
|
6016
7967
|
startSpan(options) {
|
|
6017
7968
|
const { customSamplerOptions, requestContext, metadata, tracingOptions, ...rest } = options;
|
|
6018
|
-
if (
|
|
6019
|
-
|
|
7969
|
+
if (options.parent) {
|
|
7970
|
+
if (!options.parent.isValid) {
|
|
7971
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7972
|
+
}
|
|
7973
|
+
} else {
|
|
7974
|
+
if (!this.shouldSample(customSamplerOptions)) {
|
|
7975
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7976
|
+
}
|
|
6020
7977
|
}
|
|
6021
7978
|
let traceState;
|
|
6022
7979
|
if (options.parent) {
|
|
@@ -6028,8 +7985,12 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
6028
7985
|
const mergedMetadata = metadata || tracingMetadata ? { ...metadata, ...tracingMetadata } : void 0;
|
|
6029
7986
|
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, mergedMetadata, traceState);
|
|
6030
7987
|
const tags = !options.parent ? tracingOptions?.tags : void 0;
|
|
7988
|
+
const traceId = !options.parent ? options.traceId ?? tracingOptions?.traceId : options.traceId;
|
|
7989
|
+
const parentSpanId = !options.parent ? options.parentSpanId ?? tracingOptions?.parentSpanId : options.parentSpanId;
|
|
6031
7990
|
const span = this.createSpan({
|
|
6032
7991
|
...rest,
|
|
7992
|
+
traceId,
|
|
7993
|
+
parentSpanId,
|
|
6033
7994
|
metadata: enrichedMetadata,
|
|
6034
7995
|
traceState,
|
|
6035
7996
|
tags
|
|
@@ -6042,6 +8003,37 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
6042
8003
|
}
|
|
6043
8004
|
return span;
|
|
6044
8005
|
}
|
|
8006
|
+
/**
|
|
8007
|
+
* Rebuild a span from exported data for lifecycle operations.
|
|
8008
|
+
* Used by durable execution engines (e.g., Inngest) to end/update spans
|
|
8009
|
+
* that were created in a previous durable operation.
|
|
8010
|
+
*
|
|
8011
|
+
* The rebuilt span:
|
|
8012
|
+
* - Does NOT emit SPAN_STARTED (assumes original span already did)
|
|
8013
|
+
* - Can have end(), update(), error() called on it
|
|
8014
|
+
* - Will emit SPAN_ENDED or SPAN_UPDATED when those methods are called
|
|
8015
|
+
*
|
|
8016
|
+
* @param cached - The exported span data to rebuild from
|
|
8017
|
+
* @returns A span that can have lifecycle methods called on it
|
|
8018
|
+
*/
|
|
8019
|
+
rebuildSpan(cached) {
|
|
8020
|
+
const span = this.createSpan({
|
|
8021
|
+
name: cached.name,
|
|
8022
|
+
type: cached.type,
|
|
8023
|
+
traceId: cached.traceId,
|
|
8024
|
+
spanId: cached.id,
|
|
8025
|
+
parentSpanId: cached.parentSpanId,
|
|
8026
|
+
startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
|
|
8027
|
+
input: cached.input,
|
|
8028
|
+
attributes: cached.attributes,
|
|
8029
|
+
metadata: cached.metadata,
|
|
8030
|
+
entityType: cached.entityType,
|
|
8031
|
+
entityId: cached.entityId,
|
|
8032
|
+
entityName: cached.entityName
|
|
8033
|
+
});
|
|
8034
|
+
this.wireSpanLifecycle(span);
|
|
8035
|
+
return span;
|
|
8036
|
+
}
|
|
6045
8037
|
// ============================================================================
|
|
6046
8038
|
// Configuration Management
|
|
6047
8039
|
// ============================================================================
|
|
@@ -6144,11 +8136,15 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
6144
8136
|
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
6145
8137
|
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
6146
8138
|
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
6147
|
-
|
|
8139
|
+
const hideInput = tracingOptions?.hideInput;
|
|
8140
|
+
const hideOutput = tracingOptions?.hideOutput;
|
|
8141
|
+
if (allKeys.length === 0 && !hideInput && !hideOutput) {
|
|
6148
8142
|
return void 0;
|
|
6149
8143
|
}
|
|
6150
8144
|
return {
|
|
6151
|
-
requestContextKeys: allKeys
|
|
8145
|
+
requestContextKeys: allKeys,
|
|
8146
|
+
...hideInput !== void 0 && { hideInput },
|
|
8147
|
+
...hideOutput !== void 0 && { hideOutput }
|
|
6152
8148
|
};
|
|
6153
8149
|
}
|
|
6154
8150
|
/**
|
|
@@ -6279,6 +8275,29 @@ var BaseObservabilityInstance = class extends base.MastraBase {
|
|
|
6279
8275
|
this.logger.debug(`[Observability] Initialization started [name=${this.name}]`);
|
|
6280
8276
|
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
6281
8277
|
}
|
|
8278
|
+
/**
|
|
8279
|
+
* Force flush any buffered/queued spans from all exporters and the bridge
|
|
8280
|
+
* without shutting down the observability instance.
|
|
8281
|
+
*
|
|
8282
|
+
* This is useful in serverless environments (like Vercel's fluid compute) where
|
|
8283
|
+
* you need to ensure all spans are exported before the runtime instance is
|
|
8284
|
+
* terminated, while keeping the observability system active for future requests.
|
|
8285
|
+
*/
|
|
8286
|
+
async flush() {
|
|
8287
|
+
this.logger.debug(`[Observability] Flush started [name=${this.name}]`);
|
|
8288
|
+
const flushPromises = [...this.exporters.map((e) => e.flush())];
|
|
8289
|
+
if (this.config.bridge) {
|
|
8290
|
+
flushPromises.push(this.config.bridge.flush());
|
|
8291
|
+
}
|
|
8292
|
+
const results = await Promise.allSettled(flushPromises);
|
|
8293
|
+
results.forEach((result, index) => {
|
|
8294
|
+
if (result.status === "rejected") {
|
|
8295
|
+
const targetName = index < this.exporters.length ? this.exporters[index]?.name : "bridge";
|
|
8296
|
+
this.logger.error(`[Observability] Flush error [target=${targetName}]`, result.reason);
|
|
8297
|
+
}
|
|
8298
|
+
});
|
|
8299
|
+
this.logger.debug(`[Observability] Flush completed [name=${this.name}]`);
|
|
8300
|
+
}
|
|
6282
8301
|
/**
|
|
6283
8302
|
* Shutdown Observability and clean up resources
|
|
6284
8303
|
*/
|
|
@@ -6588,6 +8607,9 @@ var Observability = class extends base.MastraBase {
|
|
|
6588
8607
|
}
|
|
6589
8608
|
}
|
|
6590
8609
|
if (config.default?.enabled) {
|
|
8610
|
+
console.warn(
|
|
8611
|
+
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter, CloudExporter, and SensitiveDataFilter instead. See https://mastra.ai/docs/observability/tracing/overview for the recommended configuration.'
|
|
8612
|
+
);
|
|
6591
8613
|
const defaultInstance = new DefaultObservabilityInstance({
|
|
6592
8614
|
serviceName: "mastra",
|
|
6593
8615
|
name: "default",
|
|
@@ -6685,19 +8707,25 @@ exports.DEFAULT_KEYS_TO_STRIP = DEFAULT_KEYS_TO_STRIP;
|
|
|
6685
8707
|
exports.DefaultExporter = DefaultExporter;
|
|
6686
8708
|
exports.DefaultObservabilityInstance = DefaultObservabilityInstance;
|
|
6687
8709
|
exports.DefaultSpan = DefaultSpan;
|
|
8710
|
+
exports.JsonExporter = JsonExporter;
|
|
6688
8711
|
exports.ModelSpanTracker = ModelSpanTracker;
|
|
6689
8712
|
exports.NoOpSpan = NoOpSpan;
|
|
6690
8713
|
exports.Observability = Observability;
|
|
6691
8714
|
exports.SamplingStrategyType = SamplingStrategyType;
|
|
6692
8715
|
exports.SensitiveDataFilter = SensitiveDataFilter;
|
|
6693
8716
|
exports.TestExporter = TestExporter;
|
|
8717
|
+
exports.TraceData = TraceData;
|
|
8718
|
+
exports.TrackingExporter = TrackingExporter;
|
|
6694
8719
|
exports.buildTracingOptions = buildTracingOptions;
|
|
8720
|
+
exports.chainFormatters = chainFormatters;
|
|
6695
8721
|
exports.deepClean = deepClean;
|
|
6696
8722
|
exports.getExternalParentId = getExternalParentId;
|
|
8723
|
+
exports.mergeSerializationOptions = mergeSerializationOptions;
|
|
6697
8724
|
exports.observabilityConfigValueSchema = observabilityConfigValueSchema;
|
|
6698
8725
|
exports.observabilityInstanceConfigSchema = observabilityInstanceConfigSchema;
|
|
6699
8726
|
exports.observabilityRegistryConfigSchema = observabilityRegistryConfigSchema;
|
|
6700
8727
|
exports.samplingStrategySchema = samplingStrategySchema;
|
|
8728
|
+
exports.serializationOptionsSchema = serializationOptionsSchema;
|
|
6701
8729
|
exports.truncateString = truncateString;
|
|
6702
8730
|
//# sourceMappingURL=index.cjs.map
|
|
6703
8731
|
//# sourceMappingURL=index.cjs.map
|