@mastra/observability 1.0.0-beta.9 → 1.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +697 -0
- package/README.md +34 -79
- package/dist/config.d.ts +110 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/base.d.ts +55 -5
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/cloud.d.ts +8 -2
- package/dist/exporters/cloud.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +8 -2
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +4 -2
- package/dist/exporters/index.d.ts.map +1 -1
- package/dist/exporters/json.d.ts +386 -0
- package/dist/exporters/json.d.ts.map +1 -0
- package/dist/exporters/span-formatters.d.ts +35 -0
- package/dist/exporters/span-formatters.d.ts.map +1 -0
- package/dist/exporters/tracking.d.ts +470 -0
- package/dist/exporters/tracking.d.ts.map +1 -0
- package/dist/index.cjs +2840 -812
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2833 -812
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +33 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/model-tracing.d.ts +6 -0
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/spans/base.d.ts +5 -0
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts.map +1 -1
- package/dist/spans/serialization.d.ts +30 -0
- package/dist/spans/serialization.d.ts.map +1 -1
- package/package.json +11 -11
package/dist/index.js
CHANGED
|
@@ -3,6 +3,9 @@ import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
|
3
3
|
import { ConsoleLogger, LogLevel, RegisteredLogger } from '@mastra/core/logger';
|
|
4
4
|
import { TracingEventType, SpanType, InternalSpans } from '@mastra/core/observability';
|
|
5
5
|
import { fetchWithRetry, getNestedValue, setNestedValue } from '@mastra/core/utils';
|
|
6
|
+
import { writeFile, readFile } from 'fs/promises';
|
|
7
|
+
import { dirname, join } from 'path';
|
|
8
|
+
import { fileURLToPath } from 'url';
|
|
6
9
|
import { TransformStream } from 'stream/web';
|
|
7
10
|
|
|
8
11
|
var __defProp = Object.defineProperty;
|
|
@@ -3837,7 +3840,7 @@ ZodNaN.create = (params) => {
|
|
|
3837
3840
|
...processCreateParams(params)
|
|
3838
3841
|
});
|
|
3839
3842
|
};
|
|
3840
|
-
var BRAND = Symbol("zod_brand");
|
|
3843
|
+
var BRAND = /* @__PURE__ */ Symbol("zod_brand");
|
|
3841
3844
|
var ZodBranded = class extends ZodType {
|
|
3842
3845
|
_parse(input) {
|
|
3843
3846
|
const { ctx } = this._processInputParams(input);
|
|
@@ -4074,6 +4077,12 @@ var samplingStrategySchema = external_exports.discriminatedUnion("type", [
|
|
|
4074
4077
|
sampler: external_exports.function().args(external_exports.any().optional()).returns(external_exports.boolean())
|
|
4075
4078
|
})
|
|
4076
4079
|
]);
|
|
4080
|
+
var serializationOptionsSchema = external_exports.object({
|
|
4081
|
+
maxStringLength: external_exports.number().int().positive().optional(),
|
|
4082
|
+
maxDepth: external_exports.number().int().positive().optional(),
|
|
4083
|
+
maxArrayLength: external_exports.number().int().positive().optional(),
|
|
4084
|
+
maxObjectKeys: external_exports.number().int().positive().optional()
|
|
4085
|
+
}).optional();
|
|
4077
4086
|
var observabilityInstanceConfigSchema = external_exports.object({
|
|
4078
4087
|
name: external_exports.string().min(1, "Name is required"),
|
|
4079
4088
|
serviceName: external_exports.string().min(1, "Service name is required"),
|
|
@@ -4082,7 +4091,8 @@ var observabilityInstanceConfigSchema = external_exports.object({
|
|
|
4082
4091
|
bridge: external_exports.any().optional(),
|
|
4083
4092
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4084
4093
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4085
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4094
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4095
|
+
serializationOptions: serializationOptionsSchema
|
|
4086
4096
|
}).refine(
|
|
4087
4097
|
(data) => {
|
|
4088
4098
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4100,7 +4110,8 @@ var observabilityConfigValueSchema = external_exports.object({
|
|
|
4100
4110
|
bridge: external_exports.any().optional(),
|
|
4101
4111
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4102
4112
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4103
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4113
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4114
|
+
serializationOptions: serializationOptionsSchema
|
|
4104
4115
|
}).refine(
|
|
4105
4116
|
(data) => {
|
|
4106
4117
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4153,12 +4164,19 @@ var observabilityRegistryConfigSchema = external_exports.object({
|
|
|
4153
4164
|
var BaseExporter = class {
|
|
4154
4165
|
/** Mastra logger instance */
|
|
4155
4166
|
logger;
|
|
4167
|
+
/** Base configuration (accessible by subclasses) */
|
|
4168
|
+
baseConfig;
|
|
4156
4169
|
/** Whether this exporter is disabled */
|
|
4157
|
-
|
|
4170
|
+
#disabled = false;
|
|
4171
|
+
/** Public getter for disabled state */
|
|
4172
|
+
get isDisabled() {
|
|
4173
|
+
return this.#disabled;
|
|
4174
|
+
}
|
|
4158
4175
|
/**
|
|
4159
4176
|
* Initialize the base exporter with logger
|
|
4160
4177
|
*/
|
|
4161
4178
|
constructor(config = {}) {
|
|
4179
|
+
this.baseConfig = config;
|
|
4162
4180
|
const logLevel = this.resolveLogLevel(config.logLevel);
|
|
4163
4181
|
this.logger = config.logger ?? new ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
4164
4182
|
}
|
|
@@ -4193,20 +4211,62 @@ var BaseExporter = class {
|
|
|
4193
4211
|
* @param reason - Reason why the exporter is disabled
|
|
4194
4212
|
*/
|
|
4195
4213
|
setDisabled(reason) {
|
|
4196
|
-
this
|
|
4214
|
+
this.#disabled = true;
|
|
4197
4215
|
this.logger.warn(`${this.name} disabled: ${reason}`);
|
|
4198
4216
|
}
|
|
4217
|
+
/**
|
|
4218
|
+
* Apply the customSpanFormatter if configured.
|
|
4219
|
+
* This is called automatically by exportTracingEvent before _exportTracingEvent.
|
|
4220
|
+
*
|
|
4221
|
+
* Supports both synchronous and asynchronous formatters. If the formatter
|
|
4222
|
+
* returns a Promise, it will be awaited.
|
|
4223
|
+
*
|
|
4224
|
+
* @param event - The incoming tracing event
|
|
4225
|
+
* @returns The (possibly modified) event to process
|
|
4226
|
+
*/
|
|
4227
|
+
async applySpanFormatter(event) {
|
|
4228
|
+
if (this.baseConfig.customSpanFormatter) {
|
|
4229
|
+
try {
|
|
4230
|
+
const formattedSpan = await this.baseConfig.customSpanFormatter(event.exportedSpan);
|
|
4231
|
+
return {
|
|
4232
|
+
...event,
|
|
4233
|
+
exportedSpan: formattedSpan
|
|
4234
|
+
};
|
|
4235
|
+
} catch (error) {
|
|
4236
|
+
this.logger.error(`${this.name}: Error in customSpanFormatter`, {
|
|
4237
|
+
error,
|
|
4238
|
+
spanId: event.exportedSpan.id,
|
|
4239
|
+
traceId: event.exportedSpan.traceId
|
|
4240
|
+
});
|
|
4241
|
+
}
|
|
4242
|
+
}
|
|
4243
|
+
return event;
|
|
4244
|
+
}
|
|
4199
4245
|
/**
|
|
4200
4246
|
* Export a tracing event
|
|
4201
4247
|
*
|
|
4202
|
-
* This method checks if the exporter is disabled
|
|
4203
|
-
*
|
|
4248
|
+
* This method checks if the exporter is disabled, applies the customSpanFormatter,
|
|
4249
|
+
* then calls _exportTracingEvent.
|
|
4250
|
+
* Subclasses should implement _exportTracingEvent instead of overriding this method.
|
|
4204
4251
|
*/
|
|
4205
4252
|
async exportTracingEvent(event) {
|
|
4206
4253
|
if (this.isDisabled) {
|
|
4207
4254
|
return;
|
|
4208
4255
|
}
|
|
4209
|
-
await this.
|
|
4256
|
+
const processedEvent = await this.applySpanFormatter(event);
|
|
4257
|
+
await this._exportTracingEvent(processedEvent);
|
|
4258
|
+
}
|
|
4259
|
+
/**
|
|
4260
|
+
* Force flush any buffered/queued spans without shutting down the exporter.
|
|
4261
|
+
*
|
|
4262
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
4263
|
+
* are exported before the runtime instance is terminated, while keeping
|
|
4264
|
+
* the exporter active for future requests.
|
|
4265
|
+
*
|
|
4266
|
+
* Default implementation is a no-op. Override to add flush logic.
|
|
4267
|
+
*/
|
|
4268
|
+
async flush() {
|
|
4269
|
+
this.logger.debug(`${this.name} flush called (no-op in base class)`);
|
|
4210
4270
|
}
|
|
4211
4271
|
/**
|
|
4212
4272
|
* Shutdown the exporter and clean up resources
|
|
@@ -4217,811 +4277,2609 @@ var BaseExporter = class {
|
|
|
4217
4277
|
this.logger.info(`${this.name} shutdown complete`);
|
|
4218
4278
|
}
|
|
4219
4279
|
};
|
|
4220
|
-
var
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4280
|
+
var TraceData = class {
|
|
4281
|
+
/** The vendor-specific root/trace object */
|
|
4282
|
+
#rootSpan;
|
|
4283
|
+
/** The span ID of the root span */
|
|
4284
|
+
#rootSpanId;
|
|
4285
|
+
/** Whether a span with isRootSpan=true has been successfully processed */
|
|
4286
|
+
#rootSpanProcessed;
|
|
4287
|
+
/** Maps eventId to vendor-specific event objects */
|
|
4288
|
+
#events;
|
|
4289
|
+
/** Maps spanId to vendor-specific span objects */
|
|
4290
|
+
#spans;
|
|
4291
|
+
/** Maps spanId to parentSpanId, representing the span hierarchy */
|
|
4292
|
+
#tree;
|
|
4293
|
+
/** Set of span IDs that have started but not yet ended */
|
|
4294
|
+
#activeSpanIds;
|
|
4295
|
+
/** Maps spanId to vendor-specific metadata */
|
|
4296
|
+
#metadata;
|
|
4297
|
+
/** Arbitrary key-value storage for per-trace data */
|
|
4298
|
+
#extraData;
|
|
4299
|
+
/** Events waiting for the root span to be processed */
|
|
4300
|
+
#waitingForRoot;
|
|
4301
|
+
/** Events waiting for specific parent spans, keyed by parentSpanId */
|
|
4302
|
+
#waitingForParent;
|
|
4303
|
+
/** When this trace data was created, used for cap enforcement */
|
|
4304
|
+
createdAt;
|
|
4305
|
+
constructor() {
|
|
4306
|
+
this.#events = /* @__PURE__ */ new Map();
|
|
4307
|
+
this.#spans = /* @__PURE__ */ new Map();
|
|
4308
|
+
this.#activeSpanIds = /* @__PURE__ */ new Set();
|
|
4309
|
+
this.#tree = /* @__PURE__ */ new Map();
|
|
4310
|
+
this.#metadata = /* @__PURE__ */ new Map();
|
|
4311
|
+
this.#extraData = /* @__PURE__ */ new Map();
|
|
4312
|
+
this.#rootSpanProcessed = false;
|
|
4313
|
+
this.#waitingForRoot = [];
|
|
4314
|
+
this.#waitingForParent = /* @__PURE__ */ new Map();
|
|
4315
|
+
this.createdAt = /* @__PURE__ */ new Date();
|
|
4247
4316
|
}
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
this.flush().catch((error) => {
|
|
4255
|
-
this.logger.error("Batch flush failed", {
|
|
4256
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4257
|
-
});
|
|
4258
|
-
});
|
|
4259
|
-
} else if (this.buffer.totalSize === 1) {
|
|
4260
|
-
this.scheduleFlush();
|
|
4261
|
-
}
|
|
4317
|
+
/**
|
|
4318
|
+
* Check if this trace has a root span registered.
|
|
4319
|
+
* @returns True if addRoot() has been called
|
|
4320
|
+
*/
|
|
4321
|
+
hasRoot() {
|
|
4322
|
+
return !!this.#rootSpanId;
|
|
4262
4323
|
}
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
this.
|
|
4324
|
+
/**
|
|
4325
|
+
* Register the root span for this trace.
|
|
4326
|
+
* @param args.rootId - The span ID of the root span
|
|
4327
|
+
* @param args.rootData - The vendor-specific root object
|
|
4328
|
+
*/
|
|
4329
|
+
addRoot(args) {
|
|
4330
|
+
this.#rootSpanId = args.rootId;
|
|
4331
|
+
this.#rootSpan = args.rootData;
|
|
4332
|
+
this.#rootSpanProcessed = true;
|
|
4270
4333
|
}
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
spanType: span.type,
|
|
4278
|
-
attributes: span.attributes ?? null,
|
|
4279
|
-
metadata: span.metadata ?? null,
|
|
4280
|
-
startedAt: span.startTime,
|
|
4281
|
-
endedAt: span.endTime ?? null,
|
|
4282
|
-
input: span.input ?? null,
|
|
4283
|
-
output: span.output ?? null,
|
|
4284
|
-
error: span.errorInfo,
|
|
4285
|
-
isEvent: span.isEvent,
|
|
4286
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
4287
|
-
updatedAt: null
|
|
4288
|
-
};
|
|
4289
|
-
return spanRecord;
|
|
4334
|
+
/**
|
|
4335
|
+
* Get the vendor-specific root object.
|
|
4336
|
+
* @returns The root object, or undefined if not yet set
|
|
4337
|
+
*/
|
|
4338
|
+
getRoot() {
|
|
4339
|
+
return this.#rootSpan;
|
|
4290
4340
|
}
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
return true;
|
|
4299
|
-
}
|
|
4300
|
-
}
|
|
4301
|
-
return false;
|
|
4341
|
+
/**
|
|
4342
|
+
* Check if a span with isRootSpan=true has been successfully processed.
|
|
4343
|
+
* Set via addRoot() or markRootSpanProcessed().
|
|
4344
|
+
* @returns True if the root span has been processed
|
|
4345
|
+
*/
|
|
4346
|
+
isRootProcessed() {
|
|
4347
|
+
return this.#rootSpanProcessed;
|
|
4302
4348
|
}
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
{
|
|
4311
|
-
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
4312
|
-
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4313
|
-
category: ErrorCategory.USER
|
|
4314
|
-
},
|
|
4315
|
-
error
|
|
4316
|
-
);
|
|
4317
|
-
this.logger.trackException(mastraError);
|
|
4318
|
-
this.logger.error("Scheduled flush failed", mastraError);
|
|
4319
|
-
});
|
|
4320
|
-
}, this.config.maxBatchWaitMs);
|
|
4349
|
+
/**
|
|
4350
|
+
* Mark that the root span has been processed.
|
|
4351
|
+
* Used by exporters with skipBuildRootTask=true where root goes through _buildSpan
|
|
4352
|
+
* instead of _buildRoot.
|
|
4353
|
+
*/
|
|
4354
|
+
markRootSpanProcessed() {
|
|
4355
|
+
this.#rootSpanProcessed = true;
|
|
4321
4356
|
}
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
}
|
|
4330
|
-
const startTime = Date.now();
|
|
4331
|
-
const spansCopy = [...this.buffer.spans];
|
|
4332
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
4333
|
-
this.resetBuffer();
|
|
4334
|
-
try {
|
|
4335
|
-
await this.batchUpload(spansCopy);
|
|
4336
|
-
const elapsed = Date.now() - startTime;
|
|
4337
|
-
this.logger.debug("Batch flushed successfully", {
|
|
4338
|
-
batchSize: spansCopy.length,
|
|
4339
|
-
flushReason,
|
|
4340
|
-
durationMs: elapsed
|
|
4341
|
-
});
|
|
4342
|
-
} catch (error) {
|
|
4343
|
-
const mastraError = new MastraError(
|
|
4344
|
-
{
|
|
4345
|
-
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
4346
|
-
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4347
|
-
category: ErrorCategory.USER,
|
|
4348
|
-
details: {
|
|
4349
|
-
droppedBatchSize: spansCopy.length
|
|
4350
|
-
}
|
|
4351
|
-
},
|
|
4352
|
-
error
|
|
4353
|
-
);
|
|
4354
|
-
this.logger.trackException(mastraError);
|
|
4355
|
-
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
4356
|
-
}
|
|
4357
|
+
/**
|
|
4358
|
+
* Store an arbitrary value in per-trace storage.
|
|
4359
|
+
* @param key - Storage key
|
|
4360
|
+
* @param value - Value to store
|
|
4361
|
+
*/
|
|
4362
|
+
setExtraValue(key, value) {
|
|
4363
|
+
this.#extraData.set(key, value);
|
|
4357
4364
|
}
|
|
4358
4365
|
/**
|
|
4359
|
-
*
|
|
4366
|
+
* Check if a key exists in per-trace storage.
|
|
4367
|
+
* @param key - Storage key
|
|
4368
|
+
* @returns True if the key exists
|
|
4360
4369
|
*/
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
Authorization: `Bearer ${this.config.accessToken}`,
|
|
4364
|
-
"Content-Type": "application/json"
|
|
4365
|
-
};
|
|
4366
|
-
const options = {
|
|
4367
|
-
method: "POST",
|
|
4368
|
-
headers,
|
|
4369
|
-
body: JSON.stringify({ spans })
|
|
4370
|
-
};
|
|
4371
|
-
await fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
4370
|
+
hasExtraValue(key) {
|
|
4371
|
+
return this.#extraData.has(key);
|
|
4372
4372
|
}
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4373
|
+
/**
|
|
4374
|
+
* Get a value from per-trace storage.
|
|
4375
|
+
* @param key - Storage key
|
|
4376
|
+
* @returns The stored value, or undefined if not found
|
|
4377
|
+
*/
|
|
4378
|
+
getExtraValue(key) {
|
|
4379
|
+
return this.#extraData.get(key);
|
|
4377
4380
|
}
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
+
// ============================================================================
|
|
4382
|
+
// Early Queue Methods
|
|
4383
|
+
// ============================================================================
|
|
4384
|
+
/**
|
|
4385
|
+
* Add an event to the waiting queue.
|
|
4386
|
+
* @param args.event - The tracing event to queue
|
|
4387
|
+
* @param args.waitingFor - 'root' or a specific parentSpanId
|
|
4388
|
+
* @param args.attempts - Optional: preserve attempts count when re-queuing
|
|
4389
|
+
* @param args.queuedAt - Optional: preserve original queue time when re-queuing
|
|
4390
|
+
*/
|
|
4391
|
+
addToWaitingQueue(args) {
|
|
4392
|
+
const queuedEvent = {
|
|
4393
|
+
event: args.event,
|
|
4394
|
+
waitingFor: args.waitingFor,
|
|
4395
|
+
attempts: args.attempts ?? 0,
|
|
4396
|
+
queuedAt: args.queuedAt ?? /* @__PURE__ */ new Date()
|
|
4397
|
+
};
|
|
4398
|
+
if (args.waitingFor === "root") {
|
|
4399
|
+
this.#waitingForRoot.push(queuedEvent);
|
|
4400
|
+
} else {
|
|
4401
|
+
const queue = this.#waitingForParent.get(args.waitingFor) ?? [];
|
|
4402
|
+
queue.push(queuedEvent);
|
|
4403
|
+
this.#waitingForParent.set(args.waitingFor, queue);
|
|
4381
4404
|
}
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4405
|
+
}
|
|
4406
|
+
/**
|
|
4407
|
+
* Get all events waiting for the root span.
|
|
4408
|
+
* Returns a copy of the internal array.
|
|
4409
|
+
*/
|
|
4410
|
+
getEventsWaitingForRoot() {
|
|
4411
|
+
return [...this.#waitingForRoot];
|
|
4412
|
+
}
|
|
4413
|
+
/**
|
|
4414
|
+
* Get all events waiting for a specific parent span.
|
|
4415
|
+
* Returns a copy of the internal array.
|
|
4416
|
+
*/
|
|
4417
|
+
getEventsWaitingFor(args) {
|
|
4418
|
+
return [...this.#waitingForParent.get(args.spanId) ?? []];
|
|
4419
|
+
}
|
|
4420
|
+
/**
|
|
4421
|
+
* Clear the waiting-for-root queue.
|
|
4422
|
+
*/
|
|
4423
|
+
clearWaitingForRoot() {
|
|
4424
|
+
this.#waitingForRoot = [];
|
|
4425
|
+
}
|
|
4426
|
+
/**
|
|
4427
|
+
* Clear the waiting queue for a specific parent span.
|
|
4428
|
+
*/
|
|
4429
|
+
clearWaitingFor(args) {
|
|
4430
|
+
this.#waitingForParent.delete(args.spanId);
|
|
4431
|
+
}
|
|
4432
|
+
/**
|
|
4433
|
+
* Get total count of events in all waiting queues.
|
|
4434
|
+
*/
|
|
4435
|
+
waitingQueueSize() {
|
|
4436
|
+
let count = this.#waitingForRoot.length;
|
|
4437
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4438
|
+
count += queue.length;
|
|
4385
4439
|
}
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
category: ErrorCategory.USER,
|
|
4398
|
-
details: {
|
|
4399
|
-
remainingEvents: this.buffer.totalSize
|
|
4400
|
-
}
|
|
4401
|
-
},
|
|
4402
|
-
error
|
|
4403
|
-
);
|
|
4404
|
-
this.logger.trackException(mastraError);
|
|
4405
|
-
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
4406
|
-
}
|
|
4440
|
+
return count;
|
|
4441
|
+
}
|
|
4442
|
+
/**
|
|
4443
|
+
* Get all queued events across all waiting queues.
|
|
4444
|
+
* Used for cleanup and logging orphaned events.
|
|
4445
|
+
* @returns Array of all queued events
|
|
4446
|
+
*/
|
|
4447
|
+
getAllQueuedEvents() {
|
|
4448
|
+
const all = [...this.#waitingForRoot];
|
|
4449
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4450
|
+
all.push(...queue);
|
|
4407
4451
|
}
|
|
4408
|
-
|
|
4452
|
+
return all;
|
|
4409
4453
|
}
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4454
|
+
// ============================================================================
|
|
4455
|
+
// Span Tree Methods
|
|
4456
|
+
// ============================================================================
|
|
4457
|
+
/**
|
|
4458
|
+
* Record the parent-child relationship for a span.
|
|
4459
|
+
* @param args.spanId - The child span ID
|
|
4460
|
+
* @param args.parentSpanId - The parent span ID, or undefined for root spans
|
|
4461
|
+
*/
|
|
4462
|
+
addBranch(args) {
|
|
4463
|
+
this.#tree.set(args.spanId, args.parentSpanId);
|
|
4415
4464
|
}
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4465
|
+
/**
|
|
4466
|
+
* Get the parent span ID for a given span.
|
|
4467
|
+
* @param args.spanId - The span ID to look up
|
|
4468
|
+
* @returns The parent span ID, or undefined if root or not found
|
|
4469
|
+
*/
|
|
4470
|
+
getParentId(args) {
|
|
4471
|
+
return this.#tree.get(args.spanId);
|
|
4472
|
+
}
|
|
4473
|
+
// ============================================================================
|
|
4474
|
+
// Span Management Methods
|
|
4475
|
+
// ============================================================================
|
|
4476
|
+
/**
|
|
4477
|
+
* Register a span and mark it as active.
|
|
4478
|
+
* @param args.spanId - The span ID
|
|
4479
|
+
* @param args.spanData - The vendor-specific span object
|
|
4480
|
+
*/
|
|
4481
|
+
addSpan(args) {
|
|
4482
|
+
this.#spans.set(args.spanId, args.spanData);
|
|
4483
|
+
this.#activeSpanIds.add(args.spanId);
|
|
4484
|
+
}
|
|
4485
|
+
/**
|
|
4486
|
+
* Check if a span exists (regardless of active state).
|
|
4487
|
+
* @param args.spanId - The span ID to check
|
|
4488
|
+
* @returns True if the span exists
|
|
4489
|
+
*/
|
|
4490
|
+
hasSpan(args) {
|
|
4491
|
+
return this.#spans.has(args.spanId);
|
|
4492
|
+
}
|
|
4493
|
+
/**
|
|
4494
|
+
* Get a span by ID.
|
|
4495
|
+
* @param args.spanId - The span ID to look up
|
|
4496
|
+
* @returns The vendor-specific span object, or undefined if not found
|
|
4497
|
+
*/
|
|
4498
|
+
getSpan(args) {
|
|
4499
|
+
return this.#spans.get(args.spanId);
|
|
4500
|
+
}
|
|
4501
|
+
/**
|
|
4502
|
+
* Mark a span as ended (no longer active).
|
|
4503
|
+
* @param args.spanId - The span ID to mark as ended
|
|
4504
|
+
*/
|
|
4505
|
+
endSpan(args) {
|
|
4506
|
+
this.#activeSpanIds.delete(args.spanId);
|
|
4507
|
+
}
|
|
4508
|
+
/**
|
|
4509
|
+
* Check if a span is currently active (started but not ended).
|
|
4510
|
+
* @param args.spanId - The span ID to check
|
|
4511
|
+
* @returns True if the span is active
|
|
4512
|
+
*/
|
|
4513
|
+
isActiveSpan(args) {
|
|
4514
|
+
return this.#activeSpanIds.has(args.spanId);
|
|
4515
|
+
}
|
|
4516
|
+
/**
|
|
4517
|
+
* Get the count of currently active spans.
|
|
4518
|
+
* @returns Number of active spans
|
|
4519
|
+
*/
|
|
4520
|
+
activeSpanCount() {
|
|
4521
|
+
return this.#activeSpanIds.size;
|
|
4522
|
+
}
|
|
4523
|
+
/**
|
|
4524
|
+
* Get all active span IDs.
|
|
4525
|
+
* @returns Array of active span IDs
|
|
4526
|
+
*/
|
|
4527
|
+
get activeSpanIds() {
|
|
4528
|
+
return [...this.#activeSpanIds];
|
|
4529
|
+
}
|
|
4530
|
+
// ============================================================================
|
|
4531
|
+
// Event Management Methods
|
|
4532
|
+
// ============================================================================
|
|
4533
|
+
/**
|
|
4534
|
+
* Register an event.
|
|
4535
|
+
* @param args.eventId - The event ID
|
|
4536
|
+
* @param args.eventData - The vendor-specific event object
|
|
4537
|
+
*/
|
|
4538
|
+
addEvent(args) {
|
|
4539
|
+
this.#events.set(args.eventId, args.eventData);
|
|
4540
|
+
}
|
|
4541
|
+
// ============================================================================
|
|
4542
|
+
// Metadata Methods
|
|
4543
|
+
// ============================================================================
|
|
4544
|
+
/**
|
|
4545
|
+
* Store vendor-specific metadata for a span.
|
|
4546
|
+
* Note: This overwrites any existing metadata for the span.
|
|
4547
|
+
* @param args.spanId - The span ID
|
|
4548
|
+
* @param args.metadata - The vendor-specific metadata
|
|
4549
|
+
*/
|
|
4550
|
+
addMetadata(args) {
|
|
4551
|
+
this.#metadata.set(args.spanId, args.metadata);
|
|
4552
|
+
}
|
|
4553
|
+
/**
|
|
4554
|
+
* Get vendor-specific metadata for a span.
|
|
4555
|
+
* @param args.spanId - The span ID
|
|
4556
|
+
* @returns The metadata, or undefined if not found
|
|
4557
|
+
*/
|
|
4558
|
+
getMetadata(args) {
|
|
4559
|
+
return this.#metadata.get(args.spanId);
|
|
4560
|
+
}
|
|
4561
|
+
// ============================================================================
|
|
4562
|
+
// Parent Lookup Methods
|
|
4563
|
+
// ============================================================================
|
|
4564
|
+
/**
|
|
4565
|
+
* Get the parent span or event for a given span.
|
|
4566
|
+
* Looks up in both spans and events maps.
|
|
4567
|
+
* @param args.span - The span to find the parent for
|
|
4568
|
+
* @returns The parent span/event object, or undefined if root or not found
|
|
4569
|
+
*/
|
|
4570
|
+
getParent(args) {
|
|
4571
|
+
const parentId = args.span.parentSpanId;
|
|
4572
|
+
if (parentId) {
|
|
4573
|
+
if (this.#spans.has(parentId)) {
|
|
4574
|
+
return this.#spans.get(parentId);
|
|
4575
|
+
}
|
|
4576
|
+
if (this.#events.has(parentId)) {
|
|
4577
|
+
return this.#events.get(parentId);
|
|
4578
|
+
}
|
|
4579
|
+
}
|
|
4580
|
+
return void 0;
|
|
4581
|
+
}
|
|
4582
|
+
/**
|
|
4583
|
+
* Get the parent span/event or fall back to the root object.
|
|
4584
|
+
* Useful for vendors that attach child spans to either parent spans or the trace root.
|
|
4585
|
+
* @param args.span - The span to find the parent for
|
|
4586
|
+
* @returns The parent span/event, the root object, or undefined
|
|
4587
|
+
*/
|
|
4588
|
+
getParentOrRoot(args) {
|
|
4589
|
+
return this.getParent(args) ?? this.getRoot();
|
|
4590
|
+
}
|
|
4591
|
+
};
|
|
4592
|
+
var DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS = 5;
|
|
4593
|
+
var DEFAULT_EARLY_QUEUE_TTL_MS = 3e4;
|
|
4594
|
+
var DEFAULT_TRACE_CLEANUP_DELAY_MS = 3e4;
|
|
4595
|
+
var DEFAULT_MAX_PENDING_CLEANUP_TRACES = 100;
|
|
4596
|
+
var DEFAULT_MAX_TOTAL_TRACES = 500;
|
|
4597
|
+
var TrackingExporter = class extends BaseExporter {
|
|
4598
|
+
/** Map of traceId to per-trace data container */
|
|
4599
|
+
#traceMap = /* @__PURE__ */ new Map();
|
|
4600
|
+
/** Flag to prevent processing during shutdown */
|
|
4601
|
+
#shutdownStarted = false;
|
|
4602
|
+
/** Flag to prevent concurrent hard cap enforcement */
|
|
4603
|
+
#hardCapEnforcementInProgress = false;
|
|
4604
|
+
/** Map of traceId to scheduled cleanup timeout */
|
|
4605
|
+
#pendingCleanups = /* @__PURE__ */ new Map();
|
|
4606
|
+
// Note: #traceMap maintains insertion order (JS Map spec), so we use
|
|
4607
|
+
// #traceMap.keys() to iterate traces oldest-first for cap enforcement.
|
|
4608
|
+
/** Subclass configuration with resolved values */
|
|
4609
|
+
config;
|
|
4610
|
+
/** Maximum attempts to process a queued event before dropping */
|
|
4611
|
+
#earlyQueueMaxAttempts;
|
|
4612
|
+
/** TTL in milliseconds for queued events */
|
|
4613
|
+
#earlyQueueTTLMs;
|
|
4614
|
+
/** Delay before cleaning up completed traces */
|
|
4615
|
+
#traceCleanupDelayMs;
|
|
4616
|
+
/** Soft cap on traces awaiting cleanup */
|
|
4617
|
+
#maxPendingCleanupTraces;
|
|
4618
|
+
/** Hard cap on total traces (will abort active spans if exceeded) */
|
|
4619
|
+
#maxTotalTraces;
|
|
4620
|
+
constructor(config) {
|
|
4621
|
+
super(config);
|
|
4622
|
+
this.config = config;
|
|
4623
|
+
this.#earlyQueueMaxAttempts = config.earlyQueueMaxAttempts ?? DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS;
|
|
4624
|
+
this.#earlyQueueTTLMs = config.earlyQueueTTLMs ?? DEFAULT_EARLY_QUEUE_TTL_MS;
|
|
4625
|
+
this.#traceCleanupDelayMs = config.traceCleanupDelayMs ?? DEFAULT_TRACE_CLEANUP_DELAY_MS;
|
|
4626
|
+
this.#maxPendingCleanupTraces = config.maxPendingCleanupTraces ?? DEFAULT_MAX_PENDING_CLEANUP_TRACES;
|
|
4627
|
+
this.#maxTotalTraces = config.maxTotalTraces ?? DEFAULT_MAX_TOTAL_TRACES;
|
|
4628
|
+
}
|
|
4629
|
+
// ============================================================================
|
|
4630
|
+
// Early Queue Processing
|
|
4631
|
+
// ============================================================================
|
|
4632
|
+
/**
|
|
4633
|
+
* Schedule async processing of events waiting for root span.
|
|
4634
|
+
* Called after root span is successfully processed.
|
|
4635
|
+
*/
|
|
4636
|
+
#scheduleProcessWaitingForRoot(traceId) {
|
|
4637
|
+
setImmediate(() => {
|
|
4638
|
+
this.#processWaitingForRoot(traceId).catch((error) => {
|
|
4639
|
+
this.logger.error(`${this.name}: Error processing waiting-for-root queue`, { error, traceId });
|
|
4640
|
+
});
|
|
4641
|
+
});
|
|
4642
|
+
}
|
|
4643
|
+
/**
|
|
4644
|
+
* Schedule async processing of events waiting for a specific parent span.
|
|
4645
|
+
* Called after a span/event is successfully created.
|
|
4646
|
+
*/
|
|
4647
|
+
#scheduleProcessWaitingFor(traceId, spanId) {
|
|
4648
|
+
setImmediate(() => {
|
|
4649
|
+
this.#processWaitingFor(traceId, spanId).catch((error) => {
|
|
4650
|
+
this.logger.error(`${this.name}: Error processing waiting queue`, { error, traceId, spanId });
|
|
4651
|
+
});
|
|
4652
|
+
});
|
|
4653
|
+
}
|
|
4654
|
+
/**
|
|
4655
|
+
* Process all events waiting for root span.
|
|
4656
|
+
*/
|
|
4657
|
+
async #processWaitingForRoot(traceId) {
|
|
4658
|
+
if (this.#shutdownStarted) return;
|
|
4659
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4660
|
+
if (!traceData) return;
|
|
4661
|
+
const queue = traceData.getEventsWaitingForRoot();
|
|
4662
|
+
if (queue.length === 0) return;
|
|
4663
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for root`, { traceId });
|
|
4664
|
+
const toKeep = [];
|
|
4665
|
+
const now = Date.now();
|
|
4666
|
+
for (const queuedEvent of queue) {
|
|
4667
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4668
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4669
|
+
traceId,
|
|
4670
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4671
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4672
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4673
|
+
attempts: queuedEvent.attempts
|
|
4674
|
+
});
|
|
4675
|
+
continue;
|
|
4676
|
+
}
|
|
4677
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4678
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4679
|
+
traceId,
|
|
4680
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4681
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4682
|
+
attempts: queuedEvent.attempts
|
|
4683
|
+
});
|
|
4684
|
+
continue;
|
|
4685
|
+
}
|
|
4686
|
+
queuedEvent.attempts++;
|
|
4687
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4688
|
+
if (!processed) {
|
|
4689
|
+
const parentId = queuedEvent.event.exportedSpan.parentSpanId;
|
|
4690
|
+
if (parentId && traceData.isRootProcessed()) {
|
|
4691
|
+
traceData.addToWaitingQueue({
|
|
4692
|
+
event: queuedEvent.event,
|
|
4693
|
+
waitingFor: parentId,
|
|
4694
|
+
attempts: queuedEvent.attempts,
|
|
4695
|
+
queuedAt: queuedEvent.queuedAt
|
|
4696
|
+
});
|
|
4697
|
+
} else {
|
|
4698
|
+
toKeep.push(queuedEvent);
|
|
4699
|
+
}
|
|
4700
|
+
}
|
|
4701
|
+
}
|
|
4702
|
+
traceData.clearWaitingForRoot();
|
|
4703
|
+
for (const event of toKeep) {
|
|
4704
|
+
traceData.addToWaitingQueue({
|
|
4705
|
+
event: event.event,
|
|
4706
|
+
waitingFor: "root",
|
|
4707
|
+
attempts: event.attempts,
|
|
4708
|
+
queuedAt: event.queuedAt
|
|
4709
|
+
});
|
|
4710
|
+
}
|
|
4711
|
+
}
|
|
4712
|
+
/**
|
|
4713
|
+
* Process events waiting for a specific parent span.
|
|
4714
|
+
*/
|
|
4715
|
+
async #processWaitingFor(traceId, spanId) {
|
|
4716
|
+
if (this.#shutdownStarted) return;
|
|
4717
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4718
|
+
if (!traceData) return;
|
|
4719
|
+
const queue = traceData.getEventsWaitingFor({ spanId });
|
|
4720
|
+
if (queue.length === 0) return;
|
|
4721
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for span`, { traceId, spanId });
|
|
4722
|
+
const toKeep = [];
|
|
4723
|
+
const now = Date.now();
|
|
4724
|
+
for (const queuedEvent of queue) {
|
|
4725
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4726
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4727
|
+
traceId,
|
|
4728
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4729
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4730
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4731
|
+
attempts: queuedEvent.attempts
|
|
4732
|
+
});
|
|
4733
|
+
continue;
|
|
4734
|
+
}
|
|
4735
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4736
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4737
|
+
traceId,
|
|
4738
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4739
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4740
|
+
attempts: queuedEvent.attempts
|
|
4741
|
+
});
|
|
4742
|
+
continue;
|
|
4743
|
+
}
|
|
4744
|
+
queuedEvent.attempts++;
|
|
4745
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4746
|
+
if (!processed) {
|
|
4747
|
+
toKeep.push(queuedEvent);
|
|
4748
|
+
}
|
|
4749
|
+
}
|
|
4750
|
+
traceData.clearWaitingFor({ spanId });
|
|
4751
|
+
for (const event of toKeep) {
|
|
4752
|
+
traceData.addToWaitingQueue({
|
|
4753
|
+
event: event.event,
|
|
4754
|
+
waitingFor: spanId,
|
|
4755
|
+
attempts: event.attempts,
|
|
4756
|
+
queuedAt: event.queuedAt
|
|
4757
|
+
});
|
|
4758
|
+
}
|
|
4759
|
+
}
|
|
4760
|
+
/**
|
|
4761
|
+
* Try to process a queued event.
|
|
4762
|
+
* Returns true if successfully processed, false if still waiting for dependencies.
|
|
4763
|
+
*/
|
|
4764
|
+
async #tryProcessQueuedEvent(queuedEvent, traceData) {
|
|
4765
|
+
const { event } = queuedEvent;
|
|
4766
|
+
const { exportedSpan } = event;
|
|
4767
|
+
const method = this.getMethod(event);
|
|
4768
|
+
try {
|
|
4769
|
+
switch (method) {
|
|
4770
|
+
case "handleEventSpan": {
|
|
4771
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4772
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
4773
|
+
if (eventData) {
|
|
4774
|
+
if (!this.skipCachingEventSpans) {
|
|
4775
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
4776
|
+
}
|
|
4777
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4778
|
+
return true;
|
|
4779
|
+
}
|
|
4780
|
+
return false;
|
|
4781
|
+
}
|
|
4782
|
+
case "handleSpanStart": {
|
|
4783
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4784
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
4785
|
+
if (spanData) {
|
|
4786
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
4787
|
+
if (exportedSpan.isRootSpan) {
|
|
4788
|
+
traceData.markRootSpanProcessed();
|
|
4789
|
+
}
|
|
4790
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4791
|
+
return true;
|
|
4792
|
+
}
|
|
4793
|
+
return false;
|
|
4794
|
+
}
|
|
4795
|
+
case "handleSpanUpdate": {
|
|
4796
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
4797
|
+
return true;
|
|
4798
|
+
}
|
|
4799
|
+
case "handleSpanEnd": {
|
|
4800
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
4801
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
4802
|
+
if (traceData.activeSpanCount() === 0) {
|
|
4803
|
+
this.#scheduleCleanup(exportedSpan.traceId);
|
|
4804
|
+
}
|
|
4805
|
+
return true;
|
|
4806
|
+
}
|
|
4807
|
+
default:
|
|
4808
|
+
return false;
|
|
4809
|
+
}
|
|
4810
|
+
} catch (error) {
|
|
4811
|
+
this.logger.error(`${this.name}: Error processing queued event`, { error, event, method });
|
|
4812
|
+
return false;
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
// ============================================================================
|
|
4816
|
+
// Delayed Cleanup
|
|
4817
|
+
// ============================================================================
|
|
4818
|
+
/**
|
|
4819
|
+
* Schedule cleanup of trace data after a delay.
|
|
4820
|
+
* Allows late-arriving data to still be processed.
|
|
4821
|
+
*/
|
|
4822
|
+
#scheduleCleanup(traceId) {
|
|
4823
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4824
|
+
this.logger.debug(`${this.name}: Scheduling cleanup in ${this.#traceCleanupDelayMs}ms`, { traceId });
|
|
4825
|
+
const timeout = setTimeout(() => {
|
|
4826
|
+
this.#pendingCleanups.delete(traceId);
|
|
4827
|
+
this.#performCleanup(traceId);
|
|
4828
|
+
}, this.#traceCleanupDelayMs);
|
|
4829
|
+
this.#pendingCleanups.set(traceId, timeout);
|
|
4830
|
+
this.#enforcePendingCleanupCap();
|
|
4831
|
+
}
|
|
4832
|
+
/**
|
|
4833
|
+
* Cancel a scheduled cleanup for a trace.
|
|
4834
|
+
*/
|
|
4835
|
+
#cancelScheduledCleanup(traceId) {
|
|
4836
|
+
const existingTimeout = this.#pendingCleanups.get(traceId);
|
|
4837
|
+
if (existingTimeout) {
|
|
4838
|
+
clearTimeout(existingTimeout);
|
|
4839
|
+
this.#pendingCleanups.delete(traceId);
|
|
4840
|
+
this.logger.debug(`${this.name}: Cancelled scheduled cleanup`, { traceId });
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4843
|
+
/**
|
|
4844
|
+
* Perform the actual cleanup of trace data.
|
|
4845
|
+
*/
|
|
4846
|
+
#performCleanup(traceId) {
|
|
4847
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4848
|
+
if (!traceData) return;
|
|
4849
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
4850
|
+
if (orphanedEvents.length > 0) {
|
|
4851
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on cleanup`, {
|
|
4852
|
+
traceId,
|
|
4853
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
4854
|
+
spanId: e.event.exportedSpan.id,
|
|
4855
|
+
waitingFor: e.waitingFor,
|
|
4856
|
+
attempts: e.attempts,
|
|
4857
|
+
queuedAt: e.queuedAt
|
|
4858
|
+
}))
|
|
4859
|
+
});
|
|
4860
|
+
}
|
|
4861
|
+
this.#traceMap.delete(traceId);
|
|
4862
|
+
this.logger.debug(`${this.name}: Cleaned up trace data`, { traceId });
|
|
4863
|
+
}
|
|
4864
|
+
// ============================================================================
|
|
4865
|
+
// Cap Enforcement
|
|
4866
|
+
// ============================================================================
|
|
4867
|
+
/**
|
|
4868
|
+
* Enforce soft cap on pending cleanup traces.
|
|
4869
|
+
* Only removes traces with activeSpanCount == 0.
|
|
4870
|
+
*/
|
|
4871
|
+
#enforcePendingCleanupCap() {
|
|
4872
|
+
if (this.#pendingCleanups.size <= this.#maxPendingCleanupTraces) {
|
|
4873
|
+
return;
|
|
4874
|
+
}
|
|
4875
|
+
const toRemove = this.#pendingCleanups.size - this.#maxPendingCleanupTraces;
|
|
4876
|
+
this.logger.warn(`${this.name}: Pending cleanup cap exceeded, force-cleaning ${toRemove} traces`, {
|
|
4877
|
+
pendingCount: this.#pendingCleanups.size,
|
|
4878
|
+
cap: this.#maxPendingCleanupTraces
|
|
4879
|
+
});
|
|
4880
|
+
let removed = 0;
|
|
4881
|
+
for (const traceId of this.#traceMap.keys()) {
|
|
4882
|
+
if (removed >= toRemove) break;
|
|
4883
|
+
if (this.#pendingCleanups.has(traceId)) {
|
|
4884
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4885
|
+
this.#performCleanup(traceId);
|
|
4886
|
+
removed++;
|
|
4887
|
+
}
|
|
4888
|
+
}
|
|
4889
|
+
}
|
|
4890
|
+
/**
|
|
4891
|
+
* Enforce hard cap on total traces.
|
|
4892
|
+
* Will kill even active traces if necessary.
|
|
4893
|
+
* Uses a flag to prevent concurrent executions when called fire-and-forget.
|
|
4894
|
+
*/
|
|
4895
|
+
async #enforceHardCap() {
|
|
4896
|
+
if (this.#traceMap.size <= this.#maxTotalTraces || this.#hardCapEnforcementInProgress) {
|
|
4897
|
+
return;
|
|
4898
|
+
}
|
|
4899
|
+
this.#hardCapEnforcementInProgress = true;
|
|
4900
|
+
try {
|
|
4901
|
+
if (this.#traceMap.size <= this.#maxTotalTraces) {
|
|
4902
|
+
return;
|
|
4903
|
+
}
|
|
4904
|
+
const toRemove = this.#traceMap.size - this.#maxTotalTraces;
|
|
4905
|
+
this.logger.warn(`${this.name}: Total trace cap exceeded, killing ${toRemove} oldest traces`, {
|
|
4906
|
+
traceCount: this.#traceMap.size,
|
|
4907
|
+
cap: this.#maxTotalTraces
|
|
4908
|
+
});
|
|
4909
|
+
const reason = {
|
|
4910
|
+
id: "TRACE_CAP_EXCEEDED",
|
|
4911
|
+
message: "Trace killed due to memory cap enforcement.",
|
|
4912
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
4913
|
+
category: "SYSTEM"
|
|
4914
|
+
};
|
|
4915
|
+
let removed = 0;
|
|
4916
|
+
for (const traceId of [...this.#traceMap.keys()]) {
|
|
4917
|
+
if (removed >= toRemove) break;
|
|
4918
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4919
|
+
if (traceData) {
|
|
4920
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
4921
|
+
const span = traceData.getSpan({ spanId });
|
|
4922
|
+
if (span) {
|
|
4923
|
+
await this._abortSpan({ span, traceData, reason });
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4927
|
+
this.#performCleanup(traceId);
|
|
4928
|
+
removed++;
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
} finally {
|
|
4932
|
+
this.#hardCapEnforcementInProgress = false;
|
|
4933
|
+
}
|
|
4934
|
+
}
|
|
4935
|
+
// ============================================================================
|
|
4936
|
+
// Lifecycle Hooks (Override in subclass)
|
|
4937
|
+
// ============================================================================
|
|
4938
|
+
/**
|
|
4939
|
+
* Hook called before processing each tracing event.
|
|
4940
|
+
* Override to transform or enrich the event before processing.
|
|
4941
|
+
*
|
|
4942
|
+
* Note: The customSpanFormatter is applied at the BaseExporter level before this hook.
|
|
4943
|
+
* Subclasses can override this to add additional pre-processing logic.
|
|
4944
|
+
*
|
|
4945
|
+
* @param event - The incoming tracing event
|
|
4946
|
+
* @returns The (possibly modified) event to process
|
|
4947
|
+
*/
|
|
4948
|
+
async _preExportTracingEvent(event) {
|
|
4949
|
+
return event;
|
|
4950
|
+
}
|
|
4951
|
+
/**
|
|
4952
|
+
* Hook called after processing each tracing event.
|
|
4953
|
+
* Override to perform post-processing actions like flushing.
|
|
4954
|
+
*/
|
|
4955
|
+
async _postExportTracingEvent() {
|
|
4956
|
+
}
|
|
4957
|
+
// ============================================================================
|
|
4958
|
+
// Behavior Flags (Override in subclass as needed)
|
|
4959
|
+
// ============================================================================
|
|
4960
|
+
/**
|
|
4961
|
+
* If true, skip calling _buildRoot and let root spans go through _buildSpan.
|
|
4962
|
+
* Use when the vendor doesn't have a separate trace/root concept.
|
|
4963
|
+
* @default false
|
|
4964
|
+
*/
|
|
4965
|
+
skipBuildRootTask = false;
|
|
4966
|
+
/**
|
|
4967
|
+
* If true, skip processing span_updated events entirely.
|
|
4968
|
+
* Use when the vendor doesn't support incremental span updates.
|
|
4969
|
+
* @default false
|
|
4970
|
+
*/
|
|
4971
|
+
skipSpanUpdateEvents = false;
|
|
4972
|
+
/**
|
|
4973
|
+
* If true, don't cache event spans in TraceData.
|
|
4974
|
+
* Use when events can't be parents of other spans.
|
|
4975
|
+
* @default false
|
|
4976
|
+
*/
|
|
4977
|
+
skipCachingEventSpans = false;
|
|
4978
|
+
getMethod(event) {
|
|
4979
|
+
if (event.exportedSpan.isEvent) {
|
|
4980
|
+
return "handleEventSpan";
|
|
4981
|
+
}
|
|
4982
|
+
const eventType = event.type;
|
|
4983
|
+
switch (eventType) {
|
|
4984
|
+
case TracingEventType.SPAN_STARTED:
|
|
4985
|
+
return "handleSpanStart";
|
|
4986
|
+
case TracingEventType.SPAN_UPDATED:
|
|
4987
|
+
return "handleSpanUpdate";
|
|
4988
|
+
case TracingEventType.SPAN_ENDED:
|
|
4989
|
+
return "handleSpanEnd";
|
|
4990
|
+
default: {
|
|
4991
|
+
const _exhaustiveCheck = eventType;
|
|
4992
|
+
throw new Error(`Unhandled event type: ${_exhaustiveCheck}`);
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
async _exportTracingEvent(event) {
|
|
4997
|
+
if (this.#shutdownStarted) {
|
|
4998
|
+
return;
|
|
4999
|
+
}
|
|
5000
|
+
const method = this.getMethod(event);
|
|
5001
|
+
if (method == "handleSpanUpdate" && this.skipSpanUpdateEvents) {
|
|
5002
|
+
return;
|
|
5003
|
+
}
|
|
5004
|
+
const traceId = event.exportedSpan.traceId;
|
|
5005
|
+
const traceData = this.getTraceData({ traceId, method });
|
|
5006
|
+
const { exportedSpan } = await this._preExportTracingEvent(event);
|
|
5007
|
+
if (!this.skipBuildRootTask && !traceData.hasRoot()) {
|
|
5008
|
+
if (exportedSpan.isRootSpan) {
|
|
5009
|
+
this.logger.debug(`${this.name}: Building root`, {
|
|
5010
|
+
traceId: exportedSpan.traceId,
|
|
5011
|
+
spanId: exportedSpan.id
|
|
5012
|
+
});
|
|
5013
|
+
const rootData = await this._buildRoot({ span: exportedSpan, traceData });
|
|
5014
|
+
if (rootData) {
|
|
5015
|
+
this.logger.debug(`${this.name}: Adding root`, {
|
|
5016
|
+
traceId: exportedSpan.traceId,
|
|
5017
|
+
spanId: exportedSpan.id
|
|
5018
|
+
});
|
|
5019
|
+
traceData.addRoot({ rootId: exportedSpan.id, rootData });
|
|
5020
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5021
|
+
}
|
|
5022
|
+
} else {
|
|
5023
|
+
this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
|
|
5024
|
+
traceId: exportedSpan.traceId,
|
|
5025
|
+
spanId: exportedSpan.id
|
|
5026
|
+
});
|
|
5027
|
+
traceData.addToWaitingQueue({ event, waitingFor: "root" });
|
|
5028
|
+
return;
|
|
5029
|
+
}
|
|
5030
|
+
}
|
|
5031
|
+
if (exportedSpan.metadata && this.name in exportedSpan.metadata) {
|
|
5032
|
+
const metadata = exportedSpan.metadata[this.name];
|
|
5033
|
+
this.logger.debug(`${this.name}: Found provider metadata in span`, {
|
|
5034
|
+
traceId: exportedSpan.traceId,
|
|
5035
|
+
spanId: exportedSpan.id,
|
|
5036
|
+
metadata
|
|
5037
|
+
});
|
|
5038
|
+
traceData.addMetadata({ spanId: exportedSpan.id, metadata });
|
|
5039
|
+
}
|
|
5040
|
+
try {
|
|
5041
|
+
switch (method) {
|
|
5042
|
+
case "handleEventSpan": {
|
|
5043
|
+
this.logger.debug(`${this.name}: handling event`, {
|
|
5044
|
+
traceId: exportedSpan.traceId,
|
|
5045
|
+
spanId: exportedSpan.id
|
|
5046
|
+
});
|
|
5047
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5048
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
5049
|
+
if (eventData) {
|
|
5050
|
+
if (!this.skipCachingEventSpans) {
|
|
5051
|
+
this.logger.debug(`${this.name}: adding event to traceData`, {
|
|
5052
|
+
traceId: exportedSpan.traceId,
|
|
5053
|
+
spanId: exportedSpan.id
|
|
5054
|
+
});
|
|
5055
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
5056
|
+
}
|
|
5057
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5058
|
+
} else {
|
|
5059
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5060
|
+
this.logger.debug(`${this.name}: adding event to waiting queue`, {
|
|
5061
|
+
traceId: exportedSpan.traceId,
|
|
5062
|
+
spanId: exportedSpan.id,
|
|
5063
|
+
waitingFor: parentId ?? "root"
|
|
5064
|
+
});
|
|
5065
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5066
|
+
}
|
|
5067
|
+
break;
|
|
5068
|
+
}
|
|
5069
|
+
case "handleSpanStart": {
|
|
5070
|
+
this.logger.debug(`${this.name}: handling span start`, {
|
|
5071
|
+
traceId: exportedSpan.traceId,
|
|
5072
|
+
spanId: exportedSpan.id
|
|
5073
|
+
});
|
|
5074
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5075
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
5076
|
+
if (spanData) {
|
|
5077
|
+
this.logger.debug(`${this.name}: adding span to traceData`, {
|
|
5078
|
+
traceId: exportedSpan.traceId,
|
|
5079
|
+
spanId: exportedSpan.id
|
|
5080
|
+
});
|
|
5081
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
5082
|
+
if (exportedSpan.isRootSpan) {
|
|
5083
|
+
traceData.markRootSpanProcessed();
|
|
5084
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5085
|
+
}
|
|
5086
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5087
|
+
} else {
|
|
5088
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5089
|
+
this.logger.debug(`${this.name}: adding span to waiting queue`, {
|
|
5090
|
+
traceId: exportedSpan.traceId,
|
|
5091
|
+
waitingFor: parentId ?? "root"
|
|
5092
|
+
});
|
|
5093
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5094
|
+
}
|
|
5095
|
+
break;
|
|
5096
|
+
}
|
|
5097
|
+
case "handleSpanUpdate":
|
|
5098
|
+
this.logger.debug(`${this.name}: handling span update`, {
|
|
5099
|
+
traceId: exportedSpan.traceId,
|
|
5100
|
+
spanId: exportedSpan.id
|
|
5101
|
+
});
|
|
5102
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
5103
|
+
break;
|
|
5104
|
+
case "handleSpanEnd":
|
|
5105
|
+
this.logger.debug(`${this.name}: handling span end`, {
|
|
5106
|
+
traceId: exportedSpan.traceId,
|
|
5107
|
+
spanId: exportedSpan.id
|
|
5108
|
+
});
|
|
5109
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
5110
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
5111
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5112
|
+
this.#scheduleCleanup(traceId);
|
|
5113
|
+
}
|
|
5114
|
+
break;
|
|
5115
|
+
}
|
|
5116
|
+
} catch (error) {
|
|
5117
|
+
this.logger.error(`${this.name}: exporter error`, { error, event, method });
|
|
5118
|
+
}
|
|
5119
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5120
|
+
this.#scheduleCleanup(traceId);
|
|
5121
|
+
}
|
|
5122
|
+
await this._postExportTracingEvent();
|
|
5123
|
+
}
|
|
5124
|
+
// ============================================================================
|
|
5125
|
+
// Protected Helpers
|
|
5126
|
+
// ============================================================================
|
|
5127
|
+
/**
|
|
5128
|
+
* Get or create the TraceData container for a trace.
|
|
5129
|
+
* Also cancels any pending cleanup since new data has arrived.
|
|
5130
|
+
*
|
|
5131
|
+
* @param args.traceId - The trace ID
|
|
5132
|
+
* @param args.method - The calling method name (for logging)
|
|
5133
|
+
* @returns The TraceData container for this trace
|
|
5134
|
+
*/
|
|
5135
|
+
getTraceData(args) {
|
|
5136
|
+
const { traceId, method } = args;
|
|
5137
|
+
this.#cancelScheduledCleanup(traceId);
|
|
5138
|
+
if (!this.#traceMap.has(traceId)) {
|
|
5139
|
+
this.#traceMap.set(traceId, new TraceData());
|
|
5140
|
+
this.logger.debug(`${this.name}: Created new trace data cache`, {
|
|
5141
|
+
traceId,
|
|
5142
|
+
method
|
|
5143
|
+
});
|
|
5144
|
+
this.#enforceHardCap().catch((error) => {
|
|
5145
|
+
this.logger.error(`${this.name}: Error enforcing hard cap`, { error });
|
|
5146
|
+
});
|
|
5147
|
+
}
|
|
5148
|
+
return this.#traceMap.get(traceId);
|
|
5149
|
+
}
|
|
5150
|
+
/**
|
|
5151
|
+
* Get the current number of traces being tracked.
|
|
5152
|
+
* @returns The trace count
|
|
5153
|
+
*/
|
|
5154
|
+
traceMapSize() {
|
|
5155
|
+
return this.#traceMap.size;
|
|
5156
|
+
}
|
|
5157
|
+
// ============================================================================
|
|
5158
|
+
// Flush and Shutdown Hooks (Override in subclass as needed)
|
|
5159
|
+
// ============================================================================
|
|
5160
|
+
/**
|
|
5161
|
+
* Hook called by flush() to perform vendor-specific flush logic.
|
|
5162
|
+
* Override to send buffered data to the vendor's API.
|
|
5163
|
+
*
|
|
5164
|
+
* Unlike _postShutdown(), this method should NOT release resources,
|
|
5165
|
+
* as the exporter will continue to be used after flushing.
|
|
5166
|
+
*/
|
|
5167
|
+
async _flush() {
|
|
5168
|
+
}
|
|
5169
|
+
/**
|
|
5170
|
+
* Force flush any buffered data without shutting down the exporter.
|
|
5171
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5172
|
+
* are exported before the runtime instance is terminated.
|
|
5173
|
+
*
|
|
5174
|
+
* Subclasses should override _flush() to implement vendor-specific flush logic.
|
|
5175
|
+
*/
|
|
5176
|
+
async flush() {
|
|
5177
|
+
if (this.isDisabled) {
|
|
5178
|
+
return;
|
|
5179
|
+
}
|
|
5180
|
+
this.logger.debug(`${this.name}: Flushing`);
|
|
5181
|
+
await this._flush();
|
|
5182
|
+
}
|
|
5183
|
+
/**
|
|
5184
|
+
* Hook called at the start of shutdown, before cancelling timers and aborting spans.
|
|
5185
|
+
* Override to perform vendor-specific pre-shutdown tasks.
|
|
5186
|
+
*/
|
|
5187
|
+
async _preShutdown() {
|
|
5188
|
+
}
|
|
5189
|
+
/**
|
|
5190
|
+
* Hook called at the end of shutdown, after all spans are aborted.
|
|
5191
|
+
* Override to perform vendor-specific cleanup (e.g., flushing).
|
|
5192
|
+
*/
|
|
5193
|
+
async _postShutdown() {
|
|
5194
|
+
}
|
|
5195
|
+
/**
|
|
5196
|
+
* Gracefully shut down the exporter.
|
|
5197
|
+
* Cancels all pending cleanup timers, aborts all active spans, and clears state.
|
|
5198
|
+
*/
|
|
5199
|
+
async shutdown() {
|
|
5200
|
+
if (this.isDisabled) {
|
|
5201
|
+
return;
|
|
5202
|
+
}
|
|
5203
|
+
this.#shutdownStarted = true;
|
|
5204
|
+
await this._preShutdown();
|
|
5205
|
+
for (const [traceId, timeout] of this.#pendingCleanups) {
|
|
5206
|
+
clearTimeout(timeout);
|
|
5207
|
+
this.logger.debug(`${this.name}: Cancelled pending cleanup on shutdown`, { traceId });
|
|
5208
|
+
}
|
|
5209
|
+
this.#pendingCleanups.clear();
|
|
5210
|
+
const reason = {
|
|
5211
|
+
id: "SHUTDOWN",
|
|
5212
|
+
message: "Observability is shutting down.",
|
|
5213
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
5214
|
+
category: "SYSTEM"
|
|
5215
|
+
};
|
|
5216
|
+
for (const [traceId, traceData] of this.#traceMap) {
|
|
5217
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
5218
|
+
if (orphanedEvents.length > 0) {
|
|
5219
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on shutdown`, {
|
|
5220
|
+
traceId,
|
|
5221
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
5222
|
+
spanId: e.event.exportedSpan.id,
|
|
5223
|
+
waitingFor: e.waitingFor,
|
|
5224
|
+
attempts: e.attempts
|
|
5225
|
+
}))
|
|
5226
|
+
});
|
|
5227
|
+
}
|
|
5228
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
5229
|
+
const span = traceData.getSpan({ spanId });
|
|
5230
|
+
if (span) {
|
|
5231
|
+
await this._abortSpan({ span, traceData, reason });
|
|
5232
|
+
}
|
|
5233
|
+
}
|
|
5234
|
+
}
|
|
5235
|
+
this.#traceMap.clear();
|
|
5236
|
+
await this._postShutdown();
|
|
5237
|
+
await super.shutdown();
|
|
5238
|
+
}
|
|
5239
|
+
};
|
|
5240
|
+
|
|
5241
|
+
// src/exporters/span-formatters.ts
|
|
5242
|
+
function chainFormatters(formatters) {
|
|
5243
|
+
return async (span) => {
|
|
5244
|
+
let currentSpan = span;
|
|
5245
|
+
for (const formatter of formatters) {
|
|
5246
|
+
currentSpan = await formatter(currentSpan);
|
|
5247
|
+
}
|
|
5248
|
+
return currentSpan;
|
|
5249
|
+
};
|
|
5250
|
+
}
|
|
5251
|
+
var CloudExporter = class extends BaseExporter {
|
|
5252
|
+
name = "mastra-cloud-observability-exporter";
|
|
5253
|
+
cloudConfig;
|
|
5254
|
+
buffer;
|
|
5255
|
+
flushTimer = null;
|
|
5256
|
+
constructor(config = {}) {
|
|
5257
|
+
super(config);
|
|
5258
|
+
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
5259
|
+
if (!accessToken) {
|
|
5260
|
+
this.setDisabled("MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.");
|
|
5261
|
+
}
|
|
5262
|
+
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
5263
|
+
this.cloudConfig = {
|
|
5264
|
+
logger: this.logger,
|
|
5265
|
+
logLevel: config.logLevel ?? LogLevel.INFO,
|
|
5266
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5267
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5268
|
+
maxRetries: config.maxRetries ?? 3,
|
|
5269
|
+
accessToken: accessToken || "",
|
|
5270
|
+
endpoint
|
|
5271
|
+
};
|
|
5272
|
+
this.buffer = {
|
|
5273
|
+
spans: [],
|
|
5274
|
+
totalSize: 0
|
|
5275
|
+
};
|
|
5276
|
+
}
|
|
5277
|
+
async _exportTracingEvent(event) {
|
|
5278
|
+
if (event.type !== TracingEventType.SPAN_ENDED) {
|
|
5279
|
+
return;
|
|
5280
|
+
}
|
|
5281
|
+
this.addToBuffer(event);
|
|
5282
|
+
if (this.shouldFlush()) {
|
|
5283
|
+
this.flush().catch((error) => {
|
|
5284
|
+
this.logger.error("Batch flush failed", {
|
|
5285
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5286
|
+
});
|
|
5287
|
+
});
|
|
5288
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5289
|
+
this.scheduleFlush();
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
addToBuffer(event) {
|
|
5293
|
+
if (this.buffer.totalSize === 0) {
|
|
5294
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5295
|
+
}
|
|
5296
|
+
const spanRecord = this.formatSpan(event.exportedSpan);
|
|
5297
|
+
this.buffer.spans.push(spanRecord);
|
|
5298
|
+
this.buffer.totalSize++;
|
|
5299
|
+
}
|
|
5300
|
+
formatSpan(span) {
|
|
5301
|
+
const spanRecord = {
|
|
5302
|
+
traceId: span.traceId,
|
|
5303
|
+
spanId: span.id,
|
|
5304
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5305
|
+
name: span.name,
|
|
5306
|
+
spanType: span.type,
|
|
5307
|
+
attributes: span.attributes ?? null,
|
|
5308
|
+
metadata: span.metadata ?? null,
|
|
5309
|
+
startedAt: span.startTime,
|
|
5310
|
+
endedAt: span.endTime ?? null,
|
|
5311
|
+
input: span.input ?? null,
|
|
5312
|
+
output: span.output ?? null,
|
|
5313
|
+
error: span.errorInfo,
|
|
5314
|
+
isEvent: span.isEvent,
|
|
5315
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
5316
|
+
updatedAt: null
|
|
5317
|
+
};
|
|
5318
|
+
return spanRecord;
|
|
5319
|
+
}
|
|
5320
|
+
shouldFlush() {
|
|
5321
|
+
if (this.buffer.totalSize >= this.cloudConfig.maxBatchSize) {
|
|
5322
|
+
return true;
|
|
5323
|
+
}
|
|
5324
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5325
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5326
|
+
if (elapsed >= this.cloudConfig.maxBatchWaitMs) {
|
|
5327
|
+
return true;
|
|
5328
|
+
}
|
|
5329
|
+
}
|
|
5330
|
+
return false;
|
|
5331
|
+
}
|
|
5332
|
+
scheduleFlush() {
|
|
5333
|
+
if (this.flushTimer) {
|
|
5334
|
+
clearTimeout(this.flushTimer);
|
|
5335
|
+
}
|
|
5336
|
+
this.flushTimer = setTimeout(() => {
|
|
5337
|
+
this.flush().catch((error) => {
|
|
5338
|
+
const mastraError = new MastraError(
|
|
5339
|
+
{
|
|
5340
|
+
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
5341
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5342
|
+
category: ErrorCategory.USER
|
|
5343
|
+
},
|
|
5344
|
+
error
|
|
5345
|
+
);
|
|
5346
|
+
this.logger.trackException(mastraError);
|
|
5347
|
+
this.logger.error("Scheduled flush failed", mastraError);
|
|
5348
|
+
});
|
|
5349
|
+
}, this.cloudConfig.maxBatchWaitMs);
|
|
5350
|
+
}
|
|
5351
|
+
async flushBuffer() {
|
|
5352
|
+
if (this.flushTimer) {
|
|
5353
|
+
clearTimeout(this.flushTimer);
|
|
5354
|
+
this.flushTimer = null;
|
|
5355
|
+
}
|
|
5356
|
+
if (this.buffer.totalSize === 0) {
|
|
5357
|
+
return;
|
|
5358
|
+
}
|
|
5359
|
+
const startTime = Date.now();
|
|
5360
|
+
const spansCopy = [...this.buffer.spans];
|
|
5361
|
+
const flushReason = this.buffer.totalSize >= this.cloudConfig.maxBatchSize ? "size" : "time";
|
|
5362
|
+
this.resetBuffer();
|
|
5363
|
+
try {
|
|
5364
|
+
await this.batchUpload(spansCopy);
|
|
5365
|
+
const elapsed = Date.now() - startTime;
|
|
5366
|
+
this.logger.debug("Batch flushed successfully", {
|
|
5367
|
+
batchSize: spansCopy.length,
|
|
5368
|
+
flushReason,
|
|
5369
|
+
durationMs: elapsed
|
|
5370
|
+
});
|
|
5371
|
+
} catch (error) {
|
|
5372
|
+
const mastraError = new MastraError(
|
|
5373
|
+
{
|
|
5374
|
+
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
5375
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5376
|
+
category: ErrorCategory.USER,
|
|
5377
|
+
details: {
|
|
5378
|
+
droppedBatchSize: spansCopy.length
|
|
5379
|
+
}
|
|
5380
|
+
},
|
|
5381
|
+
error
|
|
5382
|
+
);
|
|
5383
|
+
this.logger.trackException(mastraError);
|
|
5384
|
+
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
5385
|
+
}
|
|
5386
|
+
}
|
|
5387
|
+
/**
|
|
5388
|
+
* Uploads spans to cloud API using fetchWithRetry for all retry logic
|
|
5389
|
+
*/
|
|
5390
|
+
async batchUpload(spans) {
|
|
5391
|
+
const headers = {
|
|
5392
|
+
Authorization: `Bearer ${this.cloudConfig.accessToken}`,
|
|
5393
|
+
"Content-Type": "application/json"
|
|
5394
|
+
};
|
|
5395
|
+
const options = {
|
|
5396
|
+
method: "POST",
|
|
5397
|
+
headers,
|
|
5398
|
+
body: JSON.stringify({ spans })
|
|
5399
|
+
};
|
|
5400
|
+
await fetchWithRetry(this.cloudConfig.endpoint, options, this.cloudConfig.maxRetries);
|
|
5401
|
+
}
|
|
5402
|
+
resetBuffer() {
|
|
5403
|
+
this.buffer.spans = [];
|
|
5404
|
+
this.buffer.firstEventTime = void 0;
|
|
5405
|
+
this.buffer.totalSize = 0;
|
|
5406
|
+
}
|
|
5407
|
+
/**
|
|
5408
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
5409
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5410
|
+
* are exported before the runtime instance is terminated.
|
|
5411
|
+
*/
|
|
5412
|
+
async flush() {
|
|
5413
|
+
if (this.isDisabled) {
|
|
5414
|
+
return;
|
|
5415
|
+
}
|
|
5416
|
+
if (this.buffer.totalSize > 0) {
|
|
5417
|
+
this.logger.debug("Flushing buffered events", {
|
|
5418
|
+
bufferedEvents: this.buffer.totalSize
|
|
5419
|
+
});
|
|
5420
|
+
await this.flushBuffer();
|
|
5421
|
+
}
|
|
5422
|
+
}
|
|
5423
|
+
async shutdown() {
|
|
5424
|
+
if (this.isDisabled) {
|
|
5425
|
+
return;
|
|
5426
|
+
}
|
|
5427
|
+
if (this.flushTimer) {
|
|
5428
|
+
clearTimeout(this.flushTimer);
|
|
5429
|
+
this.flushTimer = null;
|
|
5430
|
+
}
|
|
5431
|
+
try {
|
|
5432
|
+
await this.flush();
|
|
5433
|
+
} catch (error) {
|
|
5434
|
+
const mastraError = new MastraError(
|
|
5435
|
+
{
|
|
5436
|
+
id: `CLOUD_EXPORTER_FAILED_TO_FLUSH_REMAINING_EVENTS_DURING_SHUTDOWN`,
|
|
5437
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5438
|
+
category: ErrorCategory.USER,
|
|
5439
|
+
details: {
|
|
5440
|
+
remainingEvents: this.buffer.totalSize
|
|
5441
|
+
}
|
|
5442
|
+
},
|
|
5443
|
+
error
|
|
5444
|
+
);
|
|
5445
|
+
this.logger.trackException(mastraError);
|
|
5446
|
+
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
5447
|
+
}
|
|
5448
|
+
this.logger.info("CloudExporter shutdown complete");
|
|
5449
|
+
}
|
|
5450
|
+
};
|
|
5451
|
+
var ConsoleExporter = class extends BaseExporter {
|
|
5452
|
+
name = "tracing-console-exporter";
|
|
5453
|
+
constructor(config = {}) {
|
|
5454
|
+
super(config);
|
|
5455
|
+
}
|
|
5456
|
+
async _exportTracingEvent(event) {
|
|
5457
|
+
const span = event.exportedSpan;
|
|
5458
|
+
const formatAttributes = (attributes) => {
|
|
5459
|
+
try {
|
|
5460
|
+
return JSON.stringify(attributes, null, 2);
|
|
5461
|
+
} catch (error) {
|
|
5462
|
+
const errMsg = error instanceof Error ? error.message : "Unknown formatting error";
|
|
5463
|
+
return `[Unable to serialize attributes: ${errMsg}]`;
|
|
5464
|
+
}
|
|
5465
|
+
};
|
|
5466
|
+
const formatDuration = (startTime, endTime) => {
|
|
5467
|
+
if (!endTime) return "N/A";
|
|
5468
|
+
const duration = endTime.getTime() - startTime.getTime();
|
|
5469
|
+
return `${duration}ms`;
|
|
5470
|
+
};
|
|
5471
|
+
switch (event.type) {
|
|
5472
|
+
case TracingEventType.SPAN_STARTED:
|
|
5473
|
+
this.logger.info(`\u{1F680} SPAN_STARTED`);
|
|
5474
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5475
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5476
|
+
this.logger.info(` ID: ${span.id}`);
|
|
5477
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5478
|
+
if (span.input !== void 0) {
|
|
5479
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5480
|
+
}
|
|
5481
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5482
|
+
this.logger.info("\u2500".repeat(80));
|
|
5483
|
+
break;
|
|
5484
|
+
case TracingEventType.SPAN_ENDED:
|
|
5485
|
+
const duration = formatDuration(span.startTime, span.endTime);
|
|
4446
5486
|
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
4447
5487
|
this.logger.info(` Type: ${span.type}`);
|
|
4448
5488
|
this.logger.info(` Name: ${span.name}`);
|
|
4449
5489
|
this.logger.info(` ID: ${span.id}`);
|
|
4450
|
-
this.logger.info(` Duration: ${duration}`);
|
|
5490
|
+
this.logger.info(` Duration: ${duration}`);
|
|
5491
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5492
|
+
if (span.input !== void 0) {
|
|
5493
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5494
|
+
}
|
|
5495
|
+
if (span.output !== void 0) {
|
|
5496
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5497
|
+
}
|
|
5498
|
+
if (span.errorInfo) {
|
|
5499
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5500
|
+
}
|
|
5501
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5502
|
+
this.logger.info("\u2500".repeat(80));
|
|
5503
|
+
break;
|
|
5504
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5505
|
+
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
5506
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5507
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5508
|
+
this.logger.info(` ID: ${span.id}`);
|
|
4451
5509
|
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
4452
5510
|
if (span.input !== void 0) {
|
|
4453
5511
|
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
4454
5512
|
}
|
|
4455
|
-
if (span.output !== void 0) {
|
|
4456
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5513
|
+
if (span.output !== void 0) {
|
|
5514
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5515
|
+
}
|
|
5516
|
+
if (span.errorInfo) {
|
|
5517
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5518
|
+
}
|
|
5519
|
+
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
5520
|
+
this.logger.info("\u2500".repeat(80));
|
|
5521
|
+
break;
|
|
5522
|
+
default:
|
|
5523
|
+
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
5524
|
+
}
|
|
5525
|
+
}
|
|
5526
|
+
async shutdown() {
|
|
5527
|
+
this.logger.info("ConsoleExporter shutdown");
|
|
5528
|
+
}
|
|
5529
|
+
};
|
|
5530
|
+
function resolveTracingStorageStrategy(config, observability, storageName, logger) {
|
|
5531
|
+
if (config.strategy && config.strategy !== "auto") {
|
|
5532
|
+
const hints = observability.tracingStrategy;
|
|
5533
|
+
if (hints.supported.includes(config.strategy)) {
|
|
5534
|
+
return config.strategy;
|
|
5535
|
+
}
|
|
5536
|
+
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
5537
|
+
userStrategy: config.strategy,
|
|
5538
|
+
storageAdapter: storageName,
|
|
5539
|
+
supportedStrategies: hints.supported,
|
|
5540
|
+
fallbackStrategy: hints.preferred
|
|
5541
|
+
});
|
|
5542
|
+
}
|
|
5543
|
+
return observability.tracingStrategy.preferred;
|
|
5544
|
+
}
|
|
5545
|
+
function getStringOrNull(value) {
|
|
5546
|
+
return typeof value === "string" ? value : null;
|
|
5547
|
+
}
|
|
5548
|
+
function getObjectOrNull(value) {
|
|
5549
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
5550
|
+
}
|
|
5551
|
+
var DefaultExporter = class extends BaseExporter {
|
|
5552
|
+
name = "mastra-default-observability-exporter";
|
|
5553
|
+
#storage;
|
|
5554
|
+
#observability;
|
|
5555
|
+
#config;
|
|
5556
|
+
#resolvedStrategy;
|
|
5557
|
+
buffer;
|
|
5558
|
+
#flushTimer = null;
|
|
5559
|
+
// Track all spans that have been created, persists across flushes
|
|
5560
|
+
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
5561
|
+
constructor(config = {}) {
|
|
5562
|
+
super(config);
|
|
5563
|
+
if (config === void 0) {
|
|
5564
|
+
config = {};
|
|
5565
|
+
}
|
|
5566
|
+
this.#config = {
|
|
5567
|
+
...config,
|
|
5568
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5569
|
+
maxBufferSize: config.maxBufferSize ?? 1e4,
|
|
5570
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5571
|
+
maxRetries: config.maxRetries ?? 4,
|
|
5572
|
+
retryDelayMs: config.retryDelayMs ?? 500,
|
|
5573
|
+
strategy: config.strategy ?? "auto"
|
|
5574
|
+
};
|
|
5575
|
+
this.buffer = {
|
|
5576
|
+
creates: [],
|
|
5577
|
+
updates: [],
|
|
5578
|
+
insertOnly: [],
|
|
5579
|
+
seenSpans: /* @__PURE__ */ new Set(),
|
|
5580
|
+
spanSequences: /* @__PURE__ */ new Map(),
|
|
5581
|
+
completedSpans: /* @__PURE__ */ new Set(),
|
|
5582
|
+
outOfOrderCount: 0,
|
|
5583
|
+
totalSize: 0
|
|
5584
|
+
};
|
|
5585
|
+
this.#resolvedStrategy = "batch-with-updates";
|
|
5586
|
+
}
|
|
5587
|
+
#strategyInitialized = false;
|
|
5588
|
+
/**
|
|
5589
|
+
* Initialize the exporter (called after all dependencies are ready)
|
|
5590
|
+
*/
|
|
5591
|
+
async init(options) {
|
|
5592
|
+
this.#storage = options.mastra?.getStorage();
|
|
5593
|
+
if (!this.#storage) {
|
|
5594
|
+
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
5595
|
+
return;
|
|
5596
|
+
}
|
|
5597
|
+
this.#observability = await this.#storage.getStore("observability");
|
|
5598
|
+
if (!this.#observability) {
|
|
5599
|
+
this.logger.warn("DefaultExporter disabled: Observability storage not available. Traces will not be persisted.");
|
|
5600
|
+
return;
|
|
5601
|
+
}
|
|
5602
|
+
this.initializeStrategy(this.#observability, this.#storage.constructor.name);
|
|
5603
|
+
}
|
|
5604
|
+
/**
|
|
5605
|
+
* Initialize the resolved strategy once observability store is available
|
|
5606
|
+
*/
|
|
5607
|
+
initializeStrategy(observability, storageName) {
|
|
5608
|
+
if (this.#strategyInitialized) return;
|
|
5609
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, observability, storageName, this.logger);
|
|
5610
|
+
this.#strategyInitialized = true;
|
|
5611
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
5612
|
+
strategy: this.#resolvedStrategy,
|
|
5613
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
5614
|
+
storageAdapter: storageName,
|
|
5615
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
5616
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
5617
|
+
});
|
|
5618
|
+
}
|
|
5619
|
+
/**
|
|
5620
|
+
* Builds a unique span key for tracking
|
|
5621
|
+
*/
|
|
5622
|
+
buildSpanKey(traceId, spanId) {
|
|
5623
|
+
return `${traceId}:${spanId}`;
|
|
5624
|
+
}
|
|
5625
|
+
/**
|
|
5626
|
+
* Gets the next sequence number for a span
|
|
5627
|
+
*/
|
|
5628
|
+
getNextSequence(spanKey) {
|
|
5629
|
+
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
5630
|
+
const next = current + 1;
|
|
5631
|
+
this.buffer.spanSequences.set(spanKey, next);
|
|
5632
|
+
return next;
|
|
5633
|
+
}
|
|
5634
|
+
/**
|
|
5635
|
+
* Handles out-of-order span updates by logging and skipping
|
|
5636
|
+
*/
|
|
5637
|
+
handleOutOfOrderUpdate(event) {
|
|
5638
|
+
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
5639
|
+
spanId: event.exportedSpan.id,
|
|
5640
|
+
traceId: event.exportedSpan.traceId,
|
|
5641
|
+
spanName: event.exportedSpan.name,
|
|
5642
|
+
eventType: event.type
|
|
5643
|
+
});
|
|
5644
|
+
}
|
|
5645
|
+
/**
|
|
5646
|
+
* Adds an event to the appropriate buffer based on strategy
|
|
5647
|
+
*/
|
|
5648
|
+
addToBuffer(event) {
|
|
5649
|
+
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
5650
|
+
if (this.buffer.totalSize === 0) {
|
|
5651
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5652
|
+
}
|
|
5653
|
+
switch (event.type) {
|
|
5654
|
+
case TracingEventType.SPAN_STARTED:
|
|
5655
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5656
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5657
|
+
this.buffer.creates.push(createRecord);
|
|
5658
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5659
|
+
this.allCreatedSpans.add(spanKey);
|
|
5660
|
+
}
|
|
5661
|
+
break;
|
|
5662
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5663
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5664
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5665
|
+
this.buffer.updates.push({
|
|
5666
|
+
traceId: event.exportedSpan.traceId,
|
|
5667
|
+
spanId: event.exportedSpan.id,
|
|
5668
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5669
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5670
|
+
});
|
|
5671
|
+
} else {
|
|
5672
|
+
this.handleOutOfOrderUpdate(event);
|
|
5673
|
+
this.buffer.outOfOrderCount++;
|
|
5674
|
+
}
|
|
5675
|
+
}
|
|
5676
|
+
break;
|
|
5677
|
+
case TracingEventType.SPAN_ENDED:
|
|
5678
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5679
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5680
|
+
this.buffer.updates.push({
|
|
5681
|
+
traceId: event.exportedSpan.traceId,
|
|
5682
|
+
spanId: event.exportedSpan.id,
|
|
5683
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5684
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5685
|
+
});
|
|
5686
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5687
|
+
} else if (event.exportedSpan.isEvent) {
|
|
5688
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5689
|
+
this.buffer.creates.push(createRecord);
|
|
5690
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5691
|
+
this.allCreatedSpans.add(spanKey);
|
|
5692
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5693
|
+
} else {
|
|
5694
|
+
this.handleOutOfOrderUpdate(event);
|
|
5695
|
+
this.buffer.outOfOrderCount++;
|
|
5696
|
+
}
|
|
5697
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5698
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5699
|
+
this.buffer.insertOnly.push(createRecord);
|
|
5700
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5701
|
+
this.allCreatedSpans.add(spanKey);
|
|
5702
|
+
}
|
|
5703
|
+
break;
|
|
5704
|
+
}
|
|
5705
|
+
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
5706
|
+
}
|
|
5707
|
+
/**
|
|
5708
|
+
* Checks if buffer should be flushed based on size or time triggers
|
|
5709
|
+
*/
|
|
5710
|
+
shouldFlush() {
|
|
5711
|
+
if (this.buffer.totalSize >= this.#config.maxBufferSize) {
|
|
5712
|
+
return true;
|
|
5713
|
+
}
|
|
5714
|
+
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
5715
|
+
return true;
|
|
5716
|
+
}
|
|
5717
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5718
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5719
|
+
if (elapsed >= this.#config.maxBatchWaitMs) {
|
|
5720
|
+
return true;
|
|
5721
|
+
}
|
|
5722
|
+
}
|
|
5723
|
+
return false;
|
|
5724
|
+
}
|
|
5725
|
+
/**
|
|
5726
|
+
* Resets the buffer after successful flush
|
|
5727
|
+
*/
|
|
5728
|
+
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
5729
|
+
this.buffer.creates = [];
|
|
5730
|
+
this.buffer.updates = [];
|
|
5731
|
+
this.buffer.insertOnly = [];
|
|
5732
|
+
this.buffer.seenSpans.clear();
|
|
5733
|
+
this.buffer.spanSequences.clear();
|
|
5734
|
+
this.buffer.completedSpans.clear();
|
|
5735
|
+
this.buffer.outOfOrderCount = 0;
|
|
5736
|
+
this.buffer.firstEventTime = void 0;
|
|
5737
|
+
this.buffer.totalSize = 0;
|
|
5738
|
+
for (const spanKey of completedSpansToCleanup) {
|
|
5739
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5740
|
+
}
|
|
5741
|
+
}
|
|
5742
|
+
/**
|
|
5743
|
+
* Schedules a flush using setTimeout
|
|
5744
|
+
*/
|
|
5745
|
+
scheduleFlush() {
|
|
5746
|
+
if (this.#flushTimer) {
|
|
5747
|
+
clearTimeout(this.#flushTimer);
|
|
5748
|
+
}
|
|
5749
|
+
this.#flushTimer = setTimeout(() => {
|
|
5750
|
+
this.flushBuffer().catch((error) => {
|
|
5751
|
+
this.logger.error("Scheduled flush failed", {
|
|
5752
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5753
|
+
});
|
|
5754
|
+
});
|
|
5755
|
+
}, this.#config.maxBatchWaitMs);
|
|
5756
|
+
}
|
|
5757
|
+
/**
|
|
5758
|
+
* Serializes span attributes to storage record format
|
|
5759
|
+
* Handles all Span types and their specific attributes
|
|
5760
|
+
*/
|
|
5761
|
+
serializeAttributes(span) {
|
|
5762
|
+
if (!span.attributes) {
|
|
5763
|
+
return null;
|
|
5764
|
+
}
|
|
5765
|
+
try {
|
|
5766
|
+
return JSON.parse(
|
|
5767
|
+
JSON.stringify(span.attributes, (_key, value) => {
|
|
5768
|
+
if (value instanceof Date) {
|
|
5769
|
+
return value.toISOString();
|
|
5770
|
+
}
|
|
5771
|
+
if (typeof value === "object" && value !== null) {
|
|
5772
|
+
return value;
|
|
5773
|
+
}
|
|
5774
|
+
return value;
|
|
5775
|
+
})
|
|
5776
|
+
);
|
|
5777
|
+
} catch (error) {
|
|
5778
|
+
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
5779
|
+
spanId: span.id,
|
|
5780
|
+
spanType: span.type,
|
|
5781
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5782
|
+
});
|
|
5783
|
+
return null;
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
buildCreateRecord(span) {
|
|
5787
|
+
const metadata = span.metadata ?? {};
|
|
5788
|
+
return {
|
|
5789
|
+
traceId: span.traceId,
|
|
5790
|
+
spanId: span.id,
|
|
5791
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5792
|
+
name: span.name,
|
|
5793
|
+
// Entity identification - from span
|
|
5794
|
+
entityType: span.entityType ?? null,
|
|
5795
|
+
entityId: span.entityId ?? null,
|
|
5796
|
+
entityName: span.entityName ?? null,
|
|
5797
|
+
// Identity & Tenancy - extracted from metadata if present
|
|
5798
|
+
userId: getStringOrNull(metadata.userId),
|
|
5799
|
+
organizationId: getStringOrNull(metadata.organizationId),
|
|
5800
|
+
resourceId: getStringOrNull(metadata.resourceId),
|
|
5801
|
+
// Correlation IDs - extracted from metadata if present
|
|
5802
|
+
runId: getStringOrNull(metadata.runId),
|
|
5803
|
+
sessionId: getStringOrNull(metadata.sessionId),
|
|
5804
|
+
threadId: getStringOrNull(metadata.threadId),
|
|
5805
|
+
requestId: getStringOrNull(metadata.requestId),
|
|
5806
|
+
// Deployment context - extracted from metadata if present
|
|
5807
|
+
environment: getStringOrNull(metadata.environment),
|
|
5808
|
+
source: getStringOrNull(metadata.source),
|
|
5809
|
+
serviceName: getStringOrNull(metadata.serviceName),
|
|
5810
|
+
scope: getObjectOrNull(metadata.scope),
|
|
5811
|
+
// Span data
|
|
5812
|
+
spanType: span.type,
|
|
5813
|
+
attributes: this.serializeAttributes(span),
|
|
5814
|
+
metadata: span.metadata ?? null,
|
|
5815
|
+
// Keep all metadata including extracted fields
|
|
5816
|
+
tags: span.tags ?? null,
|
|
5817
|
+
links: null,
|
|
5818
|
+
input: span.input ?? null,
|
|
5819
|
+
output: span.output ?? null,
|
|
5820
|
+
error: span.errorInfo ?? null,
|
|
5821
|
+
isEvent: span.isEvent,
|
|
5822
|
+
// Timestamps
|
|
5823
|
+
startedAt: span.startTime,
|
|
5824
|
+
endedAt: span.endTime ?? null
|
|
5825
|
+
};
|
|
5826
|
+
}
|
|
5827
|
+
buildUpdateRecord(span) {
|
|
5828
|
+
return {
|
|
5829
|
+
name: span.name,
|
|
5830
|
+
scope: null,
|
|
5831
|
+
attributes: this.serializeAttributes(span),
|
|
5832
|
+
metadata: span.metadata ?? null,
|
|
5833
|
+
links: null,
|
|
5834
|
+
endedAt: span.endTime ?? null,
|
|
5835
|
+
input: span.input,
|
|
5836
|
+
output: span.output,
|
|
5837
|
+
error: span.errorInfo ?? null
|
|
5838
|
+
};
|
|
5839
|
+
}
|
|
5840
|
+
/**
|
|
5841
|
+
* Handles realtime strategy - processes each event immediately
|
|
5842
|
+
*/
|
|
5843
|
+
async handleRealtimeEvent(event, observability) {
|
|
5844
|
+
const span = event.exportedSpan;
|
|
5845
|
+
const spanKey = this.buildSpanKey(span.traceId, span.id);
|
|
5846
|
+
if (span.isEvent) {
|
|
5847
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
5848
|
+
await observability.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5849
|
+
} else {
|
|
5850
|
+
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
5851
|
+
}
|
|
5852
|
+
} else {
|
|
5853
|
+
switch (event.type) {
|
|
5854
|
+
case TracingEventType.SPAN_STARTED:
|
|
5855
|
+
await observability.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5856
|
+
this.allCreatedSpans.add(spanKey);
|
|
5857
|
+
break;
|
|
5858
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5859
|
+
await observability.updateSpan({
|
|
5860
|
+
traceId: span.traceId,
|
|
5861
|
+
spanId: span.id,
|
|
5862
|
+
updates: this.buildUpdateRecord(span)
|
|
5863
|
+
});
|
|
5864
|
+
break;
|
|
5865
|
+
case TracingEventType.SPAN_ENDED:
|
|
5866
|
+
await observability.updateSpan({
|
|
5867
|
+
traceId: span.traceId,
|
|
5868
|
+
spanId: span.id,
|
|
5869
|
+
updates: this.buildUpdateRecord(span)
|
|
5870
|
+
});
|
|
5871
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5872
|
+
break;
|
|
5873
|
+
default:
|
|
5874
|
+
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
5875
|
+
}
|
|
5876
|
+
}
|
|
5877
|
+
}
|
|
5878
|
+
/**
|
|
5879
|
+
* Handles batch-with-updates strategy - buffers events and processes in batches
|
|
5880
|
+
*/
|
|
5881
|
+
handleBatchWithUpdatesEvent(event) {
|
|
5882
|
+
this.addToBuffer(event);
|
|
5883
|
+
if (this.shouldFlush()) {
|
|
5884
|
+
this.flushBuffer().catch((error) => {
|
|
5885
|
+
this.logger.error("Batch flush failed", {
|
|
5886
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5887
|
+
});
|
|
5888
|
+
});
|
|
5889
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5890
|
+
this.scheduleFlush();
|
|
5891
|
+
}
|
|
5892
|
+
}
|
|
5893
|
+
/**
|
|
5894
|
+
* Handles insert-only strategy - only processes SPAN_ENDED events in batches
|
|
5895
|
+
*/
|
|
5896
|
+
handleInsertOnlyEvent(event) {
|
|
5897
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
5898
|
+
this.addToBuffer(event);
|
|
5899
|
+
if (this.shouldFlush()) {
|
|
5900
|
+
this.flushBuffer().catch((error) => {
|
|
5901
|
+
this.logger.error("Batch flush failed", {
|
|
5902
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5903
|
+
});
|
|
5904
|
+
});
|
|
5905
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5906
|
+
this.scheduleFlush();
|
|
5907
|
+
}
|
|
5908
|
+
}
|
|
5909
|
+
}
|
|
5910
|
+
/**
|
|
5911
|
+
* Calculates retry delay using exponential backoff
|
|
5912
|
+
*/
|
|
5913
|
+
calculateRetryDelay(attempt) {
|
|
5914
|
+
return this.#config.retryDelayMs * Math.pow(2, attempt);
|
|
5915
|
+
}
|
|
5916
|
+
/**
|
|
5917
|
+
* Flushes the current buffer to storage with retry logic (internal implementation)
|
|
5918
|
+
*/
|
|
5919
|
+
async flushBuffer() {
|
|
5920
|
+
if (!this.#observability) {
|
|
5921
|
+
this.logger.debug("Cannot flush traces. Observability storage is not initialized");
|
|
5922
|
+
return;
|
|
5923
|
+
}
|
|
5924
|
+
if (this.#flushTimer) {
|
|
5925
|
+
clearTimeout(this.#flushTimer);
|
|
5926
|
+
this.#flushTimer = null;
|
|
5927
|
+
}
|
|
5928
|
+
if (this.buffer.totalSize === 0) {
|
|
5929
|
+
return;
|
|
5930
|
+
}
|
|
5931
|
+
const startTime = Date.now();
|
|
5932
|
+
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
5933
|
+
const bufferCopy = {
|
|
5934
|
+
creates: [...this.buffer.creates],
|
|
5935
|
+
updates: [...this.buffer.updates],
|
|
5936
|
+
insertOnly: [...this.buffer.insertOnly],
|
|
5937
|
+
seenSpans: new Set(this.buffer.seenSpans),
|
|
5938
|
+
spanSequences: new Map(this.buffer.spanSequences),
|
|
5939
|
+
completedSpans: new Set(this.buffer.completedSpans),
|
|
5940
|
+
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
5941
|
+
firstEventTime: this.buffer.firstEventTime,
|
|
5942
|
+
totalSize: this.buffer.totalSize
|
|
5943
|
+
};
|
|
5944
|
+
this.resetBuffer();
|
|
5945
|
+
await this.flushWithRetries(this.#observability, bufferCopy, 0);
|
|
5946
|
+
const elapsed = Date.now() - startTime;
|
|
5947
|
+
this.logger.debug("Batch flushed", {
|
|
5948
|
+
strategy: this.#resolvedStrategy,
|
|
5949
|
+
batchSize: bufferCopy.totalSize,
|
|
5950
|
+
flushReason,
|
|
5951
|
+
durationMs: elapsed,
|
|
5952
|
+
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
5953
|
+
});
|
|
5954
|
+
}
|
|
5955
|
+
/**
|
|
5956
|
+
* Attempts to flush with exponential backoff retry logic
|
|
5957
|
+
*/
|
|
5958
|
+
async flushWithRetries(observability, buffer, attempt) {
|
|
5959
|
+
try {
|
|
5960
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5961
|
+
if (buffer.creates.length > 0) {
|
|
5962
|
+
await observability.batchCreateSpans({ records: buffer.creates });
|
|
5963
|
+
}
|
|
5964
|
+
if (buffer.updates.length > 0) {
|
|
5965
|
+
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
5966
|
+
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
5967
|
+
this.buildSpanKey(b.traceId, b.spanId)
|
|
5968
|
+
);
|
|
5969
|
+
if (spanCompare !== 0) return spanCompare;
|
|
5970
|
+
return a.sequenceNumber - b.sequenceNumber;
|
|
5971
|
+
});
|
|
5972
|
+
await observability.batchUpdateSpans({ records: sortedUpdates });
|
|
4457
5973
|
}
|
|
4458
|
-
|
|
4459
|
-
|
|
5974
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5975
|
+
if (buffer.insertOnly.length > 0) {
|
|
5976
|
+
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
4460
5977
|
}
|
|
4461
|
-
|
|
4462
|
-
|
|
5978
|
+
}
|
|
5979
|
+
for (const spanKey of buffer.completedSpans) {
|
|
5980
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5981
|
+
}
|
|
5982
|
+
} catch (error) {
|
|
5983
|
+
if (attempt < this.#config.maxRetries) {
|
|
5984
|
+
const retryDelay = this.calculateRetryDelay(attempt);
|
|
5985
|
+
this.logger.warn("Batch flush failed, retrying", {
|
|
5986
|
+
attempt: attempt + 1,
|
|
5987
|
+
maxRetries: this.#config.maxRetries,
|
|
5988
|
+
nextRetryInMs: retryDelay,
|
|
5989
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5990
|
+
});
|
|
5991
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
5992
|
+
return this.flushWithRetries(observability, buffer, attempt + 1);
|
|
5993
|
+
} else {
|
|
5994
|
+
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
5995
|
+
finalAttempt: attempt + 1,
|
|
5996
|
+
maxRetries: this.#config.maxRetries,
|
|
5997
|
+
droppedBatchSize: buffer.totalSize,
|
|
5998
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5999
|
+
});
|
|
6000
|
+
for (const spanKey of buffer.completedSpans) {
|
|
6001
|
+
this.allCreatedSpans.delete(spanKey);
|
|
6002
|
+
}
|
|
6003
|
+
}
|
|
6004
|
+
}
|
|
6005
|
+
}
|
|
6006
|
+
async _exportTracingEvent(event) {
|
|
6007
|
+
if (!this.#observability) {
|
|
6008
|
+
this.logger.debug("Cannot store traces. Observability storage is not initialized");
|
|
6009
|
+
return;
|
|
6010
|
+
}
|
|
6011
|
+
if (!this.#strategyInitialized) {
|
|
6012
|
+
this.initializeStrategy(this.#observability, this.#storage?.constructor.name ?? "Unknown");
|
|
6013
|
+
}
|
|
6014
|
+
switch (this.#resolvedStrategy) {
|
|
6015
|
+
case "realtime":
|
|
6016
|
+
await this.handleRealtimeEvent(event, this.#observability);
|
|
6017
|
+
break;
|
|
6018
|
+
case "batch-with-updates":
|
|
6019
|
+
this.handleBatchWithUpdatesEvent(event);
|
|
4463
6020
|
break;
|
|
4464
|
-
case
|
|
4465
|
-
this.
|
|
4466
|
-
this.logger.info(` Type: ${span.type}`);
|
|
4467
|
-
this.logger.info(` Name: ${span.name}`);
|
|
4468
|
-
this.logger.info(` ID: ${span.id}`);
|
|
4469
|
-
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
4470
|
-
if (span.input !== void 0) {
|
|
4471
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
4472
|
-
}
|
|
4473
|
-
if (span.output !== void 0) {
|
|
4474
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
4475
|
-
}
|
|
4476
|
-
if (span.errorInfo) {
|
|
4477
|
-
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
4478
|
-
}
|
|
4479
|
-
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
4480
|
-
this.logger.info("\u2500".repeat(80));
|
|
6021
|
+
case "insert-only":
|
|
6022
|
+
this.handleInsertOnlyEvent(event);
|
|
4481
6023
|
break;
|
|
4482
|
-
|
|
4483
|
-
|
|
6024
|
+
}
|
|
6025
|
+
}
|
|
6026
|
+
/**
|
|
6027
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
6028
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
6029
|
+
* are exported before the runtime instance is terminated.
|
|
6030
|
+
*/
|
|
6031
|
+
async flush() {
|
|
6032
|
+
if (this.buffer.totalSize > 0) {
|
|
6033
|
+
this.logger.debug("Flushing buffered events", {
|
|
6034
|
+
bufferedEvents: this.buffer.totalSize
|
|
6035
|
+
});
|
|
6036
|
+
await this.flushBuffer();
|
|
4484
6037
|
}
|
|
4485
6038
|
}
|
|
4486
6039
|
async shutdown() {
|
|
4487
|
-
this
|
|
6040
|
+
if (this.#flushTimer) {
|
|
6041
|
+
clearTimeout(this.#flushTimer);
|
|
6042
|
+
this.#flushTimer = null;
|
|
6043
|
+
}
|
|
6044
|
+
await this.flush();
|
|
6045
|
+
this.logger.info("DefaultExporter shutdown complete");
|
|
4488
6046
|
}
|
|
4489
6047
|
};
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
userStrategy: config.strategy,
|
|
4498
|
-
storageAdapter: storageName,
|
|
4499
|
-
supportedStrategies: hints.supported,
|
|
4500
|
-
fallbackStrategy: hints.preferred
|
|
4501
|
-
});
|
|
6048
|
+
|
|
6049
|
+
// src/exporters/test.ts
|
|
6050
|
+
var TestExporter = class extends BaseExporter {
|
|
6051
|
+
name = "tracing-test-exporter";
|
|
6052
|
+
#events = [];
|
|
6053
|
+
constructor(config = {}) {
|
|
6054
|
+
super(config);
|
|
4502
6055
|
}
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
6056
|
+
async _exportTracingEvent(event) {
|
|
6057
|
+
this.#events.push(event);
|
|
6058
|
+
}
|
|
6059
|
+
clearEvents() {
|
|
6060
|
+
this.#events = [];
|
|
6061
|
+
}
|
|
6062
|
+
get events() {
|
|
6063
|
+
return this.#events;
|
|
6064
|
+
}
|
|
6065
|
+
async shutdown() {
|
|
6066
|
+
this.logger.info("TestExporter shutdown");
|
|
6067
|
+
}
|
|
6068
|
+
};
|
|
6069
|
+
var __filename$1 = fileURLToPath(import.meta.url);
|
|
6070
|
+
var __dirname$1 = dirname(__filename$1);
|
|
6071
|
+
var SNAPSHOTS_DIR = join(__dirname$1, "..", "__snapshots__");
|
|
6072
|
+
var JsonExporter = class extends BaseExporter {
|
|
6073
|
+
name = "json-exporter";
|
|
6074
|
+
/** All collected events */
|
|
6075
|
+
#events = [];
|
|
6076
|
+
/** Per-span state tracking */
|
|
6077
|
+
#spanStates = /* @__PURE__ */ new Map();
|
|
6078
|
+
/** Logs for debugging */
|
|
6079
|
+
#logs = [];
|
|
6080
|
+
/** Configuration */
|
|
4515
6081
|
#config;
|
|
4516
|
-
#resolvedStrategy;
|
|
4517
|
-
buffer;
|
|
4518
|
-
#flushTimer = null;
|
|
4519
|
-
// Track all spans that have been created, persists across flushes
|
|
4520
|
-
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
4521
6082
|
constructor(config = {}) {
|
|
4522
6083
|
super(config);
|
|
4523
|
-
if (config === void 0) {
|
|
4524
|
-
config = {};
|
|
4525
|
-
}
|
|
4526
6084
|
this.#config = {
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
maxRetries: config.maxRetries ?? 4,
|
|
4532
|
-
retryDelayMs: config.retryDelayMs ?? 500,
|
|
4533
|
-
strategy: config.strategy ?? "auto"
|
|
6085
|
+
validateLifecycle: true,
|
|
6086
|
+
storeLogs: true,
|
|
6087
|
+
jsonIndent: 2,
|
|
6088
|
+
...config
|
|
4534
6089
|
};
|
|
4535
|
-
this.buffer = {
|
|
4536
|
-
creates: [],
|
|
4537
|
-
updates: [],
|
|
4538
|
-
insertOnly: [],
|
|
4539
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
4540
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
4541
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
4542
|
-
outOfOrderCount: 0,
|
|
4543
|
-
totalSize: 0
|
|
4544
|
-
};
|
|
4545
|
-
this.#resolvedStrategy = "batch-with-updates";
|
|
4546
6090
|
}
|
|
4547
|
-
#strategyInitialized = false;
|
|
4548
6091
|
/**
|
|
4549
|
-
*
|
|
6092
|
+
* Process incoming tracing events with lifecycle tracking
|
|
4550
6093
|
*/
|
|
4551
|
-
async
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
|
|
6094
|
+
async _exportTracingEvent(event) {
|
|
6095
|
+
const span = event.exportedSpan;
|
|
6096
|
+
const spanId = span.id;
|
|
6097
|
+
const logMessage = `[JsonExporter] ${event.type}: ${span.type} "${span.name}" (entity: ${span.entityName ?? span.entityId ?? "unknown"}, trace: ${span.traceId.slice(-8)}, span: ${spanId.slice(-8)})`;
|
|
6098
|
+
if (this.#config.storeLogs) {
|
|
6099
|
+
this.#logs.push(logMessage);
|
|
6100
|
+
}
|
|
6101
|
+
const state = this.#spanStates.get(spanId) || {
|
|
6102
|
+
hasStart: false,
|
|
6103
|
+
hasEnd: false,
|
|
6104
|
+
hasUpdate: false,
|
|
6105
|
+
events: []
|
|
6106
|
+
};
|
|
6107
|
+
if (this.#config.validateLifecycle) {
|
|
6108
|
+
this.#validateLifecycle(event, state, spanId);
|
|
4556
6109
|
}
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
6110
|
+
if (event.type === TracingEventType.SPAN_STARTED) {
|
|
6111
|
+
state.hasStart = true;
|
|
6112
|
+
} else if (event.type === TracingEventType.SPAN_ENDED) {
|
|
6113
|
+
state.hasEnd = true;
|
|
6114
|
+
if (span.isEvent) {
|
|
6115
|
+
state.isEventSpan = true;
|
|
6116
|
+
}
|
|
6117
|
+
} else if (event.type === TracingEventType.SPAN_UPDATED) {
|
|
6118
|
+
state.hasUpdate = true;
|
|
4561
6119
|
}
|
|
4562
|
-
|
|
6120
|
+
state.events.push(event);
|
|
6121
|
+
this.#spanStates.set(spanId, state);
|
|
6122
|
+
this.#events.push(event);
|
|
4563
6123
|
}
|
|
4564
6124
|
/**
|
|
4565
|
-
*
|
|
6125
|
+
* Validate span lifecycle rules
|
|
4566
6126
|
*/
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
6127
|
+
#validateLifecycle(event, state, spanId) {
|
|
6128
|
+
const span = event.exportedSpan;
|
|
6129
|
+
if (event.type === TracingEventType.SPAN_STARTED) {
|
|
6130
|
+
if (state.hasStart) {
|
|
6131
|
+
this.logger.warn(`Span ${spanId} (${span.type} "${span.name}") started twice`);
|
|
6132
|
+
}
|
|
6133
|
+
} else if (event.type === TracingEventType.SPAN_ENDED) {
|
|
6134
|
+
if (span.isEvent) {
|
|
6135
|
+
if (state.hasStart) {
|
|
6136
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_STARTED`);
|
|
6137
|
+
}
|
|
6138
|
+
if (state.hasUpdate) {
|
|
6139
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_UPDATED`);
|
|
6140
|
+
}
|
|
6141
|
+
} else {
|
|
6142
|
+
if (!state.hasStart) {
|
|
6143
|
+
this.logger.warn(`Normal span ${spanId} (${span.type} "${span.name}") ended without starting`);
|
|
6144
|
+
}
|
|
6145
|
+
}
|
|
6146
|
+
}
|
|
4578
6147
|
}
|
|
6148
|
+
// ============================================================================
|
|
6149
|
+
// Query Methods
|
|
6150
|
+
// ============================================================================
|
|
4579
6151
|
/**
|
|
4580
|
-
*
|
|
6152
|
+
* Get all collected events
|
|
4581
6153
|
*/
|
|
4582
|
-
|
|
4583
|
-
return
|
|
6154
|
+
get events() {
|
|
6155
|
+
return [...this.#events];
|
|
4584
6156
|
}
|
|
4585
6157
|
/**
|
|
4586
|
-
*
|
|
6158
|
+
* Get completed spans by SpanType (e.g., 'agent_run', 'tool_call')
|
|
6159
|
+
*
|
|
6160
|
+
* @param type - The SpanType to filter by
|
|
6161
|
+
* @returns Array of completed exported spans of the specified type
|
|
4587
6162
|
*/
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
6163
|
+
getSpansByType(type) {
|
|
6164
|
+
return Array.from(this.#spanStates.values()).filter((state) => {
|
|
6165
|
+
if (!state.hasEnd) return false;
|
|
6166
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6167
|
+
return endEvent?.exportedSpan.type === type;
|
|
6168
|
+
}).map((state) => {
|
|
6169
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6170
|
+
return endEvent?.exportedSpan;
|
|
6171
|
+
}).filter((span) => span !== void 0);
|
|
4593
6172
|
}
|
|
4594
6173
|
/**
|
|
4595
|
-
*
|
|
6174
|
+
* Get events by TracingEventType (SPAN_STARTED, SPAN_UPDATED, SPAN_ENDED)
|
|
6175
|
+
*
|
|
6176
|
+
* @param type - The TracingEventType to filter by
|
|
6177
|
+
* @returns Array of events of the specified type
|
|
4596
6178
|
*/
|
|
4597
|
-
|
|
4598
|
-
this.
|
|
4599
|
-
spanId: event.exportedSpan.id,
|
|
4600
|
-
traceId: event.exportedSpan.traceId,
|
|
4601
|
-
spanName: event.exportedSpan.name,
|
|
4602
|
-
eventType: event.type
|
|
4603
|
-
});
|
|
6179
|
+
getByEventType(type) {
|
|
6180
|
+
return this.#events.filter((e) => e.type === type);
|
|
4604
6181
|
}
|
|
4605
6182
|
/**
|
|
4606
|
-
*
|
|
6183
|
+
* Get all events and spans for a specific trace
|
|
6184
|
+
*
|
|
6185
|
+
* @param traceId - The trace ID to filter by
|
|
6186
|
+
* @returns Object containing events and final spans for the trace
|
|
4607
6187
|
*/
|
|
4608
|
-
|
|
4609
|
-
const
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
}
|
|
4613
|
-
switch (event.type) {
|
|
4614
|
-
case TracingEventType.SPAN_STARTED:
|
|
4615
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4616
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4617
|
-
this.buffer.creates.push(createRecord);
|
|
4618
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4619
|
-
this.allCreatedSpans.add(spanKey);
|
|
4620
|
-
}
|
|
4621
|
-
break;
|
|
4622
|
-
case TracingEventType.SPAN_UPDATED:
|
|
4623
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4624
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4625
|
-
this.buffer.updates.push({
|
|
4626
|
-
traceId: event.exportedSpan.traceId,
|
|
4627
|
-
spanId: event.exportedSpan.id,
|
|
4628
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4629
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4630
|
-
});
|
|
4631
|
-
} else {
|
|
4632
|
-
this.handleOutOfOrderUpdate(event);
|
|
4633
|
-
this.buffer.outOfOrderCount++;
|
|
4634
|
-
}
|
|
4635
|
-
}
|
|
4636
|
-
break;
|
|
4637
|
-
case TracingEventType.SPAN_ENDED:
|
|
4638
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4639
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4640
|
-
this.buffer.updates.push({
|
|
4641
|
-
traceId: event.exportedSpan.traceId,
|
|
4642
|
-
spanId: event.exportedSpan.id,
|
|
4643
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4644
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4645
|
-
});
|
|
4646
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4647
|
-
} else if (event.exportedSpan.isEvent) {
|
|
4648
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4649
|
-
this.buffer.creates.push(createRecord);
|
|
4650
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4651
|
-
this.allCreatedSpans.add(spanKey);
|
|
4652
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4653
|
-
} else {
|
|
4654
|
-
this.handleOutOfOrderUpdate(event);
|
|
4655
|
-
this.buffer.outOfOrderCount++;
|
|
4656
|
-
}
|
|
4657
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
4658
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4659
|
-
this.buffer.insertOnly.push(createRecord);
|
|
4660
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4661
|
-
this.allCreatedSpans.add(spanKey);
|
|
4662
|
-
}
|
|
4663
|
-
break;
|
|
4664
|
-
}
|
|
4665
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
6188
|
+
getByTraceId(traceId) {
|
|
6189
|
+
const events = this.#events.filter((e) => e.exportedSpan.traceId === traceId);
|
|
6190
|
+
const spans = this.#getUniqueSpansFromEvents(events);
|
|
6191
|
+
return { events, spans };
|
|
4666
6192
|
}
|
|
4667
6193
|
/**
|
|
4668
|
-
*
|
|
6194
|
+
* Get all events for a specific span
|
|
6195
|
+
*
|
|
6196
|
+
* @param spanId - The span ID to filter by
|
|
6197
|
+
* @returns Object containing events and final span state
|
|
4669
6198
|
*/
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
4675
|
-
return true;
|
|
6199
|
+
getBySpanId(spanId) {
|
|
6200
|
+
const state = this.#spanStates.get(spanId);
|
|
6201
|
+
if (!state) {
|
|
6202
|
+
return { events: [], span: void 0, state: void 0 };
|
|
4676
6203
|
}
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
6204
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6205
|
+
const span = endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6206
|
+
return { events: state.events, span, state };
|
|
6207
|
+
}
|
|
6208
|
+
/**
|
|
6209
|
+
* Get all unique spans (returns the final state of each span)
|
|
6210
|
+
*/
|
|
6211
|
+
getAllSpans() {
|
|
6212
|
+
return Array.from(this.#spanStates.values()).map((state) => {
|
|
6213
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6214
|
+
return endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6215
|
+
}).filter((span) => span !== void 0);
|
|
6216
|
+
}
|
|
6217
|
+
/**
|
|
6218
|
+
* Get only completed spans (those that have received SPAN_ENDED)
|
|
6219
|
+
*/
|
|
6220
|
+
getCompletedSpans() {
|
|
6221
|
+
return Array.from(this.#spanStates.values()).filter((state) => state.hasEnd).map((state) => {
|
|
6222
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6223
|
+
return endEvent.exportedSpan;
|
|
6224
|
+
});
|
|
6225
|
+
}
|
|
6226
|
+
/**
|
|
6227
|
+
* Get root spans only (spans with no parent)
|
|
6228
|
+
*/
|
|
6229
|
+
getRootSpans() {
|
|
6230
|
+
return this.getAllSpans().filter((span) => span.isRootSpan);
|
|
6231
|
+
}
|
|
6232
|
+
/**
|
|
6233
|
+
* Get incomplete spans (started but not yet ended)
|
|
6234
|
+
*/
|
|
6235
|
+
getIncompleteSpans() {
|
|
6236
|
+
return Array.from(this.#spanStates.entries()).filter(([_, state]) => !state.hasEnd).map(([spanId, state]) => ({
|
|
6237
|
+
spanId,
|
|
6238
|
+
span: state.events[0]?.exportedSpan,
|
|
6239
|
+
state: {
|
|
6240
|
+
hasStart: state.hasStart,
|
|
6241
|
+
hasUpdate: state.hasUpdate,
|
|
6242
|
+
hasEnd: state.hasEnd
|
|
4681
6243
|
}
|
|
4682
|
-
}
|
|
4683
|
-
return false;
|
|
6244
|
+
}));
|
|
4684
6245
|
}
|
|
4685
6246
|
/**
|
|
4686
|
-
*
|
|
6247
|
+
* Get unique trace IDs from all collected spans
|
|
4687
6248
|
*/
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
this
|
|
4691
|
-
|
|
4692
|
-
this.buffer.seenSpans.clear();
|
|
4693
|
-
this.buffer.spanSequences.clear();
|
|
4694
|
-
this.buffer.completedSpans.clear();
|
|
4695
|
-
this.buffer.outOfOrderCount = 0;
|
|
4696
|
-
this.buffer.firstEventTime = void 0;
|
|
4697
|
-
this.buffer.totalSize = 0;
|
|
4698
|
-
for (const spanKey of completedSpansToCleanup) {
|
|
4699
|
-
this.allCreatedSpans.delete(spanKey);
|
|
6249
|
+
getTraceIds() {
|
|
6250
|
+
const traceIds = /* @__PURE__ */ new Set();
|
|
6251
|
+
for (const event of this.#events) {
|
|
6252
|
+
traceIds.add(event.exportedSpan.traceId);
|
|
4700
6253
|
}
|
|
6254
|
+
return Array.from(traceIds);
|
|
4701
6255
|
}
|
|
6256
|
+
// ============================================================================
|
|
6257
|
+
// Statistics
|
|
6258
|
+
// ============================================================================
|
|
4702
6259
|
/**
|
|
4703
|
-
*
|
|
6260
|
+
* Get comprehensive statistics about collected spans
|
|
4704
6261
|
*/
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
6262
|
+
getStatistics() {
|
|
6263
|
+
const bySpanType = {};
|
|
6264
|
+
let completedSpans = 0;
|
|
6265
|
+
let incompleteSpans = 0;
|
|
6266
|
+
for (const state of this.#spanStates.values()) {
|
|
6267
|
+
if (state.hasEnd) {
|
|
6268
|
+
completedSpans++;
|
|
6269
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6270
|
+
const spanType = endEvent?.exportedSpan.type;
|
|
6271
|
+
if (spanType) {
|
|
6272
|
+
bySpanType[spanType] = (bySpanType[spanType] || 0) + 1;
|
|
6273
|
+
}
|
|
6274
|
+
} else {
|
|
6275
|
+
incompleteSpans++;
|
|
6276
|
+
}
|
|
4708
6277
|
}
|
|
4709
|
-
|
|
4710
|
-
this.
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
6278
|
+
return {
|
|
6279
|
+
totalEvents: this.#events.length,
|
|
6280
|
+
totalSpans: this.#spanStates.size,
|
|
6281
|
+
totalTraces: this.getTraceIds().length,
|
|
6282
|
+
completedSpans,
|
|
6283
|
+
incompleteSpans,
|
|
6284
|
+
byEventType: {
|
|
6285
|
+
started: this.#events.filter((e) => e.type === TracingEventType.SPAN_STARTED).length,
|
|
6286
|
+
updated: this.#events.filter((e) => e.type === TracingEventType.SPAN_UPDATED).length,
|
|
6287
|
+
ended: this.#events.filter((e) => e.type === TracingEventType.SPAN_ENDED).length
|
|
6288
|
+
},
|
|
6289
|
+
bySpanType
|
|
6290
|
+
};
|
|
4716
6291
|
}
|
|
6292
|
+
// ============================================================================
|
|
6293
|
+
// JSON Output
|
|
6294
|
+
// ============================================================================
|
|
4717
6295
|
/**
|
|
4718
|
-
*
|
|
4719
|
-
*
|
|
6296
|
+
* Serialize all collected data to JSON string
|
|
6297
|
+
*
|
|
6298
|
+
* @param options - Serialization options
|
|
6299
|
+
* @returns JSON string of all collected data
|
|
4720
6300
|
*/
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
6301
|
+
toJSON(options) {
|
|
6302
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6303
|
+
const includeEvents = options?.includeEvents ?? true;
|
|
6304
|
+
const includeStats = options?.includeStats ?? true;
|
|
6305
|
+
const data = {
|
|
6306
|
+
spans: this.getAllSpans()
|
|
6307
|
+
};
|
|
6308
|
+
if (includeEvents) {
|
|
6309
|
+
data.events = this.#events;
|
|
4724
6310
|
}
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
JSON.stringify(span.attributes, (_key, value) => {
|
|
4728
|
-
if (value instanceof Date) {
|
|
4729
|
-
return value.toISOString();
|
|
4730
|
-
}
|
|
4731
|
-
if (typeof value === "object" && value !== null) {
|
|
4732
|
-
return value;
|
|
4733
|
-
}
|
|
4734
|
-
return value;
|
|
4735
|
-
})
|
|
4736
|
-
);
|
|
4737
|
-
} catch (error) {
|
|
4738
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
4739
|
-
spanId: span.id,
|
|
4740
|
-
spanType: span.type,
|
|
4741
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4742
|
-
});
|
|
4743
|
-
return null;
|
|
6311
|
+
if (includeStats) {
|
|
6312
|
+
data.statistics = this.getStatistics();
|
|
4744
6313
|
}
|
|
6314
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4745
6315
|
}
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
serviceName: getStringOrNull(metadata.serviceName),
|
|
4770
|
-
scope: getObjectOrNull(metadata.scope),
|
|
4771
|
-
// Span data
|
|
4772
|
-
spanType: span.type,
|
|
4773
|
-
attributes: this.serializeAttributes(span),
|
|
4774
|
-
metadata: span.metadata ?? null,
|
|
4775
|
-
// Keep all metadata including extracted fields
|
|
4776
|
-
tags: span.tags ?? null,
|
|
4777
|
-
links: null,
|
|
4778
|
-
input: span.input ?? null,
|
|
4779
|
-
output: span.output ?? null,
|
|
4780
|
-
error: span.errorInfo ?? null,
|
|
4781
|
-
isEvent: span.isEvent,
|
|
4782
|
-
// Timestamps
|
|
4783
|
-
startedAt: span.startTime,
|
|
4784
|
-
endedAt: span.endTime ?? null
|
|
6316
|
+
/**
|
|
6317
|
+
* Build a tree structure from spans, nesting children under their parents
|
|
6318
|
+
*
|
|
6319
|
+
* @returns Array of root span tree nodes (spans with no parent)
|
|
6320
|
+
*/
|
|
6321
|
+
buildSpanTree() {
|
|
6322
|
+
const spans = this.getAllSpans();
|
|
6323
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
6324
|
+
const roots = [];
|
|
6325
|
+
for (const span of spans) {
|
|
6326
|
+
nodeMap.set(span.id, { span, children: [] });
|
|
6327
|
+
}
|
|
6328
|
+
for (const span of spans) {
|
|
6329
|
+
const node = nodeMap.get(span.id);
|
|
6330
|
+
if (span.parentSpanId && nodeMap.has(span.parentSpanId)) {
|
|
6331
|
+
nodeMap.get(span.parentSpanId).children.push(node);
|
|
6332
|
+
} else {
|
|
6333
|
+
roots.push(node);
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
const sortChildren = (node) => {
|
|
6337
|
+
node.children.sort((a, b) => new Date(a.span.startTime).getTime() - new Date(b.span.startTime).getTime());
|
|
6338
|
+
node.children.forEach(sortChildren);
|
|
4785
6339
|
};
|
|
6340
|
+
roots.forEach(sortChildren);
|
|
6341
|
+
return roots;
|
|
4786
6342
|
}
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
6343
|
+
/**
|
|
6344
|
+
* Serialize spans as a tree structure to JSON string
|
|
6345
|
+
*
|
|
6346
|
+
* @param options - Serialization options
|
|
6347
|
+
* @returns JSON string with spans nested in tree format
|
|
6348
|
+
*/
|
|
6349
|
+
toTreeJSON(options) {
|
|
6350
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6351
|
+
const includeStats = options?.includeStats ?? true;
|
|
6352
|
+
const data = {
|
|
6353
|
+
tree: this.buildSpanTree()
|
|
4798
6354
|
};
|
|
6355
|
+
if (includeStats) {
|
|
6356
|
+
data.statistics = this.getStatistics();
|
|
6357
|
+
}
|
|
6358
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4799
6359
|
}
|
|
4800
6360
|
/**
|
|
4801
|
-
*
|
|
6361
|
+
* Build a normalized tree structure suitable for snapshot testing.
|
|
6362
|
+
*
|
|
6363
|
+
* Normalizations applied:
|
|
6364
|
+
* - Span IDs replaced with stable placeholders (<span-1>, <span-2>, etc.)
|
|
6365
|
+
* - Trace IDs replaced with stable placeholders (<trace-1>, <trace-2>, etc.)
|
|
6366
|
+
* - parentSpanId replaced with normalized parent ID
|
|
6367
|
+
* - Timestamps replaced with durationMs (or null if not ended)
|
|
6368
|
+
* - Empty children arrays are omitted
|
|
6369
|
+
*
|
|
6370
|
+
* @returns Array of normalized root tree nodes
|
|
4802
6371
|
*/
|
|
4803
|
-
|
|
4804
|
-
const
|
|
4805
|
-
const
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
6372
|
+
buildNormalizedTree() {
|
|
6373
|
+
const tree = this.buildSpanTree();
|
|
6374
|
+
const spanIdMap = /* @__PURE__ */ new Map();
|
|
6375
|
+
const traceIdMap = /* @__PURE__ */ new Map();
|
|
6376
|
+
const uuidMapsByKey = /* @__PURE__ */ new Map();
|
|
6377
|
+
const uuidCountersByKey = /* @__PURE__ */ new Map();
|
|
6378
|
+
let spanIdCounter = 1;
|
|
6379
|
+
let traceIdCounter = 1;
|
|
6380
|
+
const uuidRegex2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6381
|
+
const hexId32Regex = /^[0-9a-f]{32}$/i;
|
|
6382
|
+
const prefixedUuidRegex = /^([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
|
|
6383
|
+
const embeddedPrefixedUuidRegex = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
|
|
6384
|
+
const normalizeUuid = (uuid, key) => {
|
|
6385
|
+
if (!uuidMapsByKey.has(key)) {
|
|
6386
|
+
uuidMapsByKey.set(key, /* @__PURE__ */ new Map());
|
|
6387
|
+
uuidCountersByKey.set(key, 1);
|
|
4811
6388
|
}
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
6389
|
+
const keyMap = uuidMapsByKey.get(key);
|
|
6390
|
+
if (!keyMap.has(uuid)) {
|
|
6391
|
+
const counter = uuidCountersByKey.get(key);
|
|
6392
|
+
keyMap.set(uuid, `<${key}-${counter}>`);
|
|
6393
|
+
uuidCountersByKey.set(key, counter + 1);
|
|
6394
|
+
}
|
|
6395
|
+
return keyMap.get(uuid);
|
|
6396
|
+
};
|
|
6397
|
+
const normalizeValue = (value, key) => {
|
|
6398
|
+
if (value instanceof Date) {
|
|
6399
|
+
return "<date>";
|
|
6400
|
+
}
|
|
6401
|
+
if (typeof value === "string") {
|
|
6402
|
+
if (key === "traceId" && (uuidRegex2.test(value) || hexId32Regex.test(value))) {
|
|
6403
|
+
if (!traceIdMap.has(value)) {
|
|
6404
|
+
traceIdMap.set(value, `<trace-${traceIdCounter++}>`);
|
|
6405
|
+
}
|
|
6406
|
+
return traceIdMap.get(value);
|
|
6407
|
+
}
|
|
6408
|
+
if (uuidRegex2.test(value)) {
|
|
6409
|
+
return normalizeUuid(value, key ?? "uuid");
|
|
6410
|
+
}
|
|
6411
|
+
const prefixMatch = prefixedUuidRegex.exec(value);
|
|
6412
|
+
if (prefixMatch && prefixMatch[1] && prefixMatch[2]) {
|
|
6413
|
+
const prefix = prefixMatch[1];
|
|
6414
|
+
const uuid = prefixMatch[2];
|
|
6415
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6416
|
+
}
|
|
6417
|
+
if (embeddedPrefixedUuidRegex.test(value)) {
|
|
6418
|
+
embeddedPrefixedUuidRegex.lastIndex = 0;
|
|
6419
|
+
return value.replace(embeddedPrefixedUuidRegex, (_match, prefix, uuid) => {
|
|
6420
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
4830
6421
|
});
|
|
4831
|
-
|
|
4832
|
-
break;
|
|
4833
|
-
default:
|
|
4834
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
6422
|
+
}
|
|
4835
6423
|
}
|
|
4836
|
-
|
|
6424
|
+
if (Array.isArray(value)) {
|
|
6425
|
+
return value.map((v) => normalizeValue(v, key));
|
|
6426
|
+
}
|
|
6427
|
+
if (value && typeof value === "object") {
|
|
6428
|
+
const normalized = {};
|
|
6429
|
+
for (const [k, v] of Object.entries(value)) {
|
|
6430
|
+
normalized[k] = normalizeValue(v, k);
|
|
6431
|
+
}
|
|
6432
|
+
return normalized;
|
|
6433
|
+
}
|
|
6434
|
+
return value;
|
|
6435
|
+
};
|
|
6436
|
+
const assignIds = (nodes) => {
|
|
6437
|
+
for (const node of nodes) {
|
|
6438
|
+
spanIdMap.set(node.span.id, `<span-${spanIdCounter++}>`);
|
|
6439
|
+
if (!traceIdMap.has(node.span.traceId)) {
|
|
6440
|
+
traceIdMap.set(node.span.traceId, `<trace-${traceIdCounter++}>`);
|
|
6441
|
+
}
|
|
6442
|
+
assignIds(node.children);
|
|
6443
|
+
}
|
|
6444
|
+
};
|
|
6445
|
+
assignIds(tree);
|
|
6446
|
+
const normalizeNode = (node) => {
|
|
6447
|
+
const span = node.span;
|
|
6448
|
+
const completed = span.endTime !== void 0 && span.endTime !== null;
|
|
6449
|
+
const normalizedSpan = {
|
|
6450
|
+
id: spanIdMap.get(span.id),
|
|
6451
|
+
traceId: traceIdMap.get(span.traceId),
|
|
6452
|
+
name: normalizeValue(span.name, "name"),
|
|
6453
|
+
type: span.type,
|
|
6454
|
+
completed,
|
|
6455
|
+
isEvent: span.isEvent,
|
|
6456
|
+
isRootSpan: span.isRootSpan
|
|
6457
|
+
};
|
|
6458
|
+
if (span.parentSpanId && spanIdMap.has(span.parentSpanId)) {
|
|
6459
|
+
normalizedSpan.parentId = spanIdMap.get(span.parentSpanId);
|
|
6460
|
+
}
|
|
6461
|
+
if (span.entityType) {
|
|
6462
|
+
normalizedSpan.entityType = span.entityType;
|
|
6463
|
+
}
|
|
6464
|
+
if (span.entityId) {
|
|
6465
|
+
normalizedSpan.entityId = normalizeValue(span.entityId, "entityId");
|
|
6466
|
+
}
|
|
6467
|
+
if (span.attributes && Object.keys(span.attributes).length > 0) {
|
|
6468
|
+
normalizedSpan.attributes = normalizeValue(span.attributes);
|
|
6469
|
+
}
|
|
6470
|
+
if (span.metadata && Object.keys(span.metadata).length > 0) {
|
|
6471
|
+
normalizedSpan.metadata = normalizeValue(span.metadata);
|
|
6472
|
+
}
|
|
6473
|
+
if (span.input !== void 0) {
|
|
6474
|
+
normalizedSpan.input = normalizeValue(span.input);
|
|
6475
|
+
}
|
|
6476
|
+
if (span.output !== void 0) {
|
|
6477
|
+
normalizedSpan.output = normalizeValue(span.output);
|
|
6478
|
+
}
|
|
6479
|
+
if (span.errorInfo) {
|
|
6480
|
+
normalizedSpan.errorInfo = span.errorInfo;
|
|
6481
|
+
}
|
|
6482
|
+
if (span.tags && span.tags.length > 0) {
|
|
6483
|
+
normalizedSpan.tags = span.tags;
|
|
6484
|
+
}
|
|
6485
|
+
const result = { span: normalizedSpan };
|
|
6486
|
+
if (node.children.length > 0) {
|
|
6487
|
+
result.children = node.children.map(normalizeNode);
|
|
6488
|
+
}
|
|
6489
|
+
return result;
|
|
6490
|
+
};
|
|
6491
|
+
return tree.map(normalizeNode);
|
|
4837
6492
|
}
|
|
4838
6493
|
/**
|
|
4839
|
-
*
|
|
6494
|
+
* Generate an ASCII tree structure graph for debugging.
|
|
6495
|
+
* Shows span type and name in a hierarchical format.
|
|
6496
|
+
*
|
|
6497
|
+
* @param nodes - Normalized tree nodes (defaults to current normalized tree)
|
|
6498
|
+
* @returns Array of strings representing the tree structure
|
|
6499
|
+
*
|
|
6500
|
+
* @example
|
|
6501
|
+
* ```
|
|
6502
|
+
* agent_run: "agent run: 'test-agent'"
|
|
6503
|
+
* ├── processor_run: "input processor: validator"
|
|
6504
|
+
* │ └── agent_run: "agent run: 'validator-agent'"
|
|
6505
|
+
* └── model_generation: "llm: 'mock-model-id'"
|
|
6506
|
+
* ```
|
|
4840
6507
|
*/
|
|
4841
|
-
|
|
4842
|
-
this.
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
6508
|
+
generateStructureGraph(nodes) {
|
|
6509
|
+
const tree = nodes ?? this.buildNormalizedTree();
|
|
6510
|
+
const lines = [];
|
|
6511
|
+
const buildLines = (node, prefix, isLast, isRoot) => {
|
|
6512
|
+
const connector = isRoot ? "" : isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ";
|
|
6513
|
+
const line = `${prefix}${connector}${node.span.type}: "${node.span.name}"`;
|
|
6514
|
+
lines.push(line);
|
|
6515
|
+
const children = node.children ?? [];
|
|
6516
|
+
const childPrefix = isRoot ? "" : prefix + (isLast ? " " : "\u2502 ");
|
|
6517
|
+
children.forEach((child, index) => {
|
|
6518
|
+
const childIsLast = index === children.length - 1;
|
|
6519
|
+
buildLines(child, childPrefix, childIsLast, false);
|
|
4848
6520
|
});
|
|
4849
|
-
}
|
|
4850
|
-
|
|
4851
|
-
|
|
6521
|
+
};
|
|
6522
|
+
tree.forEach((rootNode, index) => {
|
|
6523
|
+
if (index > 0) {
|
|
6524
|
+
lines.push("");
|
|
6525
|
+
}
|
|
6526
|
+
buildLines(rootNode, "", true, true);
|
|
6527
|
+
});
|
|
6528
|
+
return lines;
|
|
4852
6529
|
}
|
|
4853
6530
|
/**
|
|
4854
|
-
*
|
|
6531
|
+
* Serialize spans as a normalized tree structure for snapshot testing.
|
|
6532
|
+
* Includes a __structure__ field with an ASCII tree graph for readability.
|
|
6533
|
+
*
|
|
6534
|
+
* @param options - Serialization options
|
|
6535
|
+
* @returns JSON string with normalized spans in tree format
|
|
4855
6536
|
*/
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
}
|
|
4866
|
-
|
|
4867
|
-
}
|
|
6537
|
+
toNormalizedTreeJSON(options) {
|
|
6538
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6539
|
+
const includeStructure = options?.includeStructure ?? true;
|
|
6540
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6541
|
+
if (includeStructure) {
|
|
6542
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6543
|
+
const data = {
|
|
6544
|
+
__structure__: structureGraph,
|
|
6545
|
+
spans: normalizedTree
|
|
6546
|
+
};
|
|
6547
|
+
return JSON.stringify(data, null, indent);
|
|
4868
6548
|
}
|
|
6549
|
+
return JSON.stringify(normalizedTree, null, indent);
|
|
4869
6550
|
}
|
|
4870
6551
|
/**
|
|
4871
|
-
*
|
|
6552
|
+
* Write collected data to a JSON file
|
|
6553
|
+
*
|
|
6554
|
+
* @param filePath - Path to write the JSON file
|
|
6555
|
+
* @param options - Serialization options
|
|
4872
6556
|
*/
|
|
4873
|
-
|
|
4874
|
-
|
|
6557
|
+
async writeToFile(filePath, options) {
|
|
6558
|
+
const format = options?.format ?? "flat";
|
|
6559
|
+
let json;
|
|
6560
|
+
if (format === "normalized") {
|
|
6561
|
+
json = this.toNormalizedTreeJSON({ indent: options?.indent });
|
|
6562
|
+
} else if (format === "tree") {
|
|
6563
|
+
json = this.toTreeJSON({ indent: options?.indent, includeStats: options?.includeStats });
|
|
6564
|
+
} else {
|
|
6565
|
+
json = this.toJSON(options);
|
|
6566
|
+
}
|
|
6567
|
+
await writeFile(filePath, json, "utf-8");
|
|
6568
|
+
this.logger.info(`JsonExporter: wrote ${this.#events.length} events to ${filePath}`);
|
|
4875
6569
|
}
|
|
4876
6570
|
/**
|
|
4877
|
-
*
|
|
6571
|
+
* Assert that the current normalized tree matches a snapshot file.
|
|
6572
|
+
* Throws an error with a diff if they don't match.
|
|
6573
|
+
*
|
|
6574
|
+
* The snapshot format includes:
|
|
6575
|
+
* - `__structure__`: ASCII tree graph (compared first for quick validation)
|
|
6576
|
+
* - `spans`: The normalized span tree (detailed comparison)
|
|
6577
|
+
*
|
|
6578
|
+
* Supports special markers in the snapshot:
|
|
6579
|
+
* - `{"__or__": ["value1", "value2"]}` - matches if actual equals any listed value
|
|
6580
|
+
* - `{"__any__": "string"}` - matches any string value
|
|
6581
|
+
* - `{"__any__": "number"}` - matches any number value
|
|
6582
|
+
* - `{"__any__": "boolean"}` - matches any boolean value
|
|
6583
|
+
* - `{"__any__": "object"}` - matches any object value
|
|
6584
|
+
* - `{"__any__": "array"}` - matches any array value
|
|
6585
|
+
* - `{"__any__": true}` - matches any non-null/undefined value
|
|
6586
|
+
*
|
|
6587
|
+
* Environment variables:
|
|
6588
|
+
* Use `{ updateSnapshot: true }` option to update the snapshot instead of comparing
|
|
6589
|
+
*
|
|
6590
|
+
* @param snapshotName - Name of the snapshot file (resolved relative to __snapshots__ directory)
|
|
6591
|
+
* @param options - Options for snapshot comparison
|
|
6592
|
+
* @param options.updateSnapshot - If true, update the snapshot file instead of comparing
|
|
6593
|
+
* @throws Error if the snapshot doesn't match (and updateSnapshot is false)
|
|
4878
6594
|
*/
|
|
4879
|
-
async
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
6595
|
+
async assertMatchesSnapshot(snapshotName, options) {
|
|
6596
|
+
const snapshotPath = join(SNAPSHOTS_DIR, snapshotName);
|
|
6597
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6598
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6599
|
+
const currentData = {
|
|
6600
|
+
__structure__: structureGraph,
|
|
6601
|
+
spans: normalizedTree
|
|
6602
|
+
};
|
|
6603
|
+
const currentJson = JSON.stringify(currentData, null, this.#config.jsonIndent);
|
|
6604
|
+
const shouldUpdate = options?.updateSnapshot;
|
|
6605
|
+
if (shouldUpdate) {
|
|
6606
|
+
await writeFile(snapshotPath, currentJson, "utf-8");
|
|
6607
|
+
this.logger.info(`JsonExporter: updated snapshot ${snapshotPath}`);
|
|
4889
6608
|
return;
|
|
4890
6609
|
}
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
|
|
4902
|
-
|
|
4903
|
-
}
|
|
4904
|
-
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
6610
|
+
let snapshotData;
|
|
6611
|
+
try {
|
|
6612
|
+
const snapshotContent = await readFile(snapshotPath, "utf-8");
|
|
6613
|
+
snapshotData = JSON.parse(snapshotContent);
|
|
6614
|
+
} catch {
|
|
6615
|
+
throw new Error(`Snapshot file not found: ${snapshotPath}
|
|
6616
|
+
Run with { updateSnapshot: true } to create it.`);
|
|
6617
|
+
}
|
|
6618
|
+
let expectedSpans;
|
|
6619
|
+
let expectedStructure;
|
|
6620
|
+
if (Array.isArray(snapshotData)) {
|
|
6621
|
+
expectedSpans = snapshotData;
|
|
6622
|
+
} else if (snapshotData && typeof snapshotData === "object" && "spans" in snapshotData) {
|
|
6623
|
+
expectedSpans = snapshotData.spans;
|
|
6624
|
+
expectedStructure = snapshotData.__structure__;
|
|
6625
|
+
} else {
|
|
6626
|
+
throw new Error(
|
|
6627
|
+
`Invalid snapshot format in ${snapshotPath}.
|
|
6628
|
+
Expected an array or object with 'spans' property.`
|
|
6629
|
+
);
|
|
6630
|
+
}
|
|
6631
|
+
if (expectedStructure) {
|
|
6632
|
+
const structureMismatches = this.#compareStructure(structureGraph, expectedStructure);
|
|
6633
|
+
if (structureMismatches.length > 0) {
|
|
6634
|
+
throw new Error(
|
|
6635
|
+
`Structure mismatch in snapshot:
|
|
6636
|
+
|
|
6637
|
+
Expected:
|
|
6638
|
+
${expectedStructure.join("\n")}
|
|
6639
|
+
|
|
6640
|
+
Actual:
|
|
6641
|
+
${structureGraph.join("\n")}
|
|
6642
|
+
|
|
6643
|
+
Differences:
|
|
6644
|
+
${structureMismatches.join("\n")}
|
|
6645
|
+
|
|
6646
|
+
Snapshot: ${snapshotPath}
|
|
6647
|
+
Run with { updateSnapshot: true } to update.`
|
|
6648
|
+
);
|
|
6649
|
+
}
|
|
6650
|
+
}
|
|
6651
|
+
const mismatches = [];
|
|
6652
|
+
this.#deepCompareWithMarkers(normalizedTree, expectedSpans, "$.spans", mismatches);
|
|
6653
|
+
if (mismatches.length > 0) {
|
|
6654
|
+
const mismatchDetails = mismatches.map(
|
|
6655
|
+
(m, i) => `${i + 1}. ${m.path}
|
|
6656
|
+
Expected: ${JSON.stringify(m.expected)}
|
|
6657
|
+
Actual: ${JSON.stringify(m.actual)}`
|
|
6658
|
+
).join("\n\n");
|
|
6659
|
+
throw new Error(
|
|
6660
|
+
`Snapshot has ${mismatches.length} mismatch${mismatches.length > 1 ? "es" : ""}:
|
|
6661
|
+
|
|
6662
|
+
${mismatchDetails}
|
|
6663
|
+
|
|
6664
|
+
Snapshot: ${snapshotPath}
|
|
6665
|
+
Run with { updateSnapshot: true } to update.`
|
|
6666
|
+
);
|
|
6667
|
+
}
|
|
4914
6668
|
}
|
|
4915
6669
|
/**
|
|
4916
|
-
*
|
|
6670
|
+
* Compare two structure graphs and return differences
|
|
4917
6671
|
*/
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
if (buffer.insertOnly.length > 0) {
|
|
4936
|
-
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
6672
|
+
#compareStructure(actual, expected) {
|
|
6673
|
+
const diffs = [];
|
|
6674
|
+
const maxLen = Math.max(actual.length, expected.length);
|
|
6675
|
+
for (let i = 0; i < maxLen; i++) {
|
|
6676
|
+
const actualLine = actual[i];
|
|
6677
|
+
const expectedLine = expected[i];
|
|
6678
|
+
if (actualLine !== expectedLine) {
|
|
6679
|
+
if (actualLine === void 0) {
|
|
6680
|
+
diffs.push(`Line ${i + 1}: Missing in actual`);
|
|
6681
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6682
|
+
} else if (expectedLine === void 0) {
|
|
6683
|
+
diffs.push(`Line ${i + 1}: Extra in actual`);
|
|
6684
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
6685
|
+
} else {
|
|
6686
|
+
diffs.push(`Line ${i + 1}:`);
|
|
6687
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6688
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
4937
6689
|
}
|
|
4938
6690
|
}
|
|
4939
|
-
|
|
4940
|
-
|
|
6691
|
+
}
|
|
6692
|
+
return diffs;
|
|
6693
|
+
}
|
|
6694
|
+
/**
|
|
6695
|
+
* Deep compare two values, supporting special markers like __or__ and __any__.
|
|
6696
|
+
* Collects all mismatches into the provided array.
|
|
6697
|
+
*/
|
|
6698
|
+
#deepCompareWithMarkers(actual, expected, path, mismatches) {
|
|
6699
|
+
if (this.#isOrMarker(expected)) {
|
|
6700
|
+
const allowedValues = expected.__or__;
|
|
6701
|
+
const matches = allowedValues.some((allowed) => {
|
|
6702
|
+
const tempMismatches = [];
|
|
6703
|
+
this.#deepCompareWithMarkers(actual, allowed, path, tempMismatches);
|
|
6704
|
+
return tempMismatches.length === 0;
|
|
6705
|
+
});
|
|
6706
|
+
if (!matches) {
|
|
6707
|
+
mismatches.push({ path, expected: { __or__: allowedValues }, actual });
|
|
4941
6708
|
}
|
|
4942
|
-
|
|
4943
|
-
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
6709
|
+
return;
|
|
6710
|
+
}
|
|
6711
|
+
if (this.#isAnyMarker(expected)) {
|
|
6712
|
+
const typeConstraint = expected.__any__;
|
|
6713
|
+
if (actual === null || actual === void 0) {
|
|
6714
|
+
mismatches.push({ path, expected: { __any__: typeConstraint }, actual });
|
|
6715
|
+
return;
|
|
6716
|
+
}
|
|
6717
|
+
if (typeConstraint === true) {
|
|
6718
|
+
return;
|
|
6719
|
+
}
|
|
6720
|
+
const actualType = Array.isArray(actual) ? "array" : typeof actual;
|
|
6721
|
+
if (actualType !== typeConstraint) {
|
|
6722
|
+
mismatches.push({
|
|
6723
|
+
path,
|
|
6724
|
+
expected: { __any__: typeConstraint },
|
|
6725
|
+
actual: `(${actualType}) ${JSON.stringify(actual).slice(0, 50)}...`
|
|
4959
6726
|
});
|
|
4960
|
-
for (const spanKey of buffer.completedSpans) {
|
|
4961
|
-
this.allCreatedSpans.delete(spanKey);
|
|
4962
|
-
}
|
|
4963
6727
|
}
|
|
6728
|
+
return;
|
|
4964
6729
|
}
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
6730
|
+
if (Array.isArray(expected)) {
|
|
6731
|
+
if (!Array.isArray(actual)) {
|
|
6732
|
+
mismatches.push({ path, expected, actual });
|
|
6733
|
+
return;
|
|
6734
|
+
}
|
|
6735
|
+
if (actual.length !== expected.length) {
|
|
6736
|
+
mismatches.push({ path: `${path}.length`, expected: expected.length, actual: actual.length });
|
|
6737
|
+
return;
|
|
6738
|
+
}
|
|
6739
|
+
for (let i = 0; i < expected.length; i++) {
|
|
6740
|
+
this.#deepCompareWithMarkers(actual[i], expected[i], `${path}[${i}]`, mismatches);
|
|
6741
|
+
}
|
|
4969
6742
|
return;
|
|
4970
6743
|
}
|
|
4971
|
-
if (
|
|
4972
|
-
|
|
6744
|
+
if (expected !== null && typeof expected === "object") {
|
|
6745
|
+
if (actual === null || typeof actual !== "object" || Array.isArray(actual)) {
|
|
6746
|
+
mismatches.push({ path, expected, actual });
|
|
6747
|
+
return;
|
|
6748
|
+
}
|
|
6749
|
+
const expectedObj = expected;
|
|
6750
|
+
const actualObj = actual;
|
|
6751
|
+
for (const key of Object.keys(expectedObj)) {
|
|
6752
|
+
if (this.#isMetadataKey(key)) {
|
|
6753
|
+
continue;
|
|
6754
|
+
}
|
|
6755
|
+
if (!(key in actualObj)) {
|
|
6756
|
+
if (expectedObj[key] !== void 0) {
|
|
6757
|
+
mismatches.push({ path: `${path}.${key}`, expected: expectedObj[key], actual: void 0 });
|
|
6758
|
+
}
|
|
6759
|
+
continue;
|
|
6760
|
+
}
|
|
6761
|
+
this.#deepCompareWithMarkers(actualObj[key], expectedObj[key], `${path}.${key}`, mismatches);
|
|
6762
|
+
}
|
|
6763
|
+
for (const key of Object.keys(actualObj)) {
|
|
6764
|
+
if (this.#isMetadataKey(key)) {
|
|
6765
|
+
continue;
|
|
6766
|
+
}
|
|
6767
|
+
if (!(key in expectedObj)) {
|
|
6768
|
+
if (actualObj[key] !== void 0) {
|
|
6769
|
+
mismatches.push({ path: `${path}.${key}`, expected: void 0, actual: actualObj[key] });
|
|
6770
|
+
}
|
|
6771
|
+
}
|
|
6772
|
+
}
|
|
6773
|
+
return;
|
|
4973
6774
|
}
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
await this.handleRealtimeEvent(event, this.#observability);
|
|
4977
|
-
break;
|
|
4978
|
-
case "batch-with-updates":
|
|
4979
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
4980
|
-
break;
|
|
4981
|
-
case "insert-only":
|
|
4982
|
-
this.handleInsertOnlyEvent(event);
|
|
4983
|
-
break;
|
|
6775
|
+
if (actual !== expected) {
|
|
6776
|
+
mismatches.push({ path, expected, actual });
|
|
4984
6777
|
}
|
|
4985
6778
|
}
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
6779
|
+
/**
|
|
6780
|
+
* Check if a value is an __or__ marker object
|
|
6781
|
+
*/
|
|
6782
|
+
#isOrMarker(value) {
|
|
6783
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) && "__or__" in value && Array.isArray(value.__or__);
|
|
6784
|
+
}
|
|
6785
|
+
/**
|
|
6786
|
+
* Check if a value is an __any__ marker object
|
|
6787
|
+
*/
|
|
6788
|
+
#isAnyMarker(value) {
|
|
6789
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
6790
|
+
return false;
|
|
4990
6791
|
}
|
|
4991
|
-
if (
|
|
4992
|
-
|
|
4993
|
-
remainingEvents: this.buffer.totalSize
|
|
4994
|
-
});
|
|
4995
|
-
try {
|
|
4996
|
-
await this.flush();
|
|
4997
|
-
} catch (error) {
|
|
4998
|
-
this.logger.error("Failed to flush remaining events during shutdown", {
|
|
4999
|
-
error: error instanceof Error ? error.message : String(error)
|
|
5000
|
-
});
|
|
5001
|
-
}
|
|
6792
|
+
if (!("__any__" in value)) {
|
|
6793
|
+
return false;
|
|
5002
6794
|
}
|
|
5003
|
-
|
|
6795
|
+
const constraint = value.__any__;
|
|
6796
|
+
return constraint === true || ["string", "number", "boolean", "object", "array"].includes(constraint);
|
|
5004
6797
|
}
|
|
5005
|
-
|
|
5006
|
-
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
#events = [];
|
|
5011
|
-
constructor(config = {}) {
|
|
5012
|
-
super(config);
|
|
6798
|
+
/**
|
|
6799
|
+
* Check if a key should be skipped during comparison (metadata keys like __structure__)
|
|
6800
|
+
*/
|
|
6801
|
+
#isMetadataKey(key) {
|
|
6802
|
+
return key.startsWith("__") && key.endsWith("__");
|
|
5013
6803
|
}
|
|
5014
|
-
|
|
5015
|
-
|
|
6804
|
+
/**
|
|
6805
|
+
* Custom JSON replacer to handle Date objects and other special types
|
|
6806
|
+
*/
|
|
6807
|
+
#jsonReplacer = (_key, value) => {
|
|
6808
|
+
if (value instanceof Date) {
|
|
6809
|
+
return value.toISOString();
|
|
6810
|
+
}
|
|
6811
|
+
return value;
|
|
6812
|
+
};
|
|
6813
|
+
// ============================================================================
|
|
6814
|
+
// Debugging Helpers
|
|
6815
|
+
// ============================================================================
|
|
6816
|
+
/**
|
|
6817
|
+
* Get all stored logs
|
|
6818
|
+
*/
|
|
6819
|
+
getLogs() {
|
|
6820
|
+
return [...this.#logs];
|
|
6821
|
+
}
|
|
6822
|
+
/**
|
|
6823
|
+
* Dump logs to console for debugging (uses console.error for visibility in test output)
|
|
6824
|
+
*/
|
|
6825
|
+
dumpLogs() {
|
|
6826
|
+
console.error("\n=== JsonExporter Logs ===");
|
|
6827
|
+
this.#logs.forEach((log) => console.error(log));
|
|
6828
|
+
console.error("=== End Logs ===\n");
|
|
6829
|
+
}
|
|
6830
|
+
/**
|
|
6831
|
+
* Validate final state - useful for test assertions
|
|
6832
|
+
*
|
|
6833
|
+
* @returns Object with validation results
|
|
6834
|
+
*/
|
|
6835
|
+
validateFinalState() {
|
|
6836
|
+
const traceIds = this.getTraceIds();
|
|
6837
|
+
const incompleteSpans = this.getIncompleteSpans();
|
|
6838
|
+
const singleTraceId = traceIds.length === 1;
|
|
6839
|
+
const allSpansComplete = incompleteSpans.length === 0;
|
|
6840
|
+
return {
|
|
6841
|
+
valid: singleTraceId && allSpansComplete,
|
|
6842
|
+
singleTraceId,
|
|
6843
|
+
allSpansComplete,
|
|
6844
|
+
traceIds,
|
|
6845
|
+
incompleteSpans
|
|
6846
|
+
};
|
|
5016
6847
|
}
|
|
6848
|
+
// ============================================================================
|
|
6849
|
+
// Reset & Lifecycle
|
|
6850
|
+
// ============================================================================
|
|
6851
|
+
/**
|
|
6852
|
+
* Clear all collected events and state
|
|
6853
|
+
*/
|
|
5017
6854
|
clearEvents() {
|
|
5018
6855
|
this.#events = [];
|
|
6856
|
+
this.#spanStates.clear();
|
|
6857
|
+
this.#logs = [];
|
|
5019
6858
|
}
|
|
5020
|
-
|
|
5021
|
-
|
|
6859
|
+
/**
|
|
6860
|
+
* Alias for clearEvents (compatibility with TestExporter)
|
|
6861
|
+
*/
|
|
6862
|
+
reset() {
|
|
6863
|
+
this.clearEvents();
|
|
5022
6864
|
}
|
|
5023
6865
|
async shutdown() {
|
|
5024
|
-
this.logger.info("
|
|
6866
|
+
this.logger.info("JsonExporter shutdown");
|
|
6867
|
+
}
|
|
6868
|
+
// ============================================================================
|
|
6869
|
+
// Private Helpers
|
|
6870
|
+
// ============================================================================
|
|
6871
|
+
/**
|
|
6872
|
+
* Extract unique spans from a list of events
|
|
6873
|
+
*/
|
|
6874
|
+
#getUniqueSpansFromEvents(events) {
|
|
6875
|
+
const spanMap = /* @__PURE__ */ new Map();
|
|
6876
|
+
for (const event of events) {
|
|
6877
|
+
const span = event.exportedSpan;
|
|
6878
|
+
if (event.type === TracingEventType.SPAN_ENDED || !spanMap.has(span.id)) {
|
|
6879
|
+
spanMap.set(span.id, span);
|
|
6880
|
+
}
|
|
6881
|
+
}
|
|
6882
|
+
return Array.from(spanMap.values());
|
|
5025
6883
|
}
|
|
5026
6884
|
};
|
|
5027
6885
|
|
|
@@ -5080,14 +6938,11 @@ var ModelSpanTracker = class {
|
|
|
5080
6938
|
#modelSpan;
|
|
5081
6939
|
#currentStepSpan;
|
|
5082
6940
|
#currentChunkSpan;
|
|
6941
|
+
#currentChunkType;
|
|
5083
6942
|
#accumulator = {};
|
|
5084
6943
|
#stepIndex = 0;
|
|
5085
6944
|
#chunkSequence = 0;
|
|
5086
6945
|
#completionStartTime;
|
|
5087
|
-
/** Tracks tool output accumulators by toolCallId for consolidating sub-agent streams */
|
|
5088
|
-
#toolOutputAccumulators = /* @__PURE__ */ new Map();
|
|
5089
|
-
/** Tracks toolCallIds that had streaming output (to skip redundant tool-result spans) */
|
|
5090
|
-
#streamedToolCallIds = /* @__PURE__ */ new Set();
|
|
5091
6946
|
constructor(modelSpan) {
|
|
5092
6947
|
this.#modelSpan = modelSpan;
|
|
5093
6948
|
}
|
|
@@ -5174,6 +7029,7 @@ var ModelSpanTracker = class {
|
|
|
5174
7029
|
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
5175
7030
|
*/
|
|
5176
7031
|
#endStepSpan(payload) {
|
|
7032
|
+
this.#endChunkSpan();
|
|
5177
7033
|
if (!this.#currentStepSpan) return;
|
|
5178
7034
|
const output = payload.output;
|
|
5179
7035
|
const { usage: rawUsage, ...otherOutput } = output;
|
|
@@ -5181,8 +7037,10 @@ var ModelSpanTracker = class {
|
|
|
5181
7037
|
const metadata = payload.metadata;
|
|
5182
7038
|
const usage = extractUsageMetrics(rawUsage, metadata?.providerMetadata);
|
|
5183
7039
|
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
5184
|
-
if (cleanMetadata
|
|
5185
|
-
|
|
7040
|
+
if (cleanMetadata) {
|
|
7041
|
+
for (const key of ["request", "id", "timestamp", "modelId", "modelVersion", "modelProvider"]) {
|
|
7042
|
+
delete cleanMetadata[key];
|
|
7043
|
+
}
|
|
5186
7044
|
}
|
|
5187
7045
|
this.#currentStepSpan.end({
|
|
5188
7046
|
output: otherOutput,
|
|
@@ -5203,6 +7061,7 @@ var ModelSpanTracker = class {
|
|
|
5203
7061
|
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
5204
7062
|
*/
|
|
5205
7063
|
#startChunkSpan(chunkType, initialData) {
|
|
7064
|
+
this.#endChunkSpan();
|
|
5206
7065
|
if (!this.#currentStepSpan) {
|
|
5207
7066
|
this.startStep();
|
|
5208
7067
|
}
|
|
@@ -5214,6 +7073,7 @@ var ModelSpanTracker = class {
|
|
|
5214
7073
|
sequenceNumber: this.#chunkSequence
|
|
5215
7074
|
}
|
|
5216
7075
|
});
|
|
7076
|
+
this.#currentChunkType = chunkType;
|
|
5217
7077
|
this.#accumulator = initialData || {};
|
|
5218
7078
|
}
|
|
5219
7079
|
/**
|
|
@@ -5236,13 +7096,14 @@ var ModelSpanTracker = class {
|
|
|
5236
7096
|
output: output !== void 0 ? output : this.#accumulator
|
|
5237
7097
|
});
|
|
5238
7098
|
this.#currentChunkSpan = void 0;
|
|
7099
|
+
this.#currentChunkType = void 0;
|
|
5239
7100
|
this.#accumulator = {};
|
|
5240
7101
|
this.#chunkSequence++;
|
|
5241
7102
|
}
|
|
5242
7103
|
/**
|
|
5243
7104
|
* Create an event span (for single chunks like tool-call)
|
|
5244
7105
|
*/
|
|
5245
|
-
#createEventSpan(chunkType, output) {
|
|
7106
|
+
#createEventSpan(chunkType, output, options) {
|
|
5246
7107
|
if (!this.#currentStepSpan) {
|
|
5247
7108
|
this.startStep();
|
|
5248
7109
|
}
|
|
@@ -5251,8 +7112,10 @@ var ModelSpanTracker = class {
|
|
|
5251
7112
|
type: SpanType.MODEL_CHUNK,
|
|
5252
7113
|
attributes: {
|
|
5253
7114
|
chunkType,
|
|
5254
|
-
sequenceNumber: this.#chunkSequence
|
|
7115
|
+
sequenceNumber: this.#chunkSequence,
|
|
7116
|
+
...options?.attributes
|
|
5255
7117
|
},
|
|
7118
|
+
metadata: options?.metadata,
|
|
5256
7119
|
output
|
|
5257
7120
|
});
|
|
5258
7121
|
if (span) {
|
|
@@ -5280,6 +7143,9 @@ var ModelSpanTracker = class {
|
|
|
5280
7143
|
this.#startChunkSpan("text");
|
|
5281
7144
|
break;
|
|
5282
7145
|
case "text-delta":
|
|
7146
|
+
if (this.#currentChunkType !== "text") {
|
|
7147
|
+
this.#startChunkSpan("text");
|
|
7148
|
+
}
|
|
5283
7149
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5284
7150
|
break;
|
|
5285
7151
|
case "text-end": {
|
|
@@ -5297,6 +7163,9 @@ var ModelSpanTracker = class {
|
|
|
5297
7163
|
this.#startChunkSpan("reasoning");
|
|
5298
7164
|
break;
|
|
5299
7165
|
case "reasoning-delta":
|
|
7166
|
+
if (this.#currentChunkType !== "reasoning") {
|
|
7167
|
+
this.#startChunkSpan("reasoning");
|
|
7168
|
+
}
|
|
5300
7169
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5301
7170
|
break;
|
|
5302
7171
|
case "reasoning-end": {
|
|
@@ -5343,7 +7212,7 @@ var ModelSpanTracker = class {
|
|
|
5343
7212
|
#handleObjectChunk(chunk) {
|
|
5344
7213
|
switch (chunk.type) {
|
|
5345
7214
|
case "object":
|
|
5346
|
-
if (
|
|
7215
|
+
if (this.#currentChunkType !== "object") {
|
|
5347
7216
|
this.#startChunkSpan("object");
|
|
5348
7217
|
}
|
|
5349
7218
|
break;
|
|
@@ -5353,75 +7222,27 @@ var ModelSpanTracker = class {
|
|
|
5353
7222
|
}
|
|
5354
7223
|
}
|
|
5355
7224
|
/**
|
|
5356
|
-
* Handle tool-
|
|
5357
|
-
*
|
|
7225
|
+
* Handle tool-call-approval chunks.
|
|
7226
|
+
* Creates a span for approval requests so they can be seen in traces for debugging.
|
|
5358
7227
|
*/
|
|
5359
|
-
#
|
|
5360
|
-
if (chunk.type !== "tool-
|
|
7228
|
+
#handleToolApprovalChunk(chunk) {
|
|
7229
|
+
if (chunk.type !== "tool-call-approval") return;
|
|
5361
7230
|
const payload = chunk.payload;
|
|
5362
|
-
|
|
5363
|
-
|
|
5364
|
-
if (!acc) {
|
|
5365
|
-
if (!this.#currentStepSpan) {
|
|
5366
|
-
this.startStep();
|
|
5367
|
-
}
|
|
5368
|
-
acc = {
|
|
5369
|
-
toolName: toolName || "unknown",
|
|
5370
|
-
toolCallId,
|
|
5371
|
-
text: "",
|
|
5372
|
-
reasoning: "",
|
|
5373
|
-
sequenceNumber: this.#chunkSequence++,
|
|
5374
|
-
// Name the span 'tool-result' for consistency (tool-call → tool-result)
|
|
5375
|
-
span: this.#currentStepSpan?.createChildSpan({
|
|
5376
|
-
name: `chunk: 'tool-result'`,
|
|
5377
|
-
type: SpanType.MODEL_CHUNK,
|
|
5378
|
-
attributes: {
|
|
5379
|
-
chunkType: "tool-result",
|
|
5380
|
-
sequenceNumber: this.#chunkSequence - 1
|
|
5381
|
-
}
|
|
5382
|
-
})
|
|
5383
|
-
};
|
|
5384
|
-
this.#toolOutputAccumulators.set(toolCallId, acc);
|
|
5385
|
-
}
|
|
5386
|
-
if (output && typeof output === "object" && "type" in output) {
|
|
5387
|
-
const innerType = output.type;
|
|
5388
|
-
switch (innerType) {
|
|
5389
|
-
case "text-delta":
|
|
5390
|
-
if (output.payload?.text) {
|
|
5391
|
-
acc.text += output.payload.text;
|
|
5392
|
-
}
|
|
5393
|
-
break;
|
|
5394
|
-
case "reasoning-delta":
|
|
5395
|
-
if (output.payload?.text) {
|
|
5396
|
-
acc.reasoning += output.payload.text;
|
|
5397
|
-
}
|
|
5398
|
-
break;
|
|
5399
|
-
case "finish":
|
|
5400
|
-
case "workflow-finish":
|
|
5401
|
-
this.#endToolOutputSpan(toolCallId);
|
|
5402
|
-
break;
|
|
5403
|
-
}
|
|
5404
|
-
}
|
|
5405
|
-
}
|
|
5406
|
-
/**
|
|
5407
|
-
* End a tool output span and clean up the accumulator
|
|
5408
|
-
*/
|
|
5409
|
-
#endToolOutputSpan(toolCallId) {
|
|
5410
|
-
const acc = this.#toolOutputAccumulators.get(toolCallId);
|
|
5411
|
-
if (!acc) return;
|
|
5412
|
-
const output = {
|
|
5413
|
-
toolCallId: acc.toolCallId,
|
|
5414
|
-
toolName: acc.toolName
|
|
5415
|
-
};
|
|
5416
|
-
if (acc.text) {
|
|
5417
|
-
output.text = acc.text;
|
|
7231
|
+
if (!this.#currentStepSpan) {
|
|
7232
|
+
this.startStep();
|
|
5418
7233
|
}
|
|
5419
|
-
|
|
5420
|
-
|
|
7234
|
+
const span = this.#currentStepSpan?.createEventSpan({
|
|
7235
|
+
name: `chunk: 'tool-call-approval'`,
|
|
7236
|
+
type: SpanType.MODEL_CHUNK,
|
|
7237
|
+
attributes: {
|
|
7238
|
+
chunkType: "tool-call-approval",
|
|
7239
|
+
sequenceNumber: this.#chunkSequence
|
|
7240
|
+
},
|
|
7241
|
+
output: payload
|
|
7242
|
+
});
|
|
7243
|
+
if (span) {
|
|
7244
|
+
this.#chunkSequence++;
|
|
5421
7245
|
}
|
|
5422
|
-
acc.span?.end({ output });
|
|
5423
|
-
this.#toolOutputAccumulators.delete(toolCallId);
|
|
5424
|
-
this.#streamedToolCallIds.add(toolCallId);
|
|
5425
7246
|
}
|
|
5426
7247
|
/**
|
|
5427
7248
|
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
@@ -5472,36 +7293,60 @@ var ModelSpanTracker = class {
|
|
|
5472
7293
|
case "step-finish":
|
|
5473
7294
|
this.#endStepSpan(chunk.payload);
|
|
5474
7295
|
break;
|
|
7296
|
+
// Infrastructure chunks - skip creating spans for these
|
|
7297
|
+
// They are either redundant, metadata-only, or error/control flow
|
|
5475
7298
|
case "raw":
|
|
5476
|
-
//
|
|
7299
|
+
// Redundant raw data
|
|
5477
7300
|
case "start":
|
|
7301
|
+
// Stream start marker
|
|
5478
7302
|
case "finish":
|
|
7303
|
+
// Stream finish marker (step-finish already captures this)
|
|
7304
|
+
case "response-metadata":
|
|
7305
|
+
// Response metadata (not semantic content)
|
|
7306
|
+
case "source":
|
|
7307
|
+
// Source references (metadata)
|
|
7308
|
+
case "file":
|
|
7309
|
+
// Binary file data (too large/not semantic)
|
|
7310
|
+
case "error":
|
|
7311
|
+
// Error handling
|
|
7312
|
+
case "abort":
|
|
7313
|
+
// Abort signal
|
|
7314
|
+
case "tripwire":
|
|
7315
|
+
// Processor rejection
|
|
7316
|
+
case "watch":
|
|
7317
|
+
// Internal watch event
|
|
7318
|
+
case "tool-error":
|
|
7319
|
+
// Tool error handling
|
|
7320
|
+
case "tool-call-suspended":
|
|
7321
|
+
// Suspension (not content)
|
|
7322
|
+
case "reasoning-signature":
|
|
7323
|
+
// Signature metadata
|
|
7324
|
+
case "redacted-reasoning":
|
|
7325
|
+
// Redacted content metadata
|
|
7326
|
+
case "step-output":
|
|
7327
|
+
break;
|
|
7328
|
+
case "tool-call-approval":
|
|
7329
|
+
this.#handleToolApprovalChunk(chunk);
|
|
5479
7330
|
break;
|
|
5480
7331
|
case "tool-output":
|
|
5481
|
-
this.#handleToolOutputChunk(chunk);
|
|
5482
7332
|
break;
|
|
5483
7333
|
case "tool-result": {
|
|
5484
|
-
const
|
|
5485
|
-
|
|
5486
|
-
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
|
|
5492
|
-
|
|
5493
|
-
|
|
5494
|
-
|
|
5495
|
-
|
|
5496
|
-
if (
|
|
5497
|
-
|
|
5498
|
-
|
|
5499
|
-
|
|
5500
|
-
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
5501
|
-
delete outputPayload.data;
|
|
5502
|
-
}
|
|
5503
|
-
}
|
|
5504
|
-
this.#createEventSpan(chunk.type, outputPayload);
|
|
7334
|
+
const {
|
|
7335
|
+
// Metadata - tool call context (unique to tool-result chunks)
|
|
7336
|
+
toolCallId,
|
|
7337
|
+
toolName,
|
|
7338
|
+
isError,
|
|
7339
|
+
dynamic,
|
|
7340
|
+
providerExecuted,
|
|
7341
|
+
providerMetadata,
|
|
7342
|
+
// Output - the actual result
|
|
7343
|
+
result} = chunk.payload || {};
|
|
7344
|
+
const metadata = { toolCallId, toolName };
|
|
7345
|
+
if (isError !== void 0) metadata.isError = isError;
|
|
7346
|
+
if (dynamic !== void 0) metadata.dynamic = dynamic;
|
|
7347
|
+
if (providerExecuted !== void 0) metadata.providerExecuted = providerExecuted;
|
|
7348
|
+
if (providerMetadata !== void 0) metadata.providerMetadata = providerMetadata;
|
|
7349
|
+
this.#createEventSpan(chunk.type, result, { metadata });
|
|
5505
7350
|
break;
|
|
5506
7351
|
}
|
|
5507
7352
|
}
|
|
@@ -5517,7 +7362,11 @@ var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
|
5517
7362
|
"experimental_providerMetadata",
|
|
5518
7363
|
"providerMetadata",
|
|
5519
7364
|
"steps",
|
|
5520
|
-
"tracingContext"
|
|
7365
|
+
"tracingContext",
|
|
7366
|
+
"execute",
|
|
7367
|
+
// Tool execute functions
|
|
7368
|
+
"validate"
|
|
7369
|
+
// Schema validate functions
|
|
5521
7370
|
]);
|
|
5522
7371
|
var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
5523
7372
|
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
@@ -5526,12 +7375,68 @@ var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
|
5526
7375
|
maxArrayLength: 50,
|
|
5527
7376
|
maxObjectKeys: 50
|
|
5528
7377
|
});
|
|
7378
|
+
function mergeSerializationOptions(userOptions) {
|
|
7379
|
+
if (!userOptions) {
|
|
7380
|
+
return DEFAULT_DEEP_CLEAN_OPTIONS;
|
|
7381
|
+
}
|
|
7382
|
+
return {
|
|
7383
|
+
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
7384
|
+
maxDepth: userOptions.maxDepth ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxDepth,
|
|
7385
|
+
maxStringLength: userOptions.maxStringLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxStringLength,
|
|
7386
|
+
maxArrayLength: userOptions.maxArrayLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxArrayLength,
|
|
7387
|
+
maxObjectKeys: userOptions.maxObjectKeys ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxObjectKeys
|
|
7388
|
+
};
|
|
7389
|
+
}
|
|
5529
7390
|
function truncateString(s, maxChars) {
|
|
5530
7391
|
if (s.length <= maxChars) {
|
|
5531
7392
|
return s;
|
|
5532
7393
|
}
|
|
5533
7394
|
return s.slice(0, maxChars) + "\u2026[truncated]";
|
|
5534
7395
|
}
|
|
7396
|
+
function isJsonSchema(val) {
|
|
7397
|
+
if (typeof val !== "object" || val === null) return false;
|
|
7398
|
+
if (val.$schema && typeof val.$schema === "string" && val.$schema.includes("json-schema")) {
|
|
7399
|
+
return true;
|
|
7400
|
+
}
|
|
7401
|
+
if (val.type === "object" && val.properties && typeof val.properties === "object") {
|
|
7402
|
+
return true;
|
|
7403
|
+
}
|
|
7404
|
+
return false;
|
|
7405
|
+
}
|
|
7406
|
+
function compressJsonSchema(schema, depth = 0) {
|
|
7407
|
+
if (depth > 3) {
|
|
7408
|
+
return schema.type || "object";
|
|
7409
|
+
}
|
|
7410
|
+
if (schema.type !== "object" || !schema.properties) {
|
|
7411
|
+
return schema.type || schema;
|
|
7412
|
+
}
|
|
7413
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
7414
|
+
const compressed = {};
|
|
7415
|
+
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
7416
|
+
const prop = propSchema;
|
|
7417
|
+
let value = prop.type || "unknown";
|
|
7418
|
+
if (prop.type === "object" && prop.properties) {
|
|
7419
|
+
value = compressJsonSchema(prop, depth + 1);
|
|
7420
|
+
if (required.has(key)) {
|
|
7421
|
+
compressed[key + " (required)"] = value;
|
|
7422
|
+
continue;
|
|
7423
|
+
}
|
|
7424
|
+
} else if (prop.type === "array" && prop.items) {
|
|
7425
|
+
if (prop.items.type === "object" && prop.items.properties) {
|
|
7426
|
+
value = [compressJsonSchema(prop.items, depth + 1)];
|
|
7427
|
+
} else {
|
|
7428
|
+
value = `${prop.items.type || "any"}[]`;
|
|
7429
|
+
}
|
|
7430
|
+
} else if (prop.enum) {
|
|
7431
|
+
value = prop.enum.map((v) => JSON.stringify(v)).join(" | ");
|
|
7432
|
+
}
|
|
7433
|
+
if (required.has(key) && typeof value === "string") {
|
|
7434
|
+
value += " (required)";
|
|
7435
|
+
}
|
|
7436
|
+
compressed[key] = value;
|
|
7437
|
+
}
|
|
7438
|
+
return compressed;
|
|
7439
|
+
}
|
|
5535
7440
|
function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
5536
7441
|
const { keysToStrip, maxDepth, maxStringLength, maxArrayLength, maxObjectKeys } = options;
|
|
5537
7442
|
const seen = /* @__PURE__ */ new WeakSet();
|
|
@@ -5591,6 +7496,15 @@ function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
|
5591
7496
|
if (val instanceof ArrayBuffer) {
|
|
5592
7497
|
return `[ArrayBuffer byteLength=${val.byteLength}]`;
|
|
5593
7498
|
}
|
|
7499
|
+
if (typeof val.serializeForSpan === "function") {
|
|
7500
|
+
try {
|
|
7501
|
+
return helper(val.serializeForSpan(), depth);
|
|
7502
|
+
} catch {
|
|
7503
|
+
}
|
|
7504
|
+
}
|
|
7505
|
+
if (isJsonSchema(val)) {
|
|
7506
|
+
return helper(compressJsonSchema(val), depth);
|
|
7507
|
+
}
|
|
5594
7508
|
const cleaned = {};
|
|
5595
7509
|
const entries = Object.entries(val);
|
|
5596
7510
|
let keyCount = 0;
|
|
@@ -5682,25 +7596,30 @@ var BaseSpan = class {
|
|
|
5682
7596
|
entityName;
|
|
5683
7597
|
/** Parent span ID (for root spans that are children of external spans) */
|
|
5684
7598
|
parentSpanId;
|
|
7599
|
+
/** Deep clean options for serialization */
|
|
7600
|
+
deepCleanOptions;
|
|
5685
7601
|
constructor(options, observabilityInstance) {
|
|
7602
|
+
const serializationOptions = observabilityInstance.getConfig().serializationOptions;
|
|
7603
|
+
this.deepCleanOptions = mergeSerializationOptions(serializationOptions);
|
|
5686
7604
|
this.name = options.name;
|
|
5687
7605
|
this.type = options.type;
|
|
5688
|
-
this.attributes = deepClean(options.attributes) || {};
|
|
5689
|
-
this.metadata = deepClean(options.metadata);
|
|
7606
|
+
this.attributes = deepClean(options.attributes, this.deepCleanOptions) || {};
|
|
7607
|
+
this.metadata = deepClean(options.metadata, this.deepCleanOptions);
|
|
5690
7608
|
this.parent = options.parent;
|
|
5691
|
-
this.startTime = /* @__PURE__ */ new Date();
|
|
7609
|
+
this.startTime = options.startTime ?? /* @__PURE__ */ new Date();
|
|
5692
7610
|
this.observabilityInstance = observabilityInstance;
|
|
5693
7611
|
this.isEvent = options.isEvent ?? false;
|
|
5694
7612
|
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
5695
7613
|
this.traceState = options.traceState;
|
|
5696
7614
|
this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
|
|
5697
|
-
|
|
5698
|
-
this.
|
|
5699
|
-
this.
|
|
7615
|
+
const entityParent = this.getParentSpan(false);
|
|
7616
|
+
this.entityType = options.entityType ?? entityParent?.entityType;
|
|
7617
|
+
this.entityId = options.entityId ?? entityParent?.entityId;
|
|
7618
|
+
this.entityName = options.entityName ?? entityParent?.entityName;
|
|
5700
7619
|
if (this.isEvent) {
|
|
5701
|
-
this.output = deepClean(options.output);
|
|
7620
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5702
7621
|
} else {
|
|
5703
|
-
this.input = deepClean(options.input);
|
|
7622
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5704
7623
|
}
|
|
5705
7624
|
}
|
|
5706
7625
|
createChildSpan(options) {
|
|
@@ -5723,14 +7642,25 @@ var BaseSpan = class {
|
|
|
5723
7642
|
get isRootSpan() {
|
|
5724
7643
|
return !this.parent;
|
|
5725
7644
|
}
|
|
7645
|
+
/** Get the closest parent span, optionally skipping internal spans */
|
|
7646
|
+
getParentSpan(includeInternalSpans) {
|
|
7647
|
+
if (!this.parent) {
|
|
7648
|
+
return void 0;
|
|
7649
|
+
}
|
|
7650
|
+
if (includeInternalSpans) return this.parent;
|
|
7651
|
+
if (this.parent.isInternal) return this.parent.getParentSpan(includeInternalSpans);
|
|
7652
|
+
return this.parent;
|
|
7653
|
+
}
|
|
5726
7654
|
/** Get the closest parent spanId that isn't an internal span */
|
|
5727
7655
|
getParentSpanId(includeInternalSpans) {
|
|
5728
7656
|
if (!this.parent) {
|
|
5729
7657
|
return this.parentSpanId;
|
|
5730
7658
|
}
|
|
5731
|
-
|
|
5732
|
-
if (
|
|
5733
|
-
|
|
7659
|
+
const parentSpan = this.getParentSpan(includeInternalSpans);
|
|
7660
|
+
if (parentSpan) {
|
|
7661
|
+
return parentSpan.id;
|
|
7662
|
+
}
|
|
7663
|
+
return this.parent.getParentSpanId(includeInternalSpans);
|
|
5734
7664
|
}
|
|
5735
7665
|
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
5736
7666
|
findParent(spanType) {
|
|
@@ -5745,6 +7675,8 @@ var BaseSpan = class {
|
|
|
5745
7675
|
}
|
|
5746
7676
|
/** Returns a lightweight span ready for export */
|
|
5747
7677
|
exportSpan(includeInternalSpans) {
|
|
7678
|
+
const hideInput = this.traceState?.hideInput ?? false;
|
|
7679
|
+
const hideOutput = this.traceState?.hideOutput ?? false;
|
|
5748
7680
|
return {
|
|
5749
7681
|
id: this.id,
|
|
5750
7682
|
traceId: this.traceId,
|
|
@@ -5757,8 +7689,8 @@ var BaseSpan = class {
|
|
|
5757
7689
|
metadata: this.metadata,
|
|
5758
7690
|
startTime: this.startTime,
|
|
5759
7691
|
endTime: this.endTime,
|
|
5760
|
-
input: this.input,
|
|
5761
|
-
output: this.output,
|
|
7692
|
+
input: hideInput ? void 0 : this.input,
|
|
7693
|
+
output: hideOutput ? void 0 : this.output,
|
|
5762
7694
|
errorInfo: this.errorInfo,
|
|
5763
7695
|
isEvent: this.isEvent,
|
|
5764
7696
|
isRootSpan: this.isRootSpan,
|
|
@@ -5798,6 +7730,14 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5798
7730
|
traceId;
|
|
5799
7731
|
constructor(options, observabilityInstance) {
|
|
5800
7732
|
super(options, observabilityInstance);
|
|
7733
|
+
if (options.spanId && options.traceId) {
|
|
7734
|
+
this.id = options.spanId;
|
|
7735
|
+
this.traceId = options.traceId;
|
|
7736
|
+
if (options.parentSpanId) {
|
|
7737
|
+
this.parentSpanId = options.parentSpanId;
|
|
7738
|
+
}
|
|
7739
|
+
return;
|
|
7740
|
+
}
|
|
5801
7741
|
const bridge = observabilityInstance.getBridge();
|
|
5802
7742
|
if (bridge && !this.isInternal) {
|
|
5803
7743
|
const bridgeIds = bridge.createSpan(options);
|
|
@@ -5832,13 +7772,13 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5832
7772
|
}
|
|
5833
7773
|
this.endTime = /* @__PURE__ */ new Date();
|
|
5834
7774
|
if (options?.output !== void 0) {
|
|
5835
|
-
this.output = deepClean(options.output);
|
|
7775
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5836
7776
|
}
|
|
5837
7777
|
if (options?.attributes) {
|
|
5838
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7778
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5839
7779
|
}
|
|
5840
7780
|
if (options?.metadata) {
|
|
5841
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7781
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5842
7782
|
}
|
|
5843
7783
|
}
|
|
5844
7784
|
error(options) {
|
|
@@ -5856,10 +7796,10 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5856
7796
|
message: error.message
|
|
5857
7797
|
};
|
|
5858
7798
|
if (attributes) {
|
|
5859
|
-
this.attributes = { ...this.attributes, ...deepClean(attributes) };
|
|
7799
|
+
this.attributes = { ...this.attributes, ...deepClean(attributes, this.deepCleanOptions) };
|
|
5860
7800
|
}
|
|
5861
7801
|
if (metadata) {
|
|
5862
|
-
this.metadata = { ...this.metadata, ...deepClean(metadata) };
|
|
7802
|
+
this.metadata = { ...this.metadata, ...deepClean(metadata, this.deepCleanOptions) };
|
|
5863
7803
|
}
|
|
5864
7804
|
if (endSpan) {
|
|
5865
7805
|
this.end();
|
|
@@ -5872,16 +7812,16 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5872
7812
|
return;
|
|
5873
7813
|
}
|
|
5874
7814
|
if (options.input !== void 0) {
|
|
5875
|
-
this.input = deepClean(options.input);
|
|
7815
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5876
7816
|
}
|
|
5877
7817
|
if (options.output !== void 0) {
|
|
5878
|
-
this.output = deepClean(options.output);
|
|
7818
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5879
7819
|
}
|
|
5880
7820
|
if (options.attributes) {
|
|
5881
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7821
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5882
7822
|
}
|
|
5883
7823
|
if (options.metadata) {
|
|
5884
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7824
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5885
7825
|
}
|
|
5886
7826
|
}
|
|
5887
7827
|
get isValid() {
|
|
@@ -5972,7 +7912,8 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
5972
7912
|
spanOutputProcessors: config.spanOutputProcessors ?? [],
|
|
5973
7913
|
bridge: config.bridge ?? void 0,
|
|
5974
7914
|
includeInternalSpans: config.includeInternalSpans ?? false,
|
|
5975
|
-
requestContextKeys: config.requestContextKeys ?? []
|
|
7915
|
+
requestContextKeys: config.requestContextKeys ?? [],
|
|
7916
|
+
serializationOptions: config.serializationOptions
|
|
5976
7917
|
};
|
|
5977
7918
|
if (this.config.bridge?.init) {
|
|
5978
7919
|
this.config.bridge.init({ config: this.config });
|
|
@@ -6010,11 +7951,26 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6010
7951
|
// ============================================================================
|
|
6011
7952
|
/**
|
|
6012
7953
|
* Start a new span of a specific SpanType
|
|
7954
|
+
*
|
|
7955
|
+
* Sampling Decision:
|
|
7956
|
+
* - For root spans (no parent): Perform sampling check using the configured strategy
|
|
7957
|
+
* - For child spans: Inherit the sampling decision from the parent
|
|
7958
|
+
* - If parent is a NoOpSpan (not sampled), child is also a NoOpSpan
|
|
7959
|
+
* - If parent is a valid span (sampled), child is also sampled
|
|
7960
|
+
*
|
|
7961
|
+
* This ensures trace-level sampling: either all spans in a trace are sampled or none are.
|
|
7962
|
+
* See: https://github.com/mastra-ai/mastra/issues/11504
|
|
6013
7963
|
*/
|
|
6014
7964
|
startSpan(options) {
|
|
6015
7965
|
const { customSamplerOptions, requestContext, metadata, tracingOptions, ...rest } = options;
|
|
6016
|
-
if (
|
|
6017
|
-
|
|
7966
|
+
if (options.parent) {
|
|
7967
|
+
if (!options.parent.isValid) {
|
|
7968
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7969
|
+
}
|
|
7970
|
+
} else {
|
|
7971
|
+
if (!this.shouldSample(customSamplerOptions)) {
|
|
7972
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7973
|
+
}
|
|
6018
7974
|
}
|
|
6019
7975
|
let traceState;
|
|
6020
7976
|
if (options.parent) {
|
|
@@ -6026,8 +7982,12 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6026
7982
|
const mergedMetadata = metadata || tracingMetadata ? { ...metadata, ...tracingMetadata } : void 0;
|
|
6027
7983
|
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, mergedMetadata, traceState);
|
|
6028
7984
|
const tags = !options.parent ? tracingOptions?.tags : void 0;
|
|
7985
|
+
const traceId = !options.parent ? options.traceId ?? tracingOptions?.traceId : options.traceId;
|
|
7986
|
+
const parentSpanId = !options.parent ? options.parentSpanId ?? tracingOptions?.parentSpanId : options.parentSpanId;
|
|
6029
7987
|
const span = this.createSpan({
|
|
6030
7988
|
...rest,
|
|
7989
|
+
traceId,
|
|
7990
|
+
parentSpanId,
|
|
6031
7991
|
metadata: enrichedMetadata,
|
|
6032
7992
|
traceState,
|
|
6033
7993
|
tags
|
|
@@ -6040,6 +8000,37 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6040
8000
|
}
|
|
6041
8001
|
return span;
|
|
6042
8002
|
}
|
|
8003
|
+
/**
|
|
8004
|
+
* Rebuild a span from exported data for lifecycle operations.
|
|
8005
|
+
* Used by durable execution engines (e.g., Inngest) to end/update spans
|
|
8006
|
+
* that were created in a previous durable operation.
|
|
8007
|
+
*
|
|
8008
|
+
* The rebuilt span:
|
|
8009
|
+
* - Does NOT emit SPAN_STARTED (assumes original span already did)
|
|
8010
|
+
* - Can have end(), update(), error() called on it
|
|
8011
|
+
* - Will emit SPAN_ENDED or SPAN_UPDATED when those methods are called
|
|
8012
|
+
*
|
|
8013
|
+
* @param cached - The exported span data to rebuild from
|
|
8014
|
+
* @returns A span that can have lifecycle methods called on it
|
|
8015
|
+
*/
|
|
8016
|
+
rebuildSpan(cached) {
|
|
8017
|
+
const span = this.createSpan({
|
|
8018
|
+
name: cached.name,
|
|
8019
|
+
type: cached.type,
|
|
8020
|
+
traceId: cached.traceId,
|
|
8021
|
+
spanId: cached.id,
|
|
8022
|
+
parentSpanId: cached.parentSpanId,
|
|
8023
|
+
startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
|
|
8024
|
+
input: cached.input,
|
|
8025
|
+
attributes: cached.attributes,
|
|
8026
|
+
metadata: cached.metadata,
|
|
8027
|
+
entityType: cached.entityType,
|
|
8028
|
+
entityId: cached.entityId,
|
|
8029
|
+
entityName: cached.entityName
|
|
8030
|
+
});
|
|
8031
|
+
this.wireSpanLifecycle(span);
|
|
8032
|
+
return span;
|
|
8033
|
+
}
|
|
6043
8034
|
// ============================================================================
|
|
6044
8035
|
// Configuration Management
|
|
6045
8036
|
// ============================================================================
|
|
@@ -6142,11 +8133,15 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6142
8133
|
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
6143
8134
|
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
6144
8135
|
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
6145
|
-
|
|
8136
|
+
const hideInput = tracingOptions?.hideInput;
|
|
8137
|
+
const hideOutput = tracingOptions?.hideOutput;
|
|
8138
|
+
if (allKeys.length === 0 && !hideInput && !hideOutput) {
|
|
6146
8139
|
return void 0;
|
|
6147
8140
|
}
|
|
6148
8141
|
return {
|
|
6149
|
-
requestContextKeys: allKeys
|
|
8142
|
+
requestContextKeys: allKeys,
|
|
8143
|
+
...hideInput !== void 0 && { hideInput },
|
|
8144
|
+
...hideOutput !== void 0 && { hideOutput }
|
|
6150
8145
|
};
|
|
6151
8146
|
}
|
|
6152
8147
|
/**
|
|
@@ -6277,6 +8272,29 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6277
8272
|
this.logger.debug(`[Observability] Initialization started [name=${this.name}]`);
|
|
6278
8273
|
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
6279
8274
|
}
|
|
8275
|
+
/**
|
|
8276
|
+
* Force flush any buffered/queued spans from all exporters and the bridge
|
|
8277
|
+
* without shutting down the observability instance.
|
|
8278
|
+
*
|
|
8279
|
+
* This is useful in serverless environments (like Vercel's fluid compute) where
|
|
8280
|
+
* you need to ensure all spans are exported before the runtime instance is
|
|
8281
|
+
* terminated, while keeping the observability system active for future requests.
|
|
8282
|
+
*/
|
|
8283
|
+
async flush() {
|
|
8284
|
+
this.logger.debug(`[Observability] Flush started [name=${this.name}]`);
|
|
8285
|
+
const flushPromises = [...this.exporters.map((e) => e.flush())];
|
|
8286
|
+
if (this.config.bridge) {
|
|
8287
|
+
flushPromises.push(this.config.bridge.flush());
|
|
8288
|
+
}
|
|
8289
|
+
const results = await Promise.allSettled(flushPromises);
|
|
8290
|
+
results.forEach((result, index) => {
|
|
8291
|
+
if (result.status === "rejected") {
|
|
8292
|
+
const targetName = index < this.exporters.length ? this.exporters[index]?.name : "bridge";
|
|
8293
|
+
this.logger.error(`[Observability] Flush error [target=${targetName}]`, result.reason);
|
|
8294
|
+
}
|
|
8295
|
+
});
|
|
8296
|
+
this.logger.debug(`[Observability] Flush completed [name=${this.name}]`);
|
|
8297
|
+
}
|
|
6280
8298
|
/**
|
|
6281
8299
|
* Shutdown Observability and clean up resources
|
|
6282
8300
|
*/
|
|
@@ -6586,6 +8604,9 @@ var Observability = class extends MastraBase {
|
|
|
6586
8604
|
}
|
|
6587
8605
|
}
|
|
6588
8606
|
if (config.default?.enabled) {
|
|
8607
|
+
console.warn(
|
|
8608
|
+
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter, CloudExporter, and SensitiveDataFilter instead. See https://mastra.ai/docs/observability/tracing/overview for the recommended configuration.'
|
|
8609
|
+
);
|
|
6589
8610
|
const defaultInstance = new DefaultObservabilityInstance({
|
|
6590
8611
|
serviceName: "mastra",
|
|
6591
8612
|
name: "default",
|
|
@@ -6673,6 +8694,6 @@ function buildTracingOptions(...updaters) {
|
|
|
6673
8694
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
6674
8695
|
}
|
|
6675
8696
|
|
|
6676
|
-
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, TestExporter, buildTracingOptions, deepClean, getExternalParentId, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema, truncateString };
|
|
8697
|
+
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, mergeSerializationOptions, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
6677
8698
|
//# sourceMappingURL=index.js.map
|
|
6678
8699
|
//# sourceMappingURL=index.js.map
|