@mastra/observability 0.0.0-new-button-export-20251219130424 → 0.0.0-om-20260129001517
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +746 -3
- package/README.md +34 -79
- package/dist/config.d.ts +110 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/base.d.ts +55 -5
- package/dist/exporters/base.d.ts.map +1 -1
- package/dist/exporters/cloud.d.ts +8 -2
- package/dist/exporters/cloud.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +12 -5
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/index.d.ts +4 -2
- package/dist/exporters/index.d.ts.map +1 -1
- package/dist/exporters/json.d.ts +386 -0
- package/dist/exporters/json.d.ts.map +1 -0
- package/dist/exporters/span-formatters.d.ts +35 -0
- package/dist/exporters/span-formatters.d.ts.map +1 -0
- package/dist/exporters/tracking.d.ts +470 -0
- package/dist/exporters/tracking.d.ts.map +1 -0
- package/dist/index.cjs +2880 -778
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2873 -778
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts +33 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/model-tracing.d.ts +18 -0
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/span_processors/sensitive-data-filter.d.ts.map +1 -1
- package/dist/spans/base.d.ts +12 -1
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts.map +1 -1
- package/dist/spans/serialization.d.ts +30 -0
- package/dist/spans/serialization.d.ts.map +1 -1
- package/package.json +11 -11
package/dist/index.js
CHANGED
|
@@ -3,6 +3,9 @@ import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
|
3
3
|
import { ConsoleLogger, LogLevel, RegisteredLogger } from '@mastra/core/logger';
|
|
4
4
|
import { TracingEventType, SpanType, InternalSpans } from '@mastra/core/observability';
|
|
5
5
|
import { fetchWithRetry, getNestedValue, setNestedValue } from '@mastra/core/utils';
|
|
6
|
+
import { writeFile, readFile } from 'fs/promises';
|
|
7
|
+
import { dirname, join } from 'path';
|
|
8
|
+
import { fileURLToPath } from 'url';
|
|
6
9
|
import { TransformStream } from 'stream/web';
|
|
7
10
|
|
|
8
11
|
var __defProp = Object.defineProperty;
|
|
@@ -3837,7 +3840,7 @@ ZodNaN.create = (params) => {
|
|
|
3837
3840
|
...processCreateParams(params)
|
|
3838
3841
|
});
|
|
3839
3842
|
};
|
|
3840
|
-
var BRAND = Symbol("zod_brand");
|
|
3843
|
+
var BRAND = /* @__PURE__ */ Symbol("zod_brand");
|
|
3841
3844
|
var ZodBranded = class extends ZodType {
|
|
3842
3845
|
_parse(input) {
|
|
3843
3846
|
const { ctx } = this._processInputParams(input);
|
|
@@ -4074,6 +4077,12 @@ var samplingStrategySchema = external_exports.discriminatedUnion("type", [
|
|
|
4074
4077
|
sampler: external_exports.function().args(external_exports.any().optional()).returns(external_exports.boolean())
|
|
4075
4078
|
})
|
|
4076
4079
|
]);
|
|
4080
|
+
var serializationOptionsSchema = external_exports.object({
|
|
4081
|
+
maxStringLength: external_exports.number().int().positive().optional(),
|
|
4082
|
+
maxDepth: external_exports.number().int().positive().optional(),
|
|
4083
|
+
maxArrayLength: external_exports.number().int().positive().optional(),
|
|
4084
|
+
maxObjectKeys: external_exports.number().int().positive().optional()
|
|
4085
|
+
}).optional();
|
|
4077
4086
|
var observabilityInstanceConfigSchema = external_exports.object({
|
|
4078
4087
|
name: external_exports.string().min(1, "Name is required"),
|
|
4079
4088
|
serviceName: external_exports.string().min(1, "Service name is required"),
|
|
@@ -4082,7 +4091,8 @@ var observabilityInstanceConfigSchema = external_exports.object({
|
|
|
4082
4091
|
bridge: external_exports.any().optional(),
|
|
4083
4092
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4084
4093
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4085
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4094
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4095
|
+
serializationOptions: serializationOptionsSchema
|
|
4086
4096
|
}).refine(
|
|
4087
4097
|
(data) => {
|
|
4088
4098
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4100,7 +4110,8 @@ var observabilityConfigValueSchema = external_exports.object({
|
|
|
4100
4110
|
bridge: external_exports.any().optional(),
|
|
4101
4111
|
spanOutputProcessors: external_exports.array(external_exports.any()).optional(),
|
|
4102
4112
|
includeInternalSpans: external_exports.boolean().optional(),
|
|
4103
|
-
requestContextKeys: external_exports.array(external_exports.string()).optional()
|
|
4113
|
+
requestContextKeys: external_exports.array(external_exports.string()).optional(),
|
|
4114
|
+
serializationOptions: serializationOptionsSchema
|
|
4104
4115
|
}).refine(
|
|
4105
4116
|
(data) => {
|
|
4106
4117
|
const hasExporters = data.exporters && data.exporters.length > 0;
|
|
@@ -4153,12 +4164,19 @@ var observabilityRegistryConfigSchema = external_exports.object({
|
|
|
4153
4164
|
var BaseExporter = class {
|
|
4154
4165
|
/** Mastra logger instance */
|
|
4155
4166
|
logger;
|
|
4167
|
+
/** Base configuration (accessible by subclasses) */
|
|
4168
|
+
baseConfig;
|
|
4156
4169
|
/** Whether this exporter is disabled */
|
|
4157
|
-
|
|
4170
|
+
#disabled = false;
|
|
4171
|
+
/** Public getter for disabled state */
|
|
4172
|
+
get isDisabled() {
|
|
4173
|
+
return this.#disabled;
|
|
4174
|
+
}
|
|
4158
4175
|
/**
|
|
4159
4176
|
* Initialize the base exporter with logger
|
|
4160
4177
|
*/
|
|
4161
4178
|
constructor(config = {}) {
|
|
4179
|
+
this.baseConfig = config;
|
|
4162
4180
|
const logLevel = this.resolveLogLevel(config.logLevel);
|
|
4163
4181
|
this.logger = config.logger ?? new ConsoleLogger({ level: logLevel, name: this.constructor.name });
|
|
4164
4182
|
}
|
|
@@ -4193,20 +4211,62 @@ var BaseExporter = class {
|
|
|
4193
4211
|
* @param reason - Reason why the exporter is disabled
|
|
4194
4212
|
*/
|
|
4195
4213
|
setDisabled(reason) {
|
|
4196
|
-
this
|
|
4214
|
+
this.#disabled = true;
|
|
4197
4215
|
this.logger.warn(`${this.name} disabled: ${reason}`);
|
|
4198
4216
|
}
|
|
4217
|
+
/**
|
|
4218
|
+
* Apply the customSpanFormatter if configured.
|
|
4219
|
+
* This is called automatically by exportTracingEvent before _exportTracingEvent.
|
|
4220
|
+
*
|
|
4221
|
+
* Supports both synchronous and asynchronous formatters. If the formatter
|
|
4222
|
+
* returns a Promise, it will be awaited.
|
|
4223
|
+
*
|
|
4224
|
+
* @param event - The incoming tracing event
|
|
4225
|
+
* @returns The (possibly modified) event to process
|
|
4226
|
+
*/
|
|
4227
|
+
async applySpanFormatter(event) {
|
|
4228
|
+
if (this.baseConfig.customSpanFormatter) {
|
|
4229
|
+
try {
|
|
4230
|
+
const formattedSpan = await this.baseConfig.customSpanFormatter(event.exportedSpan);
|
|
4231
|
+
return {
|
|
4232
|
+
...event,
|
|
4233
|
+
exportedSpan: formattedSpan
|
|
4234
|
+
};
|
|
4235
|
+
} catch (error) {
|
|
4236
|
+
this.logger.error(`${this.name}: Error in customSpanFormatter`, {
|
|
4237
|
+
error,
|
|
4238
|
+
spanId: event.exportedSpan.id,
|
|
4239
|
+
traceId: event.exportedSpan.traceId
|
|
4240
|
+
});
|
|
4241
|
+
}
|
|
4242
|
+
}
|
|
4243
|
+
return event;
|
|
4244
|
+
}
|
|
4199
4245
|
/**
|
|
4200
4246
|
* Export a tracing event
|
|
4201
4247
|
*
|
|
4202
|
-
* This method checks if the exporter is disabled
|
|
4203
|
-
*
|
|
4248
|
+
* This method checks if the exporter is disabled, applies the customSpanFormatter,
|
|
4249
|
+
* then calls _exportTracingEvent.
|
|
4250
|
+
* Subclasses should implement _exportTracingEvent instead of overriding this method.
|
|
4204
4251
|
*/
|
|
4205
4252
|
async exportTracingEvent(event) {
|
|
4206
4253
|
if (this.isDisabled) {
|
|
4207
4254
|
return;
|
|
4208
4255
|
}
|
|
4209
|
-
await this.
|
|
4256
|
+
const processedEvent = await this.applySpanFormatter(event);
|
|
4257
|
+
await this._exportTracingEvent(processedEvent);
|
|
4258
|
+
}
|
|
4259
|
+
/**
|
|
4260
|
+
* Force flush any buffered/queued spans without shutting down the exporter.
|
|
4261
|
+
*
|
|
4262
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
4263
|
+
* are exported before the runtime instance is terminated, while keeping
|
|
4264
|
+
* the exporter active for future requests.
|
|
4265
|
+
*
|
|
4266
|
+
* Default implementation is a no-op. Override to add flush logic.
|
|
4267
|
+
*/
|
|
4268
|
+
async flush() {
|
|
4269
|
+
this.logger.debug(`${this.name} flush called (no-op in base class)`);
|
|
4210
4270
|
}
|
|
4211
4271
|
/**
|
|
4212
4272
|
* Shutdown the exporter and clean up resources
|
|
@@ -4217,232 +4277,1212 @@ var BaseExporter = class {
|
|
|
4217
4277
|
this.logger.info(`${this.name} shutdown complete`);
|
|
4218
4278
|
}
|
|
4219
4279
|
};
|
|
4220
|
-
var
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4280
|
+
var TraceData = class {
|
|
4281
|
+
/** The vendor-specific root/trace object */
|
|
4282
|
+
#rootSpan;
|
|
4283
|
+
/** The span ID of the root span */
|
|
4284
|
+
#rootSpanId;
|
|
4285
|
+
/** Whether a span with isRootSpan=true has been successfully processed */
|
|
4286
|
+
#rootSpanProcessed;
|
|
4287
|
+
/** Maps eventId to vendor-specific event objects */
|
|
4288
|
+
#events;
|
|
4289
|
+
/** Maps spanId to vendor-specific span objects */
|
|
4290
|
+
#spans;
|
|
4291
|
+
/** Maps spanId to parentSpanId, representing the span hierarchy */
|
|
4292
|
+
#tree;
|
|
4293
|
+
/** Set of span IDs that have started but not yet ended */
|
|
4294
|
+
#activeSpanIds;
|
|
4295
|
+
/** Maps spanId to vendor-specific metadata */
|
|
4296
|
+
#metadata;
|
|
4297
|
+
/** Arbitrary key-value storage for per-trace data */
|
|
4298
|
+
#extraData;
|
|
4299
|
+
/** Events waiting for the root span to be processed */
|
|
4300
|
+
#waitingForRoot;
|
|
4301
|
+
/** Events waiting for specific parent spans, keyed by parentSpanId */
|
|
4302
|
+
#waitingForParent;
|
|
4303
|
+
/** When this trace data was created, used for cap enforcement */
|
|
4304
|
+
createdAt;
|
|
4305
|
+
constructor() {
|
|
4306
|
+
this.#events = /* @__PURE__ */ new Map();
|
|
4307
|
+
this.#spans = /* @__PURE__ */ new Map();
|
|
4308
|
+
this.#activeSpanIds = /* @__PURE__ */ new Set();
|
|
4309
|
+
this.#tree = /* @__PURE__ */ new Map();
|
|
4310
|
+
this.#metadata = /* @__PURE__ */ new Map();
|
|
4311
|
+
this.#extraData = /* @__PURE__ */ new Map();
|
|
4312
|
+
this.#rootSpanProcessed = false;
|
|
4313
|
+
this.#waitingForRoot = [];
|
|
4314
|
+
this.#waitingForParent = /* @__PURE__ */ new Map();
|
|
4315
|
+
this.createdAt = /* @__PURE__ */ new Date();
|
|
4247
4316
|
}
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
this.flush().catch((error) => {
|
|
4255
|
-
this.logger.error("Batch flush failed", {
|
|
4256
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4257
|
-
});
|
|
4258
|
-
});
|
|
4259
|
-
} else if (this.buffer.totalSize === 1) {
|
|
4260
|
-
this.scheduleFlush();
|
|
4261
|
-
}
|
|
4317
|
+
/**
|
|
4318
|
+
* Check if this trace has a root span registered.
|
|
4319
|
+
* @returns True if addRoot() has been called
|
|
4320
|
+
*/
|
|
4321
|
+
hasRoot() {
|
|
4322
|
+
return !!this.#rootSpanId;
|
|
4262
4323
|
}
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
this.
|
|
4324
|
+
/**
|
|
4325
|
+
* Register the root span for this trace.
|
|
4326
|
+
* @param args.rootId - The span ID of the root span
|
|
4327
|
+
* @param args.rootData - The vendor-specific root object
|
|
4328
|
+
*/
|
|
4329
|
+
addRoot(args) {
|
|
4330
|
+
this.#rootSpanId = args.rootId;
|
|
4331
|
+
this.#rootSpan = args.rootData;
|
|
4332
|
+
this.#rootSpanProcessed = true;
|
|
4270
4333
|
}
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
spanType: span.type,
|
|
4278
|
-
attributes: span.attributes ?? null,
|
|
4279
|
-
metadata: span.metadata ?? null,
|
|
4280
|
-
startedAt: span.startTime,
|
|
4281
|
-
endedAt: span.endTime ?? null,
|
|
4282
|
-
input: span.input ?? null,
|
|
4283
|
-
output: span.output ?? null,
|
|
4284
|
-
error: span.errorInfo,
|
|
4285
|
-
isEvent: span.isEvent,
|
|
4286
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
4287
|
-
updatedAt: null
|
|
4288
|
-
};
|
|
4289
|
-
return spanRecord;
|
|
4334
|
+
/**
|
|
4335
|
+
* Get the vendor-specific root object.
|
|
4336
|
+
* @returns The root object, or undefined if not yet set
|
|
4337
|
+
*/
|
|
4338
|
+
getRoot() {
|
|
4339
|
+
return this.#rootSpan;
|
|
4290
4340
|
}
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
return true;
|
|
4299
|
-
}
|
|
4300
|
-
}
|
|
4301
|
-
return false;
|
|
4341
|
+
/**
|
|
4342
|
+
* Check if a span with isRootSpan=true has been successfully processed.
|
|
4343
|
+
* Set via addRoot() or markRootSpanProcessed().
|
|
4344
|
+
* @returns True if the root span has been processed
|
|
4345
|
+
*/
|
|
4346
|
+
isRootProcessed() {
|
|
4347
|
+
return this.#rootSpanProcessed;
|
|
4302
4348
|
}
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
{
|
|
4311
|
-
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
4312
|
-
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4313
|
-
category: ErrorCategory.USER
|
|
4314
|
-
},
|
|
4315
|
-
error
|
|
4316
|
-
);
|
|
4317
|
-
this.logger.trackException(mastraError);
|
|
4318
|
-
this.logger.error("Scheduled flush failed", mastraError);
|
|
4319
|
-
});
|
|
4320
|
-
}, this.config.maxBatchWaitMs);
|
|
4349
|
+
/**
|
|
4350
|
+
* Mark that the root span has been processed.
|
|
4351
|
+
* Used by exporters with skipBuildRootTask=true where root goes through _buildSpan
|
|
4352
|
+
* instead of _buildRoot.
|
|
4353
|
+
*/
|
|
4354
|
+
markRootSpanProcessed() {
|
|
4355
|
+
this.#rootSpanProcessed = true;
|
|
4321
4356
|
}
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
}
|
|
4330
|
-
const startTime = Date.now();
|
|
4331
|
-
const spansCopy = [...this.buffer.spans];
|
|
4332
|
-
const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
|
|
4333
|
-
this.resetBuffer();
|
|
4334
|
-
try {
|
|
4335
|
-
await this.batchUpload(spansCopy);
|
|
4336
|
-
const elapsed = Date.now() - startTime;
|
|
4337
|
-
this.logger.debug("Batch flushed successfully", {
|
|
4338
|
-
batchSize: spansCopy.length,
|
|
4339
|
-
flushReason,
|
|
4340
|
-
durationMs: elapsed
|
|
4341
|
-
});
|
|
4342
|
-
} catch (error) {
|
|
4343
|
-
const mastraError = new MastraError(
|
|
4344
|
-
{
|
|
4345
|
-
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
4346
|
-
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4347
|
-
category: ErrorCategory.USER,
|
|
4348
|
-
details: {
|
|
4349
|
-
droppedBatchSize: spansCopy.length
|
|
4350
|
-
}
|
|
4351
|
-
},
|
|
4352
|
-
error
|
|
4353
|
-
);
|
|
4354
|
-
this.logger.trackException(mastraError);
|
|
4355
|
-
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
4356
|
-
}
|
|
4357
|
+
/**
|
|
4358
|
+
* Store an arbitrary value in per-trace storage.
|
|
4359
|
+
* @param key - Storage key
|
|
4360
|
+
* @param value - Value to store
|
|
4361
|
+
*/
|
|
4362
|
+
setExtraValue(key, value) {
|
|
4363
|
+
this.#extraData.set(key, value);
|
|
4357
4364
|
}
|
|
4358
4365
|
/**
|
|
4359
|
-
*
|
|
4366
|
+
* Check if a key exists in per-trace storage.
|
|
4367
|
+
* @param key - Storage key
|
|
4368
|
+
* @returns True if the key exists
|
|
4360
4369
|
*/
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
Authorization: `Bearer ${this.config.accessToken}`,
|
|
4364
|
-
"Content-Type": "application/json"
|
|
4365
|
-
};
|
|
4366
|
-
const options = {
|
|
4367
|
-
method: "POST",
|
|
4368
|
-
headers,
|
|
4369
|
-
body: JSON.stringify({ spans })
|
|
4370
|
-
};
|
|
4371
|
-
await fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
|
|
4370
|
+
hasExtraValue(key) {
|
|
4371
|
+
return this.#extraData.has(key);
|
|
4372
4372
|
}
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4373
|
+
/**
|
|
4374
|
+
* Get a value from per-trace storage.
|
|
4375
|
+
* @param key - Storage key
|
|
4376
|
+
* @returns The stored value, or undefined if not found
|
|
4377
|
+
*/
|
|
4378
|
+
getExtraValue(key) {
|
|
4379
|
+
return this.#extraData.get(key);
|
|
4377
4380
|
}
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
+
// ============================================================================
|
|
4382
|
+
// Early Queue Methods
|
|
4383
|
+
// ============================================================================
|
|
4384
|
+
/**
|
|
4385
|
+
* Add an event to the waiting queue.
|
|
4386
|
+
* @param args.event - The tracing event to queue
|
|
4387
|
+
* @param args.waitingFor - 'root' or a specific parentSpanId
|
|
4388
|
+
* @param args.attempts - Optional: preserve attempts count when re-queuing
|
|
4389
|
+
* @param args.queuedAt - Optional: preserve original queue time when re-queuing
|
|
4390
|
+
*/
|
|
4391
|
+
addToWaitingQueue(args) {
|
|
4392
|
+
const queuedEvent = {
|
|
4393
|
+
event: args.event,
|
|
4394
|
+
waitingFor: args.waitingFor,
|
|
4395
|
+
attempts: args.attempts ?? 0,
|
|
4396
|
+
queuedAt: args.queuedAt ?? /* @__PURE__ */ new Date()
|
|
4397
|
+
};
|
|
4398
|
+
if (args.waitingFor === "root") {
|
|
4399
|
+
this.#waitingForRoot.push(queuedEvent);
|
|
4400
|
+
} else {
|
|
4401
|
+
const queue = this.#waitingForParent.get(args.waitingFor) ?? [];
|
|
4402
|
+
queue.push(queuedEvent);
|
|
4403
|
+
this.#waitingForParent.set(args.waitingFor, queue);
|
|
4381
4404
|
}
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4405
|
+
}
|
|
4406
|
+
/**
|
|
4407
|
+
* Get all events waiting for the root span.
|
|
4408
|
+
* Returns a copy of the internal array.
|
|
4409
|
+
*/
|
|
4410
|
+
getEventsWaitingForRoot() {
|
|
4411
|
+
return [...this.#waitingForRoot];
|
|
4412
|
+
}
|
|
4413
|
+
/**
|
|
4414
|
+
* Get all events waiting for a specific parent span.
|
|
4415
|
+
* Returns a copy of the internal array.
|
|
4416
|
+
*/
|
|
4417
|
+
getEventsWaitingFor(args) {
|
|
4418
|
+
return [...this.#waitingForParent.get(args.spanId) ?? []];
|
|
4419
|
+
}
|
|
4420
|
+
/**
|
|
4421
|
+
* Clear the waiting-for-root queue.
|
|
4422
|
+
*/
|
|
4423
|
+
clearWaitingForRoot() {
|
|
4424
|
+
this.#waitingForRoot = [];
|
|
4425
|
+
}
|
|
4426
|
+
/**
|
|
4427
|
+
* Clear the waiting queue for a specific parent span.
|
|
4428
|
+
*/
|
|
4429
|
+
clearWaitingFor(args) {
|
|
4430
|
+
this.#waitingForParent.delete(args.spanId);
|
|
4431
|
+
}
|
|
4432
|
+
/**
|
|
4433
|
+
* Get total count of events in all waiting queues.
|
|
4434
|
+
*/
|
|
4435
|
+
waitingQueueSize() {
|
|
4436
|
+
let count = this.#waitingForRoot.length;
|
|
4437
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4438
|
+
count += queue.length;
|
|
4385
4439
|
}
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
category: ErrorCategory.USER,
|
|
4398
|
-
details: {
|
|
4399
|
-
remainingEvents: this.buffer.totalSize
|
|
4400
|
-
}
|
|
4401
|
-
},
|
|
4402
|
-
error
|
|
4403
|
-
);
|
|
4404
|
-
this.logger.trackException(mastraError);
|
|
4405
|
-
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
4406
|
-
}
|
|
4440
|
+
return count;
|
|
4441
|
+
}
|
|
4442
|
+
/**
|
|
4443
|
+
* Get all queued events across all waiting queues.
|
|
4444
|
+
* Used for cleanup and logging orphaned events.
|
|
4445
|
+
* @returns Array of all queued events
|
|
4446
|
+
*/
|
|
4447
|
+
getAllQueuedEvents() {
|
|
4448
|
+
const all = [...this.#waitingForRoot];
|
|
4449
|
+
for (const queue of this.#waitingForParent.values()) {
|
|
4450
|
+
all.push(...queue);
|
|
4407
4451
|
}
|
|
4408
|
-
|
|
4452
|
+
return all;
|
|
4409
4453
|
}
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4454
|
+
// ============================================================================
|
|
4455
|
+
// Span Tree Methods
|
|
4456
|
+
// ============================================================================
|
|
4457
|
+
/**
|
|
4458
|
+
* Record the parent-child relationship for a span.
|
|
4459
|
+
* @param args.spanId - The child span ID
|
|
4460
|
+
* @param args.parentSpanId - The parent span ID, or undefined for root spans
|
|
4461
|
+
*/
|
|
4462
|
+
addBranch(args) {
|
|
4463
|
+
this.#tree.set(args.spanId, args.parentSpanId);
|
|
4415
4464
|
}
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4465
|
+
/**
|
|
4466
|
+
* Get the parent span ID for a given span.
|
|
4467
|
+
* @param args.spanId - The span ID to look up
|
|
4468
|
+
* @returns The parent span ID, or undefined if root or not found
|
|
4469
|
+
*/
|
|
4470
|
+
getParentId(args) {
|
|
4471
|
+
return this.#tree.get(args.spanId);
|
|
4472
|
+
}
|
|
4473
|
+
// ============================================================================
|
|
4474
|
+
// Span Management Methods
|
|
4475
|
+
// ============================================================================
|
|
4476
|
+
/**
|
|
4477
|
+
* Register a span and mark it as active.
|
|
4478
|
+
* @param args.spanId - The span ID
|
|
4479
|
+
* @param args.spanData - The vendor-specific span object
|
|
4480
|
+
*/
|
|
4481
|
+
addSpan(args) {
|
|
4482
|
+
this.#spans.set(args.spanId, args.spanData);
|
|
4483
|
+
this.#activeSpanIds.add(args.spanId);
|
|
4484
|
+
}
|
|
4485
|
+
/**
|
|
4486
|
+
* Check if a span exists (regardless of active state).
|
|
4487
|
+
* @param args.spanId - The span ID to check
|
|
4488
|
+
* @returns True if the span exists
|
|
4489
|
+
*/
|
|
4490
|
+
hasSpan(args) {
|
|
4491
|
+
return this.#spans.has(args.spanId);
|
|
4492
|
+
}
|
|
4493
|
+
/**
|
|
4494
|
+
* Get a span by ID.
|
|
4495
|
+
* @param args.spanId - The span ID to look up
|
|
4496
|
+
* @returns The vendor-specific span object, or undefined if not found
|
|
4497
|
+
*/
|
|
4498
|
+
getSpan(args) {
|
|
4499
|
+
return this.#spans.get(args.spanId);
|
|
4500
|
+
}
|
|
4501
|
+
/**
|
|
4502
|
+
* Mark a span as ended (no longer active).
|
|
4503
|
+
* @param args.spanId - The span ID to mark as ended
|
|
4504
|
+
*/
|
|
4505
|
+
endSpan(args) {
|
|
4506
|
+
this.#activeSpanIds.delete(args.spanId);
|
|
4507
|
+
}
|
|
4508
|
+
/**
|
|
4509
|
+
* Check if a span is currently active (started but not ended).
|
|
4510
|
+
* @param args.spanId - The span ID to check
|
|
4511
|
+
* @returns True if the span is active
|
|
4512
|
+
*/
|
|
4513
|
+
isActiveSpan(args) {
|
|
4514
|
+
return this.#activeSpanIds.has(args.spanId);
|
|
4515
|
+
}
|
|
4516
|
+
/**
|
|
4517
|
+
* Get the count of currently active spans.
|
|
4518
|
+
* @returns Number of active spans
|
|
4519
|
+
*/
|
|
4520
|
+
activeSpanCount() {
|
|
4521
|
+
return this.#activeSpanIds.size;
|
|
4522
|
+
}
|
|
4523
|
+
/**
|
|
4524
|
+
* Get all active span IDs.
|
|
4525
|
+
* @returns Array of active span IDs
|
|
4526
|
+
*/
|
|
4527
|
+
get activeSpanIds() {
|
|
4528
|
+
return [...this.#activeSpanIds];
|
|
4529
|
+
}
|
|
4530
|
+
// ============================================================================
|
|
4531
|
+
// Event Management Methods
|
|
4532
|
+
// ============================================================================
|
|
4533
|
+
/**
|
|
4534
|
+
* Register an event.
|
|
4535
|
+
* @param args.eventId - The event ID
|
|
4536
|
+
* @param args.eventData - The vendor-specific event object
|
|
4537
|
+
*/
|
|
4538
|
+
addEvent(args) {
|
|
4539
|
+
this.#events.set(args.eventId, args.eventData);
|
|
4540
|
+
}
|
|
4541
|
+
// ============================================================================
|
|
4542
|
+
// Metadata Methods
|
|
4543
|
+
// ============================================================================
|
|
4544
|
+
/**
|
|
4545
|
+
* Store vendor-specific metadata for a span.
|
|
4546
|
+
* Note: This overwrites any existing metadata for the span.
|
|
4547
|
+
* @param args.spanId - The span ID
|
|
4548
|
+
* @param args.metadata - The vendor-specific metadata
|
|
4549
|
+
*/
|
|
4550
|
+
addMetadata(args) {
|
|
4551
|
+
this.#metadata.set(args.spanId, args.metadata);
|
|
4552
|
+
}
|
|
4553
|
+
/**
|
|
4554
|
+
* Get vendor-specific metadata for a span.
|
|
4555
|
+
* @param args.spanId - The span ID
|
|
4556
|
+
* @returns The metadata, or undefined if not found
|
|
4557
|
+
*/
|
|
4558
|
+
getMetadata(args) {
|
|
4559
|
+
return this.#metadata.get(args.spanId);
|
|
4560
|
+
}
|
|
4561
|
+
// ============================================================================
|
|
4562
|
+
// Parent Lookup Methods
|
|
4563
|
+
// ============================================================================
|
|
4564
|
+
/**
|
|
4565
|
+
* Get the parent span or event for a given span.
|
|
4566
|
+
* Looks up in both spans and events maps.
|
|
4567
|
+
* @param args.span - The span to find the parent for
|
|
4568
|
+
* @returns The parent span/event object, or undefined if root or not found
|
|
4569
|
+
*/
|
|
4570
|
+
getParent(args) {
|
|
4571
|
+
const parentId = args.span.parentSpanId;
|
|
4572
|
+
if (parentId) {
|
|
4573
|
+
if (this.#spans.has(parentId)) {
|
|
4574
|
+
return this.#spans.get(parentId);
|
|
4575
|
+
}
|
|
4576
|
+
if (this.#events.has(parentId)) {
|
|
4577
|
+
return this.#events.get(parentId);
|
|
4578
|
+
}
|
|
4579
|
+
}
|
|
4580
|
+
return void 0;
|
|
4581
|
+
}
|
|
4582
|
+
/**
|
|
4583
|
+
* Get the parent span/event or fall back to the root object.
|
|
4584
|
+
* Useful for vendors that attach child spans to either parent spans or the trace root.
|
|
4585
|
+
* @param args.span - The span to find the parent for
|
|
4586
|
+
* @returns The parent span/event, the root object, or undefined
|
|
4587
|
+
*/
|
|
4588
|
+
getParentOrRoot(args) {
|
|
4589
|
+
return this.getParent(args) ?? this.getRoot();
|
|
4590
|
+
}
|
|
4591
|
+
};
|
|
4592
|
+
var DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS = 5;
|
|
4593
|
+
var DEFAULT_EARLY_QUEUE_TTL_MS = 3e4;
|
|
4594
|
+
var DEFAULT_TRACE_CLEANUP_DELAY_MS = 3e4;
|
|
4595
|
+
var DEFAULT_MAX_PENDING_CLEANUP_TRACES = 100;
|
|
4596
|
+
var DEFAULT_MAX_TOTAL_TRACES = 500;
|
|
4597
|
+
var TrackingExporter = class extends BaseExporter {
|
|
4598
|
+
/** Map of traceId to per-trace data container */
|
|
4599
|
+
#traceMap = /* @__PURE__ */ new Map();
|
|
4600
|
+
/** Flag to prevent processing during shutdown */
|
|
4601
|
+
#shutdownStarted = false;
|
|
4602
|
+
/** Flag to prevent concurrent hard cap enforcement */
|
|
4603
|
+
#hardCapEnforcementInProgress = false;
|
|
4604
|
+
/** Map of traceId to scheduled cleanup timeout */
|
|
4605
|
+
#pendingCleanups = /* @__PURE__ */ new Map();
|
|
4606
|
+
// Note: #traceMap maintains insertion order (JS Map spec), so we use
|
|
4607
|
+
// #traceMap.keys() to iterate traces oldest-first for cap enforcement.
|
|
4608
|
+
/** Subclass configuration with resolved values */
|
|
4609
|
+
config;
|
|
4610
|
+
/** Maximum attempts to process a queued event before dropping */
|
|
4611
|
+
#earlyQueueMaxAttempts;
|
|
4612
|
+
/** TTL in milliseconds for queued events */
|
|
4613
|
+
#earlyQueueTTLMs;
|
|
4614
|
+
/** Delay before cleaning up completed traces */
|
|
4615
|
+
#traceCleanupDelayMs;
|
|
4616
|
+
/** Soft cap on traces awaiting cleanup */
|
|
4617
|
+
#maxPendingCleanupTraces;
|
|
4618
|
+
/** Hard cap on total traces (will abort active spans if exceeded) */
|
|
4619
|
+
#maxTotalTraces;
|
|
4620
|
+
constructor(config) {
|
|
4621
|
+
super(config);
|
|
4622
|
+
this.config = config;
|
|
4623
|
+
this.#earlyQueueMaxAttempts = config.earlyQueueMaxAttempts ?? DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS;
|
|
4624
|
+
this.#earlyQueueTTLMs = config.earlyQueueTTLMs ?? DEFAULT_EARLY_QUEUE_TTL_MS;
|
|
4625
|
+
this.#traceCleanupDelayMs = config.traceCleanupDelayMs ?? DEFAULT_TRACE_CLEANUP_DELAY_MS;
|
|
4626
|
+
this.#maxPendingCleanupTraces = config.maxPendingCleanupTraces ?? DEFAULT_MAX_PENDING_CLEANUP_TRACES;
|
|
4627
|
+
this.#maxTotalTraces = config.maxTotalTraces ?? DEFAULT_MAX_TOTAL_TRACES;
|
|
4628
|
+
}
|
|
4629
|
+
// ============================================================================
|
|
4630
|
+
// Early Queue Processing
|
|
4631
|
+
// ============================================================================
|
|
4632
|
+
/**
|
|
4633
|
+
* Schedule async processing of events waiting for root span.
|
|
4634
|
+
* Called after root span is successfully processed.
|
|
4635
|
+
*/
|
|
4636
|
+
#scheduleProcessWaitingForRoot(traceId) {
|
|
4637
|
+
setImmediate(() => {
|
|
4638
|
+
this.#processWaitingForRoot(traceId).catch((error) => {
|
|
4639
|
+
this.logger.error(`${this.name}: Error processing waiting-for-root queue`, { error, traceId });
|
|
4640
|
+
});
|
|
4641
|
+
});
|
|
4642
|
+
}
|
|
4643
|
+
/**
|
|
4644
|
+
* Schedule async processing of events waiting for a specific parent span.
|
|
4645
|
+
* Called after a span/event is successfully created.
|
|
4646
|
+
*/
|
|
4647
|
+
#scheduleProcessWaitingFor(traceId, spanId) {
|
|
4648
|
+
setImmediate(() => {
|
|
4649
|
+
this.#processWaitingFor(traceId, spanId).catch((error) => {
|
|
4650
|
+
this.logger.error(`${this.name}: Error processing waiting queue`, { error, traceId, spanId });
|
|
4651
|
+
});
|
|
4652
|
+
});
|
|
4653
|
+
}
|
|
4654
|
+
/**
|
|
4655
|
+
* Process all events waiting for root span.
|
|
4656
|
+
*/
|
|
4657
|
+
async #processWaitingForRoot(traceId) {
|
|
4658
|
+
if (this.#shutdownStarted) return;
|
|
4659
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4660
|
+
if (!traceData) return;
|
|
4661
|
+
const queue = traceData.getEventsWaitingForRoot();
|
|
4662
|
+
if (queue.length === 0) return;
|
|
4663
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for root`, { traceId });
|
|
4664
|
+
const toKeep = [];
|
|
4665
|
+
const now = Date.now();
|
|
4666
|
+
for (const queuedEvent of queue) {
|
|
4667
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4668
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4669
|
+
traceId,
|
|
4670
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4671
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4672
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4673
|
+
attempts: queuedEvent.attempts
|
|
4674
|
+
});
|
|
4675
|
+
continue;
|
|
4676
|
+
}
|
|
4677
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4678
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4679
|
+
traceId,
|
|
4680
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4681
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4682
|
+
attempts: queuedEvent.attempts
|
|
4683
|
+
});
|
|
4684
|
+
continue;
|
|
4685
|
+
}
|
|
4686
|
+
queuedEvent.attempts++;
|
|
4687
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4688
|
+
if (!processed) {
|
|
4689
|
+
const parentId = queuedEvent.event.exportedSpan.parentSpanId;
|
|
4690
|
+
if (parentId && traceData.isRootProcessed()) {
|
|
4691
|
+
traceData.addToWaitingQueue({
|
|
4692
|
+
event: queuedEvent.event,
|
|
4693
|
+
waitingFor: parentId,
|
|
4694
|
+
attempts: queuedEvent.attempts,
|
|
4695
|
+
queuedAt: queuedEvent.queuedAt
|
|
4696
|
+
});
|
|
4697
|
+
} else {
|
|
4698
|
+
toKeep.push(queuedEvent);
|
|
4699
|
+
}
|
|
4700
|
+
}
|
|
4701
|
+
}
|
|
4702
|
+
traceData.clearWaitingForRoot();
|
|
4703
|
+
for (const event of toKeep) {
|
|
4704
|
+
traceData.addToWaitingQueue({
|
|
4705
|
+
event: event.event,
|
|
4706
|
+
waitingFor: "root",
|
|
4707
|
+
attempts: event.attempts,
|
|
4708
|
+
queuedAt: event.queuedAt
|
|
4709
|
+
});
|
|
4710
|
+
}
|
|
4711
|
+
}
|
|
4712
|
+
/**
|
|
4713
|
+
* Process events waiting for a specific parent span.
|
|
4714
|
+
*/
|
|
4715
|
+
async #processWaitingFor(traceId, spanId) {
|
|
4716
|
+
if (this.#shutdownStarted) return;
|
|
4717
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4718
|
+
if (!traceData) return;
|
|
4719
|
+
const queue = traceData.getEventsWaitingFor({ spanId });
|
|
4720
|
+
if (queue.length === 0) return;
|
|
4721
|
+
this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for span`, { traceId, spanId });
|
|
4722
|
+
const toKeep = [];
|
|
4723
|
+
const now = Date.now();
|
|
4724
|
+
for (const queuedEvent of queue) {
|
|
4725
|
+
if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
|
|
4726
|
+
this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
|
|
4727
|
+
traceId,
|
|
4728
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4729
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4730
|
+
queuedAt: queuedEvent.queuedAt,
|
|
4731
|
+
attempts: queuedEvent.attempts
|
|
4732
|
+
});
|
|
4733
|
+
continue;
|
|
4734
|
+
}
|
|
4735
|
+
if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
|
|
4736
|
+
this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
|
|
4737
|
+
traceId,
|
|
4738
|
+
spanId: queuedEvent.event.exportedSpan.id,
|
|
4739
|
+
waitingFor: queuedEvent.waitingFor,
|
|
4740
|
+
attempts: queuedEvent.attempts
|
|
4741
|
+
});
|
|
4742
|
+
continue;
|
|
4743
|
+
}
|
|
4744
|
+
queuedEvent.attempts++;
|
|
4745
|
+
const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
|
|
4746
|
+
if (!processed) {
|
|
4747
|
+
toKeep.push(queuedEvent);
|
|
4748
|
+
}
|
|
4749
|
+
}
|
|
4750
|
+
traceData.clearWaitingFor({ spanId });
|
|
4751
|
+
for (const event of toKeep) {
|
|
4752
|
+
traceData.addToWaitingQueue({
|
|
4753
|
+
event: event.event,
|
|
4754
|
+
waitingFor: spanId,
|
|
4755
|
+
attempts: event.attempts,
|
|
4756
|
+
queuedAt: event.queuedAt
|
|
4757
|
+
});
|
|
4758
|
+
}
|
|
4759
|
+
}
|
|
4760
|
+
/**
|
|
4761
|
+
* Try to process a queued event.
|
|
4762
|
+
* Returns true if successfully processed, false if still waiting for dependencies.
|
|
4763
|
+
*/
|
|
4764
|
+
async #tryProcessQueuedEvent(queuedEvent, traceData) {
|
|
4765
|
+
const { event } = queuedEvent;
|
|
4766
|
+
const { exportedSpan } = event;
|
|
4767
|
+
const method = this.getMethod(event);
|
|
4768
|
+
try {
|
|
4769
|
+
switch (method) {
|
|
4770
|
+
case "handleEventSpan": {
|
|
4771
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4772
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
4773
|
+
if (eventData) {
|
|
4774
|
+
if (!this.skipCachingEventSpans) {
|
|
4775
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
4776
|
+
}
|
|
4777
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4778
|
+
return true;
|
|
4779
|
+
}
|
|
4780
|
+
return false;
|
|
4781
|
+
}
|
|
4782
|
+
case "handleSpanStart": {
|
|
4783
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
4784
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
4785
|
+
if (spanData) {
|
|
4786
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
4787
|
+
if (exportedSpan.isRootSpan) {
|
|
4788
|
+
traceData.markRootSpanProcessed();
|
|
4789
|
+
}
|
|
4790
|
+
this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
|
|
4791
|
+
return true;
|
|
4792
|
+
}
|
|
4793
|
+
return false;
|
|
4794
|
+
}
|
|
4795
|
+
case "handleSpanUpdate": {
|
|
4796
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
4797
|
+
return true;
|
|
4798
|
+
}
|
|
4799
|
+
case "handleSpanEnd": {
|
|
4800
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
4801
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
4802
|
+
if (traceData.activeSpanCount() === 0) {
|
|
4803
|
+
this.#scheduleCleanup(exportedSpan.traceId);
|
|
4804
|
+
}
|
|
4805
|
+
return true;
|
|
4806
|
+
}
|
|
4807
|
+
default:
|
|
4808
|
+
return false;
|
|
4809
|
+
}
|
|
4810
|
+
} catch (error) {
|
|
4811
|
+
this.logger.error(`${this.name}: Error processing queued event`, { error, event, method });
|
|
4812
|
+
return false;
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
// ============================================================================
|
|
4816
|
+
// Delayed Cleanup
|
|
4817
|
+
// ============================================================================
|
|
4818
|
+
/**
|
|
4819
|
+
* Schedule cleanup of trace data after a delay.
|
|
4820
|
+
* Allows late-arriving data to still be processed.
|
|
4821
|
+
*/
|
|
4822
|
+
#scheduleCleanup(traceId) {
|
|
4823
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4824
|
+
this.logger.debug(`${this.name}: Scheduling cleanup in ${this.#traceCleanupDelayMs}ms`, { traceId });
|
|
4825
|
+
const timeout = setTimeout(() => {
|
|
4826
|
+
this.#pendingCleanups.delete(traceId);
|
|
4827
|
+
this.#performCleanup(traceId);
|
|
4828
|
+
}, this.#traceCleanupDelayMs);
|
|
4829
|
+
this.#pendingCleanups.set(traceId, timeout);
|
|
4830
|
+
this.#enforcePendingCleanupCap();
|
|
4831
|
+
}
|
|
4832
|
+
/**
|
|
4833
|
+
* Cancel a scheduled cleanup for a trace.
|
|
4834
|
+
*/
|
|
4835
|
+
#cancelScheduledCleanup(traceId) {
|
|
4836
|
+
const existingTimeout = this.#pendingCleanups.get(traceId);
|
|
4837
|
+
if (existingTimeout) {
|
|
4838
|
+
clearTimeout(existingTimeout);
|
|
4839
|
+
this.#pendingCleanups.delete(traceId);
|
|
4840
|
+
this.logger.debug(`${this.name}: Cancelled scheduled cleanup`, { traceId });
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4843
|
+
/**
|
|
4844
|
+
* Perform the actual cleanup of trace data.
|
|
4845
|
+
*/
|
|
4846
|
+
#performCleanup(traceId) {
|
|
4847
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4848
|
+
if (!traceData) return;
|
|
4849
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
4850
|
+
if (orphanedEvents.length > 0) {
|
|
4851
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on cleanup`, {
|
|
4852
|
+
traceId,
|
|
4853
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
4854
|
+
spanId: e.event.exportedSpan.id,
|
|
4855
|
+
waitingFor: e.waitingFor,
|
|
4856
|
+
attempts: e.attempts,
|
|
4857
|
+
queuedAt: e.queuedAt
|
|
4858
|
+
}))
|
|
4859
|
+
});
|
|
4860
|
+
}
|
|
4861
|
+
this.#traceMap.delete(traceId);
|
|
4862
|
+
this.logger.debug(`${this.name}: Cleaned up trace data`, { traceId });
|
|
4863
|
+
}
|
|
4864
|
+
// ============================================================================
|
|
4865
|
+
// Cap Enforcement
|
|
4866
|
+
// ============================================================================
|
|
4867
|
+
/**
|
|
4868
|
+
* Enforce soft cap on pending cleanup traces.
|
|
4869
|
+
* Only removes traces with activeSpanCount == 0.
|
|
4870
|
+
*/
|
|
4871
|
+
#enforcePendingCleanupCap() {
|
|
4872
|
+
if (this.#pendingCleanups.size <= this.#maxPendingCleanupTraces) {
|
|
4873
|
+
return;
|
|
4874
|
+
}
|
|
4875
|
+
const toRemove = this.#pendingCleanups.size - this.#maxPendingCleanupTraces;
|
|
4876
|
+
this.logger.warn(`${this.name}: Pending cleanup cap exceeded, force-cleaning ${toRemove} traces`, {
|
|
4877
|
+
pendingCount: this.#pendingCleanups.size,
|
|
4878
|
+
cap: this.#maxPendingCleanupTraces
|
|
4879
|
+
});
|
|
4880
|
+
let removed = 0;
|
|
4881
|
+
for (const traceId of this.#traceMap.keys()) {
|
|
4882
|
+
if (removed >= toRemove) break;
|
|
4883
|
+
if (this.#pendingCleanups.has(traceId)) {
|
|
4884
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4885
|
+
this.#performCleanup(traceId);
|
|
4886
|
+
removed++;
|
|
4887
|
+
}
|
|
4888
|
+
}
|
|
4889
|
+
}
|
|
4890
|
+
/**
|
|
4891
|
+
* Enforce hard cap on total traces.
|
|
4892
|
+
* Will kill even active traces if necessary.
|
|
4893
|
+
* Uses a flag to prevent concurrent executions when called fire-and-forget.
|
|
4894
|
+
*/
|
|
4895
|
+
async #enforceHardCap() {
|
|
4896
|
+
if (this.#traceMap.size <= this.#maxTotalTraces || this.#hardCapEnforcementInProgress) {
|
|
4897
|
+
return;
|
|
4898
|
+
}
|
|
4899
|
+
this.#hardCapEnforcementInProgress = true;
|
|
4900
|
+
try {
|
|
4901
|
+
if (this.#traceMap.size <= this.#maxTotalTraces) {
|
|
4902
|
+
return;
|
|
4903
|
+
}
|
|
4904
|
+
const toRemove = this.#traceMap.size - this.#maxTotalTraces;
|
|
4905
|
+
this.logger.warn(`${this.name}: Total trace cap exceeded, killing ${toRemove} oldest traces`, {
|
|
4906
|
+
traceCount: this.#traceMap.size,
|
|
4907
|
+
cap: this.#maxTotalTraces
|
|
4908
|
+
});
|
|
4909
|
+
const reason = {
|
|
4910
|
+
id: "TRACE_CAP_EXCEEDED",
|
|
4911
|
+
message: "Trace killed due to memory cap enforcement.",
|
|
4912
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
4913
|
+
category: "SYSTEM"
|
|
4914
|
+
};
|
|
4915
|
+
let removed = 0;
|
|
4916
|
+
for (const traceId of [...this.#traceMap.keys()]) {
|
|
4917
|
+
if (removed >= toRemove) break;
|
|
4918
|
+
const traceData = this.#traceMap.get(traceId);
|
|
4919
|
+
if (traceData) {
|
|
4920
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
4921
|
+
const span = traceData.getSpan({ spanId });
|
|
4922
|
+
if (span) {
|
|
4923
|
+
await this._abortSpan({ span, traceData, reason });
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
this.#cancelScheduledCleanup(traceId);
|
|
4927
|
+
this.#performCleanup(traceId);
|
|
4928
|
+
removed++;
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
} finally {
|
|
4932
|
+
this.#hardCapEnforcementInProgress = false;
|
|
4933
|
+
}
|
|
4934
|
+
}
|
|
4935
|
+
// ============================================================================
|
|
4936
|
+
// Lifecycle Hooks (Override in subclass)
|
|
4937
|
+
// ============================================================================
|
|
4938
|
+
/**
|
|
4939
|
+
* Hook called before processing each tracing event.
|
|
4940
|
+
* Override to transform or enrich the event before processing.
|
|
4941
|
+
*
|
|
4942
|
+
* Note: The customSpanFormatter is applied at the BaseExporter level before this hook.
|
|
4943
|
+
* Subclasses can override this to add additional pre-processing logic.
|
|
4944
|
+
*
|
|
4945
|
+
* @param event - The incoming tracing event
|
|
4946
|
+
* @returns The (possibly modified) event to process
|
|
4947
|
+
*/
|
|
4948
|
+
async _preExportTracingEvent(event) {
|
|
4949
|
+
return event;
|
|
4950
|
+
}
|
|
4951
|
+
/**
|
|
4952
|
+
* Hook called after processing each tracing event.
|
|
4953
|
+
* Override to perform post-processing actions like flushing.
|
|
4954
|
+
*/
|
|
4955
|
+
async _postExportTracingEvent() {
|
|
4956
|
+
}
|
|
4957
|
+
// ============================================================================
|
|
4958
|
+
// Behavior Flags (Override in subclass as needed)
|
|
4959
|
+
// ============================================================================
|
|
4960
|
+
/**
|
|
4961
|
+
* If true, skip calling _buildRoot and let root spans go through _buildSpan.
|
|
4962
|
+
* Use when the vendor doesn't have a separate trace/root concept.
|
|
4963
|
+
* @default false
|
|
4964
|
+
*/
|
|
4965
|
+
skipBuildRootTask = false;
|
|
4966
|
+
/**
|
|
4967
|
+
* If true, skip processing span_updated events entirely.
|
|
4968
|
+
* Use when the vendor doesn't support incremental span updates.
|
|
4969
|
+
* @default false
|
|
4970
|
+
*/
|
|
4971
|
+
skipSpanUpdateEvents = false;
|
|
4972
|
+
/**
|
|
4973
|
+
* If true, don't cache event spans in TraceData.
|
|
4974
|
+
* Use when events can't be parents of other spans.
|
|
4975
|
+
* @default false
|
|
4976
|
+
*/
|
|
4977
|
+
skipCachingEventSpans = false;
|
|
4978
|
+
getMethod(event) {
|
|
4979
|
+
if (event.exportedSpan.isEvent) {
|
|
4980
|
+
return "handleEventSpan";
|
|
4981
|
+
}
|
|
4982
|
+
const eventType = event.type;
|
|
4983
|
+
switch (eventType) {
|
|
4984
|
+
case TracingEventType.SPAN_STARTED:
|
|
4985
|
+
return "handleSpanStart";
|
|
4986
|
+
case TracingEventType.SPAN_UPDATED:
|
|
4987
|
+
return "handleSpanUpdate";
|
|
4988
|
+
case TracingEventType.SPAN_ENDED:
|
|
4989
|
+
return "handleSpanEnd";
|
|
4990
|
+
default: {
|
|
4991
|
+
const _exhaustiveCheck = eventType;
|
|
4992
|
+
throw new Error(`Unhandled event type: ${_exhaustiveCheck}`);
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
async _exportTracingEvent(event) {
|
|
4997
|
+
if (this.#shutdownStarted) {
|
|
4998
|
+
return;
|
|
4999
|
+
}
|
|
5000
|
+
const method = this.getMethod(event);
|
|
5001
|
+
if (method == "handleSpanUpdate" && this.skipSpanUpdateEvents) {
|
|
5002
|
+
return;
|
|
5003
|
+
}
|
|
5004
|
+
const traceId = event.exportedSpan.traceId;
|
|
5005
|
+
const traceData = this.getTraceData({ traceId, method });
|
|
5006
|
+
const { exportedSpan } = await this._preExportTracingEvent(event);
|
|
5007
|
+
if (!this.skipBuildRootTask && !traceData.hasRoot()) {
|
|
5008
|
+
if (exportedSpan.isRootSpan) {
|
|
5009
|
+
this.logger.debug(`${this.name}: Building root`, {
|
|
5010
|
+
traceId: exportedSpan.traceId,
|
|
5011
|
+
spanId: exportedSpan.id
|
|
5012
|
+
});
|
|
5013
|
+
const rootData = await this._buildRoot({ span: exportedSpan, traceData });
|
|
5014
|
+
if (rootData) {
|
|
5015
|
+
this.logger.debug(`${this.name}: Adding root`, {
|
|
5016
|
+
traceId: exportedSpan.traceId,
|
|
5017
|
+
spanId: exportedSpan.id
|
|
5018
|
+
});
|
|
5019
|
+
traceData.addRoot({ rootId: exportedSpan.id, rootData });
|
|
5020
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5021
|
+
}
|
|
5022
|
+
} else {
|
|
5023
|
+
this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
|
|
5024
|
+
traceId: exportedSpan.traceId,
|
|
5025
|
+
spanId: exportedSpan.id
|
|
5026
|
+
});
|
|
5027
|
+
traceData.addToWaitingQueue({ event, waitingFor: "root" });
|
|
5028
|
+
return;
|
|
5029
|
+
}
|
|
5030
|
+
}
|
|
5031
|
+
if (exportedSpan.metadata && this.name in exportedSpan.metadata) {
|
|
5032
|
+
const metadata = exportedSpan.metadata[this.name];
|
|
5033
|
+
this.logger.debug(`${this.name}: Found provider metadata in span`, {
|
|
5034
|
+
traceId: exportedSpan.traceId,
|
|
5035
|
+
spanId: exportedSpan.id,
|
|
5036
|
+
metadata
|
|
5037
|
+
});
|
|
5038
|
+
traceData.addMetadata({ spanId: exportedSpan.id, metadata });
|
|
5039
|
+
}
|
|
5040
|
+
try {
|
|
5041
|
+
switch (method) {
|
|
5042
|
+
case "handleEventSpan": {
|
|
5043
|
+
this.logger.debug(`${this.name}: handling event`, {
|
|
5044
|
+
traceId: exportedSpan.traceId,
|
|
5045
|
+
spanId: exportedSpan.id
|
|
5046
|
+
});
|
|
5047
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5048
|
+
const eventData = await this._buildEvent({ span: exportedSpan, traceData });
|
|
5049
|
+
if (eventData) {
|
|
5050
|
+
if (!this.skipCachingEventSpans) {
|
|
5051
|
+
this.logger.debug(`${this.name}: adding event to traceData`, {
|
|
5052
|
+
traceId: exportedSpan.traceId,
|
|
5053
|
+
spanId: exportedSpan.id
|
|
5054
|
+
});
|
|
5055
|
+
traceData.addEvent({ eventId: exportedSpan.id, eventData });
|
|
5056
|
+
}
|
|
5057
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5058
|
+
} else {
|
|
5059
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5060
|
+
this.logger.debug(`${this.name}: adding event to waiting queue`, {
|
|
5061
|
+
traceId: exportedSpan.traceId,
|
|
5062
|
+
spanId: exportedSpan.id,
|
|
5063
|
+
waitingFor: parentId ?? "root"
|
|
5064
|
+
});
|
|
5065
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5066
|
+
}
|
|
5067
|
+
break;
|
|
5068
|
+
}
|
|
5069
|
+
case "handleSpanStart": {
|
|
5070
|
+
this.logger.debug(`${this.name}: handling span start`, {
|
|
5071
|
+
traceId: exportedSpan.traceId,
|
|
5072
|
+
spanId: exportedSpan.id
|
|
5073
|
+
});
|
|
5074
|
+
traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
|
|
5075
|
+
const spanData = await this._buildSpan({ span: exportedSpan, traceData });
|
|
5076
|
+
if (spanData) {
|
|
5077
|
+
this.logger.debug(`${this.name}: adding span to traceData`, {
|
|
5078
|
+
traceId: exportedSpan.traceId,
|
|
5079
|
+
spanId: exportedSpan.id
|
|
5080
|
+
});
|
|
5081
|
+
traceData.addSpan({ spanId: exportedSpan.id, spanData });
|
|
5082
|
+
if (exportedSpan.isRootSpan) {
|
|
5083
|
+
traceData.markRootSpanProcessed();
|
|
5084
|
+
this.#scheduleProcessWaitingForRoot(traceId);
|
|
5085
|
+
}
|
|
5086
|
+
this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
|
|
5087
|
+
} else {
|
|
5088
|
+
const parentId = exportedSpan.parentSpanId;
|
|
5089
|
+
this.logger.debug(`${this.name}: adding span to waiting queue`, {
|
|
5090
|
+
traceId: exportedSpan.traceId,
|
|
5091
|
+
waitingFor: parentId ?? "root"
|
|
5092
|
+
});
|
|
5093
|
+
traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
|
|
5094
|
+
}
|
|
5095
|
+
break;
|
|
5096
|
+
}
|
|
5097
|
+
case "handleSpanUpdate":
|
|
5098
|
+
this.logger.debug(`${this.name}: handling span update`, {
|
|
5099
|
+
traceId: exportedSpan.traceId,
|
|
5100
|
+
spanId: exportedSpan.id
|
|
5101
|
+
});
|
|
5102
|
+
await this._updateSpan({ span: exportedSpan, traceData });
|
|
5103
|
+
break;
|
|
5104
|
+
case "handleSpanEnd":
|
|
5105
|
+
this.logger.debug(`${this.name}: handling span end`, {
|
|
5106
|
+
traceId: exportedSpan.traceId,
|
|
5107
|
+
spanId: exportedSpan.id
|
|
5108
|
+
});
|
|
5109
|
+
traceData.endSpan({ spanId: exportedSpan.id });
|
|
5110
|
+
await this._finishSpan({ span: exportedSpan, traceData });
|
|
5111
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5112
|
+
this.#scheduleCleanup(traceId);
|
|
5113
|
+
}
|
|
5114
|
+
break;
|
|
5115
|
+
}
|
|
5116
|
+
} catch (error) {
|
|
5117
|
+
this.logger.error(`${this.name}: exporter error`, { error, event, method });
|
|
5118
|
+
}
|
|
5119
|
+
if (traceData.activeSpanCount() === 0) {
|
|
5120
|
+
this.#scheduleCleanup(traceId);
|
|
5121
|
+
}
|
|
5122
|
+
await this._postExportTracingEvent();
|
|
5123
|
+
}
|
|
5124
|
+
// ============================================================================
|
|
5125
|
+
// Protected Helpers
|
|
5126
|
+
// ============================================================================
|
|
5127
|
+
/**
|
|
5128
|
+
* Get or create the TraceData container for a trace.
|
|
5129
|
+
* Also cancels any pending cleanup since new data has arrived.
|
|
5130
|
+
*
|
|
5131
|
+
* @param args.traceId - The trace ID
|
|
5132
|
+
* @param args.method - The calling method name (for logging)
|
|
5133
|
+
* @returns The TraceData container for this trace
|
|
5134
|
+
*/
|
|
5135
|
+
getTraceData(args) {
|
|
5136
|
+
const { traceId, method } = args;
|
|
5137
|
+
this.#cancelScheduledCleanup(traceId);
|
|
5138
|
+
if (!this.#traceMap.has(traceId)) {
|
|
5139
|
+
this.#traceMap.set(traceId, new TraceData());
|
|
5140
|
+
this.logger.debug(`${this.name}: Created new trace data cache`, {
|
|
5141
|
+
traceId,
|
|
5142
|
+
method
|
|
5143
|
+
});
|
|
5144
|
+
this.#enforceHardCap().catch((error) => {
|
|
5145
|
+
this.logger.error(`${this.name}: Error enforcing hard cap`, { error });
|
|
5146
|
+
});
|
|
5147
|
+
}
|
|
5148
|
+
return this.#traceMap.get(traceId);
|
|
5149
|
+
}
|
|
5150
|
+
/**
|
|
5151
|
+
* Get the current number of traces being tracked.
|
|
5152
|
+
* @returns The trace count
|
|
5153
|
+
*/
|
|
5154
|
+
traceMapSize() {
|
|
5155
|
+
return this.#traceMap.size;
|
|
5156
|
+
}
|
|
5157
|
+
// ============================================================================
|
|
5158
|
+
// Flush and Shutdown Hooks (Override in subclass as needed)
|
|
5159
|
+
// ============================================================================
|
|
5160
|
+
/**
|
|
5161
|
+
* Hook called by flush() to perform vendor-specific flush logic.
|
|
5162
|
+
* Override to send buffered data to the vendor's API.
|
|
5163
|
+
*
|
|
5164
|
+
* Unlike _postShutdown(), this method should NOT release resources,
|
|
5165
|
+
* as the exporter will continue to be used after flushing.
|
|
5166
|
+
*/
|
|
5167
|
+
async _flush() {
|
|
5168
|
+
}
|
|
5169
|
+
/**
|
|
5170
|
+
* Force flush any buffered data without shutting down the exporter.
|
|
5171
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5172
|
+
* are exported before the runtime instance is terminated.
|
|
5173
|
+
*
|
|
5174
|
+
* Subclasses should override _flush() to implement vendor-specific flush logic.
|
|
5175
|
+
*/
|
|
5176
|
+
async flush() {
|
|
5177
|
+
if (this.isDisabled) {
|
|
5178
|
+
return;
|
|
5179
|
+
}
|
|
5180
|
+
this.logger.debug(`${this.name}: Flushing`);
|
|
5181
|
+
await this._flush();
|
|
5182
|
+
}
|
|
5183
|
+
/**
|
|
5184
|
+
* Hook called at the start of shutdown, before cancelling timers and aborting spans.
|
|
5185
|
+
* Override to perform vendor-specific pre-shutdown tasks.
|
|
5186
|
+
*/
|
|
5187
|
+
async _preShutdown() {
|
|
5188
|
+
}
|
|
5189
|
+
/**
|
|
5190
|
+
* Hook called at the end of shutdown, after all spans are aborted.
|
|
5191
|
+
* Override to perform vendor-specific cleanup (e.g., flushing).
|
|
5192
|
+
*/
|
|
5193
|
+
async _postShutdown() {
|
|
5194
|
+
}
|
|
5195
|
+
/**
|
|
5196
|
+
* Gracefully shut down the exporter.
|
|
5197
|
+
* Cancels all pending cleanup timers, aborts all active spans, and clears state.
|
|
5198
|
+
*/
|
|
5199
|
+
async shutdown() {
|
|
5200
|
+
if (this.isDisabled) {
|
|
5201
|
+
return;
|
|
5202
|
+
}
|
|
5203
|
+
this.#shutdownStarted = true;
|
|
5204
|
+
await this._preShutdown();
|
|
5205
|
+
for (const [traceId, timeout] of this.#pendingCleanups) {
|
|
5206
|
+
clearTimeout(timeout);
|
|
5207
|
+
this.logger.debug(`${this.name}: Cancelled pending cleanup on shutdown`, { traceId });
|
|
5208
|
+
}
|
|
5209
|
+
this.#pendingCleanups.clear();
|
|
5210
|
+
const reason = {
|
|
5211
|
+
id: "SHUTDOWN",
|
|
5212
|
+
message: "Observability is shutting down.",
|
|
5213
|
+
domain: "MASTRA_OBSERVABILITY",
|
|
5214
|
+
category: "SYSTEM"
|
|
5215
|
+
};
|
|
5216
|
+
for (const [traceId, traceData] of this.#traceMap) {
|
|
5217
|
+
const orphanedEvents = traceData.getAllQueuedEvents();
|
|
5218
|
+
if (orphanedEvents.length > 0) {
|
|
5219
|
+
this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on shutdown`, {
|
|
5220
|
+
traceId,
|
|
5221
|
+
orphanedEvents: orphanedEvents.map((e) => ({
|
|
5222
|
+
spanId: e.event.exportedSpan.id,
|
|
5223
|
+
waitingFor: e.waitingFor,
|
|
5224
|
+
attempts: e.attempts
|
|
5225
|
+
}))
|
|
5226
|
+
});
|
|
5227
|
+
}
|
|
5228
|
+
for (const spanId of traceData.activeSpanIds) {
|
|
5229
|
+
const span = traceData.getSpan({ spanId });
|
|
5230
|
+
if (span) {
|
|
5231
|
+
await this._abortSpan({ span, traceData, reason });
|
|
5232
|
+
}
|
|
5233
|
+
}
|
|
5234
|
+
}
|
|
5235
|
+
this.#traceMap.clear();
|
|
5236
|
+
await this._postShutdown();
|
|
5237
|
+
await super.shutdown();
|
|
5238
|
+
}
|
|
5239
|
+
};
|
|
5240
|
+
|
|
5241
|
+
// src/exporters/span-formatters.ts
|
|
5242
|
+
function chainFormatters(formatters) {
|
|
5243
|
+
return async (span) => {
|
|
5244
|
+
let currentSpan = span;
|
|
5245
|
+
for (const formatter of formatters) {
|
|
5246
|
+
currentSpan = await formatter(currentSpan);
|
|
5247
|
+
}
|
|
5248
|
+
return currentSpan;
|
|
5249
|
+
};
|
|
5250
|
+
}
|
|
5251
|
+
var CloudExporter = class extends BaseExporter {
|
|
5252
|
+
name = "mastra-cloud-observability-exporter";
|
|
5253
|
+
cloudConfig;
|
|
5254
|
+
buffer;
|
|
5255
|
+
flushTimer = null;
|
|
5256
|
+
constructor(config = {}) {
|
|
5257
|
+
super(config);
|
|
5258
|
+
const accessToken = config.accessToken ?? process.env.MASTRA_CLOUD_ACCESS_TOKEN;
|
|
5259
|
+
if (!accessToken) {
|
|
5260
|
+
this.setDisabled("MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.");
|
|
5261
|
+
}
|
|
5262
|
+
const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
|
|
5263
|
+
this.cloudConfig = {
|
|
5264
|
+
logger: this.logger,
|
|
5265
|
+
logLevel: config.logLevel ?? LogLevel.INFO,
|
|
5266
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5267
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5268
|
+
maxRetries: config.maxRetries ?? 3,
|
|
5269
|
+
accessToken: accessToken || "",
|
|
5270
|
+
endpoint
|
|
5271
|
+
};
|
|
5272
|
+
this.buffer = {
|
|
5273
|
+
spans: [],
|
|
5274
|
+
totalSize: 0
|
|
5275
|
+
};
|
|
5276
|
+
}
|
|
5277
|
+
async _exportTracingEvent(event) {
|
|
5278
|
+
if (event.type !== TracingEventType.SPAN_ENDED) {
|
|
5279
|
+
return;
|
|
5280
|
+
}
|
|
5281
|
+
this.addToBuffer(event);
|
|
5282
|
+
if (this.shouldFlush()) {
|
|
5283
|
+
this.flush().catch((error) => {
|
|
5284
|
+
this.logger.error("Batch flush failed", {
|
|
5285
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5286
|
+
});
|
|
5287
|
+
});
|
|
5288
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5289
|
+
this.scheduleFlush();
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
addToBuffer(event) {
|
|
5293
|
+
if (this.buffer.totalSize === 0) {
|
|
5294
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5295
|
+
}
|
|
5296
|
+
const spanRecord = this.formatSpan(event.exportedSpan);
|
|
5297
|
+
this.buffer.spans.push(spanRecord);
|
|
5298
|
+
this.buffer.totalSize++;
|
|
5299
|
+
}
|
|
5300
|
+
formatSpan(span) {
|
|
5301
|
+
const spanRecord = {
|
|
5302
|
+
traceId: span.traceId,
|
|
5303
|
+
spanId: span.id,
|
|
5304
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5305
|
+
name: span.name,
|
|
5306
|
+
spanType: span.type,
|
|
5307
|
+
attributes: span.attributes ?? null,
|
|
5308
|
+
metadata: span.metadata ?? null,
|
|
5309
|
+
startedAt: span.startTime,
|
|
5310
|
+
endedAt: span.endTime ?? null,
|
|
5311
|
+
input: span.input ?? null,
|
|
5312
|
+
output: span.output ?? null,
|
|
5313
|
+
error: span.errorInfo,
|
|
5314
|
+
isEvent: span.isEvent,
|
|
5315
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
5316
|
+
updatedAt: null
|
|
5317
|
+
};
|
|
5318
|
+
return spanRecord;
|
|
5319
|
+
}
|
|
5320
|
+
shouldFlush() {
|
|
5321
|
+
if (this.buffer.totalSize >= this.cloudConfig.maxBatchSize) {
|
|
5322
|
+
return true;
|
|
5323
|
+
}
|
|
5324
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5325
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5326
|
+
if (elapsed >= this.cloudConfig.maxBatchWaitMs) {
|
|
5327
|
+
return true;
|
|
5328
|
+
}
|
|
5329
|
+
}
|
|
5330
|
+
return false;
|
|
5331
|
+
}
|
|
5332
|
+
scheduleFlush() {
|
|
5333
|
+
if (this.flushTimer) {
|
|
5334
|
+
clearTimeout(this.flushTimer);
|
|
5335
|
+
}
|
|
5336
|
+
this.flushTimer = setTimeout(() => {
|
|
5337
|
+
this.flush().catch((error) => {
|
|
5338
|
+
const mastraError = new MastraError(
|
|
5339
|
+
{
|
|
5340
|
+
id: `CLOUD_EXPORTER_FAILED_TO_SCHEDULE_FLUSH`,
|
|
5341
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5342
|
+
category: ErrorCategory.USER
|
|
5343
|
+
},
|
|
5344
|
+
error
|
|
5345
|
+
);
|
|
5346
|
+
this.logger.trackException(mastraError);
|
|
5347
|
+
this.logger.error("Scheduled flush failed", mastraError);
|
|
5348
|
+
});
|
|
5349
|
+
}, this.cloudConfig.maxBatchWaitMs);
|
|
5350
|
+
}
|
|
5351
|
+
async flushBuffer() {
|
|
5352
|
+
if (this.flushTimer) {
|
|
5353
|
+
clearTimeout(this.flushTimer);
|
|
5354
|
+
this.flushTimer = null;
|
|
5355
|
+
}
|
|
5356
|
+
if (this.buffer.totalSize === 0) {
|
|
5357
|
+
return;
|
|
5358
|
+
}
|
|
5359
|
+
const startTime = Date.now();
|
|
5360
|
+
const spansCopy = [...this.buffer.spans];
|
|
5361
|
+
const flushReason = this.buffer.totalSize >= this.cloudConfig.maxBatchSize ? "size" : "time";
|
|
5362
|
+
this.resetBuffer();
|
|
5363
|
+
try {
|
|
5364
|
+
await this.batchUpload(spansCopy);
|
|
5365
|
+
const elapsed = Date.now() - startTime;
|
|
5366
|
+
this.logger.debug("Batch flushed successfully", {
|
|
5367
|
+
batchSize: spansCopy.length,
|
|
5368
|
+
flushReason,
|
|
5369
|
+
durationMs: elapsed
|
|
5370
|
+
});
|
|
5371
|
+
} catch (error) {
|
|
5372
|
+
const mastraError = new MastraError(
|
|
5373
|
+
{
|
|
5374
|
+
id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD`,
|
|
5375
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5376
|
+
category: ErrorCategory.USER,
|
|
5377
|
+
details: {
|
|
5378
|
+
droppedBatchSize: spansCopy.length
|
|
5379
|
+
}
|
|
5380
|
+
},
|
|
5381
|
+
error
|
|
5382
|
+
);
|
|
5383
|
+
this.logger.trackException(mastraError);
|
|
5384
|
+
this.logger.error("Batch upload failed after all retries, dropping batch", mastraError);
|
|
5385
|
+
}
|
|
5386
|
+
}
|
|
5387
|
+
/**
|
|
5388
|
+
* Uploads spans to cloud API using fetchWithRetry for all retry logic
|
|
5389
|
+
*/
|
|
5390
|
+
async batchUpload(spans) {
|
|
5391
|
+
const headers = {
|
|
5392
|
+
Authorization: `Bearer ${this.cloudConfig.accessToken}`,
|
|
5393
|
+
"Content-Type": "application/json"
|
|
5394
|
+
};
|
|
5395
|
+
const options = {
|
|
5396
|
+
method: "POST",
|
|
5397
|
+
headers,
|
|
5398
|
+
body: JSON.stringify({ spans })
|
|
5399
|
+
};
|
|
5400
|
+
await fetchWithRetry(this.cloudConfig.endpoint, options, this.cloudConfig.maxRetries);
|
|
5401
|
+
}
|
|
5402
|
+
resetBuffer() {
|
|
5403
|
+
this.buffer.spans = [];
|
|
5404
|
+
this.buffer.firstEventTime = void 0;
|
|
5405
|
+
this.buffer.totalSize = 0;
|
|
5406
|
+
}
|
|
5407
|
+
/**
|
|
5408
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
5409
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
5410
|
+
* are exported before the runtime instance is terminated.
|
|
5411
|
+
*/
|
|
5412
|
+
async flush() {
|
|
5413
|
+
if (this.isDisabled) {
|
|
5414
|
+
return;
|
|
5415
|
+
}
|
|
5416
|
+
if (this.buffer.totalSize > 0) {
|
|
5417
|
+
this.logger.debug("Flushing buffered events", {
|
|
5418
|
+
bufferedEvents: this.buffer.totalSize
|
|
5419
|
+
});
|
|
5420
|
+
await this.flushBuffer();
|
|
5421
|
+
}
|
|
5422
|
+
}
|
|
5423
|
+
async shutdown() {
|
|
5424
|
+
if (this.isDisabled) {
|
|
5425
|
+
return;
|
|
5426
|
+
}
|
|
5427
|
+
if (this.flushTimer) {
|
|
5428
|
+
clearTimeout(this.flushTimer);
|
|
5429
|
+
this.flushTimer = null;
|
|
5430
|
+
}
|
|
5431
|
+
try {
|
|
5432
|
+
await this.flush();
|
|
5433
|
+
} catch (error) {
|
|
5434
|
+
const mastraError = new MastraError(
|
|
5435
|
+
{
|
|
5436
|
+
id: `CLOUD_EXPORTER_FAILED_TO_FLUSH_REMAINING_EVENTS_DURING_SHUTDOWN`,
|
|
5437
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
5438
|
+
category: ErrorCategory.USER,
|
|
5439
|
+
details: {
|
|
5440
|
+
remainingEvents: this.buffer.totalSize
|
|
5441
|
+
}
|
|
5442
|
+
},
|
|
5443
|
+
error
|
|
5444
|
+
);
|
|
5445
|
+
this.logger.trackException(mastraError);
|
|
5446
|
+
this.logger.error("Failed to flush remaining events during shutdown", mastraError);
|
|
5447
|
+
}
|
|
5448
|
+
this.logger.info("CloudExporter shutdown complete");
|
|
5449
|
+
}
|
|
5450
|
+
};
|
|
5451
|
+
var ConsoleExporter = class extends BaseExporter {
|
|
5452
|
+
name = "tracing-console-exporter";
|
|
5453
|
+
constructor(config = {}) {
|
|
5454
|
+
super(config);
|
|
5455
|
+
}
|
|
5456
|
+
async _exportTracingEvent(event) {
|
|
5457
|
+
const span = event.exportedSpan;
|
|
5458
|
+
const formatAttributes = (attributes) => {
|
|
5459
|
+
try {
|
|
5460
|
+
return JSON.stringify(attributes, null, 2);
|
|
5461
|
+
} catch (error) {
|
|
5462
|
+
const errMsg = error instanceof Error ? error.message : "Unknown formatting error";
|
|
5463
|
+
return `[Unable to serialize attributes: ${errMsg}]`;
|
|
5464
|
+
}
|
|
5465
|
+
};
|
|
5466
|
+
const formatDuration = (startTime, endTime) => {
|
|
5467
|
+
if (!endTime) return "N/A";
|
|
5468
|
+
const duration = endTime.getTime() - startTime.getTime();
|
|
5469
|
+
return `${duration}ms`;
|
|
5470
|
+
};
|
|
5471
|
+
switch (event.type) {
|
|
5472
|
+
case TracingEventType.SPAN_STARTED:
|
|
5473
|
+
this.logger.info(`\u{1F680} SPAN_STARTED`);
|
|
5474
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5475
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5476
|
+
this.logger.info(` ID: ${span.id}`);
|
|
5477
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5478
|
+
if (span.input !== void 0) {
|
|
5479
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5480
|
+
}
|
|
5481
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5482
|
+
this.logger.info("\u2500".repeat(80));
|
|
5483
|
+
break;
|
|
5484
|
+
case TracingEventType.SPAN_ENDED:
|
|
5485
|
+
const duration = formatDuration(span.startTime, span.endTime);
|
|
4446
5486
|
this.logger.info(`\u2705 SPAN_ENDED`);
|
|
4447
5487
|
this.logger.info(` Type: ${span.type}`);
|
|
4448
5488
|
this.logger.info(` Name: ${span.name}`);
|
|
@@ -4452,542 +5492,1394 @@ var ConsoleExporter = class extends BaseExporter {
|
|
|
4452
5492
|
if (span.input !== void 0) {
|
|
4453
5493
|
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
4454
5494
|
}
|
|
4455
|
-
if (span.output !== void 0) {
|
|
4456
|
-
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5495
|
+
if (span.output !== void 0) {
|
|
5496
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5497
|
+
}
|
|
5498
|
+
if (span.errorInfo) {
|
|
5499
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5500
|
+
}
|
|
5501
|
+
this.logger.info(` Attributes: ${formatAttributes(span.attributes)}`);
|
|
5502
|
+
this.logger.info("\u2500".repeat(80));
|
|
5503
|
+
break;
|
|
5504
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5505
|
+
this.logger.info(`\u{1F4DD} SPAN_UPDATED`);
|
|
5506
|
+
this.logger.info(` Type: ${span.type}`);
|
|
5507
|
+
this.logger.info(` Name: ${span.name}`);
|
|
5508
|
+
this.logger.info(` ID: ${span.id}`);
|
|
5509
|
+
this.logger.info(` Trace ID: ${span.traceId}`);
|
|
5510
|
+
if (span.input !== void 0) {
|
|
5511
|
+
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5512
|
+
}
|
|
5513
|
+
if (span.output !== void 0) {
|
|
5514
|
+
this.logger.info(` Output: ${formatAttributes(span.output)}`);
|
|
5515
|
+
}
|
|
5516
|
+
if (span.errorInfo) {
|
|
5517
|
+
this.logger.info(` Error: ${formatAttributes(span.errorInfo)}`);
|
|
5518
|
+
}
|
|
5519
|
+
this.logger.info(` Updated Attributes: ${formatAttributes(span.attributes)}`);
|
|
5520
|
+
this.logger.info("\u2500".repeat(80));
|
|
5521
|
+
break;
|
|
5522
|
+
default:
|
|
5523
|
+
this.logger.warn(`Tracing event type not implemented: ${event.type}`);
|
|
5524
|
+
}
|
|
5525
|
+
}
|
|
5526
|
+
async shutdown() {
|
|
5527
|
+
this.logger.info("ConsoleExporter shutdown");
|
|
5528
|
+
}
|
|
5529
|
+
};
|
|
5530
|
+
function resolveTracingStorageStrategy(config, observability, storageName, logger) {
|
|
5531
|
+
if (config.strategy && config.strategy !== "auto") {
|
|
5532
|
+
const hints = observability.tracingStrategy;
|
|
5533
|
+
if (hints.supported.includes(config.strategy)) {
|
|
5534
|
+
return config.strategy;
|
|
5535
|
+
}
|
|
5536
|
+
logger.warn("User-specified tracing strategy not supported by storage adapter, falling back to auto-selection", {
|
|
5537
|
+
userStrategy: config.strategy,
|
|
5538
|
+
storageAdapter: storageName,
|
|
5539
|
+
supportedStrategies: hints.supported,
|
|
5540
|
+
fallbackStrategy: hints.preferred
|
|
5541
|
+
});
|
|
5542
|
+
}
|
|
5543
|
+
return observability.tracingStrategy.preferred;
|
|
5544
|
+
}
|
|
5545
|
+
function getStringOrNull(value) {
|
|
5546
|
+
return typeof value === "string" ? value : null;
|
|
5547
|
+
}
|
|
5548
|
+
function getObjectOrNull(value) {
|
|
5549
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
5550
|
+
}
|
|
5551
|
+
var DefaultExporter = class extends BaseExporter {
|
|
5552
|
+
name = "mastra-default-observability-exporter";
|
|
5553
|
+
#storage;
|
|
5554
|
+
#observability;
|
|
5555
|
+
#config;
|
|
5556
|
+
#resolvedStrategy;
|
|
5557
|
+
buffer;
|
|
5558
|
+
#flushTimer = null;
|
|
5559
|
+
// Track all spans that have been created, persists across flushes
|
|
5560
|
+
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
5561
|
+
constructor(config = {}) {
|
|
5562
|
+
super(config);
|
|
5563
|
+
if (config === void 0) {
|
|
5564
|
+
config = {};
|
|
5565
|
+
}
|
|
5566
|
+
this.#config = {
|
|
5567
|
+
...config,
|
|
5568
|
+
maxBatchSize: config.maxBatchSize ?? 1e3,
|
|
5569
|
+
maxBufferSize: config.maxBufferSize ?? 1e4,
|
|
5570
|
+
maxBatchWaitMs: config.maxBatchWaitMs ?? 5e3,
|
|
5571
|
+
maxRetries: config.maxRetries ?? 4,
|
|
5572
|
+
retryDelayMs: config.retryDelayMs ?? 500,
|
|
5573
|
+
strategy: config.strategy ?? "auto"
|
|
5574
|
+
};
|
|
5575
|
+
this.buffer = {
|
|
5576
|
+
creates: [],
|
|
5577
|
+
updates: [],
|
|
5578
|
+
insertOnly: [],
|
|
5579
|
+
seenSpans: /* @__PURE__ */ new Set(),
|
|
5580
|
+
spanSequences: /* @__PURE__ */ new Map(),
|
|
5581
|
+
completedSpans: /* @__PURE__ */ new Set(),
|
|
5582
|
+
outOfOrderCount: 0,
|
|
5583
|
+
totalSize: 0
|
|
5584
|
+
};
|
|
5585
|
+
this.#resolvedStrategy = "batch-with-updates";
|
|
5586
|
+
}
|
|
5587
|
+
#strategyInitialized = false;
|
|
5588
|
+
/**
|
|
5589
|
+
* Initialize the exporter (called after all dependencies are ready)
|
|
5590
|
+
*/
|
|
5591
|
+
async init(options) {
|
|
5592
|
+
this.#storage = options.mastra?.getStorage();
|
|
5593
|
+
if (!this.#storage) {
|
|
5594
|
+
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
5595
|
+
return;
|
|
5596
|
+
}
|
|
5597
|
+
this.#observability = await this.#storage.getStore("observability");
|
|
5598
|
+
if (!this.#observability) {
|
|
5599
|
+
this.logger.warn("DefaultExporter disabled: Observability storage not available. Traces will not be persisted.");
|
|
5600
|
+
return;
|
|
5601
|
+
}
|
|
5602
|
+
this.initializeStrategy(this.#observability, this.#storage.constructor.name);
|
|
5603
|
+
}
|
|
5604
|
+
/**
|
|
5605
|
+
* Initialize the resolved strategy once observability store is available
|
|
5606
|
+
*/
|
|
5607
|
+
initializeStrategy(observability, storageName) {
|
|
5608
|
+
if (this.#strategyInitialized) return;
|
|
5609
|
+
this.#resolvedStrategy = resolveTracingStorageStrategy(this.#config, observability, storageName, this.logger);
|
|
5610
|
+
this.#strategyInitialized = true;
|
|
5611
|
+
this.logger.debug("tracing storage exporter initialized", {
|
|
5612
|
+
strategy: this.#resolvedStrategy,
|
|
5613
|
+
source: this.#config.strategy !== "auto" ? "user" : "auto",
|
|
5614
|
+
storageAdapter: storageName,
|
|
5615
|
+
maxBatchSize: this.#config.maxBatchSize,
|
|
5616
|
+
maxBatchWaitMs: this.#config.maxBatchWaitMs
|
|
5617
|
+
});
|
|
5618
|
+
}
|
|
5619
|
+
/**
|
|
5620
|
+
* Builds a unique span key for tracking
|
|
5621
|
+
*/
|
|
5622
|
+
buildSpanKey(traceId, spanId) {
|
|
5623
|
+
return `${traceId}:${spanId}`;
|
|
5624
|
+
}
|
|
5625
|
+
/**
|
|
5626
|
+
* Gets the next sequence number for a span
|
|
5627
|
+
*/
|
|
5628
|
+
getNextSequence(spanKey) {
|
|
5629
|
+
const current = this.buffer.spanSequences.get(spanKey) || 0;
|
|
5630
|
+
const next = current + 1;
|
|
5631
|
+
this.buffer.spanSequences.set(spanKey, next);
|
|
5632
|
+
return next;
|
|
5633
|
+
}
|
|
5634
|
+
/**
|
|
5635
|
+
* Handles out-of-order span updates by logging and skipping
|
|
5636
|
+
*/
|
|
5637
|
+
handleOutOfOrderUpdate(event) {
|
|
5638
|
+
this.logger.warn("Out-of-order span update detected - skipping event", {
|
|
5639
|
+
spanId: event.exportedSpan.id,
|
|
5640
|
+
traceId: event.exportedSpan.traceId,
|
|
5641
|
+
spanName: event.exportedSpan.name,
|
|
5642
|
+
eventType: event.type
|
|
5643
|
+
});
|
|
5644
|
+
}
|
|
5645
|
+
/**
|
|
5646
|
+
* Adds an event to the appropriate buffer based on strategy
|
|
5647
|
+
*/
|
|
5648
|
+
addToBuffer(event) {
|
|
5649
|
+
const spanKey = this.buildSpanKey(event.exportedSpan.traceId, event.exportedSpan.id);
|
|
5650
|
+
if (this.buffer.totalSize === 0) {
|
|
5651
|
+
this.buffer.firstEventTime = /* @__PURE__ */ new Date();
|
|
5652
|
+
}
|
|
5653
|
+
switch (event.type) {
|
|
5654
|
+
case TracingEventType.SPAN_STARTED:
|
|
5655
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5656
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5657
|
+
this.buffer.creates.push(createRecord);
|
|
5658
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5659
|
+
this.allCreatedSpans.add(spanKey);
|
|
5660
|
+
}
|
|
5661
|
+
break;
|
|
5662
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5663
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5664
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5665
|
+
this.buffer.updates.push({
|
|
5666
|
+
traceId: event.exportedSpan.traceId,
|
|
5667
|
+
spanId: event.exportedSpan.id,
|
|
5668
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5669
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5670
|
+
});
|
|
5671
|
+
} else {
|
|
5672
|
+
this.handleOutOfOrderUpdate(event);
|
|
5673
|
+
this.buffer.outOfOrderCount++;
|
|
5674
|
+
}
|
|
5675
|
+
}
|
|
5676
|
+
break;
|
|
5677
|
+
case TracingEventType.SPAN_ENDED:
|
|
5678
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5679
|
+
if (this.allCreatedSpans.has(spanKey)) {
|
|
5680
|
+
this.buffer.updates.push({
|
|
5681
|
+
traceId: event.exportedSpan.traceId,
|
|
5682
|
+
spanId: event.exportedSpan.id,
|
|
5683
|
+
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
5684
|
+
sequenceNumber: this.getNextSequence(spanKey)
|
|
5685
|
+
});
|
|
5686
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5687
|
+
} else if (event.exportedSpan.isEvent) {
|
|
5688
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5689
|
+
this.buffer.creates.push(createRecord);
|
|
5690
|
+
this.buffer.seenSpans.add(spanKey);
|
|
5691
|
+
this.allCreatedSpans.add(spanKey);
|
|
5692
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5693
|
+
} else {
|
|
5694
|
+
this.handleOutOfOrderUpdate(event);
|
|
5695
|
+
this.buffer.outOfOrderCount++;
|
|
5696
|
+
}
|
|
5697
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5698
|
+
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
5699
|
+
this.buffer.insertOnly.push(createRecord);
|
|
5700
|
+
this.buffer.completedSpans.add(spanKey);
|
|
5701
|
+
this.allCreatedSpans.add(spanKey);
|
|
4457
5702
|
}
|
|
4458
|
-
|
|
4459
|
-
|
|
5703
|
+
break;
|
|
5704
|
+
}
|
|
5705
|
+
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
5706
|
+
}
|
|
5707
|
+
/**
|
|
5708
|
+
* Checks if buffer should be flushed based on size or time triggers
|
|
5709
|
+
*/
|
|
5710
|
+
shouldFlush() {
|
|
5711
|
+
if (this.buffer.totalSize >= this.#config.maxBufferSize) {
|
|
5712
|
+
return true;
|
|
5713
|
+
}
|
|
5714
|
+
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
5715
|
+
return true;
|
|
5716
|
+
}
|
|
5717
|
+
if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
|
|
5718
|
+
const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
|
|
5719
|
+
if (elapsed >= this.#config.maxBatchWaitMs) {
|
|
5720
|
+
return true;
|
|
5721
|
+
}
|
|
5722
|
+
}
|
|
5723
|
+
return false;
|
|
5724
|
+
}
|
|
5725
|
+
/**
|
|
5726
|
+
* Resets the buffer after successful flush
|
|
5727
|
+
*/
|
|
5728
|
+
resetBuffer(completedSpansToCleanup = /* @__PURE__ */ new Set()) {
|
|
5729
|
+
this.buffer.creates = [];
|
|
5730
|
+
this.buffer.updates = [];
|
|
5731
|
+
this.buffer.insertOnly = [];
|
|
5732
|
+
this.buffer.seenSpans.clear();
|
|
5733
|
+
this.buffer.spanSequences.clear();
|
|
5734
|
+
this.buffer.completedSpans.clear();
|
|
5735
|
+
this.buffer.outOfOrderCount = 0;
|
|
5736
|
+
this.buffer.firstEventTime = void 0;
|
|
5737
|
+
this.buffer.totalSize = 0;
|
|
5738
|
+
for (const spanKey of completedSpansToCleanup) {
|
|
5739
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5740
|
+
}
|
|
5741
|
+
}
|
|
5742
|
+
/**
|
|
5743
|
+
* Schedules a flush using setTimeout
|
|
5744
|
+
*/
|
|
5745
|
+
scheduleFlush() {
|
|
5746
|
+
if (this.#flushTimer) {
|
|
5747
|
+
clearTimeout(this.#flushTimer);
|
|
5748
|
+
}
|
|
5749
|
+
this.#flushTimer = setTimeout(() => {
|
|
5750
|
+
this.flushBuffer().catch((error) => {
|
|
5751
|
+
this.logger.error("Scheduled flush failed", {
|
|
5752
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5753
|
+
});
|
|
5754
|
+
});
|
|
5755
|
+
}, this.#config.maxBatchWaitMs);
|
|
5756
|
+
}
|
|
5757
|
+
/**
|
|
5758
|
+
* Serializes span attributes to storage record format
|
|
5759
|
+
* Handles all Span types and their specific attributes
|
|
5760
|
+
*/
|
|
5761
|
+
serializeAttributes(span) {
|
|
5762
|
+
if (!span.attributes) {
|
|
5763
|
+
return null;
|
|
5764
|
+
}
|
|
5765
|
+
try {
|
|
5766
|
+
return JSON.parse(
|
|
5767
|
+
JSON.stringify(span.attributes, (_key, value) => {
|
|
5768
|
+
if (value instanceof Date) {
|
|
5769
|
+
return value.toISOString();
|
|
5770
|
+
}
|
|
5771
|
+
if (typeof value === "object" && value !== null) {
|
|
5772
|
+
return value;
|
|
5773
|
+
}
|
|
5774
|
+
return value;
|
|
5775
|
+
})
|
|
5776
|
+
);
|
|
5777
|
+
} catch (error) {
|
|
5778
|
+
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
5779
|
+
spanId: span.id,
|
|
5780
|
+
spanType: span.type,
|
|
5781
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5782
|
+
});
|
|
5783
|
+
return null;
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
buildCreateRecord(span) {
|
|
5787
|
+
const metadata = span.metadata ?? {};
|
|
5788
|
+
return {
|
|
5789
|
+
traceId: span.traceId,
|
|
5790
|
+
spanId: span.id,
|
|
5791
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
5792
|
+
name: span.name,
|
|
5793
|
+
// Entity identification - from span
|
|
5794
|
+
entityType: span.entityType ?? null,
|
|
5795
|
+
entityId: span.entityId ?? null,
|
|
5796
|
+
entityName: span.entityName ?? null,
|
|
5797
|
+
// Identity & Tenancy - extracted from metadata if present
|
|
5798
|
+
userId: getStringOrNull(metadata.userId),
|
|
5799
|
+
organizationId: getStringOrNull(metadata.organizationId),
|
|
5800
|
+
resourceId: getStringOrNull(metadata.resourceId),
|
|
5801
|
+
// Correlation IDs - extracted from metadata if present
|
|
5802
|
+
runId: getStringOrNull(metadata.runId),
|
|
5803
|
+
sessionId: getStringOrNull(metadata.sessionId),
|
|
5804
|
+
threadId: getStringOrNull(metadata.threadId),
|
|
5805
|
+
requestId: getStringOrNull(metadata.requestId),
|
|
5806
|
+
// Deployment context - extracted from metadata if present
|
|
5807
|
+
environment: getStringOrNull(metadata.environment),
|
|
5808
|
+
source: getStringOrNull(metadata.source),
|
|
5809
|
+
serviceName: getStringOrNull(metadata.serviceName),
|
|
5810
|
+
scope: getObjectOrNull(metadata.scope),
|
|
5811
|
+
// Span data
|
|
5812
|
+
spanType: span.type,
|
|
5813
|
+
attributes: this.serializeAttributes(span),
|
|
5814
|
+
metadata: span.metadata ?? null,
|
|
5815
|
+
// Keep all metadata including extracted fields
|
|
5816
|
+
tags: span.tags ?? null,
|
|
5817
|
+
links: null,
|
|
5818
|
+
input: span.input ?? null,
|
|
5819
|
+
output: span.output ?? null,
|
|
5820
|
+
error: span.errorInfo ?? null,
|
|
5821
|
+
isEvent: span.isEvent,
|
|
5822
|
+
// Timestamps
|
|
5823
|
+
startedAt: span.startTime,
|
|
5824
|
+
endedAt: span.endTime ?? null
|
|
5825
|
+
};
|
|
5826
|
+
}
|
|
5827
|
+
buildUpdateRecord(span) {
|
|
5828
|
+
return {
|
|
5829
|
+
name: span.name,
|
|
5830
|
+
scope: null,
|
|
5831
|
+
attributes: this.serializeAttributes(span),
|
|
5832
|
+
metadata: span.metadata ?? null,
|
|
5833
|
+
links: null,
|
|
5834
|
+
endedAt: span.endTime ?? null,
|
|
5835
|
+
input: span.input,
|
|
5836
|
+
output: span.output,
|
|
5837
|
+
error: span.errorInfo ?? null
|
|
5838
|
+
};
|
|
5839
|
+
}
|
|
5840
|
+
/**
|
|
5841
|
+
* Handles realtime strategy - processes each event immediately
|
|
5842
|
+
*/
|
|
5843
|
+
async handleRealtimeEvent(event, observability) {
|
|
5844
|
+
const span = event.exportedSpan;
|
|
5845
|
+
const spanKey = this.buildSpanKey(span.traceId, span.id);
|
|
5846
|
+
if (span.isEvent) {
|
|
5847
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
5848
|
+
await observability.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5849
|
+
} else {
|
|
5850
|
+
this.logger.warn(`Tracing event type not implemented for event spans: ${event.type}`);
|
|
5851
|
+
}
|
|
5852
|
+
} else {
|
|
5853
|
+
switch (event.type) {
|
|
5854
|
+
case TracingEventType.SPAN_STARTED:
|
|
5855
|
+
await observability.createSpan({ span: this.buildCreateRecord(event.exportedSpan) });
|
|
5856
|
+
this.allCreatedSpans.add(spanKey);
|
|
5857
|
+
break;
|
|
5858
|
+
case TracingEventType.SPAN_UPDATED:
|
|
5859
|
+
await observability.updateSpan({
|
|
5860
|
+
traceId: span.traceId,
|
|
5861
|
+
spanId: span.id,
|
|
5862
|
+
updates: this.buildUpdateRecord(span)
|
|
5863
|
+
});
|
|
5864
|
+
break;
|
|
5865
|
+
case TracingEventType.SPAN_ENDED:
|
|
5866
|
+
await observability.updateSpan({
|
|
5867
|
+
traceId: span.traceId,
|
|
5868
|
+
spanId: span.id,
|
|
5869
|
+
updates: this.buildUpdateRecord(span)
|
|
5870
|
+
});
|
|
5871
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5872
|
+
break;
|
|
5873
|
+
default:
|
|
5874
|
+
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
5875
|
+
}
|
|
5876
|
+
}
|
|
5877
|
+
}
|
|
5878
|
+
/**
|
|
5879
|
+
* Handles batch-with-updates strategy - buffers events and processes in batches
|
|
5880
|
+
*/
|
|
5881
|
+
handleBatchWithUpdatesEvent(event) {
|
|
5882
|
+
this.addToBuffer(event);
|
|
5883
|
+
if (this.shouldFlush()) {
|
|
5884
|
+
this.flushBuffer().catch((error) => {
|
|
5885
|
+
this.logger.error("Batch flush failed", {
|
|
5886
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5887
|
+
});
|
|
5888
|
+
});
|
|
5889
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5890
|
+
this.scheduleFlush();
|
|
5891
|
+
}
|
|
5892
|
+
}
|
|
5893
|
+
/**
|
|
5894
|
+
* Handles insert-only strategy - only processes SPAN_ENDED events in batches
|
|
5895
|
+
*/
|
|
5896
|
+
handleInsertOnlyEvent(event) {
|
|
5897
|
+
if (event.type === TracingEventType.SPAN_ENDED) {
|
|
5898
|
+
this.addToBuffer(event);
|
|
5899
|
+
if (this.shouldFlush()) {
|
|
5900
|
+
this.flushBuffer().catch((error) => {
|
|
5901
|
+
this.logger.error("Batch flush failed", {
|
|
5902
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5903
|
+
});
|
|
5904
|
+
});
|
|
5905
|
+
} else if (this.buffer.totalSize === 1) {
|
|
5906
|
+
this.scheduleFlush();
|
|
5907
|
+
}
|
|
5908
|
+
}
|
|
5909
|
+
}
|
|
5910
|
+
/**
|
|
5911
|
+
* Calculates retry delay using exponential backoff
|
|
5912
|
+
*/
|
|
5913
|
+
calculateRetryDelay(attempt) {
|
|
5914
|
+
return this.#config.retryDelayMs * Math.pow(2, attempt);
|
|
5915
|
+
}
|
|
5916
|
+
/**
|
|
5917
|
+
* Flushes the current buffer to storage with retry logic (internal implementation)
|
|
5918
|
+
*/
|
|
5919
|
+
async flushBuffer() {
|
|
5920
|
+
if (!this.#observability) {
|
|
5921
|
+
this.logger.debug("Cannot flush traces. Observability storage is not initialized");
|
|
5922
|
+
return;
|
|
5923
|
+
}
|
|
5924
|
+
if (this.#flushTimer) {
|
|
5925
|
+
clearTimeout(this.#flushTimer);
|
|
5926
|
+
this.#flushTimer = null;
|
|
5927
|
+
}
|
|
5928
|
+
if (this.buffer.totalSize === 0) {
|
|
5929
|
+
return;
|
|
5930
|
+
}
|
|
5931
|
+
const startTime = Date.now();
|
|
5932
|
+
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
5933
|
+
const bufferCopy = {
|
|
5934
|
+
creates: [...this.buffer.creates],
|
|
5935
|
+
updates: [...this.buffer.updates],
|
|
5936
|
+
insertOnly: [...this.buffer.insertOnly],
|
|
5937
|
+
seenSpans: new Set(this.buffer.seenSpans),
|
|
5938
|
+
spanSequences: new Map(this.buffer.spanSequences),
|
|
5939
|
+
completedSpans: new Set(this.buffer.completedSpans),
|
|
5940
|
+
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
5941
|
+
firstEventTime: this.buffer.firstEventTime,
|
|
5942
|
+
totalSize: this.buffer.totalSize
|
|
5943
|
+
};
|
|
5944
|
+
this.resetBuffer();
|
|
5945
|
+
await this.flushWithRetries(this.#observability, bufferCopy, 0);
|
|
5946
|
+
const elapsed = Date.now() - startTime;
|
|
5947
|
+
this.logger.debug("Batch flushed", {
|
|
5948
|
+
strategy: this.#resolvedStrategy,
|
|
5949
|
+
batchSize: bufferCopy.totalSize,
|
|
5950
|
+
flushReason,
|
|
5951
|
+
durationMs: elapsed,
|
|
5952
|
+
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
5953
|
+
});
|
|
5954
|
+
}
|
|
5955
|
+
/**
|
|
5956
|
+
* Attempts to flush with exponential backoff retry logic
|
|
5957
|
+
*/
|
|
5958
|
+
async flushWithRetries(observability, buffer, attempt) {
|
|
5959
|
+
try {
|
|
5960
|
+
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
5961
|
+
if (buffer.creates.length > 0) {
|
|
5962
|
+
await observability.batchCreateSpans({ records: buffer.creates });
|
|
4460
5963
|
}
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
if (span.input !== void 0) {
|
|
4471
|
-
this.logger.info(` Input: ${formatAttributes(span.input)}`);
|
|
5964
|
+
if (buffer.updates.length > 0) {
|
|
5965
|
+
const sortedUpdates = buffer.updates.sort((a, b) => {
|
|
5966
|
+
const spanCompare = this.buildSpanKey(a.traceId, a.spanId).localeCompare(
|
|
5967
|
+
this.buildSpanKey(b.traceId, b.spanId)
|
|
5968
|
+
);
|
|
5969
|
+
if (spanCompare !== 0) return spanCompare;
|
|
5970
|
+
return a.sequenceNumber - b.sequenceNumber;
|
|
5971
|
+
});
|
|
5972
|
+
await observability.batchUpdateSpans({ records: sortedUpdates });
|
|
4472
5973
|
}
|
|
4473
|
-
|
|
4474
|
-
|
|
5974
|
+
} else if (this.#resolvedStrategy === "insert-only") {
|
|
5975
|
+
if (buffer.insertOnly.length > 0) {
|
|
5976
|
+
await observability.batchCreateSpans({ records: buffer.insertOnly });
|
|
4475
5977
|
}
|
|
4476
|
-
|
|
4477
|
-
|
|
5978
|
+
}
|
|
5979
|
+
for (const spanKey of buffer.completedSpans) {
|
|
5980
|
+
this.allCreatedSpans.delete(spanKey);
|
|
5981
|
+
}
|
|
5982
|
+
} catch (error) {
|
|
5983
|
+
if (attempt < this.#config.maxRetries) {
|
|
5984
|
+
const retryDelay = this.calculateRetryDelay(attempt);
|
|
5985
|
+
this.logger.warn("Batch flush failed, retrying", {
|
|
5986
|
+
attempt: attempt + 1,
|
|
5987
|
+
maxRetries: this.#config.maxRetries,
|
|
5988
|
+
nextRetryInMs: retryDelay,
|
|
5989
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5990
|
+
});
|
|
5991
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
5992
|
+
return this.flushWithRetries(observability, buffer, attempt + 1);
|
|
5993
|
+
} else {
|
|
5994
|
+
this.logger.error("Batch flush failed after all retries, dropping batch", {
|
|
5995
|
+
finalAttempt: attempt + 1,
|
|
5996
|
+
maxRetries: this.#config.maxRetries,
|
|
5997
|
+
droppedBatchSize: buffer.totalSize,
|
|
5998
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5999
|
+
});
|
|
6000
|
+
for (const spanKey of buffer.completedSpans) {
|
|
6001
|
+
this.allCreatedSpans.delete(spanKey);
|
|
4478
6002
|
}
|
|
4479
|
-
|
|
4480
|
-
|
|
6003
|
+
}
|
|
6004
|
+
}
|
|
6005
|
+
}
|
|
6006
|
+
async _exportTracingEvent(event) {
|
|
6007
|
+
if (!this.#observability) {
|
|
6008
|
+
this.logger.debug("Cannot store traces. Observability storage is not initialized");
|
|
6009
|
+
return;
|
|
6010
|
+
}
|
|
6011
|
+
if (!this.#strategyInitialized) {
|
|
6012
|
+
this.initializeStrategy(this.#observability, this.#storage?.constructor.name ?? "Unknown");
|
|
6013
|
+
}
|
|
6014
|
+
switch (this.#resolvedStrategy) {
|
|
6015
|
+
case "realtime":
|
|
6016
|
+
await this.handleRealtimeEvent(event, this.#observability);
|
|
4481
6017
|
break;
|
|
4482
|
-
|
|
4483
|
-
this.
|
|
6018
|
+
case "batch-with-updates":
|
|
6019
|
+
this.handleBatchWithUpdatesEvent(event);
|
|
6020
|
+
break;
|
|
6021
|
+
case "insert-only":
|
|
6022
|
+
this.handleInsertOnlyEvent(event);
|
|
6023
|
+
break;
|
|
6024
|
+
}
|
|
6025
|
+
}
|
|
6026
|
+
/**
|
|
6027
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
6028
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
6029
|
+
* are exported before the runtime instance is terminated.
|
|
6030
|
+
*/
|
|
6031
|
+
async flush() {
|
|
6032
|
+
if (this.buffer.totalSize > 0) {
|
|
6033
|
+
this.logger.debug("Flushing buffered events", {
|
|
6034
|
+
bufferedEvents: this.buffer.totalSize
|
|
6035
|
+
});
|
|
6036
|
+
await this.flushBuffer();
|
|
4484
6037
|
}
|
|
4485
6038
|
}
|
|
4486
6039
|
async shutdown() {
|
|
4487
|
-
this
|
|
6040
|
+
if (this.#flushTimer) {
|
|
6041
|
+
clearTimeout(this.#flushTimer);
|
|
6042
|
+
this.#flushTimer = null;
|
|
6043
|
+
}
|
|
6044
|
+
await this.flush();
|
|
6045
|
+
this.logger.info("DefaultExporter shutdown complete");
|
|
4488
6046
|
}
|
|
4489
6047
|
};
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
userStrategy: config.strategy,
|
|
4498
|
-
storageAdapter: storage.constructor.name,
|
|
4499
|
-
supportedStrategies: hints.supported,
|
|
4500
|
-
fallbackStrategy: hints.preferred
|
|
4501
|
-
});
|
|
6048
|
+
|
|
6049
|
+
// src/exporters/test.ts
|
|
6050
|
+
var TestExporter = class extends BaseExporter {
|
|
6051
|
+
name = "tracing-test-exporter";
|
|
6052
|
+
#events = [];
|
|
6053
|
+
constructor(config = {}) {
|
|
6054
|
+
super(config);
|
|
4502
6055
|
}
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
6056
|
+
async _exportTracingEvent(event) {
|
|
6057
|
+
this.#events.push(event);
|
|
6058
|
+
}
|
|
6059
|
+
clearEvents() {
|
|
6060
|
+
this.#events = [];
|
|
6061
|
+
}
|
|
6062
|
+
get events() {
|
|
6063
|
+
return this.#events;
|
|
6064
|
+
}
|
|
6065
|
+
async shutdown() {
|
|
6066
|
+
this.logger.info("TestExporter shutdown");
|
|
6067
|
+
}
|
|
6068
|
+
};
|
|
6069
|
+
var __filename$1 = fileURLToPath(import.meta.url);
|
|
6070
|
+
var __dirname$1 = dirname(__filename$1);
|
|
6071
|
+
var SNAPSHOTS_DIR = join(__dirname$1, "..", "__snapshots__");
|
|
6072
|
+
var JsonExporter = class extends BaseExporter {
|
|
6073
|
+
name = "json-exporter";
|
|
6074
|
+
/** All collected events */
|
|
6075
|
+
#events = [];
|
|
6076
|
+
/** Per-span state tracking */
|
|
6077
|
+
#spanStates = /* @__PURE__ */ new Map();
|
|
6078
|
+
/** Logs for debugging */
|
|
6079
|
+
#logs = [];
|
|
6080
|
+
/** Configuration */
|
|
4508
6081
|
#config;
|
|
4509
|
-
#resolvedStrategy;
|
|
4510
|
-
buffer;
|
|
4511
|
-
#flushTimer = null;
|
|
4512
|
-
// Track all spans that have been created, persists across flushes
|
|
4513
|
-
allCreatedSpans = /* @__PURE__ */ new Set();
|
|
4514
6082
|
constructor(config = {}) {
|
|
4515
6083
|
super(config);
|
|
4516
|
-
if (config === void 0) {
|
|
4517
|
-
config = {};
|
|
4518
|
-
}
|
|
4519
6084
|
this.#config = {
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
maxRetries: config.maxRetries ?? 4,
|
|
4525
|
-
retryDelayMs: config.retryDelayMs ?? 500,
|
|
4526
|
-
strategy: config.strategy ?? "auto"
|
|
4527
|
-
};
|
|
4528
|
-
this.buffer = {
|
|
4529
|
-
creates: [],
|
|
4530
|
-
updates: [],
|
|
4531
|
-
insertOnly: [],
|
|
4532
|
-
seenSpans: /* @__PURE__ */ new Set(),
|
|
4533
|
-
spanSequences: /* @__PURE__ */ new Map(),
|
|
4534
|
-
completedSpans: /* @__PURE__ */ new Set(),
|
|
4535
|
-
outOfOrderCount: 0,
|
|
4536
|
-
totalSize: 0
|
|
6085
|
+
validateLifecycle: true,
|
|
6086
|
+
storeLogs: true,
|
|
6087
|
+
jsonIndent: 2,
|
|
6088
|
+
...config
|
|
4537
6089
|
};
|
|
4538
|
-
this.#resolvedStrategy = "batch-with-updates";
|
|
4539
6090
|
}
|
|
4540
|
-
#strategyInitialized = false;
|
|
4541
6091
|
/**
|
|
4542
|
-
*
|
|
6092
|
+
* Process incoming tracing events with lifecycle tracking
|
|
4543
6093
|
*/
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
6094
|
+
async _exportTracingEvent(event) {
|
|
6095
|
+
const span = event.exportedSpan;
|
|
6096
|
+
const spanId = span.id;
|
|
6097
|
+
const logMessage = `[JsonExporter] ${event.type}: ${span.type} "${span.name}" (entity: ${span.entityName ?? span.entityId ?? "unknown"}, trace: ${span.traceId.slice(-8)}, span: ${spanId.slice(-8)})`;
|
|
6098
|
+
if (this.#config.storeLogs) {
|
|
6099
|
+
this.#logs.push(logMessage);
|
|
6100
|
+
}
|
|
6101
|
+
const state = this.#spanStates.get(spanId) || {
|
|
6102
|
+
hasStart: false,
|
|
6103
|
+
hasEnd: false,
|
|
6104
|
+
hasUpdate: false,
|
|
6105
|
+
events: []
|
|
6106
|
+
};
|
|
6107
|
+
if (this.#config.validateLifecycle) {
|
|
6108
|
+
this.#validateLifecycle(event, state, spanId);
|
|
4549
6109
|
}
|
|
4550
|
-
|
|
6110
|
+
if (event.type === TracingEventType.SPAN_STARTED) {
|
|
6111
|
+
state.hasStart = true;
|
|
6112
|
+
} else if (event.type === TracingEventType.SPAN_ENDED) {
|
|
6113
|
+
state.hasEnd = true;
|
|
6114
|
+
if (span.isEvent) {
|
|
6115
|
+
state.isEventSpan = true;
|
|
6116
|
+
}
|
|
6117
|
+
} else if (event.type === TracingEventType.SPAN_UPDATED) {
|
|
6118
|
+
state.hasUpdate = true;
|
|
6119
|
+
}
|
|
6120
|
+
state.events.push(event);
|
|
6121
|
+
this.#spanStates.set(spanId, state);
|
|
6122
|
+
this.#events.push(event);
|
|
4551
6123
|
}
|
|
4552
6124
|
/**
|
|
4553
|
-
*
|
|
6125
|
+
* Validate span lifecycle rules
|
|
4554
6126
|
*/
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
6127
|
+
#validateLifecycle(event, state, spanId) {
|
|
6128
|
+
const span = event.exportedSpan;
|
|
6129
|
+
if (event.type === TracingEventType.SPAN_STARTED) {
|
|
6130
|
+
if (state.hasStart) {
|
|
6131
|
+
this.logger.warn(`Span ${spanId} (${span.type} "${span.name}") started twice`);
|
|
6132
|
+
}
|
|
6133
|
+
} else if (event.type === TracingEventType.SPAN_ENDED) {
|
|
6134
|
+
if (span.isEvent) {
|
|
6135
|
+
if (state.hasStart) {
|
|
6136
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_STARTED`);
|
|
6137
|
+
}
|
|
6138
|
+
if (state.hasUpdate) {
|
|
6139
|
+
this.logger.warn(`Event span ${spanId} (${span.type} "${span.name}") incorrectly received SPAN_UPDATED`);
|
|
6140
|
+
}
|
|
6141
|
+
} else {
|
|
6142
|
+
if (!state.hasStart) {
|
|
6143
|
+
this.logger.warn(`Normal span ${spanId} (${span.type} "${span.name}") ended without starting`);
|
|
6144
|
+
}
|
|
6145
|
+
}
|
|
6146
|
+
}
|
|
4566
6147
|
}
|
|
6148
|
+
// ============================================================================
|
|
6149
|
+
// Query Methods
|
|
6150
|
+
// ============================================================================
|
|
4567
6151
|
/**
|
|
4568
|
-
*
|
|
6152
|
+
* Get all collected events
|
|
4569
6153
|
*/
|
|
4570
|
-
|
|
4571
|
-
return
|
|
6154
|
+
get events() {
|
|
6155
|
+
return [...this.#events];
|
|
4572
6156
|
}
|
|
4573
6157
|
/**
|
|
4574
|
-
*
|
|
6158
|
+
* Get completed spans by SpanType (e.g., 'agent_run', 'tool_call')
|
|
6159
|
+
*
|
|
6160
|
+
* @param type - The SpanType to filter by
|
|
6161
|
+
* @returns Array of completed exported spans of the specified type
|
|
4575
6162
|
*/
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
6163
|
+
getSpansByType(type) {
|
|
6164
|
+
return Array.from(this.#spanStates.values()).filter((state) => {
|
|
6165
|
+
if (!state.hasEnd) return false;
|
|
6166
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6167
|
+
return endEvent?.exportedSpan.type === type;
|
|
6168
|
+
}).map((state) => {
|
|
6169
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6170
|
+
return endEvent?.exportedSpan;
|
|
6171
|
+
}).filter((span) => span !== void 0);
|
|
4581
6172
|
}
|
|
4582
6173
|
/**
|
|
4583
|
-
*
|
|
6174
|
+
* Get events by TracingEventType (SPAN_STARTED, SPAN_UPDATED, SPAN_ENDED)
|
|
6175
|
+
*
|
|
6176
|
+
* @param type - The TracingEventType to filter by
|
|
6177
|
+
* @returns Array of events of the specified type
|
|
4584
6178
|
*/
|
|
4585
|
-
|
|
4586
|
-
this.
|
|
4587
|
-
spanId: event.exportedSpan.id,
|
|
4588
|
-
traceId: event.exportedSpan.traceId,
|
|
4589
|
-
spanName: event.exportedSpan.name,
|
|
4590
|
-
eventType: event.type
|
|
4591
|
-
});
|
|
6179
|
+
getByEventType(type) {
|
|
6180
|
+
return this.#events.filter((e) => e.type === type);
|
|
4592
6181
|
}
|
|
4593
6182
|
/**
|
|
4594
|
-
*
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4604
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4605
|
-
this.buffer.creates.push(createRecord);
|
|
4606
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4607
|
-
this.allCreatedSpans.add(spanKey);
|
|
4608
|
-
}
|
|
4609
|
-
break;
|
|
4610
|
-
case TracingEventType.SPAN_UPDATED:
|
|
4611
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4612
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4613
|
-
this.buffer.updates.push({
|
|
4614
|
-
traceId: event.exportedSpan.traceId,
|
|
4615
|
-
spanId: event.exportedSpan.id,
|
|
4616
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4617
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4618
|
-
});
|
|
4619
|
-
} else {
|
|
4620
|
-
this.handleOutOfOrderUpdate(event);
|
|
4621
|
-
this.buffer.outOfOrderCount++;
|
|
4622
|
-
}
|
|
4623
|
-
}
|
|
4624
|
-
break;
|
|
4625
|
-
case TracingEventType.SPAN_ENDED:
|
|
4626
|
-
if (this.#resolvedStrategy === "batch-with-updates") {
|
|
4627
|
-
if (this.allCreatedSpans.has(spanKey)) {
|
|
4628
|
-
this.buffer.updates.push({
|
|
4629
|
-
traceId: event.exportedSpan.traceId,
|
|
4630
|
-
spanId: event.exportedSpan.id,
|
|
4631
|
-
updates: this.buildUpdateRecord(event.exportedSpan),
|
|
4632
|
-
sequenceNumber: this.getNextSequence(spanKey)
|
|
4633
|
-
});
|
|
4634
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4635
|
-
} else if (event.exportedSpan.isEvent) {
|
|
4636
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4637
|
-
this.buffer.creates.push(createRecord);
|
|
4638
|
-
this.buffer.seenSpans.add(spanKey);
|
|
4639
|
-
this.allCreatedSpans.add(spanKey);
|
|
4640
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4641
|
-
} else {
|
|
4642
|
-
this.handleOutOfOrderUpdate(event);
|
|
4643
|
-
this.buffer.outOfOrderCount++;
|
|
4644
|
-
}
|
|
4645
|
-
} else if (this.#resolvedStrategy === "insert-only") {
|
|
4646
|
-
const createRecord = this.buildCreateRecord(event.exportedSpan);
|
|
4647
|
-
this.buffer.insertOnly.push(createRecord);
|
|
4648
|
-
this.buffer.completedSpans.add(spanKey);
|
|
4649
|
-
this.allCreatedSpans.add(spanKey);
|
|
4650
|
-
}
|
|
4651
|
-
break;
|
|
4652
|
-
}
|
|
4653
|
-
this.buffer.totalSize = this.buffer.creates.length + this.buffer.updates.length + this.buffer.insertOnly.length;
|
|
6183
|
+
* Get all events and spans for a specific trace
|
|
6184
|
+
*
|
|
6185
|
+
* @param traceId - The trace ID to filter by
|
|
6186
|
+
* @returns Object containing events and final spans for the trace
|
|
6187
|
+
*/
|
|
6188
|
+
getByTraceId(traceId) {
|
|
6189
|
+
const events = this.#events.filter((e) => e.exportedSpan.traceId === traceId);
|
|
6190
|
+
const spans = this.#getUniqueSpansFromEvents(events);
|
|
6191
|
+
return { events, spans };
|
|
4654
6192
|
}
|
|
4655
6193
|
/**
|
|
4656
|
-
*
|
|
6194
|
+
* Get all events for a specific span
|
|
6195
|
+
*
|
|
6196
|
+
* @param spanId - The span ID to filter by
|
|
6197
|
+
* @returns Object containing events and final span state
|
|
4657
6198
|
*/
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
if (this.buffer.totalSize >= this.#config.maxBatchSize) {
|
|
4663
|
-
return true;
|
|
6199
|
+
getBySpanId(spanId) {
|
|
6200
|
+
const state = this.#spanStates.get(spanId);
|
|
6201
|
+
if (!state) {
|
|
6202
|
+
return { events: [], span: void 0, state: void 0 };
|
|
4664
6203
|
}
|
|
4665
|
-
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
6204
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6205
|
+
const span = endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6206
|
+
return { events: state.events, span, state };
|
|
6207
|
+
}
|
|
6208
|
+
/**
|
|
6209
|
+
* Get all unique spans (returns the final state of each span)
|
|
6210
|
+
*/
|
|
6211
|
+
getAllSpans() {
|
|
6212
|
+
return Array.from(this.#spanStates.values()).map((state) => {
|
|
6213
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6214
|
+
return endEvent?.exportedSpan ?? state.events[state.events.length - 1]?.exportedSpan;
|
|
6215
|
+
}).filter((span) => span !== void 0);
|
|
6216
|
+
}
|
|
6217
|
+
/**
|
|
6218
|
+
* Get only completed spans (those that have received SPAN_ENDED)
|
|
6219
|
+
*/
|
|
6220
|
+
getCompletedSpans() {
|
|
6221
|
+
return Array.from(this.#spanStates.values()).filter((state) => state.hasEnd).map((state) => {
|
|
6222
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6223
|
+
return endEvent.exportedSpan;
|
|
6224
|
+
});
|
|
6225
|
+
}
|
|
6226
|
+
/**
|
|
6227
|
+
* Get root spans only (spans with no parent)
|
|
6228
|
+
*/
|
|
6229
|
+
getRootSpans() {
|
|
6230
|
+
return this.getAllSpans().filter((span) => span.isRootSpan);
|
|
6231
|
+
}
|
|
6232
|
+
/**
|
|
6233
|
+
* Get incomplete spans (started but not yet ended)
|
|
6234
|
+
*/
|
|
6235
|
+
getIncompleteSpans() {
|
|
6236
|
+
return Array.from(this.#spanStates.entries()).filter(([_, state]) => !state.hasEnd).map(([spanId, state]) => ({
|
|
6237
|
+
spanId,
|
|
6238
|
+
span: state.events[0]?.exportedSpan,
|
|
6239
|
+
state: {
|
|
6240
|
+
hasStart: state.hasStart,
|
|
6241
|
+
hasUpdate: state.hasUpdate,
|
|
6242
|
+
hasEnd: state.hasEnd
|
|
4669
6243
|
}
|
|
4670
|
-
}
|
|
4671
|
-
return false;
|
|
6244
|
+
}));
|
|
4672
6245
|
}
|
|
4673
6246
|
/**
|
|
4674
|
-
*
|
|
6247
|
+
* Get unique trace IDs from all collected spans
|
|
4675
6248
|
*/
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
this
|
|
4679
|
-
|
|
4680
|
-
this.buffer.seenSpans.clear();
|
|
4681
|
-
this.buffer.spanSequences.clear();
|
|
4682
|
-
this.buffer.completedSpans.clear();
|
|
4683
|
-
this.buffer.outOfOrderCount = 0;
|
|
4684
|
-
this.buffer.firstEventTime = void 0;
|
|
4685
|
-
this.buffer.totalSize = 0;
|
|
4686
|
-
for (const spanKey of completedSpansToCleanup) {
|
|
4687
|
-
this.allCreatedSpans.delete(spanKey);
|
|
6249
|
+
getTraceIds() {
|
|
6250
|
+
const traceIds = /* @__PURE__ */ new Set();
|
|
6251
|
+
for (const event of this.#events) {
|
|
6252
|
+
traceIds.add(event.exportedSpan.traceId);
|
|
4688
6253
|
}
|
|
6254
|
+
return Array.from(traceIds);
|
|
4689
6255
|
}
|
|
6256
|
+
// ============================================================================
|
|
6257
|
+
// Statistics
|
|
6258
|
+
// ============================================================================
|
|
4690
6259
|
/**
|
|
4691
|
-
*
|
|
6260
|
+
* Get comprehensive statistics about collected spans
|
|
4692
6261
|
*/
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
6262
|
+
getStatistics() {
|
|
6263
|
+
const bySpanType = {};
|
|
6264
|
+
let completedSpans = 0;
|
|
6265
|
+
let incompleteSpans = 0;
|
|
6266
|
+
for (const state of this.#spanStates.values()) {
|
|
6267
|
+
if (state.hasEnd) {
|
|
6268
|
+
completedSpans++;
|
|
6269
|
+
const endEvent = state.events.find((e) => e.type === TracingEventType.SPAN_ENDED);
|
|
6270
|
+
const spanType = endEvent?.exportedSpan.type;
|
|
6271
|
+
if (spanType) {
|
|
6272
|
+
bySpanType[spanType] = (bySpanType[spanType] || 0) + 1;
|
|
6273
|
+
}
|
|
6274
|
+
} else {
|
|
6275
|
+
incompleteSpans++;
|
|
6276
|
+
}
|
|
4696
6277
|
}
|
|
4697
|
-
|
|
4698
|
-
this.
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
6278
|
+
return {
|
|
6279
|
+
totalEvents: this.#events.length,
|
|
6280
|
+
totalSpans: this.#spanStates.size,
|
|
6281
|
+
totalTraces: this.getTraceIds().length,
|
|
6282
|
+
completedSpans,
|
|
6283
|
+
incompleteSpans,
|
|
6284
|
+
byEventType: {
|
|
6285
|
+
started: this.#events.filter((e) => e.type === TracingEventType.SPAN_STARTED).length,
|
|
6286
|
+
updated: this.#events.filter((e) => e.type === TracingEventType.SPAN_UPDATED).length,
|
|
6287
|
+
ended: this.#events.filter((e) => e.type === TracingEventType.SPAN_ENDED).length
|
|
6288
|
+
},
|
|
6289
|
+
bySpanType
|
|
6290
|
+
};
|
|
4704
6291
|
}
|
|
6292
|
+
// ============================================================================
|
|
6293
|
+
// JSON Output
|
|
6294
|
+
// ============================================================================
|
|
4705
6295
|
/**
|
|
4706
|
-
*
|
|
4707
|
-
*
|
|
6296
|
+
* Serialize all collected data to JSON string
|
|
6297
|
+
*
|
|
6298
|
+
* @param options - Serialization options
|
|
6299
|
+
* @returns JSON string of all collected data
|
|
4708
6300
|
*/
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
6301
|
+
toJSON(options) {
|
|
6302
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6303
|
+
const includeEvents = options?.includeEvents ?? true;
|
|
6304
|
+
const includeStats = options?.includeStats ?? true;
|
|
6305
|
+
const data = {
|
|
6306
|
+
spans: this.getAllSpans()
|
|
6307
|
+
};
|
|
6308
|
+
if (includeEvents) {
|
|
6309
|
+
data.events = this.#events;
|
|
4712
6310
|
}
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
JSON.stringify(span.attributes, (_key, value) => {
|
|
4716
|
-
if (value instanceof Date) {
|
|
4717
|
-
return value.toISOString();
|
|
4718
|
-
}
|
|
4719
|
-
if (typeof value === "object" && value !== null) {
|
|
4720
|
-
return value;
|
|
4721
|
-
}
|
|
4722
|
-
return value;
|
|
4723
|
-
})
|
|
4724
|
-
);
|
|
4725
|
-
} catch (error) {
|
|
4726
|
-
this.logger.warn("Failed to serialize span attributes, storing as null", {
|
|
4727
|
-
spanId: span.id,
|
|
4728
|
-
spanType: span.type,
|
|
4729
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4730
|
-
});
|
|
4731
|
-
return null;
|
|
6311
|
+
if (includeStats) {
|
|
6312
|
+
data.statistics = this.getStatistics();
|
|
4732
6313
|
}
|
|
6314
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4733
6315
|
}
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
6316
|
+
/**
|
|
6317
|
+
* Build a tree structure from spans, nesting children under their parents
|
|
6318
|
+
*
|
|
6319
|
+
* @returns Array of root span tree nodes (spans with no parent)
|
|
6320
|
+
*/
|
|
6321
|
+
buildSpanTree() {
|
|
6322
|
+
const spans = this.getAllSpans();
|
|
6323
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
6324
|
+
const roots = [];
|
|
6325
|
+
for (const span of spans) {
|
|
6326
|
+
nodeMap.set(span.id, { span, children: [] });
|
|
6327
|
+
}
|
|
6328
|
+
for (const span of spans) {
|
|
6329
|
+
const node = nodeMap.get(span.id);
|
|
6330
|
+
if (span.parentSpanId && nodeMap.has(span.parentSpanId)) {
|
|
6331
|
+
nodeMap.get(span.parentSpanId).children.push(node);
|
|
6332
|
+
} else {
|
|
6333
|
+
roots.push(node);
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
const sortChildren = (node) => {
|
|
6337
|
+
node.children.sort((a, b) => new Date(a.span.startTime).getTime() - new Date(b.span.startTime).getTime());
|
|
6338
|
+
node.children.forEach(sortChildren);
|
|
4751
6339
|
};
|
|
6340
|
+
roots.forEach(sortChildren);
|
|
6341
|
+
return roots;
|
|
4752
6342
|
}
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
6343
|
+
/**
|
|
6344
|
+
* Serialize spans as a tree structure to JSON string
|
|
6345
|
+
*
|
|
6346
|
+
* @param options - Serialization options
|
|
6347
|
+
* @returns JSON string with spans nested in tree format
|
|
6348
|
+
*/
|
|
6349
|
+
toTreeJSON(options) {
|
|
6350
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6351
|
+
const includeStats = options?.includeStats ?? true;
|
|
6352
|
+
const data = {
|
|
6353
|
+
tree: this.buildSpanTree()
|
|
4764
6354
|
};
|
|
6355
|
+
if (includeStats) {
|
|
6356
|
+
data.statistics = this.getStatistics();
|
|
6357
|
+
}
|
|
6358
|
+
return JSON.stringify(data, this.#jsonReplacer, indent);
|
|
4765
6359
|
}
|
|
4766
6360
|
/**
|
|
4767
|
-
*
|
|
6361
|
+
* Build a normalized tree structure suitable for snapshot testing.
|
|
6362
|
+
*
|
|
6363
|
+
* Normalizations applied:
|
|
6364
|
+
* - Span IDs replaced with stable placeholders (<span-1>, <span-2>, etc.)
|
|
6365
|
+
* - Trace IDs replaced with stable placeholders (<trace-1>, <trace-2>, etc.)
|
|
6366
|
+
* - parentSpanId replaced with normalized parent ID
|
|
6367
|
+
* - Timestamps replaced with durationMs (or null if not ended)
|
|
6368
|
+
* - Empty children arrays are omitted
|
|
6369
|
+
*
|
|
6370
|
+
* @returns Array of normalized root tree nodes
|
|
4768
6371
|
*/
|
|
4769
|
-
|
|
4770
|
-
const
|
|
4771
|
-
const
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
6372
|
+
buildNormalizedTree() {
|
|
6373
|
+
const tree = this.buildSpanTree();
|
|
6374
|
+
const spanIdMap = /* @__PURE__ */ new Map();
|
|
6375
|
+
const traceIdMap = /* @__PURE__ */ new Map();
|
|
6376
|
+
const uuidMapsByKey = /* @__PURE__ */ new Map();
|
|
6377
|
+
const uuidCountersByKey = /* @__PURE__ */ new Map();
|
|
6378
|
+
let spanIdCounter = 1;
|
|
6379
|
+
let traceIdCounter = 1;
|
|
6380
|
+
const uuidRegex2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6381
|
+
const hexId32Regex = /^[0-9a-f]{32}$/i;
|
|
6382
|
+
const prefixedUuidRegex = /^([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i;
|
|
6383
|
+
const embeddedPrefixedUuidRegex = /([a-z_]+)_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
|
|
6384
|
+
const normalizeUuid = (uuid, key) => {
|
|
6385
|
+
if (!uuidMapsByKey.has(key)) {
|
|
6386
|
+
uuidMapsByKey.set(key, /* @__PURE__ */ new Map());
|
|
6387
|
+
uuidCountersByKey.set(key, 1);
|
|
4777
6388
|
}
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
6389
|
+
const keyMap = uuidMapsByKey.get(key);
|
|
6390
|
+
if (!keyMap.has(uuid)) {
|
|
6391
|
+
const counter = uuidCountersByKey.get(key);
|
|
6392
|
+
keyMap.set(uuid, `<${key}-${counter}>`);
|
|
6393
|
+
uuidCountersByKey.set(key, counter + 1);
|
|
6394
|
+
}
|
|
6395
|
+
return keyMap.get(uuid);
|
|
6396
|
+
};
|
|
6397
|
+
const normalizeValue = (value, key) => {
|
|
6398
|
+
if (value instanceof Date) {
|
|
6399
|
+
return "<date>";
|
|
6400
|
+
}
|
|
6401
|
+
if (typeof value === "string") {
|
|
6402
|
+
if (key === "traceId" && (uuidRegex2.test(value) || hexId32Regex.test(value))) {
|
|
6403
|
+
if (!traceIdMap.has(value)) {
|
|
6404
|
+
traceIdMap.set(value, `<trace-${traceIdCounter++}>`);
|
|
6405
|
+
}
|
|
6406
|
+
return traceIdMap.get(value);
|
|
6407
|
+
}
|
|
6408
|
+
if (uuidRegex2.test(value)) {
|
|
6409
|
+
return normalizeUuid(value, key ?? "uuid");
|
|
6410
|
+
}
|
|
6411
|
+
const prefixMatch = prefixedUuidRegex.exec(value);
|
|
6412
|
+
if (prefixMatch && prefixMatch[1] && prefixMatch[2]) {
|
|
6413
|
+
const prefix = prefixMatch[1];
|
|
6414
|
+
const uuid = prefixMatch[2];
|
|
6415
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
6416
|
+
}
|
|
6417
|
+
if (embeddedPrefixedUuidRegex.test(value)) {
|
|
6418
|
+
embeddedPrefixedUuidRegex.lastIndex = 0;
|
|
6419
|
+
return value.replace(embeddedPrefixedUuidRegex, (_match, prefix, uuid) => {
|
|
6420
|
+
return `${prefix}_${normalizeUuid(uuid, prefix)}`;
|
|
4796
6421
|
});
|
|
4797
|
-
|
|
4798
|
-
break;
|
|
4799
|
-
default:
|
|
4800
|
-
this.logger.warn(`Tracing event type not implemented for span spans: ${event.type}`);
|
|
6422
|
+
}
|
|
4801
6423
|
}
|
|
4802
|
-
|
|
6424
|
+
if (Array.isArray(value)) {
|
|
6425
|
+
return value.map((v) => normalizeValue(v, key));
|
|
6426
|
+
}
|
|
6427
|
+
if (value && typeof value === "object") {
|
|
6428
|
+
const normalized = {};
|
|
6429
|
+
for (const [k, v] of Object.entries(value)) {
|
|
6430
|
+
normalized[k] = normalizeValue(v, k);
|
|
6431
|
+
}
|
|
6432
|
+
return normalized;
|
|
6433
|
+
}
|
|
6434
|
+
return value;
|
|
6435
|
+
};
|
|
6436
|
+
const assignIds = (nodes) => {
|
|
6437
|
+
for (const node of nodes) {
|
|
6438
|
+
spanIdMap.set(node.span.id, `<span-${spanIdCounter++}>`);
|
|
6439
|
+
if (!traceIdMap.has(node.span.traceId)) {
|
|
6440
|
+
traceIdMap.set(node.span.traceId, `<trace-${traceIdCounter++}>`);
|
|
6441
|
+
}
|
|
6442
|
+
assignIds(node.children);
|
|
6443
|
+
}
|
|
6444
|
+
};
|
|
6445
|
+
assignIds(tree);
|
|
6446
|
+
const normalizeNode = (node) => {
|
|
6447
|
+
const span = node.span;
|
|
6448
|
+
const completed = span.endTime !== void 0 && span.endTime !== null;
|
|
6449
|
+
const normalizedSpan = {
|
|
6450
|
+
id: spanIdMap.get(span.id),
|
|
6451
|
+
traceId: traceIdMap.get(span.traceId),
|
|
6452
|
+
name: normalizeValue(span.name, "name"),
|
|
6453
|
+
type: span.type,
|
|
6454
|
+
completed,
|
|
6455
|
+
isEvent: span.isEvent,
|
|
6456
|
+
isRootSpan: span.isRootSpan
|
|
6457
|
+
};
|
|
6458
|
+
if (span.parentSpanId && spanIdMap.has(span.parentSpanId)) {
|
|
6459
|
+
normalizedSpan.parentId = spanIdMap.get(span.parentSpanId);
|
|
6460
|
+
}
|
|
6461
|
+
if (span.entityType) {
|
|
6462
|
+
normalizedSpan.entityType = span.entityType;
|
|
6463
|
+
}
|
|
6464
|
+
if (span.entityId) {
|
|
6465
|
+
normalizedSpan.entityId = normalizeValue(span.entityId, "entityId");
|
|
6466
|
+
}
|
|
6467
|
+
if (span.attributes && Object.keys(span.attributes).length > 0) {
|
|
6468
|
+
normalizedSpan.attributes = normalizeValue(span.attributes);
|
|
6469
|
+
}
|
|
6470
|
+
if (span.metadata && Object.keys(span.metadata).length > 0) {
|
|
6471
|
+
normalizedSpan.metadata = normalizeValue(span.metadata);
|
|
6472
|
+
}
|
|
6473
|
+
if (span.input !== void 0) {
|
|
6474
|
+
normalizedSpan.input = normalizeValue(span.input);
|
|
6475
|
+
}
|
|
6476
|
+
if (span.output !== void 0) {
|
|
6477
|
+
normalizedSpan.output = normalizeValue(span.output);
|
|
6478
|
+
}
|
|
6479
|
+
if (span.errorInfo) {
|
|
6480
|
+
normalizedSpan.errorInfo = span.errorInfo;
|
|
6481
|
+
}
|
|
6482
|
+
if (span.tags && span.tags.length > 0) {
|
|
6483
|
+
normalizedSpan.tags = span.tags;
|
|
6484
|
+
}
|
|
6485
|
+
const result = { span: normalizedSpan };
|
|
6486
|
+
if (node.children.length > 0) {
|
|
6487
|
+
result.children = node.children.map(normalizeNode);
|
|
6488
|
+
}
|
|
6489
|
+
return result;
|
|
6490
|
+
};
|
|
6491
|
+
return tree.map(normalizeNode);
|
|
4803
6492
|
}
|
|
4804
6493
|
/**
|
|
4805
|
-
*
|
|
6494
|
+
* Generate an ASCII tree structure graph for debugging.
|
|
6495
|
+
* Shows span type and name in a hierarchical format.
|
|
6496
|
+
*
|
|
6497
|
+
* @param nodes - Normalized tree nodes (defaults to current normalized tree)
|
|
6498
|
+
* @returns Array of strings representing the tree structure
|
|
6499
|
+
*
|
|
6500
|
+
* @example
|
|
6501
|
+
* ```
|
|
6502
|
+
* agent_run: "agent run: 'test-agent'"
|
|
6503
|
+
* ├── processor_run: "input processor: validator"
|
|
6504
|
+
* │ └── agent_run: "agent run: 'validator-agent'"
|
|
6505
|
+
* └── model_generation: "llm: 'mock-model-id'"
|
|
6506
|
+
* ```
|
|
4806
6507
|
*/
|
|
4807
|
-
|
|
4808
|
-
this.
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
6508
|
+
generateStructureGraph(nodes) {
|
|
6509
|
+
const tree = nodes ?? this.buildNormalizedTree();
|
|
6510
|
+
const lines = [];
|
|
6511
|
+
const buildLines = (node, prefix, isLast, isRoot) => {
|
|
6512
|
+
const connector = isRoot ? "" : isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ";
|
|
6513
|
+
const line = `${prefix}${connector}${node.span.type}: "${node.span.name}"`;
|
|
6514
|
+
lines.push(line);
|
|
6515
|
+
const children = node.children ?? [];
|
|
6516
|
+
const childPrefix = isRoot ? "" : prefix + (isLast ? " " : "\u2502 ");
|
|
6517
|
+
children.forEach((child, index) => {
|
|
6518
|
+
const childIsLast = index === children.length - 1;
|
|
6519
|
+
buildLines(child, childPrefix, childIsLast, false);
|
|
4814
6520
|
});
|
|
4815
|
-
}
|
|
4816
|
-
|
|
4817
|
-
|
|
6521
|
+
};
|
|
6522
|
+
tree.forEach((rootNode, index) => {
|
|
6523
|
+
if (index > 0) {
|
|
6524
|
+
lines.push("");
|
|
6525
|
+
}
|
|
6526
|
+
buildLines(rootNode, "", true, true);
|
|
6527
|
+
});
|
|
6528
|
+
return lines;
|
|
4818
6529
|
}
|
|
4819
6530
|
/**
|
|
4820
|
-
*
|
|
6531
|
+
* Serialize spans as a normalized tree structure for snapshot testing.
|
|
6532
|
+
* Includes a __structure__ field with an ASCII tree graph for readability.
|
|
6533
|
+
*
|
|
6534
|
+
* @param options - Serialization options
|
|
6535
|
+
* @returns JSON string with normalized spans in tree format
|
|
4821
6536
|
*/
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
}
|
|
4832
|
-
|
|
4833
|
-
}
|
|
6537
|
+
toNormalizedTreeJSON(options) {
|
|
6538
|
+
const indent = options?.indent ?? this.#config.jsonIndent;
|
|
6539
|
+
const includeStructure = options?.includeStructure ?? true;
|
|
6540
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6541
|
+
if (includeStructure) {
|
|
6542
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6543
|
+
const data = {
|
|
6544
|
+
__structure__: structureGraph,
|
|
6545
|
+
spans: normalizedTree
|
|
6546
|
+
};
|
|
6547
|
+
return JSON.stringify(data, null, indent);
|
|
4834
6548
|
}
|
|
6549
|
+
return JSON.stringify(normalizedTree, null, indent);
|
|
4835
6550
|
}
|
|
4836
6551
|
/**
|
|
4837
|
-
*
|
|
6552
|
+
* Write collected data to a JSON file
|
|
6553
|
+
*
|
|
6554
|
+
* @param filePath - Path to write the JSON file
|
|
6555
|
+
* @param options - Serialization options
|
|
4838
6556
|
*/
|
|
4839
|
-
|
|
4840
|
-
|
|
6557
|
+
async writeToFile(filePath, options) {
|
|
6558
|
+
const format = options?.format ?? "flat";
|
|
6559
|
+
let json;
|
|
6560
|
+
if (format === "normalized") {
|
|
6561
|
+
json = this.toNormalizedTreeJSON({ indent: options?.indent });
|
|
6562
|
+
} else if (format === "tree") {
|
|
6563
|
+
json = this.toTreeJSON({ indent: options?.indent, includeStats: options?.includeStats });
|
|
6564
|
+
} else {
|
|
6565
|
+
json = this.toJSON(options);
|
|
6566
|
+
}
|
|
6567
|
+
await writeFile(filePath, json, "utf-8");
|
|
6568
|
+
this.logger.info(`JsonExporter: wrote ${this.#events.length} events to ${filePath}`);
|
|
4841
6569
|
}
|
|
4842
6570
|
/**
|
|
4843
|
-
*
|
|
6571
|
+
* Assert that the current normalized tree matches a snapshot file.
|
|
6572
|
+
* Throws an error with a diff if they don't match.
|
|
6573
|
+
*
|
|
6574
|
+
* The snapshot format includes:
|
|
6575
|
+
* - `__structure__`: ASCII tree graph (compared first for quick validation)
|
|
6576
|
+
* - `spans`: The normalized span tree (detailed comparison)
|
|
6577
|
+
*
|
|
6578
|
+
* Supports special markers in the snapshot:
|
|
6579
|
+
* - `{"__or__": ["value1", "value2"]}` - matches if actual equals any listed value
|
|
6580
|
+
* - `{"__any__": "string"}` - matches any string value
|
|
6581
|
+
* - `{"__any__": "number"}` - matches any number value
|
|
6582
|
+
* - `{"__any__": "boolean"}` - matches any boolean value
|
|
6583
|
+
* - `{"__any__": "object"}` - matches any object value
|
|
6584
|
+
* - `{"__any__": "array"}` - matches any array value
|
|
6585
|
+
* - `{"__any__": true}` - matches any non-null/undefined value
|
|
6586
|
+
*
|
|
6587
|
+
* Environment variables:
|
|
6588
|
+
* Use `{ updateSnapshot: true }` option to update the snapshot instead of comparing
|
|
6589
|
+
*
|
|
6590
|
+
* @param snapshotName - Name of the snapshot file (resolved relative to __snapshots__ directory)
|
|
6591
|
+
* @param options - Options for snapshot comparison
|
|
6592
|
+
* @param options.updateSnapshot - If true, update the snapshot file instead of comparing
|
|
6593
|
+
* @throws Error if the snapshot doesn't match (and updateSnapshot is false)
|
|
4844
6594
|
*/
|
|
4845
|
-
async
|
|
4846
|
-
|
|
4847
|
-
|
|
6595
|
+
async assertMatchesSnapshot(snapshotName, options) {
|
|
6596
|
+
const snapshotPath = join(SNAPSHOTS_DIR, snapshotName);
|
|
6597
|
+
const normalizedTree = this.buildNormalizedTree();
|
|
6598
|
+
const structureGraph = this.generateStructureGraph(normalizedTree);
|
|
6599
|
+
const currentData = {
|
|
6600
|
+
__structure__: structureGraph,
|
|
6601
|
+
spans: normalizedTree
|
|
6602
|
+
};
|
|
6603
|
+
const currentJson = JSON.stringify(currentData, null, this.#config.jsonIndent);
|
|
6604
|
+
const shouldUpdate = options?.updateSnapshot;
|
|
6605
|
+
if (shouldUpdate) {
|
|
6606
|
+
await writeFile(snapshotPath, currentJson, "utf-8");
|
|
6607
|
+
this.logger.info(`JsonExporter: updated snapshot ${snapshotPath}`);
|
|
4848
6608
|
return;
|
|
4849
6609
|
}
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
6610
|
+
let snapshotData;
|
|
6611
|
+
try {
|
|
6612
|
+
const snapshotContent = await readFile(snapshotPath, "utf-8");
|
|
6613
|
+
snapshotData = JSON.parse(snapshotContent);
|
|
6614
|
+
} catch {
|
|
6615
|
+
throw new Error(`Snapshot file not found: ${snapshotPath}
|
|
6616
|
+
Run with { updateSnapshot: true } to create it.`);
|
|
6617
|
+
}
|
|
6618
|
+
let expectedSpans;
|
|
6619
|
+
let expectedStructure;
|
|
6620
|
+
if (Array.isArray(snapshotData)) {
|
|
6621
|
+
expectedSpans = snapshotData;
|
|
6622
|
+
} else if (snapshotData && typeof snapshotData === "object" && "spans" in snapshotData) {
|
|
6623
|
+
expectedSpans = snapshotData.spans;
|
|
6624
|
+
expectedStructure = snapshotData.__structure__;
|
|
6625
|
+
} else {
|
|
6626
|
+
throw new Error(
|
|
6627
|
+
`Invalid snapshot format in ${snapshotPath}.
|
|
6628
|
+
Expected an array or object with 'spans' property.`
|
|
6629
|
+
);
|
|
6630
|
+
}
|
|
6631
|
+
if (expectedStructure) {
|
|
6632
|
+
const structureMismatches = this.#compareStructure(structureGraph, expectedStructure);
|
|
6633
|
+
if (structureMismatches.length > 0) {
|
|
6634
|
+
throw new Error(
|
|
6635
|
+
`Structure mismatch in snapshot:
|
|
6636
|
+
|
|
6637
|
+
Expected:
|
|
6638
|
+
${expectedStructure.join("\n")}
|
|
6639
|
+
|
|
6640
|
+
Actual:
|
|
6641
|
+
${structureGraph.join("\n")}
|
|
6642
|
+
|
|
6643
|
+
Differences:
|
|
6644
|
+
${structureMismatches.join("\n")}
|
|
6645
|
+
|
|
6646
|
+
Snapshot: ${snapshotPath}
|
|
6647
|
+
Run with { updateSnapshot: true } to update.`
|
|
6648
|
+
);
|
|
6649
|
+
}
|
|
6650
|
+
}
|
|
6651
|
+
const mismatches = [];
|
|
6652
|
+
this.#deepCompareWithMarkers(normalizedTree, expectedSpans, "$.spans", mismatches);
|
|
6653
|
+
if (mismatches.length > 0) {
|
|
6654
|
+
const mismatchDetails = mismatches.map(
|
|
6655
|
+
(m, i) => `${i + 1}. ${m.path}
|
|
6656
|
+
Expected: ${JSON.stringify(m.expected)}
|
|
6657
|
+
Actual: ${JSON.stringify(m.actual)}`
|
|
6658
|
+
).join("\n\n");
|
|
6659
|
+
throw new Error(
|
|
6660
|
+
`Snapshot has ${mismatches.length} mismatch${mismatches.length > 1 ? "es" : ""}:
|
|
6661
|
+
|
|
6662
|
+
${mismatchDetails}
|
|
6663
|
+
|
|
6664
|
+
Snapshot: ${snapshotPath}
|
|
6665
|
+
Run with { updateSnapshot: true } to update.`
|
|
6666
|
+
);
|
|
4856
6667
|
}
|
|
4857
|
-
const startTime = Date.now();
|
|
4858
|
-
const flushReason = this.buffer.totalSize >= this.#config.maxBufferSize ? "overflow" : this.buffer.totalSize >= this.#config.maxBatchSize ? "size" : "time";
|
|
4859
|
-
const bufferCopy = {
|
|
4860
|
-
creates: [...this.buffer.creates],
|
|
4861
|
-
updates: [...this.buffer.updates],
|
|
4862
|
-
insertOnly: [...this.buffer.insertOnly],
|
|
4863
|
-
seenSpans: new Set(this.buffer.seenSpans),
|
|
4864
|
-
spanSequences: new Map(this.buffer.spanSequences),
|
|
4865
|
-
completedSpans: new Set(this.buffer.completedSpans),
|
|
4866
|
-
outOfOrderCount: this.buffer.outOfOrderCount,
|
|
4867
|
-
firstEventTime: this.buffer.firstEventTime,
|
|
4868
|
-
totalSize: this.buffer.totalSize
|
|
4869
|
-
};
|
|
4870
|
-
this.resetBuffer();
|
|
4871
|
-
await this.flushWithRetries(this.#storage, bufferCopy, 0);
|
|
4872
|
-
const elapsed = Date.now() - startTime;
|
|
4873
|
-
this.logger.debug("Batch flushed", {
|
|
4874
|
-
strategy: this.#resolvedStrategy,
|
|
4875
|
-
batchSize: bufferCopy.totalSize,
|
|
4876
|
-
flushReason,
|
|
4877
|
-
durationMs: elapsed,
|
|
4878
|
-
outOfOrderCount: bufferCopy.outOfOrderCount > 0 ? bufferCopy.outOfOrderCount : void 0
|
|
4879
|
-
});
|
|
4880
6668
|
}
|
|
4881
6669
|
/**
|
|
4882
|
-
*
|
|
6670
|
+
* Compare two structure graphs and return differences
|
|
4883
6671
|
*/
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
if (buffer.insertOnly.length > 0) {
|
|
4902
|
-
await storage.batchCreateSpans({ records: buffer.insertOnly });
|
|
6672
|
+
#compareStructure(actual, expected) {
|
|
6673
|
+
const diffs = [];
|
|
6674
|
+
const maxLen = Math.max(actual.length, expected.length);
|
|
6675
|
+
for (let i = 0; i < maxLen; i++) {
|
|
6676
|
+
const actualLine = actual[i];
|
|
6677
|
+
const expectedLine = expected[i];
|
|
6678
|
+
if (actualLine !== expectedLine) {
|
|
6679
|
+
if (actualLine === void 0) {
|
|
6680
|
+
diffs.push(`Line ${i + 1}: Missing in actual`);
|
|
6681
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6682
|
+
} else if (expectedLine === void 0) {
|
|
6683
|
+
diffs.push(`Line ${i + 1}: Extra in actual`);
|
|
6684
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
6685
|
+
} else {
|
|
6686
|
+
diffs.push(`Line ${i + 1}:`);
|
|
6687
|
+
diffs.push(` Expected: ${expectedLine}`);
|
|
6688
|
+
diffs.push(` Actual: ${actualLine}`);
|
|
4903
6689
|
}
|
|
4904
6690
|
}
|
|
4905
|
-
|
|
4906
|
-
|
|
6691
|
+
}
|
|
6692
|
+
return diffs;
|
|
6693
|
+
}
|
|
6694
|
+
/**
|
|
6695
|
+
* Deep compare two values, supporting special markers like __or__ and __any__.
|
|
6696
|
+
* Collects all mismatches into the provided array.
|
|
6697
|
+
*/
|
|
6698
|
+
#deepCompareWithMarkers(actual, expected, path, mismatches) {
|
|
6699
|
+
if (this.#isOrMarker(expected)) {
|
|
6700
|
+
const allowedValues = expected.__or__;
|
|
6701
|
+
const matches = allowedValues.some((allowed) => {
|
|
6702
|
+
const tempMismatches = [];
|
|
6703
|
+
this.#deepCompareWithMarkers(actual, allowed, path, tempMismatches);
|
|
6704
|
+
return tempMismatches.length === 0;
|
|
6705
|
+
});
|
|
6706
|
+
if (!matches) {
|
|
6707
|
+
mismatches.push({ path, expected: { __or__: allowedValues }, actual });
|
|
4907
6708
|
}
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
6709
|
+
return;
|
|
6710
|
+
}
|
|
6711
|
+
if (this.#isAnyMarker(expected)) {
|
|
6712
|
+
const typeConstraint = expected.__any__;
|
|
6713
|
+
if (actual === null || actual === void 0) {
|
|
6714
|
+
mismatches.push({ path, expected: { __any__: typeConstraint }, actual });
|
|
6715
|
+
return;
|
|
6716
|
+
}
|
|
6717
|
+
if (typeConstraint === true) {
|
|
6718
|
+
return;
|
|
6719
|
+
}
|
|
6720
|
+
const actualType = Array.isArray(actual) ? "array" : typeof actual;
|
|
6721
|
+
if (actualType !== typeConstraint) {
|
|
6722
|
+
mismatches.push({
|
|
6723
|
+
path,
|
|
6724
|
+
expected: { __any__: typeConstraint },
|
|
6725
|
+
actual: `(${actualType}) ${JSON.stringify(actual).slice(0, 50)}...`
|
|
4925
6726
|
});
|
|
4926
|
-
for (const spanKey of buffer.completedSpans) {
|
|
4927
|
-
this.allCreatedSpans.delete(spanKey);
|
|
4928
|
-
}
|
|
4929
6727
|
}
|
|
6728
|
+
return;
|
|
4930
6729
|
}
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
6730
|
+
if (Array.isArray(expected)) {
|
|
6731
|
+
if (!Array.isArray(actual)) {
|
|
6732
|
+
mismatches.push({ path, expected, actual });
|
|
6733
|
+
return;
|
|
6734
|
+
}
|
|
6735
|
+
if (actual.length !== expected.length) {
|
|
6736
|
+
mismatches.push({ path: `${path}.length`, expected: expected.length, actual: actual.length });
|
|
6737
|
+
return;
|
|
6738
|
+
}
|
|
6739
|
+
for (let i = 0; i < expected.length; i++) {
|
|
6740
|
+
this.#deepCompareWithMarkers(actual[i], expected[i], `${path}[${i}]`, mismatches);
|
|
6741
|
+
}
|
|
4935
6742
|
return;
|
|
4936
6743
|
}
|
|
4937
|
-
if (
|
|
4938
|
-
|
|
6744
|
+
if (expected !== null && typeof expected === "object") {
|
|
6745
|
+
if (actual === null || typeof actual !== "object" || Array.isArray(actual)) {
|
|
6746
|
+
mismatches.push({ path, expected, actual });
|
|
6747
|
+
return;
|
|
6748
|
+
}
|
|
6749
|
+
const expectedObj = expected;
|
|
6750
|
+
const actualObj = actual;
|
|
6751
|
+
for (const key of Object.keys(expectedObj)) {
|
|
6752
|
+
if (this.#isMetadataKey(key)) {
|
|
6753
|
+
continue;
|
|
6754
|
+
}
|
|
6755
|
+
if (!(key in actualObj)) {
|
|
6756
|
+
if (expectedObj[key] !== void 0) {
|
|
6757
|
+
mismatches.push({ path: `${path}.${key}`, expected: expectedObj[key], actual: void 0 });
|
|
6758
|
+
}
|
|
6759
|
+
continue;
|
|
6760
|
+
}
|
|
6761
|
+
this.#deepCompareWithMarkers(actualObj[key], expectedObj[key], `${path}.${key}`, mismatches);
|
|
6762
|
+
}
|
|
6763
|
+
for (const key of Object.keys(actualObj)) {
|
|
6764
|
+
if (this.#isMetadataKey(key)) {
|
|
6765
|
+
continue;
|
|
6766
|
+
}
|
|
6767
|
+
if (!(key in expectedObj)) {
|
|
6768
|
+
if (actualObj[key] !== void 0) {
|
|
6769
|
+
mismatches.push({ path: `${path}.${key}`, expected: void 0, actual: actualObj[key] });
|
|
6770
|
+
}
|
|
6771
|
+
}
|
|
6772
|
+
}
|
|
6773
|
+
return;
|
|
4939
6774
|
}
|
|
4940
|
-
|
|
4941
|
-
|
|
4942
|
-
await this.handleRealtimeEvent(event, this.#storage);
|
|
4943
|
-
break;
|
|
4944
|
-
case "batch-with-updates":
|
|
4945
|
-
this.handleBatchWithUpdatesEvent(event);
|
|
4946
|
-
break;
|
|
4947
|
-
case "insert-only":
|
|
4948
|
-
this.handleInsertOnlyEvent(event);
|
|
4949
|
-
break;
|
|
6775
|
+
if (actual !== expected) {
|
|
6776
|
+
mismatches.push({ path, expected, actual });
|
|
4950
6777
|
}
|
|
4951
6778
|
}
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
6779
|
+
/**
|
|
6780
|
+
* Check if a value is an __or__ marker object
|
|
6781
|
+
*/
|
|
6782
|
+
#isOrMarker(value) {
|
|
6783
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) && "__or__" in value && Array.isArray(value.__or__);
|
|
6784
|
+
}
|
|
6785
|
+
/**
|
|
6786
|
+
* Check if a value is an __any__ marker object
|
|
6787
|
+
*/
|
|
6788
|
+
#isAnyMarker(value) {
|
|
6789
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
6790
|
+
return false;
|
|
4956
6791
|
}
|
|
4957
|
-
if (
|
|
4958
|
-
|
|
4959
|
-
remainingEvents: this.buffer.totalSize
|
|
4960
|
-
});
|
|
4961
|
-
try {
|
|
4962
|
-
await this.flush();
|
|
4963
|
-
} catch (error) {
|
|
4964
|
-
this.logger.error("Failed to flush remaining events during shutdown", {
|
|
4965
|
-
error: error instanceof Error ? error.message : String(error)
|
|
4966
|
-
});
|
|
4967
|
-
}
|
|
6792
|
+
if (!("__any__" in value)) {
|
|
6793
|
+
return false;
|
|
4968
6794
|
}
|
|
4969
|
-
|
|
6795
|
+
const constraint = value.__any__;
|
|
6796
|
+
return constraint === true || ["string", "number", "boolean", "object", "array"].includes(constraint);
|
|
4970
6797
|
}
|
|
4971
|
-
|
|
4972
|
-
|
|
4973
|
-
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
#events = [];
|
|
4977
|
-
constructor(config = {}) {
|
|
4978
|
-
super(config);
|
|
6798
|
+
/**
|
|
6799
|
+
* Check if a key should be skipped during comparison (metadata keys like __structure__)
|
|
6800
|
+
*/
|
|
6801
|
+
#isMetadataKey(key) {
|
|
6802
|
+
return key.startsWith("__") && key.endsWith("__");
|
|
4979
6803
|
}
|
|
4980
|
-
|
|
4981
|
-
|
|
6804
|
+
/**
|
|
6805
|
+
* Custom JSON replacer to handle Date objects and other special types
|
|
6806
|
+
*/
|
|
6807
|
+
#jsonReplacer = (_key, value) => {
|
|
6808
|
+
if (value instanceof Date) {
|
|
6809
|
+
return value.toISOString();
|
|
6810
|
+
}
|
|
6811
|
+
return value;
|
|
6812
|
+
};
|
|
6813
|
+
// ============================================================================
|
|
6814
|
+
// Debugging Helpers
|
|
6815
|
+
// ============================================================================
|
|
6816
|
+
/**
|
|
6817
|
+
* Get all stored logs
|
|
6818
|
+
*/
|
|
6819
|
+
getLogs() {
|
|
6820
|
+
return [...this.#logs];
|
|
6821
|
+
}
|
|
6822
|
+
/**
|
|
6823
|
+
* Dump logs to console for debugging (uses console.error for visibility in test output)
|
|
6824
|
+
*/
|
|
6825
|
+
dumpLogs() {
|
|
6826
|
+
console.error("\n=== JsonExporter Logs ===");
|
|
6827
|
+
this.#logs.forEach((log) => console.error(log));
|
|
6828
|
+
console.error("=== End Logs ===\n");
|
|
6829
|
+
}
|
|
6830
|
+
/**
|
|
6831
|
+
* Validate final state - useful for test assertions
|
|
6832
|
+
*
|
|
6833
|
+
* @returns Object with validation results
|
|
6834
|
+
*/
|
|
6835
|
+
validateFinalState() {
|
|
6836
|
+
const traceIds = this.getTraceIds();
|
|
6837
|
+
const incompleteSpans = this.getIncompleteSpans();
|
|
6838
|
+
const singleTraceId = traceIds.length === 1;
|
|
6839
|
+
const allSpansComplete = incompleteSpans.length === 0;
|
|
6840
|
+
return {
|
|
6841
|
+
valid: singleTraceId && allSpansComplete,
|
|
6842
|
+
singleTraceId,
|
|
6843
|
+
allSpansComplete,
|
|
6844
|
+
traceIds,
|
|
6845
|
+
incompleteSpans
|
|
6846
|
+
};
|
|
4982
6847
|
}
|
|
6848
|
+
// ============================================================================
|
|
6849
|
+
// Reset & Lifecycle
|
|
6850
|
+
// ============================================================================
|
|
6851
|
+
/**
|
|
6852
|
+
* Clear all collected events and state
|
|
6853
|
+
*/
|
|
4983
6854
|
clearEvents() {
|
|
4984
6855
|
this.#events = [];
|
|
6856
|
+
this.#spanStates.clear();
|
|
6857
|
+
this.#logs = [];
|
|
4985
6858
|
}
|
|
4986
|
-
|
|
4987
|
-
|
|
6859
|
+
/**
|
|
6860
|
+
* Alias for clearEvents (compatibility with TestExporter)
|
|
6861
|
+
*/
|
|
6862
|
+
reset() {
|
|
6863
|
+
this.clearEvents();
|
|
4988
6864
|
}
|
|
4989
6865
|
async shutdown() {
|
|
4990
|
-
this.logger.info("
|
|
6866
|
+
this.logger.info("JsonExporter shutdown");
|
|
6867
|
+
}
|
|
6868
|
+
// ============================================================================
|
|
6869
|
+
// Private Helpers
|
|
6870
|
+
// ============================================================================
|
|
6871
|
+
/**
|
|
6872
|
+
* Extract unique spans from a list of events
|
|
6873
|
+
*/
|
|
6874
|
+
#getUniqueSpansFromEvents(events) {
|
|
6875
|
+
const spanMap = /* @__PURE__ */ new Map();
|
|
6876
|
+
for (const event of events) {
|
|
6877
|
+
const span = event.exportedSpan;
|
|
6878
|
+
if (event.type === TracingEventType.SPAN_ENDED || !spanMap.has(span.id)) {
|
|
6879
|
+
spanMap.set(span.id, span);
|
|
6880
|
+
}
|
|
6881
|
+
}
|
|
6882
|
+
return Array.from(spanMap.values());
|
|
4991
6883
|
}
|
|
4992
6884
|
};
|
|
4993
6885
|
|
|
@@ -5046,14 +6938,11 @@ var ModelSpanTracker = class {
|
|
|
5046
6938
|
#modelSpan;
|
|
5047
6939
|
#currentStepSpan;
|
|
5048
6940
|
#currentChunkSpan;
|
|
6941
|
+
#currentChunkType;
|
|
5049
6942
|
#accumulator = {};
|
|
5050
6943
|
#stepIndex = 0;
|
|
5051
6944
|
#chunkSequence = 0;
|
|
5052
6945
|
#completionStartTime;
|
|
5053
|
-
/** Tracks tool output accumulators by toolCallId for consolidating sub-agent streams */
|
|
5054
|
-
#toolOutputAccumulators = /* @__PURE__ */ new Map();
|
|
5055
|
-
/** Tracks toolCallIds that had streaming output (to skip redundant tool-result spans) */
|
|
5056
|
-
#streamedToolCallIds = /* @__PURE__ */ new Set();
|
|
5057
6946
|
constructor(modelSpan) {
|
|
5058
6947
|
this.#modelSpan = modelSpan;
|
|
5059
6948
|
}
|
|
@@ -5100,9 +6989,14 @@ var ModelSpanTracker = class {
|
|
|
5100
6989
|
this.#modelSpan?.update(options);
|
|
5101
6990
|
}
|
|
5102
6991
|
/**
|
|
5103
|
-
* Start a new Model execution step
|
|
6992
|
+
* Start a new Model execution step.
|
|
6993
|
+
* This should be called at the beginning of LLM execution to capture accurate startTime.
|
|
6994
|
+
* The step-start chunk payload can be passed later via updateStep() if needed.
|
|
5104
6995
|
*/
|
|
5105
|
-
|
|
6996
|
+
startStep(payload) {
|
|
6997
|
+
if (this.#currentStepSpan) {
|
|
6998
|
+
return;
|
|
6999
|
+
}
|
|
5106
7000
|
this.#currentStepSpan = this.#modelSpan?.createChildSpan({
|
|
5107
7001
|
name: `step: ${this.#stepIndex}`,
|
|
5108
7002
|
type: SpanType.MODEL_STEP,
|
|
@@ -5115,10 +7009,27 @@ var ModelSpanTracker = class {
|
|
|
5115
7009
|
});
|
|
5116
7010
|
this.#chunkSequence = 0;
|
|
5117
7011
|
}
|
|
7012
|
+
/**
|
|
7013
|
+
* Update the current step span with additional payload data.
|
|
7014
|
+
* Called when step-start chunk arrives with request/warnings info.
|
|
7015
|
+
*/
|
|
7016
|
+
updateStep(payload) {
|
|
7017
|
+
if (!this.#currentStepSpan || !payload) {
|
|
7018
|
+
return;
|
|
7019
|
+
}
|
|
7020
|
+
this.#currentStepSpan.update({
|
|
7021
|
+
input: payload.request,
|
|
7022
|
+
attributes: {
|
|
7023
|
+
...payload.messageId ? { messageId: payload.messageId } : {},
|
|
7024
|
+
...payload.warnings?.length ? { warnings: payload.warnings } : {}
|
|
7025
|
+
}
|
|
7026
|
+
});
|
|
7027
|
+
}
|
|
5118
7028
|
/**
|
|
5119
7029
|
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
5120
7030
|
*/
|
|
5121
7031
|
#endStepSpan(payload) {
|
|
7032
|
+
this.#endChunkSpan();
|
|
5122
7033
|
if (!this.#currentStepSpan) return;
|
|
5123
7034
|
const output = payload.output;
|
|
5124
7035
|
const { usage: rawUsage, ...otherOutput } = output;
|
|
@@ -5126,8 +7037,10 @@ var ModelSpanTracker = class {
|
|
|
5126
7037
|
const metadata = payload.metadata;
|
|
5127
7038
|
const usage = extractUsageMetrics(rawUsage, metadata?.providerMetadata);
|
|
5128
7039
|
const cleanMetadata = metadata ? { ...metadata } : void 0;
|
|
5129
|
-
if (cleanMetadata
|
|
5130
|
-
|
|
7040
|
+
if (cleanMetadata) {
|
|
7041
|
+
for (const key of ["request", "id", "timestamp", "modelId", "modelVersion", "modelProvider"]) {
|
|
7042
|
+
delete cleanMetadata[key];
|
|
7043
|
+
}
|
|
5131
7044
|
}
|
|
5132
7045
|
this.#currentStepSpan.end({
|
|
5133
7046
|
output: otherOutput,
|
|
@@ -5148,8 +7061,9 @@ var ModelSpanTracker = class {
|
|
|
5148
7061
|
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
5149
7062
|
*/
|
|
5150
7063
|
#startChunkSpan(chunkType, initialData) {
|
|
7064
|
+
this.#endChunkSpan();
|
|
5151
7065
|
if (!this.#currentStepSpan) {
|
|
5152
|
-
this
|
|
7066
|
+
this.startStep();
|
|
5153
7067
|
}
|
|
5154
7068
|
this.#currentChunkSpan = this.#currentStepSpan?.createChildSpan({
|
|
5155
7069
|
name: `chunk: '${chunkType}'`,
|
|
@@ -5159,6 +7073,7 @@ var ModelSpanTracker = class {
|
|
|
5159
7073
|
sequenceNumber: this.#chunkSequence
|
|
5160
7074
|
}
|
|
5161
7075
|
});
|
|
7076
|
+
this.#currentChunkType = chunkType;
|
|
5162
7077
|
this.#accumulator = initialData || {};
|
|
5163
7078
|
}
|
|
5164
7079
|
/**
|
|
@@ -5181,23 +7096,26 @@ var ModelSpanTracker = class {
|
|
|
5181
7096
|
output: output !== void 0 ? output : this.#accumulator
|
|
5182
7097
|
});
|
|
5183
7098
|
this.#currentChunkSpan = void 0;
|
|
7099
|
+
this.#currentChunkType = void 0;
|
|
5184
7100
|
this.#accumulator = {};
|
|
5185
7101
|
this.#chunkSequence++;
|
|
5186
7102
|
}
|
|
5187
7103
|
/**
|
|
5188
7104
|
* Create an event span (for single chunks like tool-call)
|
|
5189
7105
|
*/
|
|
5190
|
-
#createEventSpan(chunkType, output) {
|
|
7106
|
+
#createEventSpan(chunkType, output, options) {
|
|
5191
7107
|
if (!this.#currentStepSpan) {
|
|
5192
|
-
this
|
|
7108
|
+
this.startStep();
|
|
5193
7109
|
}
|
|
5194
7110
|
const span = this.#currentStepSpan?.createEventSpan({
|
|
5195
7111
|
name: `chunk: '${chunkType}'`,
|
|
5196
7112
|
type: SpanType.MODEL_CHUNK,
|
|
5197
7113
|
attributes: {
|
|
5198
7114
|
chunkType,
|
|
5199
|
-
sequenceNumber: this.#chunkSequence
|
|
7115
|
+
sequenceNumber: this.#chunkSequence,
|
|
7116
|
+
...options?.attributes
|
|
5200
7117
|
},
|
|
7118
|
+
metadata: options?.metadata,
|
|
5201
7119
|
output
|
|
5202
7120
|
});
|
|
5203
7121
|
if (span) {
|
|
@@ -5225,6 +7143,9 @@ var ModelSpanTracker = class {
|
|
|
5225
7143
|
this.#startChunkSpan("text");
|
|
5226
7144
|
break;
|
|
5227
7145
|
case "text-delta":
|
|
7146
|
+
if (this.#currentChunkType !== "text") {
|
|
7147
|
+
this.#startChunkSpan("text");
|
|
7148
|
+
}
|
|
5228
7149
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5229
7150
|
break;
|
|
5230
7151
|
case "text-end": {
|
|
@@ -5242,6 +7163,9 @@ var ModelSpanTracker = class {
|
|
|
5242
7163
|
this.#startChunkSpan("reasoning");
|
|
5243
7164
|
break;
|
|
5244
7165
|
case "reasoning-delta":
|
|
7166
|
+
if (this.#currentChunkType !== "reasoning") {
|
|
7167
|
+
this.#startChunkSpan("reasoning");
|
|
7168
|
+
}
|
|
5245
7169
|
this.#appendToAccumulator("text", chunk.payload.text);
|
|
5246
7170
|
break;
|
|
5247
7171
|
case "reasoning-end": {
|
|
@@ -5288,7 +7212,7 @@ var ModelSpanTracker = class {
|
|
|
5288
7212
|
#handleObjectChunk(chunk) {
|
|
5289
7213
|
switch (chunk.type) {
|
|
5290
7214
|
case "object":
|
|
5291
|
-
if (
|
|
7215
|
+
if (this.#currentChunkType !== "object") {
|
|
5292
7216
|
this.#startChunkSpan("object");
|
|
5293
7217
|
}
|
|
5294
7218
|
break;
|
|
@@ -5298,75 +7222,27 @@ var ModelSpanTracker = class {
|
|
|
5298
7222
|
}
|
|
5299
7223
|
}
|
|
5300
7224
|
/**
|
|
5301
|
-
* Handle tool-
|
|
5302
|
-
*
|
|
7225
|
+
* Handle tool-call-approval chunks.
|
|
7226
|
+
* Creates a span for approval requests so they can be seen in traces for debugging.
|
|
5303
7227
|
*/
|
|
5304
|
-
#
|
|
5305
|
-
if (chunk.type !== "tool-
|
|
7228
|
+
#handleToolApprovalChunk(chunk) {
|
|
7229
|
+
if (chunk.type !== "tool-call-approval") return;
|
|
5306
7230
|
const payload = chunk.payload;
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
if (!acc) {
|
|
5310
|
-
if (!this.#currentStepSpan) {
|
|
5311
|
-
this.#startStepSpan();
|
|
5312
|
-
}
|
|
5313
|
-
acc = {
|
|
5314
|
-
toolName: toolName || "unknown",
|
|
5315
|
-
toolCallId,
|
|
5316
|
-
text: "",
|
|
5317
|
-
reasoning: "",
|
|
5318
|
-
sequenceNumber: this.#chunkSequence++,
|
|
5319
|
-
// Name the span 'tool-result' for consistency (tool-call → tool-result)
|
|
5320
|
-
span: this.#currentStepSpan?.createChildSpan({
|
|
5321
|
-
name: `chunk: 'tool-result'`,
|
|
5322
|
-
type: SpanType.MODEL_CHUNK,
|
|
5323
|
-
attributes: {
|
|
5324
|
-
chunkType: "tool-result",
|
|
5325
|
-
sequenceNumber: this.#chunkSequence - 1
|
|
5326
|
-
}
|
|
5327
|
-
})
|
|
5328
|
-
};
|
|
5329
|
-
this.#toolOutputAccumulators.set(toolCallId, acc);
|
|
5330
|
-
}
|
|
5331
|
-
if (output && typeof output === "object" && "type" in output) {
|
|
5332
|
-
const innerType = output.type;
|
|
5333
|
-
switch (innerType) {
|
|
5334
|
-
case "text-delta":
|
|
5335
|
-
if (output.payload?.text) {
|
|
5336
|
-
acc.text += output.payload.text;
|
|
5337
|
-
}
|
|
5338
|
-
break;
|
|
5339
|
-
case "reasoning-delta":
|
|
5340
|
-
if (output.payload?.text) {
|
|
5341
|
-
acc.reasoning += output.payload.text;
|
|
5342
|
-
}
|
|
5343
|
-
break;
|
|
5344
|
-
case "finish":
|
|
5345
|
-
case "workflow-finish":
|
|
5346
|
-
this.#endToolOutputSpan(toolCallId);
|
|
5347
|
-
break;
|
|
5348
|
-
}
|
|
5349
|
-
}
|
|
5350
|
-
}
|
|
5351
|
-
/**
|
|
5352
|
-
* End a tool output span and clean up the accumulator
|
|
5353
|
-
*/
|
|
5354
|
-
#endToolOutputSpan(toolCallId) {
|
|
5355
|
-
const acc = this.#toolOutputAccumulators.get(toolCallId);
|
|
5356
|
-
if (!acc) return;
|
|
5357
|
-
const output = {
|
|
5358
|
-
toolCallId: acc.toolCallId,
|
|
5359
|
-
toolName: acc.toolName
|
|
5360
|
-
};
|
|
5361
|
-
if (acc.text) {
|
|
5362
|
-
output.text = acc.text;
|
|
7231
|
+
if (!this.#currentStepSpan) {
|
|
7232
|
+
this.startStep();
|
|
5363
7233
|
}
|
|
5364
|
-
|
|
5365
|
-
|
|
7234
|
+
const span = this.#currentStepSpan?.createEventSpan({
|
|
7235
|
+
name: `chunk: 'tool-call-approval'`,
|
|
7236
|
+
type: SpanType.MODEL_CHUNK,
|
|
7237
|
+
attributes: {
|
|
7238
|
+
chunkType: "tool-call-approval",
|
|
7239
|
+
sequenceNumber: this.#chunkSequence
|
|
7240
|
+
},
|
|
7241
|
+
output: payload
|
|
7242
|
+
});
|
|
7243
|
+
if (span) {
|
|
7244
|
+
this.#chunkSequence++;
|
|
5366
7245
|
}
|
|
5367
|
-
acc.span?.end({ output });
|
|
5368
|
-
this.#toolOutputAccumulators.delete(toolCallId);
|
|
5369
|
-
this.#streamedToolCallIds.add(toolCallId);
|
|
5370
7246
|
}
|
|
5371
7247
|
/**
|
|
5372
7248
|
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
@@ -5408,41 +7284,69 @@ var ModelSpanTracker = class {
|
|
|
5408
7284
|
this.#handleObjectChunk(chunk);
|
|
5409
7285
|
break;
|
|
5410
7286
|
case "step-start":
|
|
5411
|
-
this.#
|
|
7287
|
+
if (this.#currentStepSpan) {
|
|
7288
|
+
this.updateStep(chunk.payload);
|
|
7289
|
+
} else {
|
|
7290
|
+
this.startStep(chunk.payload);
|
|
7291
|
+
}
|
|
5412
7292
|
break;
|
|
5413
7293
|
case "step-finish":
|
|
5414
7294
|
this.#endStepSpan(chunk.payload);
|
|
5415
7295
|
break;
|
|
7296
|
+
// Infrastructure chunks - skip creating spans for these
|
|
7297
|
+
// They are either redundant, metadata-only, or error/control flow
|
|
5416
7298
|
case "raw":
|
|
5417
|
-
//
|
|
7299
|
+
// Redundant raw data
|
|
5418
7300
|
case "start":
|
|
7301
|
+
// Stream start marker
|
|
5419
7302
|
case "finish":
|
|
7303
|
+
// Stream finish marker (step-finish already captures this)
|
|
7304
|
+
case "response-metadata":
|
|
7305
|
+
// Response metadata (not semantic content)
|
|
7306
|
+
case "source":
|
|
7307
|
+
// Source references (metadata)
|
|
7308
|
+
case "file":
|
|
7309
|
+
// Binary file data (too large/not semantic)
|
|
7310
|
+
case "error":
|
|
7311
|
+
// Error handling
|
|
7312
|
+
case "abort":
|
|
7313
|
+
// Abort signal
|
|
7314
|
+
case "tripwire":
|
|
7315
|
+
// Processor rejection
|
|
7316
|
+
case "watch":
|
|
7317
|
+
// Internal watch event
|
|
7318
|
+
case "tool-error":
|
|
7319
|
+
// Tool error handling
|
|
7320
|
+
case "tool-call-suspended":
|
|
7321
|
+
// Suspension (not content)
|
|
7322
|
+
case "reasoning-signature":
|
|
7323
|
+
// Signature metadata
|
|
7324
|
+
case "redacted-reasoning":
|
|
7325
|
+
// Redacted content metadata
|
|
7326
|
+
case "step-output":
|
|
7327
|
+
break;
|
|
7328
|
+
case "tool-call-approval":
|
|
7329
|
+
this.#handleToolApprovalChunk(chunk);
|
|
5420
7330
|
break;
|
|
5421
7331
|
case "tool-output":
|
|
5422
|
-
this.#handleToolOutputChunk(chunk);
|
|
5423
7332
|
break;
|
|
5424
7333
|
case "tool-result": {
|
|
5425
|
-
const
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
if (
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
|
|
5442
|
-
delete outputPayload.data;
|
|
5443
|
-
}
|
|
5444
|
-
}
|
|
5445
|
-
this.#createEventSpan(chunk.type, outputPayload);
|
|
7334
|
+
const {
|
|
7335
|
+
// Metadata - tool call context (unique to tool-result chunks)
|
|
7336
|
+
toolCallId,
|
|
7337
|
+
toolName,
|
|
7338
|
+
isError,
|
|
7339
|
+
dynamic,
|
|
7340
|
+
providerExecuted,
|
|
7341
|
+
providerMetadata,
|
|
7342
|
+
// Output - the actual result
|
|
7343
|
+
result} = chunk.payload || {};
|
|
7344
|
+
const metadata = { toolCallId, toolName };
|
|
7345
|
+
if (isError !== void 0) metadata.isError = isError;
|
|
7346
|
+
if (dynamic !== void 0) metadata.dynamic = dynamic;
|
|
7347
|
+
if (providerExecuted !== void 0) metadata.providerExecuted = providerExecuted;
|
|
7348
|
+
if (providerMetadata !== void 0) metadata.providerMetadata = providerMetadata;
|
|
7349
|
+
this.#createEventSpan(chunk.type, result, { metadata });
|
|
5446
7350
|
break;
|
|
5447
7351
|
}
|
|
5448
7352
|
}
|
|
@@ -5458,7 +7362,11 @@ var DEFAULT_KEYS_TO_STRIP = /* @__PURE__ */ new Set([
|
|
|
5458
7362
|
"experimental_providerMetadata",
|
|
5459
7363
|
"providerMetadata",
|
|
5460
7364
|
"steps",
|
|
5461
|
-
"tracingContext"
|
|
7365
|
+
"tracingContext",
|
|
7366
|
+
"execute",
|
|
7367
|
+
// Tool execute functions
|
|
7368
|
+
"validate"
|
|
7369
|
+
// Schema validate functions
|
|
5462
7370
|
]);
|
|
5463
7371
|
var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
5464
7372
|
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
@@ -5467,12 +7375,68 @@ var DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
|
|
|
5467
7375
|
maxArrayLength: 50,
|
|
5468
7376
|
maxObjectKeys: 50
|
|
5469
7377
|
});
|
|
7378
|
+
function mergeSerializationOptions(userOptions) {
|
|
7379
|
+
if (!userOptions) {
|
|
7380
|
+
return DEFAULT_DEEP_CLEAN_OPTIONS;
|
|
7381
|
+
}
|
|
7382
|
+
return {
|
|
7383
|
+
keysToStrip: DEFAULT_KEYS_TO_STRIP,
|
|
7384
|
+
maxDepth: userOptions.maxDepth ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxDepth,
|
|
7385
|
+
maxStringLength: userOptions.maxStringLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxStringLength,
|
|
7386
|
+
maxArrayLength: userOptions.maxArrayLength ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxArrayLength,
|
|
7387
|
+
maxObjectKeys: userOptions.maxObjectKeys ?? DEFAULT_DEEP_CLEAN_OPTIONS.maxObjectKeys
|
|
7388
|
+
};
|
|
7389
|
+
}
|
|
5470
7390
|
function truncateString(s, maxChars) {
|
|
5471
7391
|
if (s.length <= maxChars) {
|
|
5472
7392
|
return s;
|
|
5473
7393
|
}
|
|
5474
7394
|
return s.slice(0, maxChars) + "\u2026[truncated]";
|
|
5475
7395
|
}
|
|
7396
|
+
function isJsonSchema(val) {
|
|
7397
|
+
if (typeof val !== "object" || val === null) return false;
|
|
7398
|
+
if (val.$schema && typeof val.$schema === "string" && val.$schema.includes("json-schema")) {
|
|
7399
|
+
return true;
|
|
7400
|
+
}
|
|
7401
|
+
if (val.type === "object" && val.properties && typeof val.properties === "object") {
|
|
7402
|
+
return true;
|
|
7403
|
+
}
|
|
7404
|
+
return false;
|
|
7405
|
+
}
|
|
7406
|
+
function compressJsonSchema(schema, depth = 0) {
|
|
7407
|
+
if (depth > 3) {
|
|
7408
|
+
return schema.type || "object";
|
|
7409
|
+
}
|
|
7410
|
+
if (schema.type !== "object" || !schema.properties) {
|
|
7411
|
+
return schema.type || schema;
|
|
7412
|
+
}
|
|
7413
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
7414
|
+
const compressed = {};
|
|
7415
|
+
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
7416
|
+
const prop = propSchema;
|
|
7417
|
+
let value = prop.type || "unknown";
|
|
7418
|
+
if (prop.type === "object" && prop.properties) {
|
|
7419
|
+
value = compressJsonSchema(prop, depth + 1);
|
|
7420
|
+
if (required.has(key)) {
|
|
7421
|
+
compressed[key + " (required)"] = value;
|
|
7422
|
+
continue;
|
|
7423
|
+
}
|
|
7424
|
+
} else if (prop.type === "array" && prop.items) {
|
|
7425
|
+
if (prop.items.type === "object" && prop.items.properties) {
|
|
7426
|
+
value = [compressJsonSchema(prop.items, depth + 1)];
|
|
7427
|
+
} else {
|
|
7428
|
+
value = `${prop.items.type || "any"}[]`;
|
|
7429
|
+
}
|
|
7430
|
+
} else if (prop.enum) {
|
|
7431
|
+
value = prop.enum.map((v) => JSON.stringify(v)).join(" | ");
|
|
7432
|
+
}
|
|
7433
|
+
if (required.has(key) && typeof value === "string") {
|
|
7434
|
+
value += " (required)";
|
|
7435
|
+
}
|
|
7436
|
+
compressed[key] = value;
|
|
7437
|
+
}
|
|
7438
|
+
return compressed;
|
|
7439
|
+
}
|
|
5476
7440
|
function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
5477
7441
|
const { keysToStrip, maxDepth, maxStringLength, maxArrayLength, maxObjectKeys } = options;
|
|
5478
7442
|
const seen = /* @__PURE__ */ new WeakSet();
|
|
@@ -5532,6 +7496,15 @@ function deepClean(value, options = DEFAULT_DEEP_CLEAN_OPTIONS) {
|
|
|
5532
7496
|
if (val instanceof ArrayBuffer) {
|
|
5533
7497
|
return `[ArrayBuffer byteLength=${val.byteLength}]`;
|
|
5534
7498
|
}
|
|
7499
|
+
if (typeof val.serializeForSpan === "function") {
|
|
7500
|
+
try {
|
|
7501
|
+
return helper(val.serializeForSpan(), depth);
|
|
7502
|
+
} catch {
|
|
7503
|
+
}
|
|
7504
|
+
}
|
|
7505
|
+
if (isJsonSchema(val)) {
|
|
7506
|
+
return helper(compressJsonSchema(val), depth);
|
|
7507
|
+
}
|
|
5535
7508
|
const cleaned = {};
|
|
5536
7509
|
const entries = Object.entries(val);
|
|
5537
7510
|
let keyCount = 0;
|
|
@@ -5615,24 +7588,38 @@ var BaseSpan = class {
|
|
|
5615
7588
|
metadata;
|
|
5616
7589
|
tags;
|
|
5617
7590
|
traceState;
|
|
7591
|
+
/** Entity type that created the span (e.g., agent, workflow) */
|
|
7592
|
+
entityType;
|
|
7593
|
+
/** Entity ID that created the span */
|
|
7594
|
+
entityId;
|
|
7595
|
+
/** Entity name that created the span */
|
|
7596
|
+
entityName;
|
|
5618
7597
|
/** Parent span ID (for root spans that are children of external spans) */
|
|
5619
7598
|
parentSpanId;
|
|
7599
|
+
/** Deep clean options for serialization */
|
|
7600
|
+
deepCleanOptions;
|
|
5620
7601
|
constructor(options, observabilityInstance) {
|
|
7602
|
+
const serializationOptions = observabilityInstance.getConfig().serializationOptions;
|
|
7603
|
+
this.deepCleanOptions = mergeSerializationOptions(serializationOptions);
|
|
5621
7604
|
this.name = options.name;
|
|
5622
7605
|
this.type = options.type;
|
|
5623
|
-
this.attributes = deepClean(options.attributes) || {};
|
|
5624
|
-
this.metadata = deepClean(options.metadata);
|
|
7606
|
+
this.attributes = deepClean(options.attributes, this.deepCleanOptions) || {};
|
|
7607
|
+
this.metadata = deepClean(options.metadata, this.deepCleanOptions);
|
|
5625
7608
|
this.parent = options.parent;
|
|
5626
|
-
this.startTime = /* @__PURE__ */ new Date();
|
|
7609
|
+
this.startTime = options.startTime ?? /* @__PURE__ */ new Date();
|
|
5627
7610
|
this.observabilityInstance = observabilityInstance;
|
|
5628
7611
|
this.isEvent = options.isEvent ?? false;
|
|
5629
7612
|
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
5630
7613
|
this.traceState = options.traceState;
|
|
5631
7614
|
this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
|
|
7615
|
+
const entityParent = this.getParentSpan(false);
|
|
7616
|
+
this.entityType = options.entityType ?? entityParent?.entityType;
|
|
7617
|
+
this.entityId = options.entityId ?? entityParent?.entityId;
|
|
7618
|
+
this.entityName = options.entityName ?? entityParent?.entityName;
|
|
5632
7619
|
if (this.isEvent) {
|
|
5633
|
-
this.output = deepClean(options.output);
|
|
7620
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5634
7621
|
} else {
|
|
5635
|
-
this.input = deepClean(options.input);
|
|
7622
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5636
7623
|
}
|
|
5637
7624
|
}
|
|
5638
7625
|
createChildSpan(options) {
|
|
@@ -5655,14 +7642,25 @@ var BaseSpan = class {
|
|
|
5655
7642
|
get isRootSpan() {
|
|
5656
7643
|
return !this.parent;
|
|
5657
7644
|
}
|
|
7645
|
+
/** Get the closest parent span, optionally skipping internal spans */
|
|
7646
|
+
getParentSpan(includeInternalSpans) {
|
|
7647
|
+
if (!this.parent) {
|
|
7648
|
+
return void 0;
|
|
7649
|
+
}
|
|
7650
|
+
if (includeInternalSpans) return this.parent;
|
|
7651
|
+
if (this.parent.isInternal) return this.parent.getParentSpan(includeInternalSpans);
|
|
7652
|
+
return this.parent;
|
|
7653
|
+
}
|
|
5658
7654
|
/** Get the closest parent spanId that isn't an internal span */
|
|
5659
7655
|
getParentSpanId(includeInternalSpans) {
|
|
5660
7656
|
if (!this.parent) {
|
|
5661
7657
|
return this.parentSpanId;
|
|
5662
7658
|
}
|
|
5663
|
-
|
|
5664
|
-
if (
|
|
5665
|
-
|
|
7659
|
+
const parentSpan = this.getParentSpan(includeInternalSpans);
|
|
7660
|
+
if (parentSpan) {
|
|
7661
|
+
return parentSpan.id;
|
|
7662
|
+
}
|
|
7663
|
+
return this.parent.getParentSpanId(includeInternalSpans);
|
|
5666
7664
|
}
|
|
5667
7665
|
/** Find the closest parent span of a specific type by walking up the parent chain */
|
|
5668
7666
|
findParent(spanType) {
|
|
@@ -5677,17 +7675,22 @@ var BaseSpan = class {
|
|
|
5677
7675
|
}
|
|
5678
7676
|
/** Returns a lightweight span ready for export */
|
|
5679
7677
|
exportSpan(includeInternalSpans) {
|
|
7678
|
+
const hideInput = this.traceState?.hideInput ?? false;
|
|
7679
|
+
const hideOutput = this.traceState?.hideOutput ?? false;
|
|
5680
7680
|
return {
|
|
5681
7681
|
id: this.id,
|
|
5682
7682
|
traceId: this.traceId,
|
|
5683
7683
|
name: this.name,
|
|
5684
7684
|
type: this.type,
|
|
7685
|
+
entityType: this.entityType,
|
|
7686
|
+
entityId: this.entityId,
|
|
7687
|
+
entityName: this.entityName,
|
|
5685
7688
|
attributes: this.attributes,
|
|
5686
7689
|
metadata: this.metadata,
|
|
5687
7690
|
startTime: this.startTime,
|
|
5688
7691
|
endTime: this.endTime,
|
|
5689
|
-
input: this.input,
|
|
5690
|
-
output: this.output,
|
|
7692
|
+
input: hideInput ? void 0 : this.input,
|
|
7693
|
+
output: hideOutput ? void 0 : this.output,
|
|
5691
7694
|
errorInfo: this.errorInfo,
|
|
5692
7695
|
isEvent: this.isEvent,
|
|
5693
7696
|
isRootSpan: this.isRootSpan,
|
|
@@ -5727,6 +7730,14 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5727
7730
|
traceId;
|
|
5728
7731
|
constructor(options, observabilityInstance) {
|
|
5729
7732
|
super(options, observabilityInstance);
|
|
7733
|
+
if (options.spanId && options.traceId) {
|
|
7734
|
+
this.id = options.spanId;
|
|
7735
|
+
this.traceId = options.traceId;
|
|
7736
|
+
if (options.parentSpanId) {
|
|
7737
|
+
this.parentSpanId = options.parentSpanId;
|
|
7738
|
+
}
|
|
7739
|
+
return;
|
|
7740
|
+
}
|
|
5730
7741
|
const bridge = observabilityInstance.getBridge();
|
|
5731
7742
|
if (bridge && !this.isInternal) {
|
|
5732
7743
|
const bridgeIds = bridge.createSpan(options);
|
|
@@ -5761,13 +7772,13 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5761
7772
|
}
|
|
5762
7773
|
this.endTime = /* @__PURE__ */ new Date();
|
|
5763
7774
|
if (options?.output !== void 0) {
|
|
5764
|
-
this.output = deepClean(options.output);
|
|
7775
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5765
7776
|
}
|
|
5766
7777
|
if (options?.attributes) {
|
|
5767
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7778
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5768
7779
|
}
|
|
5769
7780
|
if (options?.metadata) {
|
|
5770
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7781
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5771
7782
|
}
|
|
5772
7783
|
}
|
|
5773
7784
|
error(options) {
|
|
@@ -5785,10 +7796,10 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5785
7796
|
message: error.message
|
|
5786
7797
|
};
|
|
5787
7798
|
if (attributes) {
|
|
5788
|
-
this.attributes = { ...this.attributes, ...deepClean(attributes) };
|
|
7799
|
+
this.attributes = { ...this.attributes, ...deepClean(attributes, this.deepCleanOptions) };
|
|
5789
7800
|
}
|
|
5790
7801
|
if (metadata) {
|
|
5791
|
-
this.metadata = { ...this.metadata, ...deepClean(metadata) };
|
|
7802
|
+
this.metadata = { ...this.metadata, ...deepClean(metadata, this.deepCleanOptions) };
|
|
5792
7803
|
}
|
|
5793
7804
|
if (endSpan) {
|
|
5794
7805
|
this.end();
|
|
@@ -5801,16 +7812,16 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
5801
7812
|
return;
|
|
5802
7813
|
}
|
|
5803
7814
|
if (options.input !== void 0) {
|
|
5804
|
-
this.input = deepClean(options.input);
|
|
7815
|
+
this.input = deepClean(options.input, this.deepCleanOptions);
|
|
5805
7816
|
}
|
|
5806
7817
|
if (options.output !== void 0) {
|
|
5807
|
-
this.output = deepClean(options.output);
|
|
7818
|
+
this.output = deepClean(options.output, this.deepCleanOptions);
|
|
5808
7819
|
}
|
|
5809
7820
|
if (options.attributes) {
|
|
5810
|
-
this.attributes = { ...this.attributes, ...deepClean(options.attributes) };
|
|
7821
|
+
this.attributes = { ...this.attributes, ...deepClean(options.attributes, this.deepCleanOptions) };
|
|
5811
7822
|
}
|
|
5812
7823
|
if (options.metadata) {
|
|
5813
|
-
this.metadata = { ...this.metadata, ...deepClean(options.metadata) };
|
|
7824
|
+
this.metadata = { ...this.metadata, ...deepClean(options.metadata, this.deepCleanOptions) };
|
|
5814
7825
|
}
|
|
5815
7826
|
}
|
|
5816
7827
|
get isValid() {
|
|
@@ -5901,7 +7912,8 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
5901
7912
|
spanOutputProcessors: config.spanOutputProcessors ?? [],
|
|
5902
7913
|
bridge: config.bridge ?? void 0,
|
|
5903
7914
|
includeInternalSpans: config.includeInternalSpans ?? false,
|
|
5904
|
-
requestContextKeys: config.requestContextKeys ?? []
|
|
7915
|
+
requestContextKeys: config.requestContextKeys ?? [],
|
|
7916
|
+
serializationOptions: config.serializationOptions
|
|
5905
7917
|
};
|
|
5906
7918
|
if (this.config.bridge?.init) {
|
|
5907
7919
|
this.config.bridge.init({ config: this.config });
|
|
@@ -5939,11 +7951,26 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
5939
7951
|
// ============================================================================
|
|
5940
7952
|
/**
|
|
5941
7953
|
* Start a new span of a specific SpanType
|
|
7954
|
+
*
|
|
7955
|
+
* Sampling Decision:
|
|
7956
|
+
* - For root spans (no parent): Perform sampling check using the configured strategy
|
|
7957
|
+
* - For child spans: Inherit the sampling decision from the parent
|
|
7958
|
+
* - If parent is a NoOpSpan (not sampled), child is also a NoOpSpan
|
|
7959
|
+
* - If parent is a valid span (sampled), child is also sampled
|
|
7960
|
+
*
|
|
7961
|
+
* This ensures trace-level sampling: either all spans in a trace are sampled or none are.
|
|
7962
|
+
* See: https://github.com/mastra-ai/mastra/issues/11504
|
|
5942
7963
|
*/
|
|
5943
7964
|
startSpan(options) {
|
|
5944
7965
|
const { customSamplerOptions, requestContext, metadata, tracingOptions, ...rest } = options;
|
|
5945
|
-
if (
|
|
5946
|
-
|
|
7966
|
+
if (options.parent) {
|
|
7967
|
+
if (!options.parent.isValid) {
|
|
7968
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7969
|
+
}
|
|
7970
|
+
} else {
|
|
7971
|
+
if (!this.shouldSample(customSamplerOptions)) {
|
|
7972
|
+
return new NoOpSpan({ ...rest, metadata }, this);
|
|
7973
|
+
}
|
|
5947
7974
|
}
|
|
5948
7975
|
let traceState;
|
|
5949
7976
|
if (options.parent) {
|
|
@@ -5955,8 +7982,12 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
5955
7982
|
const mergedMetadata = metadata || tracingMetadata ? { ...metadata, ...tracingMetadata } : void 0;
|
|
5956
7983
|
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, mergedMetadata, traceState);
|
|
5957
7984
|
const tags = !options.parent ? tracingOptions?.tags : void 0;
|
|
7985
|
+
const traceId = !options.parent ? options.traceId ?? tracingOptions?.traceId : options.traceId;
|
|
7986
|
+
const parentSpanId = !options.parent ? options.parentSpanId ?? tracingOptions?.parentSpanId : options.parentSpanId;
|
|
5958
7987
|
const span = this.createSpan({
|
|
5959
7988
|
...rest,
|
|
7989
|
+
traceId,
|
|
7990
|
+
parentSpanId,
|
|
5960
7991
|
metadata: enrichedMetadata,
|
|
5961
7992
|
traceState,
|
|
5962
7993
|
tags
|
|
@@ -5969,6 +8000,37 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
5969
8000
|
}
|
|
5970
8001
|
return span;
|
|
5971
8002
|
}
|
|
8003
|
+
/**
|
|
8004
|
+
* Rebuild a span from exported data for lifecycle operations.
|
|
8005
|
+
* Used by durable execution engines (e.g., Inngest) to end/update spans
|
|
8006
|
+
* that were created in a previous durable operation.
|
|
8007
|
+
*
|
|
8008
|
+
* The rebuilt span:
|
|
8009
|
+
* - Does NOT emit SPAN_STARTED (assumes original span already did)
|
|
8010
|
+
* - Can have end(), update(), error() called on it
|
|
8011
|
+
* - Will emit SPAN_ENDED or SPAN_UPDATED when those methods are called
|
|
8012
|
+
*
|
|
8013
|
+
* @param cached - The exported span data to rebuild from
|
|
8014
|
+
* @returns A span that can have lifecycle methods called on it
|
|
8015
|
+
*/
|
|
8016
|
+
rebuildSpan(cached) {
|
|
8017
|
+
const span = this.createSpan({
|
|
8018
|
+
name: cached.name,
|
|
8019
|
+
type: cached.type,
|
|
8020
|
+
traceId: cached.traceId,
|
|
8021
|
+
spanId: cached.id,
|
|
8022
|
+
parentSpanId: cached.parentSpanId,
|
|
8023
|
+
startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
|
|
8024
|
+
input: cached.input,
|
|
8025
|
+
attributes: cached.attributes,
|
|
8026
|
+
metadata: cached.metadata,
|
|
8027
|
+
entityType: cached.entityType,
|
|
8028
|
+
entityId: cached.entityId,
|
|
8029
|
+
entityName: cached.entityName
|
|
8030
|
+
});
|
|
8031
|
+
this.wireSpanLifecycle(span);
|
|
8032
|
+
return span;
|
|
8033
|
+
}
|
|
5972
8034
|
// ============================================================================
|
|
5973
8035
|
// Configuration Management
|
|
5974
8036
|
// ============================================================================
|
|
@@ -6071,11 +8133,15 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6071
8133
|
const configuredKeys = this.config.requestContextKeys ?? [];
|
|
6072
8134
|
const additionalKeys = tracingOptions?.requestContextKeys ?? [];
|
|
6073
8135
|
const allKeys = [...configuredKeys, ...additionalKeys];
|
|
6074
|
-
|
|
8136
|
+
const hideInput = tracingOptions?.hideInput;
|
|
8137
|
+
const hideOutput = tracingOptions?.hideOutput;
|
|
8138
|
+
if (allKeys.length === 0 && !hideInput && !hideOutput) {
|
|
6075
8139
|
return void 0;
|
|
6076
8140
|
}
|
|
6077
8141
|
return {
|
|
6078
|
-
requestContextKeys: allKeys
|
|
8142
|
+
requestContextKeys: allKeys,
|
|
8143
|
+
...hideInput !== void 0 && { hideInput },
|
|
8144
|
+
...hideOutput !== void 0 && { hideOutput }
|
|
6079
8145
|
};
|
|
6080
8146
|
}
|
|
6081
8147
|
/**
|
|
@@ -6206,6 +8272,29 @@ var BaseObservabilityInstance = class extends MastraBase {
|
|
|
6206
8272
|
this.logger.debug(`[Observability] Initialization started [name=${this.name}]`);
|
|
6207
8273
|
this.logger.info(`[Observability] Initialized successfully [name=${this.name}]`);
|
|
6208
8274
|
}
|
|
8275
|
+
/**
|
|
8276
|
+
* Force flush any buffered/queued spans from all exporters and the bridge
|
|
8277
|
+
* without shutting down the observability instance.
|
|
8278
|
+
*
|
|
8279
|
+
* This is useful in serverless environments (like Vercel's fluid compute) where
|
|
8280
|
+
* you need to ensure all spans are exported before the runtime instance is
|
|
8281
|
+
* terminated, while keeping the observability system active for future requests.
|
|
8282
|
+
*/
|
|
8283
|
+
async flush() {
|
|
8284
|
+
this.logger.debug(`[Observability] Flush started [name=${this.name}]`);
|
|
8285
|
+
const flushPromises = [...this.exporters.map((e) => e.flush())];
|
|
8286
|
+
if (this.config.bridge) {
|
|
8287
|
+
flushPromises.push(this.config.bridge.flush());
|
|
8288
|
+
}
|
|
8289
|
+
const results = await Promise.allSettled(flushPromises);
|
|
8290
|
+
results.forEach((result, index) => {
|
|
8291
|
+
if (result.status === "rejected") {
|
|
8292
|
+
const targetName = index < this.exporters.length ? this.exporters[index]?.name : "bridge";
|
|
8293
|
+
this.logger.error(`[Observability] Flush error [target=${targetName}]`, result.reason);
|
|
8294
|
+
}
|
|
8295
|
+
});
|
|
8296
|
+
this.logger.debug(`[Observability] Flush completed [name=${this.name}]`);
|
|
8297
|
+
}
|
|
6209
8298
|
/**
|
|
6210
8299
|
* Shutdown Observability and clean up resources
|
|
6211
8300
|
*/
|
|
@@ -6380,6 +8469,9 @@ var SensitiveDataFilter = class {
|
|
|
6380
8469
|
return "[Circular Reference]";
|
|
6381
8470
|
}
|
|
6382
8471
|
seen.add(obj);
|
|
8472
|
+
if (obj instanceof Date) {
|
|
8473
|
+
return obj;
|
|
8474
|
+
}
|
|
6383
8475
|
if (Array.isArray(obj)) {
|
|
6384
8476
|
return obj.map((item) => this.deepFilter(item, seen));
|
|
6385
8477
|
}
|
|
@@ -6512,6 +8604,9 @@ var Observability = class extends MastraBase {
|
|
|
6512
8604
|
}
|
|
6513
8605
|
}
|
|
6514
8606
|
if (config.default?.enabled) {
|
|
8607
|
+
console.warn(
|
|
8608
|
+
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter, CloudExporter, and SensitiveDataFilter instead. See https://mastra.ai/docs/observability/tracing/overview for the recommended configuration.'
|
|
8609
|
+
);
|
|
6515
8610
|
const defaultInstance = new DefaultObservabilityInstance({
|
|
6516
8611
|
serviceName: "mastra",
|
|
6517
8612
|
name: "default",
|
|
@@ -6599,6 +8694,6 @@ function buildTracingOptions(...updaters) {
|
|
|
6599
8694
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
6600
8695
|
}
|
|
6601
8696
|
|
|
6602
|
-
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, TestExporter, buildTracingOptions, deepClean, getExternalParentId, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema, truncateString };
|
|
8697
|
+
export { BaseExporter, BaseObservabilityInstance, BaseSpan, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, ModelSpanTracker, NoOpSpan, Observability, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, mergeSerializationOptions, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
6603
8698
|
//# sourceMappingURL=index.js.map
|
|
6604
8699
|
//# sourceMappingURL=index.js.map
|