@observyze/sdk 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +145 -198
- package/dist/{chunk-FQBYUOJB.mjs → chunk-53PSEBAD.mjs} +67 -21
- package/dist/{index-IZdiaORv.d.mts → index-ClE8H5jj.d.mts} +264 -95
- package/dist/{index-IZdiaORv.d.ts → index-ClE8H5jj.d.ts} +264 -95
- package/dist/index.d.mts +4 -5
- package/dist/index.d.ts +4 -5
- package/dist/index.js +1087 -344
- package/dist/index.mjs +1012 -321
- package/dist/opentelemetry/index.d.mts +1 -2
- package/dist/opentelemetry/index.d.ts +1 -2
- package/dist/opentelemetry/index.js +50 -23
- package/dist/opentelemetry/index.mjs +1 -1
- package/package.json +60 -57
- package/dist/chunk-YSYKPKKX.mjs +0 -357
- package/dist/index--b41_E-1.d.mts +0 -440
- package/dist/index--b41_E-1.d.ts +0 -440
- package/dist/index-Cr-FN-y5.d.mts +0 -405
- package/dist/index-Cr-FN-y5.d.ts +0 -405
- package/dist/index-D4UXMom5.d.mts +0 -429
- package/dist/index-D4UXMom5.d.ts +0 -429
package/dist/index.js
CHANGED
|
@@ -30,15 +30,21 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
+
ExecutionBudget: () => ExecutionBudget,
|
|
34
|
+
ExecutionBudgetExceededError: () => ExecutionBudgetExceededError,
|
|
33
35
|
ObservyzeClient: () => ObservyzeClient,
|
|
34
36
|
ObservyzeSpanExporter: () => ObservyzeSpanExporter,
|
|
35
37
|
Span: () => Span,
|
|
36
|
-
SpanType: () =>
|
|
38
|
+
SpanType: () => SpanType,
|
|
37
39
|
Trace: () => Trace,
|
|
38
|
-
TraceStatus: () =>
|
|
40
|
+
TraceStatus: () => TraceStatus,
|
|
39
41
|
wrap: () => wrap,
|
|
40
42
|
wrapAnthropic: () => wrapAnthropic,
|
|
41
|
-
|
|
43
|
+
wrapGemini: () => wrapGemini,
|
|
44
|
+
wrapLangChain: () => wrapLangChain,
|
|
45
|
+
wrapLlamaIndex: () => wrapLlamaIndex,
|
|
46
|
+
wrapOpenAI: () => wrapOpenAI,
|
|
47
|
+
wrapVercelAI: () => wrapVercelAI
|
|
42
48
|
});
|
|
43
49
|
module.exports = __toCommonJS(index_exports);
|
|
44
50
|
|
|
@@ -46,22 +52,61 @@ module.exports = __toCommonJS(index_exports);
|
|
|
46
52
|
var import_debug3 = __toESM(require("debug"));
|
|
47
53
|
|
|
48
54
|
// src/types.ts
|
|
49
|
-
var
|
|
55
|
+
var SpanType = /* @__PURE__ */ ((SpanType3) => {
|
|
56
|
+
SpanType3["LLM"] = "llm";
|
|
57
|
+
SpanType3["TOOL"] = "tool";
|
|
58
|
+
SpanType3["AGENT"] = "agent";
|
|
59
|
+
SpanType3["CHAIN"] = "chain";
|
|
60
|
+
SpanType3["RETRIEVAL"] = "retrieval";
|
|
61
|
+
SpanType3["EMBEDDING"] = "embedding";
|
|
62
|
+
SpanType3["CUSTOM"] = "custom";
|
|
63
|
+
return SpanType3;
|
|
64
|
+
})(SpanType || {});
|
|
65
|
+
var TraceStatus = /* @__PURE__ */ ((TraceStatus2) => {
|
|
66
|
+
TraceStatus2["SUCCESS"] = "success";
|
|
67
|
+
TraceStatus2["ERROR"] = "error";
|
|
68
|
+
TraceStatus2["TIMEOUT"] = "timeout";
|
|
69
|
+
TraceStatus2["RUNNING"] = "running";
|
|
70
|
+
return TraceStatus2;
|
|
71
|
+
})(TraceStatus || {});
|
|
50
72
|
|
|
51
73
|
// src/trace.ts
|
|
52
74
|
var import_crypto = require("crypto");
|
|
53
75
|
function generateId() {
|
|
54
76
|
return `${Date.now()}-${(0, import_crypto.randomUUID)().substring(0, 8)}`;
|
|
55
77
|
}
|
|
78
|
+
var SAFE_OPERATIONAL_METADATA = /* @__PURE__ */ new Set([
|
|
79
|
+
"provider",
|
|
80
|
+
"model",
|
|
81
|
+
"temperature",
|
|
82
|
+
"max_tokens",
|
|
83
|
+
"max_completion_tokens",
|
|
84
|
+
"max_output_tokens",
|
|
85
|
+
"latency_ms",
|
|
86
|
+
"streaming",
|
|
87
|
+
"stream_completed",
|
|
88
|
+
"output_truncated",
|
|
89
|
+
"token_usage_source",
|
|
90
|
+
"cost_source",
|
|
91
|
+
"known_pricing",
|
|
92
|
+
"source",
|
|
93
|
+
"lifecycle",
|
|
94
|
+
"invocation_type"
|
|
95
|
+
]);
|
|
96
|
+
function isSafeOperationalValue(value) {
|
|
97
|
+
return value === null || ["string", "number", "boolean"].includes(typeof value);
|
|
98
|
+
}
|
|
56
99
|
var Span = class {
|
|
57
100
|
data;
|
|
58
101
|
startTime;
|
|
59
|
-
|
|
102
|
+
captureContent;
|
|
103
|
+
constructor(name, type, parentSpanId, captureContent = true) {
|
|
60
104
|
this.startTime = Date.now();
|
|
105
|
+
this.captureContent = captureContent;
|
|
61
106
|
this.data = {
|
|
62
107
|
span_id: generateId(),
|
|
63
108
|
parent_span_id: parentSpanId,
|
|
64
|
-
name,
|
|
109
|
+
name: captureContent ? name : "[CONTENT_CAPTURE_DISABLED]",
|
|
65
110
|
type,
|
|
66
111
|
start_time: new Date(this.startTime),
|
|
67
112
|
end_time: new Date(this.startTime),
|
|
@@ -76,14 +121,14 @@ var Span = class {
|
|
|
76
121
|
* Set the input data for this span
|
|
77
122
|
*/
|
|
78
123
|
setInput(input) {
|
|
79
|
-
this.data.input = input;
|
|
124
|
+
if (this.captureContent) this.data.input = input;
|
|
80
125
|
return this;
|
|
81
126
|
}
|
|
82
127
|
/**
|
|
83
128
|
* Set the output data for this span
|
|
84
129
|
*/
|
|
85
130
|
setOutput(output) {
|
|
86
|
-
this.data.output = output;
|
|
131
|
+
if (this.captureContent) this.data.output = output;
|
|
87
132
|
return this;
|
|
88
133
|
}
|
|
89
134
|
/**
|
|
@@ -91,9 +136,9 @@ var Span = class {
|
|
|
91
136
|
*/
|
|
92
137
|
setError(error) {
|
|
93
138
|
this.data.error = {
|
|
94
|
-
message: error.message,
|
|
95
|
-
stack: error.stack,
|
|
96
|
-
code: error.code
|
|
139
|
+
message: this.captureContent ? error.message : "Error details omitted because content capture is disabled",
|
|
140
|
+
stack: this.captureContent ? error.stack : void 0,
|
|
141
|
+
code: this.captureContent ? error.code : void 0
|
|
97
142
|
};
|
|
98
143
|
return this;
|
|
99
144
|
}
|
|
@@ -101,14 +146,16 @@ var Span = class {
|
|
|
101
146
|
* Set metadata for this span
|
|
102
147
|
*/
|
|
103
148
|
setMetadata(key, value) {
|
|
104
|
-
this.
|
|
149
|
+
if (this.captureContent || SAFE_OPERATIONAL_METADATA.has(key) && isSafeOperationalValue(value)) {
|
|
150
|
+
this.data.metadata[key] = value;
|
|
151
|
+
}
|
|
105
152
|
return this;
|
|
106
153
|
}
|
|
107
154
|
/**
|
|
108
155
|
* Set multiple metadata fields at once
|
|
109
156
|
*/
|
|
110
157
|
setMetadataAll(metadata) {
|
|
111
|
-
|
|
158
|
+
for (const [key, value] of Object.entries(metadata)) this.setMetadata(key, value);
|
|
112
159
|
return this;
|
|
113
160
|
}
|
|
114
161
|
/**
|
|
@@ -144,14 +191,16 @@ var Trace = class {
|
|
|
144
191
|
startTime;
|
|
145
192
|
spans = [];
|
|
146
193
|
ended = false;
|
|
147
|
-
|
|
194
|
+
captureContent;
|
|
195
|
+
constructor(name, organizationId, projectId, captureContent = true) {
|
|
148
196
|
this.startTime = Date.now();
|
|
197
|
+
this.captureContent = captureContent;
|
|
149
198
|
this.data = {
|
|
150
199
|
trace_id: generateId(),
|
|
151
200
|
organization_id: organizationId,
|
|
152
201
|
project_id: projectId,
|
|
153
|
-
name,
|
|
154
|
-
status:
|
|
202
|
+
name: captureContent ? name : "[CONTENT_CAPTURE_DISABLED]",
|
|
203
|
+
status: "running" /* RUNNING */,
|
|
155
204
|
start_time: new Date(this.startTime),
|
|
156
205
|
end_time: new Date(this.startTime),
|
|
157
206
|
// Will be updated on end()
|
|
@@ -168,7 +217,7 @@ var Trace = class {
|
|
|
168
217
|
if (this.ended) {
|
|
169
218
|
throw new Error("Cannot start span on an ended trace");
|
|
170
219
|
}
|
|
171
|
-
const span = new Span(name, type, parentSpanId);
|
|
220
|
+
const span = new Span(name, type, parentSpanId, this.captureContent);
|
|
172
221
|
this.spans.push(span);
|
|
173
222
|
return span;
|
|
174
223
|
}
|
|
@@ -176,20 +225,23 @@ var Trace = class {
|
|
|
176
225
|
* Add metadata to the trace
|
|
177
226
|
*/
|
|
178
227
|
setMetadata(key, value) {
|
|
179
|
-
this.
|
|
228
|
+
if (this.captureContent || SAFE_OPERATIONAL_METADATA.has(key) && isSafeOperationalValue(value)) {
|
|
229
|
+
this.data.metadata[key] = value;
|
|
230
|
+
}
|
|
180
231
|
return this;
|
|
181
232
|
}
|
|
182
233
|
/**
|
|
183
234
|
* Set multiple metadata fields at once
|
|
184
235
|
*/
|
|
185
236
|
setMetadataAll(metadata) {
|
|
186
|
-
|
|
237
|
+
for (const [key, value] of Object.entries(metadata)) this.setMetadata(key, value);
|
|
187
238
|
return this;
|
|
188
239
|
}
|
|
189
240
|
/**
|
|
190
241
|
* Add tags to the trace
|
|
191
242
|
*/
|
|
192
243
|
addTag(tag) {
|
|
244
|
+
if (!this.captureContent) return this;
|
|
193
245
|
if (!this.data.tags.includes(tag)) {
|
|
194
246
|
this.data.tags.push(tag);
|
|
195
247
|
}
|
|
@@ -206,20 +258,20 @@ var Trace = class {
|
|
|
206
258
|
* Set the user ID associated with this trace
|
|
207
259
|
*/
|
|
208
260
|
setUserId(userId) {
|
|
209
|
-
this.data.user_id = userId;
|
|
261
|
+
if (this.captureContent) this.data.user_id = userId;
|
|
210
262
|
return this;
|
|
211
263
|
}
|
|
212
264
|
/**
|
|
213
265
|
* Set the session ID associated with this trace
|
|
214
266
|
*/
|
|
215
267
|
setSessionId(sessionId) {
|
|
216
|
-
this.data.session_id = sessionId;
|
|
268
|
+
if (this.captureContent) this.data.session_id = sessionId;
|
|
217
269
|
return this;
|
|
218
270
|
}
|
|
219
271
|
/**
|
|
220
272
|
* End the trace with a final status
|
|
221
273
|
*/
|
|
222
|
-
end(status =
|
|
274
|
+
end(status = "success" /* SUCCESS */) {
|
|
223
275
|
if (this.ended) {
|
|
224
276
|
return;
|
|
225
277
|
}
|
|
@@ -253,8 +305,11 @@ var Trace = class {
|
|
|
253
305
|
// src/instrumentation/openai.ts
|
|
254
306
|
var import_debug = __toESM(require("debug"));
|
|
255
307
|
var log = (0, import_debug.default)("observyze:sdk");
|
|
308
|
+
var WRAPPED = /* @__PURE__ */ Symbol.for("observyze.openai.wrapped");
|
|
309
|
+
var MAX_CAPTURED_STREAM_CHARS = 1e6;
|
|
256
310
|
function wrapOpenAI(client, nwClient) {
|
|
257
311
|
const anyClient = client;
|
|
312
|
+
if (anyClient[WRAPPED]) return client;
|
|
258
313
|
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
259
314
|
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/openai");
|
|
260
315
|
if (!isAlreadyRedirected) {
|
|
@@ -280,7 +335,7 @@ function wrapOpenAI(client, nwClient) {
|
|
|
280
335
|
provider: "openai",
|
|
281
336
|
model: params.model
|
|
282
337
|
});
|
|
283
|
-
const span = trace.startSpan("chat.completions.create",
|
|
338
|
+
const span = trace.startSpan("chat.completions.create", "llm" /* LLM */);
|
|
284
339
|
span.setMetadata("model", params.model);
|
|
285
340
|
span.setMetadata("provider", "openai");
|
|
286
341
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -310,38 +365,64 @@ function wrapOpenAI(client, nwClient) {
|
|
|
310
365
|
output: completionResponse.usage.completion_tokens,
|
|
311
366
|
total: completionResponse.usage.total_tokens
|
|
312
367
|
});
|
|
368
|
+
span.setMetadata("token_usage_source", "provider");
|
|
369
|
+
} else {
|
|
370
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
313
371
|
}
|
|
314
372
|
span.setMetadata("latency_ms", latency);
|
|
315
373
|
span.end();
|
|
316
|
-
trace.end();
|
|
374
|
+
trace.end("success" /* SUCCESS */);
|
|
317
375
|
return response;
|
|
318
376
|
} catch (error) {
|
|
319
377
|
const latency = Date.now() - startTime;
|
|
320
378
|
span.setMetadata("latency_ms", latency);
|
|
321
379
|
span.setError(error);
|
|
322
380
|
span.end();
|
|
323
|
-
trace.end();
|
|
381
|
+
trace.end("error" /* ERROR */);
|
|
324
382
|
throw error;
|
|
325
383
|
}
|
|
326
384
|
};
|
|
385
|
+
Object.defineProperty(anyClient, WRAPPED, { value: true, enumerable: false });
|
|
327
386
|
return client;
|
|
328
387
|
}
|
|
329
388
|
function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
330
389
|
const bufferedChunks = [];
|
|
331
390
|
let streamId = "";
|
|
332
391
|
let streamModel = "";
|
|
392
|
+
let inputTokens = 0;
|
|
393
|
+
let outputTokens = 0;
|
|
394
|
+
let outputTruncated = false;
|
|
395
|
+
let capturedChars = 0;
|
|
333
396
|
return {
|
|
334
397
|
[Symbol.asyncIterator]: async function* () {
|
|
398
|
+
let completed = false;
|
|
399
|
+
let failure;
|
|
335
400
|
try {
|
|
336
401
|
for await (const chunk of stream) {
|
|
337
402
|
if (chunk.id) streamId = chunk.id;
|
|
338
403
|
if (chunk.model) streamModel = chunk.model;
|
|
339
404
|
const delta = chunk.choices[0]?.delta;
|
|
340
405
|
if (delta?.content) {
|
|
341
|
-
|
|
406
|
+
if (capturedChars < MAX_CAPTURED_STREAM_CHARS) {
|
|
407
|
+
const captured = delta.content.slice(0, MAX_CAPTURED_STREAM_CHARS - capturedChars);
|
|
408
|
+
bufferedChunks.push(captured);
|
|
409
|
+
capturedChars += captured.length;
|
|
410
|
+
if (captured.length < delta.content.length) outputTruncated = true;
|
|
411
|
+
} else {
|
|
412
|
+
outputTruncated = true;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (chunk.usage) {
|
|
416
|
+
inputTokens = Math.max(inputTokens, chunk.usage.prompt_tokens || 0);
|
|
417
|
+
outputTokens = Math.max(outputTokens, chunk.usage.completion_tokens || 0);
|
|
342
418
|
}
|
|
343
419
|
yield chunk;
|
|
344
420
|
}
|
|
421
|
+
completed = true;
|
|
422
|
+
} catch (error) {
|
|
423
|
+
failure = error;
|
|
424
|
+
throw error;
|
|
425
|
+
} finally {
|
|
345
426
|
const latency = Date.now() - startTime;
|
|
346
427
|
const completeOutput = bufferedChunks.join("");
|
|
347
428
|
span.setOutput({
|
|
@@ -351,15 +432,21 @@ function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
|
351
432
|
});
|
|
352
433
|
span.setMetadata("latency_ms", latency);
|
|
353
434
|
span.setMetadata("streaming", true);
|
|
435
|
+
span.setMetadata("stream_completed", completed);
|
|
436
|
+
span.setMetadata("output_truncated", outputTruncated);
|
|
437
|
+
if (inputTokens > 0 || outputTokens > 0) {
|
|
438
|
+
span.setTokens({ input: inputTokens, output: outputTokens, total: inputTokens + outputTokens });
|
|
439
|
+
span.setMetadata("token_usage_source", "provider");
|
|
440
|
+
} else {
|
|
441
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
442
|
+
}
|
|
443
|
+
if (failure) {
|
|
444
|
+
span.setError(failure);
|
|
445
|
+
} else if (!completed) {
|
|
446
|
+
span.setError(new Error("Stream consumption ended before provider completion"));
|
|
447
|
+
}
|
|
354
448
|
span.end();
|
|
355
|
-
trace.end();
|
|
356
|
-
} catch (error) {
|
|
357
|
-
const latency = Date.now() - startTime;
|
|
358
|
-
span.setMetadata("latency_ms", latency);
|
|
359
|
-
span.setError(error);
|
|
360
|
-
span.end();
|
|
361
|
-
trace.end();
|
|
362
|
-
throw error;
|
|
449
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
363
450
|
}
|
|
364
451
|
}
|
|
365
452
|
};
|
|
@@ -368,8 +455,11 @@ function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
|
368
455
|
// src/instrumentation/anthropic.ts
|
|
369
456
|
var import_debug2 = __toESM(require("debug"));
|
|
370
457
|
var log2 = (0, import_debug2.default)("observyze:sdk");
|
|
458
|
+
var WRAPPED2 = /* @__PURE__ */ Symbol.for("observyze.anthropic.wrapped");
|
|
459
|
+
var MAX_CAPTURED_STREAM_CHARS2 = 1e6;
|
|
371
460
|
function wrapAnthropic(client, nwClient) {
|
|
372
461
|
const anyClient = client;
|
|
462
|
+
if (anyClient[WRAPPED2]) return client;
|
|
373
463
|
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
374
464
|
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/anthropic");
|
|
375
465
|
if (!isAlreadyRedirected) {
|
|
@@ -395,7 +485,7 @@ function wrapAnthropic(client, nwClient) {
|
|
|
395
485
|
provider: "anthropic",
|
|
396
486
|
model: params.model
|
|
397
487
|
});
|
|
398
|
-
const span = trace.startSpan("messages.create",
|
|
488
|
+
const span = trace.startSpan("messages.create", "llm" /* LLM */);
|
|
399
489
|
span.setMetadata("model", params.model);
|
|
400
490
|
span.setMetadata("provider", "anthropic");
|
|
401
491
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -429,20 +519,24 @@ function wrapAnthropic(client, nwClient) {
|
|
|
429
519
|
output: messageResponse.usage.output_tokens,
|
|
430
520
|
total: messageResponse.usage.input_tokens + messageResponse.usage.output_tokens
|
|
431
521
|
});
|
|
522
|
+
span.setMetadata("token_usage_source", "provider");
|
|
523
|
+
} else {
|
|
524
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
432
525
|
}
|
|
433
526
|
span.setMetadata("latency_ms", latency);
|
|
434
527
|
span.end();
|
|
435
|
-
trace.end();
|
|
528
|
+
trace.end("success" /* SUCCESS */);
|
|
436
529
|
return response;
|
|
437
530
|
} catch (error) {
|
|
438
531
|
const latency = Date.now() - startTime;
|
|
439
532
|
span.setMetadata("latency_ms", latency);
|
|
440
533
|
span.setError(error);
|
|
441
534
|
span.end();
|
|
442
|
-
trace.end();
|
|
535
|
+
trace.end("error" /* ERROR */);
|
|
443
536
|
throw error;
|
|
444
537
|
}
|
|
445
538
|
};
|
|
539
|
+
Object.defineProperty(anyClient, WRAPPED2, { value: true, enumerable: false });
|
|
446
540
|
return client;
|
|
447
541
|
}
|
|
448
542
|
function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
@@ -452,8 +546,12 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
452
546
|
let stopReason = null;
|
|
453
547
|
let inputTokens = 0;
|
|
454
548
|
let outputTokens = 0;
|
|
549
|
+
let capturedChars = 0;
|
|
550
|
+
let outputTruncated = false;
|
|
455
551
|
return {
|
|
456
552
|
[Symbol.asyncIterator]: async function* () {
|
|
553
|
+
let completed = false;
|
|
554
|
+
let failure;
|
|
457
555
|
try {
|
|
458
556
|
for await (const event of stream) {
|
|
459
557
|
if (event.type === "message_start" && event.message) {
|
|
@@ -464,7 +562,14 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
464
562
|
}
|
|
465
563
|
}
|
|
466
564
|
if (event.type === "content_block_delta" && event.delta?.text) {
|
|
467
|
-
|
|
565
|
+
if (capturedChars < MAX_CAPTURED_STREAM_CHARS2) {
|
|
566
|
+
const captured = event.delta.text.slice(0, MAX_CAPTURED_STREAM_CHARS2 - capturedChars);
|
|
567
|
+
bufferedChunks.push(captured);
|
|
568
|
+
capturedChars += captured.length;
|
|
569
|
+
if (captured.length < event.delta.text.length) outputTruncated = true;
|
|
570
|
+
} else {
|
|
571
|
+
outputTruncated = true;
|
|
572
|
+
}
|
|
468
573
|
}
|
|
469
574
|
if (event.type === "message_delta" && event.delta) {
|
|
470
575
|
if (event.delta.stop_reason) {
|
|
@@ -476,6 +581,11 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
476
581
|
}
|
|
477
582
|
yield event;
|
|
478
583
|
}
|
|
584
|
+
completed = true;
|
|
585
|
+
} catch (error) {
|
|
586
|
+
failure = error;
|
|
587
|
+
throw error;
|
|
588
|
+
} finally {
|
|
479
589
|
const latency = Date.now() - startTime;
|
|
480
590
|
const completeOutput = bufferedChunks.join("");
|
|
481
591
|
span.setOutput({
|
|
@@ -490,52 +600,747 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
490
600
|
output: outputTokens,
|
|
491
601
|
total: inputTokens + outputTokens
|
|
492
602
|
});
|
|
603
|
+
span.setMetadata("token_usage_source", "provider");
|
|
604
|
+
} else {
|
|
605
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
493
606
|
}
|
|
494
607
|
span.setMetadata("latency_ms", latency);
|
|
495
608
|
span.setMetadata("streaming", true);
|
|
609
|
+
span.setMetadata("stream_completed", completed);
|
|
610
|
+
span.setMetadata("output_truncated", outputTruncated);
|
|
611
|
+
if (failure) {
|
|
612
|
+
span.setError(failure);
|
|
613
|
+
} else if (!completed) {
|
|
614
|
+
span.setError(new Error("Stream consumption ended before provider completion"));
|
|
615
|
+
}
|
|
496
616
|
span.end();
|
|
497
|
-
trace.end();
|
|
617
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// src/instrumentation/langchain.ts
|
|
624
|
+
var import_async_hooks = require("async_hooks");
|
|
625
|
+
var WRAPPED3 = /* @__PURE__ */ Symbol.for("observyze.langchain.wrapped");
|
|
626
|
+
var nestedInstrumentation = new import_async_hooks.AsyncLocalStorage();
|
|
627
|
+
var MAX_CAPTURED_STREAM_CHARS3 = 1e6;
|
|
628
|
+
function runnableName(runnable) {
|
|
629
|
+
return String(
|
|
630
|
+
runnable.name || runnable.lc_namespace?.join(".") || runnable.constructor?.name || "Runnable"
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
function extractUsage(output) {
|
|
634
|
+
const usage2 = output?.usage_metadata || output?.response_metadata?.tokenUsage || output?.usage;
|
|
635
|
+
if (!usage2) return null;
|
|
636
|
+
const input = Number(usage2.input_tokens ?? usage2.promptTokens ?? usage2.prompt_tokens ?? 0);
|
|
637
|
+
const outputTokens = Number(usage2.output_tokens ?? usage2.completionTokens ?? usage2.completion_tokens ?? 0);
|
|
638
|
+
const total = Number(usage2.total_tokens ?? usage2.totalTokens ?? input + outputTokens);
|
|
639
|
+
return input > 0 || outputTokens > 0 || total > 0 ? { input, output: outputTokens, total } : null;
|
|
640
|
+
}
|
|
641
|
+
function chunkText(chunk) {
|
|
642
|
+
if (typeof chunk === "string") return chunk;
|
|
643
|
+
if (typeof chunk?.content === "string") return chunk.content;
|
|
644
|
+
if (typeof chunk?.text === "string") return chunk.text;
|
|
645
|
+
return "";
|
|
646
|
+
}
|
|
647
|
+
function wrapLangChain(runnable, nwClient) {
|
|
648
|
+
const target = runnable;
|
|
649
|
+
if (target[WRAPPED3]) return runnable;
|
|
650
|
+
const name = runnableName(runnable);
|
|
651
|
+
const originalInvoke = runnable.invoke.bind(runnable);
|
|
652
|
+
target.invoke = async (input, config) => {
|
|
653
|
+
if (nestedInstrumentation.getStore()) return originalInvoke(input, config);
|
|
654
|
+
const trace = nwClient.startTrace(`langchain.${name}.invoke`, { provider: "langchain", runnable: name });
|
|
655
|
+
const span = trace.startSpan(`${name}.invoke`, "chain" /* CHAIN */);
|
|
656
|
+
span.setInput(input);
|
|
657
|
+
try {
|
|
658
|
+
const output = await nestedInstrumentation.run(true, () => originalInvoke(input, config));
|
|
659
|
+
span.setOutput(output);
|
|
660
|
+
const usage2 = extractUsage(output);
|
|
661
|
+
if (usage2) {
|
|
662
|
+
span.setTokens(usage2);
|
|
663
|
+
span.setMetadata("token_usage_source", "provider");
|
|
664
|
+
} else {
|
|
665
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
666
|
+
}
|
|
667
|
+
span.end();
|
|
668
|
+
trace.end("success" /* SUCCESS */);
|
|
669
|
+
return output;
|
|
670
|
+
} catch (error) {
|
|
671
|
+
span.setError(error);
|
|
672
|
+
span.end();
|
|
673
|
+
trace.end("error" /* ERROR */);
|
|
674
|
+
throw error;
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
if (typeof runnable.stream === "function") {
|
|
678
|
+
const originalStream = runnable.stream.bind(runnable);
|
|
679
|
+
target.stream = async (input, config) => {
|
|
680
|
+
if (nestedInstrumentation.getStore()) return originalStream(input, config);
|
|
681
|
+
const trace = nwClient.startTrace(`langchain.${name}.stream`, { provider: "langchain", runnable: name });
|
|
682
|
+
const span = trace.startSpan(`${name}.stream`, "chain" /* CHAIN */);
|
|
683
|
+
span.setInput(input);
|
|
684
|
+
let source;
|
|
685
|
+
try {
|
|
686
|
+
source = await nestedInstrumentation.run(true, () => originalStream(input, config));
|
|
498
687
|
} catch (error) {
|
|
499
|
-
const latency = Date.now() - startTime;
|
|
500
|
-
span.setMetadata("latency_ms", latency);
|
|
501
688
|
span.setError(error);
|
|
502
689
|
span.end();
|
|
503
|
-
trace.end();
|
|
690
|
+
trace.end("error" /* ERROR */);
|
|
691
|
+
throw error;
|
|
692
|
+
}
|
|
693
|
+
return {
|
|
694
|
+
[Symbol.asyncIterator]: async function* () {
|
|
695
|
+
let output = "";
|
|
696
|
+
let completed = false;
|
|
697
|
+
let truncated = false;
|
|
698
|
+
let failure;
|
|
699
|
+
try {
|
|
700
|
+
for await (const chunk of source) {
|
|
701
|
+
const text = chunkText(chunk);
|
|
702
|
+
if (output.length < MAX_CAPTURED_STREAM_CHARS3) {
|
|
703
|
+
const captured = text.slice(0, MAX_CAPTURED_STREAM_CHARS3 - output.length);
|
|
704
|
+
output += captured;
|
|
705
|
+
if (captured.length < text.length) truncated = true;
|
|
706
|
+
} else if (text) {
|
|
707
|
+
truncated = true;
|
|
708
|
+
}
|
|
709
|
+
yield chunk;
|
|
710
|
+
}
|
|
711
|
+
completed = true;
|
|
712
|
+
} catch (error) {
|
|
713
|
+
failure = error;
|
|
714
|
+
throw error;
|
|
715
|
+
} finally {
|
|
716
|
+
span.setOutput({ content: output });
|
|
717
|
+
span.setMetadata("streaming", true);
|
|
718
|
+
span.setMetadata("stream_completed", completed);
|
|
719
|
+
span.setMetadata("output_truncated", truncated);
|
|
720
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
721
|
+
if (failure) span.setError(failure);
|
|
722
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before runnable completion"));
|
|
723
|
+
span.end();
|
|
724
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
Object.defineProperty(target, WRAPPED3, { value: true, enumerable: false });
|
|
731
|
+
return runnable;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// src/instrumentation/gemini.ts
|
|
735
|
+
var WRAPPED4 = /* @__PURE__ */ Symbol.for("observyze.gemini.wrapped");
|
|
736
|
+
var MAX_CAPTURED_STREAM_CHARS4 = 1e6;
|
|
737
|
+
function modelName(target, request) {
|
|
738
|
+
return String(request?.model || target?.model || target?.modelName || "unknown");
|
|
739
|
+
}
|
|
740
|
+
function responseText(response) {
|
|
741
|
+
try {
|
|
742
|
+
if (typeof response?.text === "function") return String(response.text());
|
|
743
|
+
if (typeof response?.text === "string") return response.text;
|
|
744
|
+
const parts = response?.candidates?.[0]?.content?.parts;
|
|
745
|
+
return Array.isArray(parts) ? parts.map((part) => part?.text || "").join("") : "";
|
|
746
|
+
} catch {
|
|
747
|
+
return "";
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
function usage(response) {
|
|
751
|
+
const value = response?.usageMetadata || response?.usage_metadata || response?.usage;
|
|
752
|
+
if (!value) return null;
|
|
753
|
+
const input = Number(value.promptTokenCount ?? value.inputTokens ?? value.input_tokens ?? 0);
|
|
754
|
+
const output = Number(value.candidatesTokenCount ?? value.outputTokens ?? value.output_tokens ?? 0);
|
|
755
|
+
const total = Number(value.totalTokenCount ?? value.totalTokens ?? value.total_tokens ?? input + output);
|
|
756
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
757
|
+
}
|
|
758
|
+
function instrumentMethod(target, method, nwClient) {
|
|
759
|
+
if (typeof target?.[method] !== "function") return;
|
|
760
|
+
const original = target[method].bind(target);
|
|
761
|
+
target[method] = async (...args) => {
|
|
762
|
+
const request = args[0];
|
|
763
|
+
const model = modelName(target, request);
|
|
764
|
+
const trace = nwClient.startTrace(`gemini.${method}`, { provider: "google", model });
|
|
765
|
+
const span = trace.startSpan(method, "llm" /* LLM */);
|
|
766
|
+
span.setMetadata("provider", "google");
|
|
767
|
+
span.setMetadata("model", model);
|
|
768
|
+
span.setInput(request);
|
|
769
|
+
try {
|
|
770
|
+
const result = await original(...args);
|
|
771
|
+
const stream = result?.stream || (result?.[Symbol.asyncIterator] ? result : null);
|
|
772
|
+
if (stream?.[Symbol.asyncIterator]) {
|
|
773
|
+
const wrappedStream = wrapGeminiStream(stream, result?.response, span, trace, model);
|
|
774
|
+
if (result?.stream) {
|
|
775
|
+
return new Proxy(result, {
|
|
776
|
+
get(target2, property, receiver) {
|
|
777
|
+
return property === "stream" ? wrappedStream : Reflect.get(target2, property, receiver);
|
|
778
|
+
}
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
return wrappedStream;
|
|
782
|
+
}
|
|
783
|
+
const resolved = result?.response ? await result.response : result;
|
|
784
|
+
span.setOutput({
|
|
785
|
+
text: responseText(resolved),
|
|
786
|
+
finish_reason: resolved?.candidates?.[0]?.finishReason
|
|
787
|
+
});
|
|
788
|
+
const tokenUsage = usage(resolved);
|
|
789
|
+
if (tokenUsage) {
|
|
790
|
+
span.setTokens(tokenUsage);
|
|
791
|
+
span.setMetadata("token_usage_source", "provider");
|
|
792
|
+
} else {
|
|
793
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
794
|
+
}
|
|
795
|
+
span.end();
|
|
796
|
+
trace.end("success" /* SUCCESS */);
|
|
797
|
+
return result;
|
|
798
|
+
} catch (error) {
|
|
799
|
+
span.setError(error);
|
|
800
|
+
span.end();
|
|
801
|
+
trace.end("error" /* ERROR */);
|
|
802
|
+
throw error;
|
|
803
|
+
}
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
function wrapGeminiStream(stream, finalResponse, span, trace, model) {
|
|
807
|
+
return {
|
|
808
|
+
[Symbol.asyncIterator]: async function* () {
|
|
809
|
+
let text = "";
|
|
810
|
+
let completed = false;
|
|
811
|
+
let truncated = false;
|
|
812
|
+
let failure;
|
|
813
|
+
let latestUsage = null;
|
|
814
|
+
try {
|
|
815
|
+
for await (const chunk of stream) {
|
|
816
|
+
const chunkText2 = responseText(chunk);
|
|
817
|
+
if (text.length < MAX_CAPTURED_STREAM_CHARS4) {
|
|
818
|
+
const captured = chunkText2.slice(0, MAX_CAPTURED_STREAM_CHARS4 - text.length);
|
|
819
|
+
text += captured;
|
|
820
|
+
if (captured.length < chunkText2.length) truncated = true;
|
|
821
|
+
} else if (chunkText2) {
|
|
822
|
+
truncated = true;
|
|
823
|
+
}
|
|
824
|
+
latestUsage = usage(chunk) || latestUsage;
|
|
825
|
+
yield chunk;
|
|
826
|
+
}
|
|
827
|
+
completed = true;
|
|
828
|
+
} catch (error) {
|
|
829
|
+
failure = error;
|
|
504
830
|
throw error;
|
|
831
|
+
} finally {
|
|
832
|
+
if (completed && finalResponse) {
|
|
833
|
+
try {
|
|
834
|
+
const resolved = await finalResponse;
|
|
835
|
+
latestUsage = usage(resolved) || latestUsage;
|
|
836
|
+
if (!text) text = responseText(resolved).slice(0, MAX_CAPTURED_STREAM_CHARS4);
|
|
837
|
+
} catch (error) {
|
|
838
|
+
failure = error;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
span.setOutput({ model, text });
|
|
842
|
+
span.setMetadata("streaming", true);
|
|
843
|
+
span.setMetadata("stream_completed", completed);
|
|
844
|
+
span.setMetadata("output_truncated", truncated);
|
|
845
|
+
if (latestUsage) {
|
|
846
|
+
span.setTokens(latestUsage);
|
|
847
|
+
span.setMetadata("token_usage_source", "provider");
|
|
848
|
+
} else {
|
|
849
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
850
|
+
}
|
|
851
|
+
if (failure) span.setError(failure);
|
|
852
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before Gemini completed"));
|
|
853
|
+
span.end();
|
|
854
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
505
855
|
}
|
|
506
856
|
}
|
|
507
857
|
};
|
|
508
858
|
}
|
|
859
|
+
function wrapGemini(client, nwClient) {
|
|
860
|
+
const target = client.models || client;
|
|
861
|
+
if (target[WRAPPED4]) return client;
|
|
862
|
+
instrumentMethod(target, "generateContent", nwClient);
|
|
863
|
+
instrumentMethod(target, "generateContentStream", nwClient);
|
|
864
|
+
Object.defineProperty(target, WRAPPED4, { value: true, enumerable: false });
|
|
865
|
+
return client;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// src/instrumentation/vercel-ai.ts
|
|
869
|
+
var MAX_CAPTURED_STREAM_CHARS5 = 1e6;
|
|
870
|
+
function modelName2(request, response) {
|
|
871
|
+
let requestModelId;
|
|
872
|
+
try {
|
|
873
|
+
requestModelId = typeof request?.model?.modelId === "function" ? request.model.modelId() : request?.model?.modelId;
|
|
874
|
+
} catch {
|
|
875
|
+
requestModelId = void 0;
|
|
876
|
+
}
|
|
877
|
+
return String(
|
|
878
|
+
response?.response?.modelId || requestModelId || request?.model || "unknown"
|
|
879
|
+
);
|
|
880
|
+
}
|
|
881
|
+
async function resolveUsage(value) {
|
|
882
|
+
const usage2 = await Promise.resolve(value?.usage).catch(() => null);
|
|
883
|
+
if (!usage2) return null;
|
|
884
|
+
const input = Number(usage2.inputTokens ?? usage2.promptTokens ?? usage2.input_tokens ?? 0);
|
|
885
|
+
const output = Number(usage2.outputTokens ?? usage2.completionTokens ?? usage2.output_tokens ?? 0);
|
|
886
|
+
const total = Number(usage2.totalTokens ?? usage2.total_tokens ?? input + output);
|
|
887
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
888
|
+
}
|
|
889
|
+
function wrapVercelAI(sdk, nwClient) {
|
|
890
|
+
const wrapped = { ...sdk };
|
|
891
|
+
const generateText = sdk.generateText;
|
|
892
|
+
const streamText = sdk.streamText;
|
|
893
|
+
if (typeof generateText === "function") {
|
|
894
|
+
wrapped.generateText = async (...args) => {
|
|
895
|
+
const request = args[0];
|
|
896
|
+
const trace = nwClient.startTrace("vercel-ai.generateText", { provider: "vercel-ai", model: modelName2(request) });
|
|
897
|
+
const span = trace.startSpan("generateText", "llm" /* LLM */);
|
|
898
|
+
span.setMetadata("provider", "vercel-ai");
|
|
899
|
+
span.setMetadata("model", modelName2(request));
|
|
900
|
+
span.setInput(request);
|
|
901
|
+
try {
|
|
902
|
+
const result = await generateText.apply(sdk, args);
|
|
903
|
+
span.setOutput({ text: result?.text, finish_reason: result?.finishReason });
|
|
904
|
+
const tokens = await resolveUsage(result);
|
|
905
|
+
if (tokens) {
|
|
906
|
+
span.setTokens(tokens);
|
|
907
|
+
span.setMetadata("token_usage_source", "provider");
|
|
908
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
909
|
+
span.end();
|
|
910
|
+
trace.end("success" /* SUCCESS */);
|
|
911
|
+
return result;
|
|
912
|
+
} catch (error) {
|
|
913
|
+
span.setError(error);
|
|
914
|
+
span.end();
|
|
915
|
+
trace.end("error" /* ERROR */);
|
|
916
|
+
throw error;
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
}
|
|
920
|
+
if (typeof streamText === "function") {
|
|
921
|
+
wrapped.streamText = (...args) => {
|
|
922
|
+
const request = args[0];
|
|
923
|
+
const trace = nwClient.startTrace("vercel-ai.streamText", { provider: "vercel-ai", model: modelName2(request) });
|
|
924
|
+
const span = trace.startSpan("streamText", "llm" /* LLM */);
|
|
925
|
+
span.setMetadata("provider", "vercel-ai");
|
|
926
|
+
span.setMetadata("model", modelName2(request));
|
|
927
|
+
span.setInput(request);
|
|
928
|
+
let result;
|
|
929
|
+
try {
|
|
930
|
+
result = streamText.apply(sdk, args);
|
|
931
|
+
} catch (error) {
|
|
932
|
+
span.setError(error);
|
|
933
|
+
span.end();
|
|
934
|
+
trace.end("error" /* ERROR */);
|
|
935
|
+
throw error;
|
|
936
|
+
}
|
|
937
|
+
if (!result?.textStream?.[Symbol.asyncIterator]) {
|
|
938
|
+
span.setError(new Error("Vercel AI streamText returned no textStream"));
|
|
939
|
+
span.end();
|
|
940
|
+
trace.end("error" /* ERROR */);
|
|
941
|
+
return result;
|
|
942
|
+
}
|
|
943
|
+
const source = result.textStream;
|
|
944
|
+
const textStream = {
|
|
945
|
+
[Symbol.asyncIterator]: async function* () {
|
|
946
|
+
let text = "";
|
|
947
|
+
let completed = false;
|
|
948
|
+
let truncated = false;
|
|
949
|
+
let failure;
|
|
950
|
+
try {
|
|
951
|
+
for await (const chunk of source) {
|
|
952
|
+
if (text.length < MAX_CAPTURED_STREAM_CHARS5) {
|
|
953
|
+
const captured = String(chunk).slice(0, MAX_CAPTURED_STREAM_CHARS5 - text.length);
|
|
954
|
+
text += captured;
|
|
955
|
+
if (captured.length < String(chunk).length) truncated = true;
|
|
956
|
+
} else if (chunk) truncated = true;
|
|
957
|
+
yield chunk;
|
|
958
|
+
}
|
|
959
|
+
completed = true;
|
|
960
|
+
} catch (error) {
|
|
961
|
+
failure = error;
|
|
962
|
+
throw error;
|
|
963
|
+
} finally {
|
|
964
|
+
span.setOutput({ text });
|
|
965
|
+
span.setMetadata("streaming", true);
|
|
966
|
+
span.setMetadata("stream_completed", completed);
|
|
967
|
+
span.setMetadata("output_truncated", truncated);
|
|
968
|
+
const tokens = completed ? await resolveUsage(result) : null;
|
|
969
|
+
if (tokens) {
|
|
970
|
+
span.setTokens(tokens);
|
|
971
|
+
span.setMetadata("token_usage_source", "provider");
|
|
972
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
973
|
+
if (failure) span.setError(failure);
|
|
974
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before Vercel AI completed"));
|
|
975
|
+
span.end();
|
|
976
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
};
|
|
980
|
+
return new Proxy(result, {
|
|
981
|
+
get(target, property, receiver) {
|
|
982
|
+
return property === "textStream" ? textStream : Reflect.get(target, property, receiver);
|
|
983
|
+
}
|
|
984
|
+
});
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
return wrapped;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
// src/instrumentation/llamaindex.ts
|
|
991
|
+
var WRAPPED5 = /* @__PURE__ */ Symbol.for("observyze.llamaindex.wrapped");
|
|
992
|
+
function outputValue(response) {
|
|
993
|
+
if (typeof response?.response === "string") return { text: response.response };
|
|
994
|
+
if (typeof response?.message?.content === "string") return { text: response.message.content };
|
|
995
|
+
if (typeof response === "string") return { text: response };
|
|
996
|
+
if (typeof response?.toString === "function" && response.toString !== Object.prototype.toString) {
|
|
997
|
+
try {
|
|
998
|
+
return { text: String(response.toString()) };
|
|
999
|
+
} catch {
|
|
1000
|
+
return { response_type: response?.constructor?.name || typeof response };
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
return { response_type: response?.constructor?.name || typeof response };
|
|
1004
|
+
}
|
|
1005
|
+
function extractUsage2(response) {
|
|
1006
|
+
const usage2 = response?.usage || response?.raw?.usage || response?.message?.additionalKwargs?.usage;
|
|
1007
|
+
if (!usage2) return null;
|
|
1008
|
+
const input = Number(usage2.prompt_tokens ?? usage2.input_tokens ?? usage2.inputTokens ?? 0);
|
|
1009
|
+
const output = Number(usage2.completion_tokens ?? usage2.output_tokens ?? usage2.outputTokens ?? 0);
|
|
1010
|
+
const total = Number(usage2.total_tokens ?? usage2.totalTokens ?? input + output);
|
|
1011
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
1012
|
+
}
|
|
1013
|
+
function instrument(target, method, nwClient) {
|
|
1014
|
+
if (typeof target?.[method] !== "function") return;
|
|
1015
|
+
const original = target[method].bind(target);
|
|
1016
|
+
target[method] = async (...args) => {
|
|
1017
|
+
const trace = nwClient.startTrace(`llamaindex.${method}`, { provider: "llamaindex" });
|
|
1018
|
+
const span = trace.startSpan(method, method === "query" ? "retrieval" /* RETRIEVAL */ : "llm" /* LLM */);
|
|
1019
|
+
span.setMetadata("provider", "llamaindex");
|
|
1020
|
+
span.setInput(args[0]);
|
|
1021
|
+
try {
|
|
1022
|
+
const result = await original(...args);
|
|
1023
|
+
span.setOutput(outputValue(result));
|
|
1024
|
+
const tokens = extractUsage2(result);
|
|
1025
|
+
if (tokens) {
|
|
1026
|
+
span.setTokens(tokens);
|
|
1027
|
+
span.setMetadata("token_usage_source", "provider");
|
|
1028
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
1029
|
+
span.end();
|
|
1030
|
+
trace.end("success" /* SUCCESS */);
|
|
1031
|
+
return result;
|
|
1032
|
+
} catch (error) {
|
|
1033
|
+
span.setError(error);
|
|
1034
|
+
span.end();
|
|
1035
|
+
trace.end("error" /* ERROR */);
|
|
1036
|
+
throw error;
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
}
|
|
1040
|
+
function wrapLlamaIndex(engine, nwClient) {
|
|
1041
|
+
const target = engine;
|
|
1042
|
+
if (target[WRAPPED5]) return engine;
|
|
1043
|
+
instrument(target, "query", nwClient);
|
|
1044
|
+
instrument(target, "chat", nwClient);
|
|
1045
|
+
Object.defineProperty(target, WRAPPED5, { value: true, enumerable: false });
|
|
1046
|
+
return engine;
|
|
1047
|
+
}
|
|
509
1048
|
|
|
510
1049
|
// src/instrumentation/index.ts
|
|
511
1050
|
function wrap(client, nwClient) {
|
|
512
|
-
|
|
1051
|
+
const candidate = client;
|
|
1052
|
+
if (candidate?.chat?.completions && typeof candidate.chat.completions.create === "function") {
|
|
513
1053
|
return wrapOpenAI(client, nwClient);
|
|
514
1054
|
}
|
|
515
|
-
if (
|
|
1055
|
+
if (candidate?.messages && typeof candidate.messages.create === "function") {
|
|
516
1056
|
return wrapAnthropic(client, nwClient);
|
|
517
1057
|
}
|
|
1058
|
+
if (typeof candidate?.generateContent === "function" || typeof candidate?.generateContentStream === "function" || typeof candidate?.models?.generateContent === "function" || typeof candidate?.models?.generateContentStream === "function") {
|
|
1059
|
+
return wrapGemini(client, nwClient);
|
|
1060
|
+
}
|
|
1061
|
+
if (typeof candidate?.generateText === "function" || typeof candidate?.streamText === "function") {
|
|
1062
|
+
return wrapVercelAI(client, nwClient);
|
|
1063
|
+
}
|
|
1064
|
+
if (typeof candidate?.invoke === "function") {
|
|
1065
|
+
return wrapLangChain(client, nwClient);
|
|
1066
|
+
}
|
|
1067
|
+
if (typeof candidate?.query === "function" || typeof candidate?.chat === "function") {
|
|
1068
|
+
return wrapLlamaIndex(client, nwClient);
|
|
1069
|
+
}
|
|
518
1070
|
throw new Error(
|
|
519
|
-
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic"
|
|
1071
|
+
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic, Gemini, Vercel AI SDK, LangChain, and LlamaIndex"
|
|
520
1072
|
);
|
|
521
1073
|
}
|
|
522
1074
|
|
|
1075
|
+
// src/execution-budget.ts
|
|
1076
|
+
var ExecutionBudgetExceededError = class extends Error {
|
|
1077
|
+
constructor(dimension, message) {
|
|
1078
|
+
super(message);
|
|
1079
|
+
this.dimension = dimension;
|
|
1080
|
+
this.name = "ExecutionBudgetExceededError";
|
|
1081
|
+
}
|
|
1082
|
+
dimension;
|
|
1083
|
+
code = "OBSERVYZE_EXECUTION_BUDGET_EXCEEDED";
|
|
1084
|
+
};
|
|
1085
|
+
var ExecutionBudget = class {
|
|
1086
|
+
constructor(options) {
|
|
1087
|
+
this.options = options;
|
|
1088
|
+
if (!Number.isInteger(options.maxCalls) || options.maxCalls < 1 || options.maxCalls > 1e5) {
|
|
1089
|
+
throw new Error("ExecutionBudget maxCalls must be an integer between 1 and 100000");
|
|
1090
|
+
}
|
|
1091
|
+
if (!Number.isInteger(options.timeoutMs) || options.timeoutMs < 100 || options.timeoutMs > 864e5) {
|
|
1092
|
+
throw new Error("ExecutionBudget timeoutMs must be an integer between 100 and 86400000");
|
|
1093
|
+
}
|
|
1094
|
+
if (options.maxTokens !== void 0 && (!Number.isInteger(options.maxTokens) || options.maxTokens < 1 || options.maxTokens > 1e10)) {
|
|
1095
|
+
throw new Error("ExecutionBudget maxTokens must be an integer between 1 and 10000000000");
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
options;
|
|
1099
|
+
startedAt = Date.now();
|
|
1100
|
+
controller = new AbortController();
|
|
1101
|
+
callsUsed = 0;
|
|
1102
|
+
tokensUsed = 0;
|
|
1103
|
+
get signal() {
|
|
1104
|
+
return this.controller.signal;
|
|
1105
|
+
}
|
|
1106
|
+
get stats() {
|
|
1107
|
+
const elapsedMs = Date.now() - this.startedAt;
|
|
1108
|
+
return {
|
|
1109
|
+
callsUsed: this.callsUsed,
|
|
1110
|
+
tokensUsed: this.tokensUsed,
|
|
1111
|
+
elapsedMs,
|
|
1112
|
+
remainingCalls: Math.max(0, this.options.maxCalls - this.callsUsed),
|
|
1113
|
+
remainingTokens: this.options.maxTokens === void 0 ? null : Math.max(0, this.options.maxTokens - this.tokensUsed),
|
|
1114
|
+
remainingMs: Math.max(0, this.options.timeoutMs - elapsedMs),
|
|
1115
|
+
aborted: this.controller.signal.aborted
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
abort(reason = "Execution budget aborted by the application") {
|
|
1119
|
+
if (!this.controller.signal.aborted) this.controller.abort(new Error(reason));
|
|
1120
|
+
}
|
|
1121
|
+
consumeTokens(tokens) {
|
|
1122
|
+
if (!Number.isInteger(tokens) || tokens < 0) {
|
|
1123
|
+
throw new Error("ExecutionBudget token usage must be a non-negative integer");
|
|
1124
|
+
}
|
|
1125
|
+
this.assertTime();
|
|
1126
|
+
if (this.options.maxTokens !== void 0 && this.tokensUsed + tokens > this.options.maxTokens) {
|
|
1127
|
+
this.abort("Execution token budget exceeded");
|
|
1128
|
+
throw new ExecutionBudgetExceededError(
|
|
1129
|
+
"tokens",
|
|
1130
|
+
`Execution token budget exceeded (${this.tokensUsed + tokens}/${this.options.maxTokens})`
|
|
1131
|
+
);
|
|
1132
|
+
}
|
|
1133
|
+
this.tokensUsed += tokens;
|
|
1134
|
+
}
|
|
1135
|
+
async run(operation, reservedTokens = 0) {
|
|
1136
|
+
this.assertTime();
|
|
1137
|
+
if (this.controller.signal.aborted) {
|
|
1138
|
+
throw new ExecutionBudgetExceededError("time", "Execution budget is already aborted");
|
|
1139
|
+
}
|
|
1140
|
+
if (this.callsUsed >= this.options.maxCalls) {
|
|
1141
|
+
this.abort("Execution call budget exceeded");
|
|
1142
|
+
throw new ExecutionBudgetExceededError(
|
|
1143
|
+
"calls",
|
|
1144
|
+
`Execution call budget exceeded (${this.callsUsed}/${this.options.maxCalls})`
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
this.consumeTokens(reservedTokens);
|
|
1148
|
+
this.callsUsed += 1;
|
|
1149
|
+
const remainingMs = this.options.timeoutMs - (Date.now() - this.startedAt);
|
|
1150
|
+
if (remainingMs <= 0) this.assertTime();
|
|
1151
|
+
let timeout;
|
|
1152
|
+
const deadline = new Promise((_resolve, reject) => {
|
|
1153
|
+
timeout = setTimeout(() => {
|
|
1154
|
+
this.abort("Execution time budget exceeded");
|
|
1155
|
+
reject(new ExecutionBudgetExceededError(
|
|
1156
|
+
"time",
|
|
1157
|
+
`Execution time budget exceeded (${this.options.timeoutMs}ms)`
|
|
1158
|
+
));
|
|
1159
|
+
}, remainingMs);
|
|
1160
|
+
});
|
|
1161
|
+
try {
|
|
1162
|
+
return await Promise.race([operation(this.controller.signal), deadline]);
|
|
1163
|
+
} finally {
|
|
1164
|
+
if (timeout) clearTimeout(timeout);
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
assertTime() {
|
|
1168
|
+
if (Date.now() - this.startedAt >= this.options.timeoutMs) {
|
|
1169
|
+
this.abort("Execution time budget exceeded");
|
|
1170
|
+
throw new ExecutionBudgetExceededError(
|
|
1171
|
+
"time",
|
|
1172
|
+
`Execution time budget exceeded (${this.options.timeoutMs}ms)`
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
};
|
|
1177
|
+
|
|
523
1178
|
// src/client.ts
|
|
524
1179
|
var import_fs = __toESM(require("fs"));
|
|
525
1180
|
var import_path = __toESM(require("path"));
|
|
526
1181
|
var log3 = (0, import_debug3.default)("observyze:sdk");
|
|
1182
|
+
var MAX_HISTORY_FILE_BYTES = 50 * 1024 * 1024;
|
|
1183
|
+
var MAX_HISTORY_TRACES = 1e4;
|
|
1184
|
+
var MAX_GUARDRAIL_REQUEST_BYTES = 1024 * 1024;
|
|
1185
|
+
var ALLOWED_EVALUATION_SOURCES = /* @__PURE__ */ new Set(["live", "consensus", "nli_fast_path", "error", "disabled", "fallback"]);
|
|
1186
|
+
var SAFE_IMPORTED_METADATA = /* @__PURE__ */ new Set([
|
|
1187
|
+
"provider",
|
|
1188
|
+
"model",
|
|
1189
|
+
"temperature",
|
|
1190
|
+
"max_tokens",
|
|
1191
|
+
"max_completion_tokens",
|
|
1192
|
+
"max_output_tokens",
|
|
1193
|
+
"latency_ms",
|
|
1194
|
+
"streaming",
|
|
1195
|
+
"stream_completed",
|
|
1196
|
+
"output_truncated",
|
|
1197
|
+
"token_usage_source",
|
|
1198
|
+
"cost_source",
|
|
1199
|
+
"known_pricing",
|
|
1200
|
+
"source",
|
|
1201
|
+
"lifecycle",
|
|
1202
|
+
"invocation_type"
|
|
1203
|
+
]);
|
|
1204
|
+
function luhnValid(candidate) {
|
|
1205
|
+
const digits = candidate.replace(/[^0-9]/g, "");
|
|
1206
|
+
if (digits.length < 13 || digits.length > 19 || /^(\d)\1+$/.test(digits)) return false;
|
|
1207
|
+
let sum = 0;
|
|
1208
|
+
let shouldDouble = false;
|
|
1209
|
+
for (let index = digits.length - 1; index >= 0; index -= 1) {
|
|
1210
|
+
let digit = Number(digits[index]);
|
|
1211
|
+
if (shouldDouble) {
|
|
1212
|
+
digit *= 2;
|
|
1213
|
+
if (digit > 9) digit -= 9;
|
|
1214
|
+
}
|
|
1215
|
+
sum += digit;
|
|
1216
|
+
shouldDouble = !shouldDouble;
|
|
1217
|
+
}
|
|
1218
|
+
return sum % 10 === 0;
|
|
1219
|
+
}
|
|
1220
|
+
var PII_PATTERNS = [
|
|
1221
|
+
{ name: "email", pattern: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/g, replacement: "[EMAIL_REDACTED]" },
|
|
1222
|
+
{ name: "jwt", pattern: /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, replacement: "[JWT_REDACTED]" },
|
|
1223
|
+
{ name: "authorization", pattern: /\b(?:Bearer|Token|Basic)\s+[A-Za-z0-9\-._~+/]+=*/gi, replacement: "[AUTH_TOKEN_REDACTED]" },
|
|
1224
|
+
{ name: "aws_access_key", pattern: /\b(?:AKIA|ASIA|AROA|ANPA|ANVA|AIDA)[A-Z0-9]{16}\b/g, replacement: "[AWS_KEY_REDACTED]" },
|
|
1225
|
+
{ name: "api_key", pattern: /(?<![A-Za-z0-9_])(?:(?:sk|pk)-[A-Za-z0-9-]{20,}|ob_[A-Za-z0-9]{20,}|claude-[A-Za-z0-9-]{20,})(?![A-Za-z0-9_])/g, replacement: "[API_KEY_REDACTED]" },
|
|
1226
|
+
{ name: "github_token", pattern: /\b(?:ghp|gho|ghu|ghs|github_pat)_[A-Za-z0-9_]{20,}\b/g, replacement: "[API_KEY_REDACTED]" },
|
|
1227
|
+
{ name: "ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/g, replacement: "[SSN_REDACTED]" },
|
|
1228
|
+
{ name: "credit_card_luhn", pattern: /\b(?:\d[ -]?){12,18}\d\b/g, replacement: (match) => luhnValid(match) ? "[CC_REDACTED]" : match },
|
|
1229
|
+
{ name: "phone_us", pattern: /(?<!\d)(?:\+?1[-.\\s]?)?\(?\d{3}\)?[-.\\s]?\d{3}[-.\\s]?\d{4}(?!\d)/g, replacement: "[PHONE_REDACTED]" },
|
|
1230
|
+
{ name: "phone_international", pattern: /(?<!\d)\+(?:[0-9][().\s-]?){7,15}[0-9](?!\d)/g, replacement: "[PHONE_REDACTED]" },
|
|
1231
|
+
{ name: "zip_context", pattern: /(?:zip\s*(?:code)?[\s:]*)\b\d{5}(?:-\d{4})?\b/gi, replacement: "zip: [ZIP_REDACTED]" },
|
|
1232
|
+
{ name: "ipv4", pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, replacement: "[IP_REDACTED]" },
|
|
1233
|
+
{ name: "ipv6", pattern: /\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, replacement: "[IP_REDACTED]" }
|
|
1234
|
+
];
|
|
1235
|
+
var SENSITIVE_KEY_SEGMENTS = /* @__PURE__ */ new Set([
|
|
1236
|
+
"password",
|
|
1237
|
+
"passwd",
|
|
1238
|
+
"secret",
|
|
1239
|
+
"token",
|
|
1240
|
+
"apikey",
|
|
1241
|
+
"key",
|
|
1242
|
+
"authorization",
|
|
1243
|
+
"credential",
|
|
1244
|
+
"private",
|
|
1245
|
+
"ssn",
|
|
1246
|
+
"social",
|
|
1247
|
+
"dob",
|
|
1248
|
+
"birth",
|
|
1249
|
+
"passport",
|
|
1250
|
+
"credit",
|
|
1251
|
+
"card",
|
|
1252
|
+
"cvv",
|
|
1253
|
+
"cvc",
|
|
1254
|
+
"pin",
|
|
1255
|
+
"bank",
|
|
1256
|
+
"routing",
|
|
1257
|
+
"account"
|
|
1258
|
+
]);
|
|
1259
|
+
function isSensitiveKey(key) {
|
|
1260
|
+
const normalized = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/-/g, "_");
|
|
1261
|
+
if (SENSITIVE_KEY_SEGMENTS.has(normalized)) return true;
|
|
1262
|
+
if (/^(?:first|last|given|family|full|person)_?name$/.test(normalized)) return true;
|
|
1263
|
+
return normalized.split("_").some((segment) => SENSITIVE_KEY_SEGMENTS.has(segment)) && /(?:password|passwd|secret|token|api_?key|authorization|credential|private_?key|ssn|social_?security|credit_?card|card_?number|cvv|cvc|bank_?account|routing_?number|passport|date_?of_?birth)/.test(normalized);
|
|
1264
|
+
}
|
|
1265
|
+
function redactValue(value, depth = 0) {
|
|
1266
|
+
if (depth > 16) return "[MAX_DEPTH_EXCEEDED]";
|
|
1267
|
+
if (typeof value === "string") {
|
|
1268
|
+
let redacted = value;
|
|
1269
|
+
for (const { pattern, replacement } of PII_PATTERNS) {
|
|
1270
|
+
pattern.lastIndex = 0;
|
|
1271
|
+
redacted = typeof replacement === "string" ? redacted.replace(pattern, replacement) : redacted.replace(pattern, replacement);
|
|
1272
|
+
}
|
|
1273
|
+
return redacted;
|
|
1274
|
+
}
|
|
1275
|
+
if (Array.isArray(value)) return value.map((item) => redactValue(item, depth + 1));
|
|
1276
|
+
if (value instanceof Date) return value;
|
|
1277
|
+
if (value instanceof Map) return redactValue(Object.fromEntries(value.entries()), depth + 1);
|
|
1278
|
+
if (value !== null && typeof value === "object") {
|
|
1279
|
+
const redacted = {};
|
|
1280
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
1281
|
+
redacted[key] = isSensitiveKey(key) ? "[REDACTED]" : redactValue(nested, depth + 1);
|
|
1282
|
+
}
|
|
1283
|
+
return redacted;
|
|
1284
|
+
}
|
|
1285
|
+
return value;
|
|
1286
|
+
}
|
|
1287
|
+
function operationalMetadata(value) {
|
|
1288
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
1289
|
+
return Object.fromEntries(Object.entries(value).filter(
|
|
1290
|
+
([key, item]) => SAFE_IMPORTED_METADATA.has(key) && (item === null || ["string", "number", "boolean"].includes(typeof item))
|
|
1291
|
+
));
|
|
1292
|
+
}
|
|
1293
|
+
function metadataOnlyImportedTrace(trace) {
|
|
1294
|
+
return {
|
|
1295
|
+
trace_id: trace?.trace_id,
|
|
1296
|
+
organization_id: trace?.organization_id,
|
|
1297
|
+
project_id: trace?.project_id,
|
|
1298
|
+
name: "[CONTENT_CAPTURE_DISABLED]",
|
|
1299
|
+
status: trace?.status,
|
|
1300
|
+
start_time: trace?.start_time,
|
|
1301
|
+
end_time: trace?.end_time,
|
|
1302
|
+
duration_ms: trace?.duration_ms,
|
|
1303
|
+
metadata: operationalMetadata(trace?.metadata),
|
|
1304
|
+
spans: Array.isArray(trace?.spans) ? trace.spans.map((span) => ({
|
|
1305
|
+
span_id: span?.span_id,
|
|
1306
|
+
parent_span_id: span?.parent_span_id,
|
|
1307
|
+
name: "[CONTENT_CAPTURE_DISABLED]",
|
|
1308
|
+
type: span?.type,
|
|
1309
|
+
start_time: span?.start_time,
|
|
1310
|
+
end_time: span?.end_time,
|
|
1311
|
+
duration_ms: span?.duration_ms,
|
|
1312
|
+
input: null,
|
|
1313
|
+
output: null,
|
|
1314
|
+
...span?.error ? { error: { message: "Error details omitted because content capture is disabled" } } : {},
|
|
1315
|
+
metadata: operationalMetadata(span?.metadata),
|
|
1316
|
+
tokens: span?.tokens || span?.tokens_used
|
|
1317
|
+
})) : [],
|
|
1318
|
+
tags: [],
|
|
1319
|
+
cost: trace?.cost
|
|
1320
|
+
};
|
|
1321
|
+
}
|
|
1322
|
+
var IngestionRequestError = class extends Error {
|
|
1323
|
+
constructor(message, retryable, retryAfterMs) {
|
|
1324
|
+
super(message);
|
|
1325
|
+
this.retryable = retryable;
|
|
1326
|
+
this.retryAfterMs = retryAfterMs;
|
|
1327
|
+
this.name = "IngestionRequestError";
|
|
1328
|
+
}
|
|
1329
|
+
retryable;
|
|
1330
|
+
retryAfterMs;
|
|
1331
|
+
};
|
|
527
1332
|
var DEFAULT_CONFIG = {
|
|
528
|
-
endpoint: "
|
|
1333
|
+
endpoint: "https://api.observyze.com",
|
|
529
1334
|
batchSize: 100,
|
|
530
1335
|
flushInterval: 5e3,
|
|
531
|
-
|
|
1336
|
+
requestTimeoutMs: 1e4,
|
|
532
1337
|
debug: false,
|
|
533
1338
|
dryRun: false,
|
|
534
1339
|
enablePiiRedaction: true,
|
|
1340
|
+
captureContent: true,
|
|
535
1341
|
hallucinationThreshold: 0.8,
|
|
536
1342
|
safetyThreshold: 0.9,
|
|
537
1343
|
confidenceThreshold: 0.4,
|
|
538
|
-
evalEndpoint: process.env.EVAL_ENDPOINT || (process.env.NODE_ENV === "production" ? "https://api.observyze.com" : "http://localhost:3001"),
|
|
539
1344
|
enableCircuitBreaker: true,
|
|
540
1345
|
failClosed: true,
|
|
541
1346
|
enableProxyRedirect: true
|
|
@@ -547,10 +1352,46 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
547
1352
|
isShuttingDown = false;
|
|
548
1353
|
MAX_QUEUE_SIZE = 1e3;
|
|
549
1354
|
RETRY_DELAYS = [1e3, 2e3, 4e3, 8e3, 16e3, 3e4];
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
1355
|
+
static async readBoundedResponse(response, maxBytes = 64 * 1024) {
|
|
1356
|
+
if (!response.body) {
|
|
1357
|
+
if (typeof response.text === "function") return String(await response.text()).slice(0, maxBytes);
|
|
1358
|
+
if (typeof response.json === "function") return JSON.stringify(await response.json()).slice(0, maxBytes);
|
|
1359
|
+
return "";
|
|
1360
|
+
}
|
|
1361
|
+
const reader = response.body.getReader();
|
|
1362
|
+
const chunks = [];
|
|
1363
|
+
let size = 0;
|
|
1364
|
+
while (true) {
|
|
1365
|
+
const { done, value } = await reader.read();
|
|
1366
|
+
if (done) break;
|
|
1367
|
+
const remaining = maxBytes - size;
|
|
1368
|
+
if (remaining <= 0) {
|
|
1369
|
+
await reader.cancel();
|
|
1370
|
+
break;
|
|
1371
|
+
}
|
|
1372
|
+
chunks.push(value.byteLength > remaining ? value.slice(0, remaining) : value);
|
|
1373
|
+
size += Math.min(value.byteLength, remaining);
|
|
1374
|
+
if (value.byteLength > remaining) {
|
|
1375
|
+
await reader.cancel();
|
|
1376
|
+
break;
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
const merged = new Uint8Array(size);
|
|
1380
|
+
let offset = 0;
|
|
1381
|
+
for (const chunk of chunks) {
|
|
1382
|
+
merged.set(chunk, offset);
|
|
1383
|
+
offset += chunk.byteLength;
|
|
1384
|
+
}
|
|
1385
|
+
return new TextDecoder().decode(merged);
|
|
1386
|
+
}
|
|
1387
|
+
static retryAfterMs(response) {
|
|
1388
|
+
const value = response.headers?.get?.("retry-after");
|
|
1389
|
+
if (!value) return void 0;
|
|
1390
|
+
const seconds = Number(value);
|
|
1391
|
+
if (Number.isFinite(seconds)) return Math.min(6e4, Math.max(0, seconds * 1e3));
|
|
1392
|
+
const date = Date.parse(value);
|
|
1393
|
+
return Number.isFinite(date) ? Math.min(6e4, Math.max(0, date - Date.now())) : void 0;
|
|
1394
|
+
}
|
|
554
1395
|
static parseApiError(_response, body) {
|
|
555
1396
|
try {
|
|
556
1397
|
const parsed = JSON.parse(body);
|
|
@@ -561,18 +1402,9 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
561
1402
|
message: error.message || body.slice(0, 200)
|
|
562
1403
|
};
|
|
563
1404
|
} catch {
|
|
564
|
-
return {
|
|
565
|
-
traceId: "unknown",
|
|
566
|
-
code: "UNKNOWN_ERROR",
|
|
567
|
-
message: body.slice(0, 200)
|
|
568
|
-
};
|
|
1405
|
+
return { traceId: "unknown", code: "UNKNOWN_ERROR", message: body.slice(0, 200) };
|
|
569
1406
|
}
|
|
570
1407
|
}
|
|
571
|
-
/**
|
|
572
|
-
* Format an API error into a user-friendly message with trace_id for correlation.
|
|
573
|
-
* Example output:
|
|
574
|
-
* "Observyze API error (401 [ref: err_a1b2c3d4]): MISSING_PROVIDER_KEY — No API key configured..."
|
|
575
|
-
*/
|
|
576
1408
|
static formatApiError(response, body) {
|
|
577
1409
|
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, body);
|
|
578
1410
|
const prefix = traceId !== "unknown" ? ` [ref: ${traceId}]` : "";
|
|
@@ -582,11 +1414,27 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
582
1414
|
if (!config.apiKey) {
|
|
583
1415
|
throw new Error("Observyze SDK: apiKey is required");
|
|
584
1416
|
}
|
|
1417
|
+
if (config.batchSize !== void 0 && (!Number.isInteger(config.batchSize) || config.batchSize < 1 || config.batchSize > 100)) {
|
|
1418
|
+
throw new Error("Observyze SDK: batchSize must be an integer between 1 and 100");
|
|
1419
|
+
}
|
|
1420
|
+
if (config.flushInterval !== void 0 && (!Number.isFinite(config.flushInterval) || config.flushInterval < 100 || config.flushInterval > 36e5)) {
|
|
1421
|
+
throw new Error("Observyze SDK: flushInterval must be between 100 and 3600000 milliseconds");
|
|
1422
|
+
}
|
|
1423
|
+
if (config.requestTimeoutMs !== void 0 && (!Number.isFinite(config.requestTimeoutMs) || config.requestTimeoutMs < 100 || config.requestTimeoutMs > 12e4)) {
|
|
1424
|
+
throw new Error("Observyze SDK: requestTimeoutMs must be between 100 and 120000 milliseconds");
|
|
1425
|
+
}
|
|
1426
|
+
try {
|
|
1427
|
+
const endpoint = new URL(config.endpoint || String(DEFAULT_CONFIG.endpoint));
|
|
1428
|
+
if (!["http:", "https:"].includes(endpoint.protocol)) throw new Error("invalid protocol");
|
|
1429
|
+
} catch {
|
|
1430
|
+
throw new Error("Observyze SDK: endpoint must be a valid HTTP(S) URL");
|
|
1431
|
+
}
|
|
585
1432
|
this.config = {
|
|
586
1433
|
...DEFAULT_CONFIG,
|
|
587
1434
|
...config,
|
|
588
1435
|
organizationId: config.organizationId || "",
|
|
589
|
-
projectId: config.projectId || ""
|
|
1436
|
+
projectId: config.projectId || "",
|
|
1437
|
+
evalEndpoint: config.evalEndpoint || config.endpoint || String(DEFAULT_CONFIG.endpoint)
|
|
590
1438
|
};
|
|
591
1439
|
this.startFlushTimer();
|
|
592
1440
|
if (this.config.debug) {
|
|
@@ -605,13 +1453,14 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
605
1453
|
const trace = new Trace(
|
|
606
1454
|
name,
|
|
607
1455
|
this.config.organizationId,
|
|
608
|
-
this.config.projectId
|
|
1456
|
+
this.config.projectId,
|
|
1457
|
+
this.config.captureContent
|
|
609
1458
|
);
|
|
610
1459
|
if (metadata) {
|
|
611
1460
|
trace.setMetadataAll(metadata);
|
|
612
1461
|
}
|
|
613
1462
|
const originalEnd = trace.end.bind(trace);
|
|
614
|
-
trace.end = (status =
|
|
1463
|
+
trace.end = (status = "success" /* SUCCESS */) => {
|
|
615
1464
|
originalEnd(status);
|
|
616
1465
|
this.bufferTrace(trace);
|
|
617
1466
|
};
|
|
@@ -662,7 +1511,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
662
1511
|
}
|
|
663
1512
|
}
|
|
664
1513
|
/**
|
|
665
|
-
* Flush all buffered traces to
|
|
1514
|
+
* Flush all buffered traces to Observyze
|
|
666
1515
|
*/
|
|
667
1516
|
async flush() {
|
|
668
1517
|
if (this.traceBuffer.length === 0) {
|
|
@@ -705,37 +1554,52 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
705
1554
|
let lastError = null;
|
|
706
1555
|
for (let attempt = 0; attempt < this.RETRY_DELAYS.length + 1; attempt++) {
|
|
707
1556
|
try {
|
|
708
|
-
const
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
1557
|
+
const controller = new AbortController();
|
|
1558
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1559
|
+
let response;
|
|
1560
|
+
try {
|
|
1561
|
+
response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
1562
|
+
method: "POST",
|
|
1563
|
+
headers: {
|
|
1564
|
+
"Content-Type": "application/json",
|
|
1565
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1566
|
+
"x-api-key": this.config.apiKey
|
|
1567
|
+
},
|
|
1568
|
+
body: JSON.stringify({
|
|
1569
|
+
traces: traces.map((trace) => {
|
|
1570
|
+
const json = trace.toJSON();
|
|
1571
|
+
return this.config.enablePiiRedaction ? this.sanitizePII(json) : json;
|
|
1572
|
+
})
|
|
1573
|
+
}),
|
|
1574
|
+
signal: controller.signal
|
|
1575
|
+
});
|
|
1576
|
+
} finally {
|
|
1577
|
+
clearTimeout(timeout);
|
|
1578
|
+
}
|
|
724
1579
|
if (!response.ok) {
|
|
725
|
-
const errorBody = await
|
|
1580
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
726
1581
|
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
727
|
-
|
|
1582
|
+
const retryable = [408, 425, 429].includes(response.status) || response.status >= 500;
|
|
1583
|
+
throw new IngestionRequestError(
|
|
1584
|
+
`[Observyze SDK] Trace ingestion failed. ${formatted}`,
|
|
1585
|
+
retryable,
|
|
1586
|
+
_ObservyzeClient.retryAfterMs(response)
|
|
1587
|
+
);
|
|
728
1588
|
}
|
|
729
1589
|
if (this.config.debug) {
|
|
730
1590
|
log3(`[Observyze SDK] Successfully sent ${traces.length} traces${attempt > 0 ? ` (after ${attempt} retries)` : ""}`);
|
|
731
1591
|
}
|
|
1592
|
+
if (response.body) await response.body.cancel().catch(() => void 0);
|
|
732
1593
|
return;
|
|
733
1594
|
} catch (error) {
|
|
734
1595
|
lastError = error;
|
|
735
|
-
|
|
1596
|
+
const retryable = !(error instanceof IngestionRequestError) || error.retryable;
|
|
1597
|
+
if (!retryable || attempt >= this.RETRY_DELAYS.length) {
|
|
736
1598
|
break;
|
|
737
1599
|
}
|
|
738
|
-
const
|
|
1600
|
+
const configuredDelay = error instanceof IngestionRequestError ? error.retryAfterMs : void 0;
|
|
1601
|
+
const baseDelay = configuredDelay ?? this.RETRY_DELAYS[attempt];
|
|
1602
|
+
const delay = Math.min(6e4, Math.round(baseDelay * (0.8 + Math.random() * 0.4)));
|
|
739
1603
|
if (this.config.debug) {
|
|
740
1604
|
log3.extend("warn")(`[Observyze SDK] Attempt ${attempt + 1} failed, retrying in ${delay}ms...`, error);
|
|
741
1605
|
}
|
|
@@ -781,22 +1645,21 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
781
1645
|
return { ...this.config };
|
|
782
1646
|
}
|
|
783
1647
|
/**
|
|
784
|
-
* Wrap
|
|
785
|
-
*
|
|
1648
|
+
* Wrap a supported LLM client or framework boundary to enable auto-instrumentation.
|
|
1649
|
+
*
|
|
1650
|
+
* Supports: OpenAI, Anthropic, Google Gemini, Vercel AI SDK, LangChain, LlamaIndex.
|
|
1651
|
+
*
|
|
786
1652
|
* @example
|
|
787
1653
|
* ```typescript
|
|
788
1654
|
* import OpenAI from 'openai'
|
|
789
1655
|
* import { ObservyzeClient } from '@observyze/sdk'
|
|
790
|
-
*
|
|
1656
|
+
*
|
|
791
1657
|
* const nw = new ObservyzeClient({ apiKey: 'your-api-key' })
|
|
792
|
-
* const openai = new OpenAI({ apiKey: 'openai-key' })
|
|
793
|
-
*
|
|
794
|
-
* // Wrap the client to enable auto-instrumentation
|
|
795
|
-
* nw.wrap(openai)
|
|
796
|
-
*
|
|
1658
|
+
* const openai = nw.wrap(new OpenAI({ apiKey: 'openai-key' }))
|
|
1659
|
+
*
|
|
797
1660
|
* // All calls are now automatically traced
|
|
798
1661
|
* const response = await openai.chat.completions.create({
|
|
799
|
-
* model: 'gpt-
|
|
1662
|
+
* model: 'gpt-4o',
|
|
800
1663
|
* messages: [{ role: 'user', content: 'Hello!' }]
|
|
801
1664
|
* })
|
|
802
1665
|
* ```
|
|
@@ -804,27 +1667,19 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
804
1667
|
wrap(client) {
|
|
805
1668
|
return wrap(client, this);
|
|
806
1669
|
}
|
|
1670
|
+
/** Create an active call/token/time budget for one agent execution. */
|
|
1671
|
+
createExecutionBudget(options) {
|
|
1672
|
+
return new ExecutionBudget(options);
|
|
1673
|
+
}
|
|
807
1674
|
/**
|
|
808
1675
|
* Verify that the SDK can reach Observyze and send traces end-to-end.
|
|
809
1676
|
*
|
|
810
|
-
* This is the definitive "is my integration working?" test for SDK users.
|
|
811
|
-
* It sends a real test trace through the EXACT same pipeline used by
|
|
812
|
-
* `flush()` / the wrapped LLM clients (same endpoint, apiKey, retry logic),
|
|
813
|
-
* so a successful call proves the whole chain works from your code:
|
|
814
|
-
* - apiKey is valid and authorized
|
|
815
|
-
* - endpoint is reachable from your environment
|
|
816
|
-
* - organization / project resolution works
|
|
817
|
-
* - the ingest pipeline accepts and stores traces
|
|
818
|
-
*
|
|
819
1677
|
* @example
|
|
820
1678
|
* ```typescript
|
|
821
1679
|
* const nw = new ObservyzeClient({ apiKey: process.env.OBSERVYZE_API_KEY })
|
|
822
1680
|
* const result = await nw.testConnection()
|
|
823
|
-
* // { ok: true, traceId: '
|
|
1681
|
+
* // { ok: true, traceId: '...', message: 'Connection successful...' }
|
|
824
1682
|
* ```
|
|
825
|
-
*
|
|
826
|
-
* The returned traceId can be searched in the Observyze dashboard (Traces →
|
|
827
|
-
* search the trace name "Observyze Connection Test") to confirm it landed.
|
|
828
1683
|
*/
|
|
829
1684
|
async testConnection() {
|
|
830
1685
|
if (this.config.dryRun) {
|
|
@@ -836,9 +1691,10 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
836
1691
|
const trace = new Trace(
|
|
837
1692
|
"Observyze Connection Test",
|
|
838
1693
|
this.config.organizationId,
|
|
839
|
-
this.config.projectId
|
|
1694
|
+
this.config.projectId,
|
|
1695
|
+
this.config.captureContent
|
|
840
1696
|
);
|
|
841
|
-
const span = trace.startSpan("connection-test",
|
|
1697
|
+
const span = trace.startSpan("connection-test", "llm" /* LLM */);
|
|
842
1698
|
span.setInput({ prompt: "Observyze SDK connection test" });
|
|
843
1699
|
span.setOutput({ response: "Connection successful" });
|
|
844
1700
|
span.setTokens({ input: 5, output: 4, total: 9 });
|
|
@@ -846,7 +1702,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
846
1702
|
span.end();
|
|
847
1703
|
trace.setMetadata("source", "sdk-test-connection");
|
|
848
1704
|
trace.addTag("setup-test");
|
|
849
|
-
trace.end(
|
|
1705
|
+
trace.end("success" /* SUCCESS */);
|
|
850
1706
|
try {
|
|
851
1707
|
await this.sendWithRetry([trace]);
|
|
852
1708
|
return {
|
|
@@ -858,8 +1714,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
858
1714
|
const rawMessage = error?.message || String(error);
|
|
859
1715
|
const statusMatch = rawMessage.match(/\((\d{3})/);
|
|
860
1716
|
const isNetworkFailure = !statusMatch && /fetch|network|ENOTFOUND|ECONNREFUSED|ETIMEDOUT/i.test(rawMessage);
|
|
861
|
-
const
|
|
862
|
-
const hint = isNetworkFailure && this.config.endpoint === defaultEndpoint ? ` You are using the default endpoint (${defaultEndpoint}). For production, set endpoint: "https://api.observyze.com" in the ObservyzeClient config, then re-run.` : isNetworkFailure ? " Check that your endpoint is reachable from this environment (firewalls, proxies, DNS) and that you are not using a local endpoint in production." : "";
|
|
1717
|
+
const hint = isNetworkFailure ? " Check that your endpoint is reachable from this environment (firewalls, proxies, DNS)." : "";
|
|
863
1718
|
return {
|
|
864
1719
|
ok: false,
|
|
865
1720
|
...statusMatch ? { status: parseInt(statusMatch[1], 10) } : {},
|
|
@@ -868,8 +1723,8 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
868
1723
|
}
|
|
869
1724
|
}
|
|
870
1725
|
/**
|
|
871
|
-
* Sync local agent .history file to Observyze cloud
|
|
872
|
-
* Parses JSON/NDJSON agent history and sends to ingestion endpoint.
|
|
1726
|
+
* Sync a local agent .history file to Observyze cloud.
|
|
1727
|
+
* Parses JSON/NDJSON agent history and sends to the ingestion endpoint.
|
|
873
1728
|
*/
|
|
874
1729
|
async syncLocalHistory(filePath) {
|
|
875
1730
|
try {
|
|
@@ -880,6 +1735,11 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
880
1735
|
if (!import_fs.default.existsSync(fullPath)) {
|
|
881
1736
|
throw new Error(`History file not found: ${fullPath}`);
|
|
882
1737
|
}
|
|
1738
|
+
const fileStats = import_fs.default.statSync(fullPath);
|
|
1739
|
+
if (!fileStats.isFile()) throw new Error("History path must reference a regular file");
|
|
1740
|
+
if (fileStats.size > MAX_HISTORY_FILE_BYTES) {
|
|
1741
|
+
throw new Error(`History file exceeds the ${MAX_HISTORY_FILE_BYTES} byte limit`);
|
|
1742
|
+
}
|
|
883
1743
|
const content = import_fs.default.readFileSync(fullPath, "utf-8");
|
|
884
1744
|
let items = [];
|
|
885
1745
|
try {
|
|
@@ -890,21 +1750,36 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
890
1750
|
if (!Array.isArray(items)) {
|
|
891
1751
|
items = [items];
|
|
892
1752
|
}
|
|
1753
|
+
if (items.length > MAX_HISTORY_TRACES) {
|
|
1754
|
+
throw new Error(`History import exceeds the ${MAX_HISTORY_TRACES} trace limit`);
|
|
1755
|
+
}
|
|
893
1756
|
if (this.config.debug) {
|
|
894
1757
|
log3(`[Observyze SDK] Syncing ${items.length} traces from ${filePath}`);
|
|
895
1758
|
}
|
|
896
1759
|
for (let i = 0; i < items.length; i += this.config.batchSize) {
|
|
897
1760
|
const batch = items.slice(i, i + this.config.batchSize);
|
|
898
|
-
const
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
|
|
1761
|
+
const privacyBoundBatch = this.config.captureContent ? batch : batch.map(metadataOnlyImportedTrace);
|
|
1762
|
+
const controller = new AbortController();
|
|
1763
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1764
|
+
let response;
|
|
1765
|
+
try {
|
|
1766
|
+
response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
1767
|
+
method: "POST",
|
|
1768
|
+
headers: {
|
|
1769
|
+
"Content-Type": "application/json",
|
|
1770
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1771
|
+
"x-api-key": this.config.apiKey
|
|
1772
|
+
},
|
|
1773
|
+
body: JSON.stringify({
|
|
1774
|
+
traces: this.config.enablePiiRedaction ? this.sanitizePII(privacyBoundBatch) : privacyBoundBatch
|
|
1775
|
+
}),
|
|
1776
|
+
signal: controller.signal
|
|
1777
|
+
});
|
|
1778
|
+
} finally {
|
|
1779
|
+
clearTimeout(timeout);
|
|
1780
|
+
}
|
|
906
1781
|
if (!response.ok) {
|
|
907
|
-
const errorBody = await
|
|
1782
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
908
1783
|
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
909
1784
|
throw new Error(`[Observyze SDK] Local history sync failed. ${formatted}`);
|
|
910
1785
|
}
|
|
@@ -918,219 +1793,106 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
918
1793
|
}
|
|
919
1794
|
}
|
|
920
1795
|
/**
|
|
921
|
-
* Industry-grade PII
|
|
922
|
-
*
|
|
923
|
-
* Recursively scrubs PII from trace data before transmission to the cloud.
|
|
924
|
-
* Coverage: emails, JWTs, bearer tokens, AWS/API keys, SSNs, credit cards,
|
|
925
|
-
* phone numbers (US + E.164), IPv4/IPv6, passport numbers, ZIP codes, plus
|
|
926
|
-
* key-based redaction for sensitive JSON fields (password, token, api_key, etc.).
|
|
927
|
-
*
|
|
928
|
-
* Design:
|
|
929
|
-
* - Pure function, never mutates the original object
|
|
930
|
-
* - Depth-limited to 16 levels to prevent stack overflow on deep agent outputs
|
|
931
|
-
* - Key-aware: sensitive key names are fully redacted regardless of value format
|
|
1796
|
+
* Industry-grade PII redaction.
|
|
1797
|
+
* Recursively scrubs PII from trace data before transmission.
|
|
932
1798
|
*/
|
|
933
|
-
static PII_PATTERNS = [
|
|
934
|
-
{ pattern: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/gi, label: "[EMAIL_REDACTED]" },
|
|
935
|
-
{ pattern: /\beyJ[A-Za-z0-9_\-]+\.eyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\b/g, label: "[JWT_REDACTED]" },
|
|
936
|
-
{ pattern: /\b(Bearer|Token|Basic)\s+[A-Za-z0-9\-.~+\/]+=*\b/gi, label: "[AUTH_TOKEN_REDACTED]" },
|
|
937
|
-
{ pattern: /\b(AKIA|ASIA|AROA|ANPA|ANVA|AIDA)[A-Z0-9]{16}\b/g, label: "[AWS_KEY_REDACTED]" },
|
|
938
|
-
{ pattern: /\b(sk-[A-Za-z0-9\-]{20,}|pk-[A-Za-z0-9\-]{20,}|ob_[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{36,}|gho_[A-Za-z0-9]{36,})\b/g, label: "[API_KEY_REDACTED]" },
|
|
939
|
-
{ pattern: /\b\d{3}-\d{2}-\d{4}\b/g, label: "[SSN_REDACTED]" },
|
|
940
|
-
{ pattern: /\b(?:\d[ \-]?){13,18}\d\b/g, label: "[CC_REDACTED]" },
|
|
941
|
-
{ pattern: /(?:\+1[\s.\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b/g, label: "[PHONE_REDACTED]" },
|
|
942
|
-
{ pattern: /\+\d{1,3}[\s.\-]?\(?\d{1,4}\)?[\s.\-]?\d{1,4}[\s.\-]?\d{1,9}/g, label: "[PHONE_REDACTED]" },
|
|
943
|
-
{ pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, label: "[IP_REDACTED]" },
|
|
944
|
-
{ pattern: /\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, label: "[IP_REDACTED]" }
|
|
945
|
-
];
|
|
946
|
-
static SENSITIVE_KEYS = /* @__PURE__ */ new Set([
|
|
947
|
-
"password",
|
|
948
|
-
"passwd",
|
|
949
|
-
"secret",
|
|
950
|
-
"token",
|
|
951
|
-
"apikey",
|
|
952
|
-
"api_key",
|
|
953
|
-
"accesstoken",
|
|
954
|
-
"access_token",
|
|
955
|
-
"refreshtoken",
|
|
956
|
-
"refresh_token",
|
|
957
|
-
"authorization",
|
|
958
|
-
"auth",
|
|
959
|
-
"credential",
|
|
960
|
-
"credentials",
|
|
961
|
-
"private_key",
|
|
962
|
-
"privatekey",
|
|
963
|
-
"client_secret",
|
|
964
|
-
"clientsecret",
|
|
965
|
-
"ssn",
|
|
966
|
-
"social_security",
|
|
967
|
-
"dob",
|
|
968
|
-
"date_of_birth",
|
|
969
|
-
"dateofbirth",
|
|
970
|
-
"passport",
|
|
971
|
-
"passport_number",
|
|
972
|
-
"credit_card",
|
|
973
|
-
"creditcard",
|
|
974
|
-
"card_number",
|
|
975
|
-
"cardnumber",
|
|
976
|
-
"cvv",
|
|
977
|
-
"cvc",
|
|
978
|
-
"pin",
|
|
979
|
-
"bank_account",
|
|
980
|
-
"routing_number"
|
|
981
|
-
]);
|
|
982
|
-
static isSensitiveKey(key) {
|
|
983
|
-
const normalized = key.toLowerCase().replace(/-/g, "_");
|
|
984
|
-
if (_ObservyzeClient.SENSITIVE_KEYS.has(normalized)) return true;
|
|
985
|
-
const segments = normalized.split("_");
|
|
986
|
-
for (const segment of segments) {
|
|
987
|
-
if (_ObservyzeClient.SENSITIVE_KEYS.has(segment)) return true;
|
|
988
|
-
}
|
|
989
|
-
return false;
|
|
990
|
-
}
|
|
991
1799
|
sanitizePII(data, depth = 0) {
|
|
992
|
-
|
|
993
|
-
if (data === null || data === void 0) return data;
|
|
994
|
-
if (typeof data === "string") {
|
|
995
|
-
let result = data;
|
|
996
|
-
for (const { pattern, label } of _ObservyzeClient.PII_PATTERNS) {
|
|
997
|
-
pattern.lastIndex = 0;
|
|
998
|
-
result = result.replace(pattern, label);
|
|
999
|
-
}
|
|
1000
|
-
return result;
|
|
1001
|
-
}
|
|
1002
|
-
if (typeof data === "number" || typeof data === "boolean") return data;
|
|
1003
|
-
if (Array.isArray(data)) return data.map((item) => this.sanitizePII(item, depth + 1));
|
|
1004
|
-
if (typeof data === "object") {
|
|
1005
|
-
const sanitized = {};
|
|
1006
|
-
for (const [k, v] of Object.entries(data)) {
|
|
1007
|
-
sanitized[k] = _ObservyzeClient.isSensitiveKey(k) ? "[REDACTED]" : this.sanitizePII(v, depth + 1);
|
|
1008
|
-
}
|
|
1009
|
-
return sanitized;
|
|
1010
|
-
}
|
|
1011
|
-
return data;
|
|
1800
|
+
return redactValue(data, depth);
|
|
1012
1801
|
}
|
|
1013
1802
|
/**
|
|
1014
|
-
*
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1017
|
-
*
|
|
1803
|
+
* Explicitly evaluate content before an application action.
|
|
1804
|
+
* Requires an active Observyze subscription with guardrails enabled.
|
|
1805
|
+
*
|
|
1018
1806
|
* Returns GuardrailResult with score: null when evaluation couldn't be performed.
|
|
1019
|
-
* In failClosed mode, null scores
|
|
1020
|
-
*
|
|
1807
|
+
* In failClosed mode (default), null scores block execution.
|
|
1808
|
+
*
|
|
1809
|
+
* @example
|
|
1810
|
+
* ```typescript
|
|
1811
|
+
* const result = await nw.checkGuardrails(llmOutput)
|
|
1812
|
+
* if (!result.pass) {
|
|
1813
|
+
* throw new Error('Guardrail blocked: ' + result.reason)
|
|
1814
|
+
* }
|
|
1815
|
+
* ```
|
|
1021
1816
|
*/
|
|
1022
1817
|
async checkGuardrails(content) {
|
|
1023
1818
|
if (!this.config.enableCircuitBreaker) {
|
|
1024
1819
|
return { pass: true, score: 0, safetyScore: 0, evaluationSource: "disabled" };
|
|
1025
1820
|
}
|
|
1821
|
+
if (!this.config.captureContent) {
|
|
1822
|
+
const reason = "Guardrail content dispatch is disabled because captureContent is false";
|
|
1823
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "disabled", fallbackReason: reason, reason } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "disabled", fallbackReason: reason };
|
|
1824
|
+
}
|
|
1026
1825
|
try {
|
|
1027
1826
|
if (this.config.debug) {
|
|
1028
|
-
log3(`[Observyze Guardrail] Analyzing payload
|
|
1827
|
+
log3(`[Observyze Guardrail] Analyzing payload...`);
|
|
1029
1828
|
}
|
|
1030
1829
|
const evalEndpoint = this.config.evalEndpoint;
|
|
1031
|
-
const
|
|
1032
|
-
const
|
|
1033
|
-
const
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1830
|
+
const protectedContent = this.config.enablePiiRedaction ? this.sanitizePII(content) : content;
|
|
1831
|
+
const payload = typeof protectedContent === "string" ? { text: protectedContent, organization_id: this.config.organizationId } : { trace: protectedContent, organization_id: this.config.organizationId };
|
|
1832
|
+
const serializedPayload = JSON.stringify(payload);
|
|
1833
|
+
if (Buffer.byteLength(serializedPayload, "utf8") > MAX_GUARDRAIL_REQUEST_BYTES) {
|
|
1834
|
+
throw new Error(`Guardrail payload exceeds the ${MAX_GUARDRAIL_REQUEST_BYTES} byte limit`);
|
|
1835
|
+
}
|
|
1836
|
+
const evaluate = async (type) => {
|
|
1837
|
+
const controller = new AbortController();
|
|
1838
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1839
|
+
try {
|
|
1840
|
+
const response = await fetch(`${evalEndpoint}/api/v1/evaluate/${type}`, {
|
|
1841
|
+
method: "POST",
|
|
1842
|
+
headers: {
|
|
1843
|
+
"Content-Type": "application/json",
|
|
1844
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1845
|
+
"x-api-key": this.config.apiKey
|
|
1846
|
+
},
|
|
1847
|
+
body: serializedPayload,
|
|
1848
|
+
signal: controller.signal
|
|
1849
|
+
});
|
|
1850
|
+
if (response.ok) {
|
|
1851
|
+
const text = await _ObservyzeClient.readBoundedResponse(response);
|
|
1852
|
+
const parsed = JSON.parse(text);
|
|
1853
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
1854
|
+
}
|
|
1855
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
1050
1856
|
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, errorBody);
|
|
1051
1857
|
if (this.config.debug) {
|
|
1052
|
-
log3.extend("warn")(`[Observyze Guardrail]
|
|
1858
|
+
log3.extend("warn")(`[Observyze Guardrail] ${type} eval returned ${response.status} [${code}] (ref: ${traceId}): ${message}`);
|
|
1053
1859
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
clearTimeout(timeout);
|
|
1057
|
-
if (fetchError.name === "AbortError") {
|
|
1860
|
+
return null;
|
|
1861
|
+
} catch (fetchError) {
|
|
1058
1862
|
if (this.config.debug) {
|
|
1059
|
-
|
|
1863
|
+
const reason = fetchError?.name === "AbortError" ? `timed out after ${this.config.requestTimeoutMs}ms` : `failed: ${fetchError?.message || "unknown error"}`;
|
|
1864
|
+
log3.extend("warn")(`[Observyze Guardrail] ${type} evaluation ${reason}`);
|
|
1060
1865
|
}
|
|
1061
|
-
|
|
1062
|
-
|
|
1866
|
+
return null;
|
|
1867
|
+
} finally {
|
|
1868
|
+
clearTimeout(timeout);
|
|
1063
1869
|
}
|
|
1870
|
+
};
|
|
1871
|
+
const [hallucinationResult, safetyResult] = await Promise.all([
|
|
1872
|
+
evaluate("hallucination"),
|
|
1873
|
+
evaluate("safety")
|
|
1874
|
+
]);
|
|
1875
|
+
const unavailableReason = "One or more required guardrail evaluations were unavailable";
|
|
1876
|
+
if (!hallucinationResult || !safetyResult) {
|
|
1877
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: unavailableReason, reason: `${unavailableReason}. Fail-closed: execution blocked.` } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: unavailableReason };
|
|
1064
1878
|
}
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
confidence
|
|
1074
|
-
if (hallucinationScore === null) {
|
|
1075
|
-
if (this.config.failClosed) {
|
|
1076
|
-
if (this.config.debug) {
|
|
1077
|
-
log3.extend("warn")("[Observyze Guardrail] Eval returned null score \u2014 failing closed (blocking)");
|
|
1078
|
-
}
|
|
1079
|
-
return {
|
|
1080
|
-
pass: false,
|
|
1081
|
-
score: null,
|
|
1082
|
-
confidence: null,
|
|
1083
|
-
safetyScore: null,
|
|
1084
|
-
evaluationSource: "error",
|
|
1085
|
-
fallbackReason: evalResult.message || "Evaluation failed to produce a score",
|
|
1086
|
-
reason: "Evaluation service failed to produce a score. Fail-closed: execution blocked."
|
|
1087
|
-
};
|
|
1088
|
-
}
|
|
1089
|
-
if (this.config.debug) {
|
|
1090
|
-
log3("[Observyze Guardrail] Eval returned null score \u2014 allowing (fail-open)");
|
|
1091
|
-
}
|
|
1092
|
-
return {
|
|
1093
|
-
pass: true,
|
|
1094
|
-
score: null,
|
|
1095
|
-
confidence: null,
|
|
1096
|
-
safetyScore: null,
|
|
1097
|
-
evaluationSource: "error",
|
|
1098
|
-
fallbackReason: evalResult.message || "Evaluation failed to produce a score"
|
|
1099
|
-
};
|
|
1100
|
-
}
|
|
1101
|
-
} else if (this.config.failClosed) {
|
|
1102
|
-
if (this.config.debug) {
|
|
1103
|
-
log3.extend("warn")("[Observyze Guardrail] Eval unavailable \u2014 failing closed (blocking)");
|
|
1104
|
-
}
|
|
1105
|
-
return {
|
|
1106
|
-
pass: false,
|
|
1107
|
-
score: null,
|
|
1108
|
-
confidence: null,
|
|
1109
|
-
safetyScore: null,
|
|
1110
|
-
evaluationSource: "error",
|
|
1111
|
-
fallbackReason: "Evaluation service unreachable",
|
|
1112
|
-
reason: "Evaluation service unreachable. Fail-closed: execution blocked."
|
|
1113
|
-
};
|
|
1114
|
-
} else {
|
|
1115
|
-
if (this.config.debug) {
|
|
1116
|
-
log3("[Observyze Guardrail] Eval unavailable \u2014 allowing (fail-open)");
|
|
1117
|
-
}
|
|
1118
|
-
return {
|
|
1119
|
-
pass: true,
|
|
1120
|
-
score: null,
|
|
1121
|
-
confidence: null,
|
|
1122
|
-
safetyScore: null,
|
|
1123
|
-
evaluationSource: "error",
|
|
1124
|
-
fallbackReason: "Evaluation service unreachable"
|
|
1125
|
-
};
|
|
1879
|
+
const validScore = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1 ? value : null;
|
|
1880
|
+
const hallucinationScore = validScore(hallucinationResult.score ?? hallucinationResult.hallucination_score);
|
|
1881
|
+
const safetyScore = validScore(safetyResult.score ?? safetyResult.safety_score);
|
|
1882
|
+
const rawSource = hallucinationResult.evaluation_source ?? safetyResult.evaluation_source ?? "live";
|
|
1883
|
+
const evaluationSource = ALLOWED_EVALUATION_SOURCES.has(rawSource) ? rawSource : "live";
|
|
1884
|
+
const confidence = validScore(hallucinationResult.confidence ?? safetyResult.confidence);
|
|
1885
|
+
if (hallucinationScore === null || safetyScore === null) {
|
|
1886
|
+
const reason = hallucinationResult.message || safetyResult.message || "Evaluation failed to produce a score";
|
|
1887
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: reason, reason: `${reason}. Fail-closed: execution blocked.` } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: reason };
|
|
1126
1888
|
}
|
|
1127
1889
|
const hallThreshold = this.config.hallucinationThreshold;
|
|
1128
1890
|
const safeThreshold = this.config.safetyThreshold;
|
|
1129
1891
|
const confThreshold = this.config.confidenceThreshold;
|
|
1130
1892
|
if (confidence !== null && confidence < confThreshold) {
|
|
1131
|
-
if (hallucinationScore >= hallThreshold) {
|
|
1893
|
+
if (hallucinationScore >= hallThreshold || safetyScore >= safeThreshold) {
|
|
1132
1894
|
if (this.config.debug) {
|
|
1133
|
-
log3.extend("warn")(`[Observyze Guardrail] High score
|
|
1895
|
+
log3.extend("warn")(`[Observyze Guardrail] High score but low confidence (${confidence.toFixed(2)}). Alerting only.`);
|
|
1134
1896
|
}
|
|
1135
1897
|
return {
|
|
1136
1898
|
pass: true,
|
|
@@ -1138,7 +1900,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
1138
1900
|
confidence,
|
|
1139
1901
|
safetyScore,
|
|
1140
1902
|
evaluationSource,
|
|
1141
|
-
reason: `
|
|
1903
|
+
reason: `Risk score exceeded a threshold but confidence ${confidence.toFixed(2)} is below ${confThreshold.toFixed(2)}. Execution allowed with alert.`
|
|
1142
1904
|
};
|
|
1143
1905
|
}
|
|
1144
1906
|
}
|
|
@@ -1178,9 +1940,8 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
1178
1940
|
}
|
|
1179
1941
|
}
|
|
1180
1942
|
/**
|
|
1181
|
-
*
|
|
1182
|
-
*
|
|
1183
|
-
* Pauses execution if hallucination score >= hallucinationThreshold and requests human review.
|
|
1943
|
+
* Execute an application action only after an explicit pre-execution
|
|
1944
|
+
* guardrail check passes.
|
|
1184
1945
|
* @throws Error when execution is blocked by circuit breaker
|
|
1185
1946
|
*/
|
|
1186
1947
|
async executeWithCircuitBreaker(agentExecution, traceContext) {
|
|
@@ -1189,7 +1950,9 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
1189
1950
|
}
|
|
1190
1951
|
const guardResult = await this.checkGuardrails(traceContext || "execution context");
|
|
1191
1952
|
if (!guardResult.pass) {
|
|
1192
|
-
const error = new Error(
|
|
1953
|
+
const error = new Error(
|
|
1954
|
+
`[Observyze] Execution Blocked by Autonomous Circuit Breaker. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}. Reason: ${guardResult.reason}. Human approval required before agent can continue.`
|
|
1955
|
+
);
|
|
1193
1956
|
if (this.config.debug) {
|
|
1194
1957
|
log3.extend("error")("[Observyze CircuitBreaker] Execution blocked:", error.message);
|
|
1195
1958
|
}
|
|
@@ -1200,32 +1963,6 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
1200
1963
|
}
|
|
1201
1964
|
return await agentExecution();
|
|
1202
1965
|
}
|
|
1203
|
-
/**
|
|
1204
|
-
* Phase 4: Bug Bounty Protocol (Automated) (Requirement 4.2)
|
|
1205
|
-
* Automatically shard persistent failure cases to external security researcher endpoints (e.g. HackerOne wrapper)
|
|
1206
|
-
*/
|
|
1207
|
-
async reportBugBounty(traceId, securityEndpoint, failureContext) {
|
|
1208
|
-
try {
|
|
1209
|
-
if (this.config.debug) {
|
|
1210
|
-
log3(`[Observyze SDK] Sharding persistent failure case ${traceId} to Bug Bounty Protocol endpoint...`);
|
|
1211
|
-
}
|
|
1212
|
-
await fetch(securityEndpoint, {
|
|
1213
|
-
method: "POST",
|
|
1214
|
-
headers: { "Content-Type": "application/json" },
|
|
1215
|
-
body: JSON.stringify({
|
|
1216
|
-
alert: "persistent_failure_sharded",
|
|
1217
|
-
trace_id: traceId,
|
|
1218
|
-
context: failureContext,
|
|
1219
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1220
|
-
})
|
|
1221
|
-
});
|
|
1222
|
-
if (this.config.debug) {
|
|
1223
|
-
log3(`[Observyze SDK] Bug Bounty payload successfully transmitted.`);
|
|
1224
|
-
}
|
|
1225
|
-
} catch (err) {
|
|
1226
|
-
log3.extend("error")("[Observyze Bug Bounty] Failed to shard failure case:", err);
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
1966
|
};
|
|
1230
1967
|
|
|
1231
1968
|
// src/opentelemetry/exporter.ts
|
|
@@ -1298,9 +2035,9 @@ var ObservyzeSpanExporter = class {
|
|
|
1298
2035
|
const statusCode = status?.code;
|
|
1299
2036
|
if (statusCode === 2) {
|
|
1300
2037
|
oSpan.setError(new Error(status?.message || "OTel span error"));
|
|
1301
|
-
trace.end(
|
|
2038
|
+
trace.end("error" /* ERROR */);
|
|
1302
2039
|
} else {
|
|
1303
|
-
trace.end(
|
|
2040
|
+
trace.end("success" /* SUCCESS */);
|
|
1304
2041
|
}
|
|
1305
2042
|
}
|
|
1306
2043
|
resultCallback({ code: 0 });
|
|
@@ -1333,6 +2070,8 @@ var ObservyzeSpanExporter = class {
|
|
|
1333
2070
|
};
|
|
1334
2071
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1335
2072
|
0 && (module.exports = {
|
|
2073
|
+
ExecutionBudget,
|
|
2074
|
+
ExecutionBudgetExceededError,
|
|
1336
2075
|
ObservyzeClient,
|
|
1337
2076
|
ObservyzeSpanExporter,
|
|
1338
2077
|
Span,
|
|
@@ -1341,5 +2080,9 @@ var ObservyzeSpanExporter = class {
|
|
|
1341
2080
|
TraceStatus,
|
|
1342
2081
|
wrap,
|
|
1343
2082
|
wrapAnthropic,
|
|
1344
|
-
|
|
2083
|
+
wrapGemini,
|
|
2084
|
+
wrapLangChain,
|
|
2085
|
+
wrapLlamaIndex,
|
|
2086
|
+
wrapOpenAI,
|
|
2087
|
+
wrapVercelAI
|
|
1345
2088
|
});
|