@observyze/sdk 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +145 -198
- package/dist/{chunk-FQBYUOJB.mjs → chunk-53PSEBAD.mjs} +67 -21
- package/dist/index-ClE8H5jj.d.mts +646 -0
- package/dist/index-ClE8H5jj.d.ts +646 -0
- package/dist/index.d.mts +4 -5
- package/dist/index.d.ts +4 -5
- package/dist/index.js +1132 -326
- package/dist/index.mjs +1057 -303
- package/dist/opentelemetry/index.d.mts +1 -2
- package/dist/opentelemetry/index.d.ts +1 -2
- package/dist/opentelemetry/index.js +50 -23
- package/dist/opentelemetry/index.mjs +1 -1
- package/package.json +60 -57
- package/dist/chunk-YSYKPKKX.mjs +0 -357
- package/dist/index--b41_E-1.d.mts +0 -440
- package/dist/index--b41_E-1.d.ts +0 -440
- package/dist/index-Cr-FN-y5.d.mts +0 -405
- package/dist/index-Cr-FN-y5.d.ts +0 -405
- package/dist/index-D4UXMom5.d.mts +0 -429
- package/dist/index-D4UXMom5.d.ts +0 -429
package/dist/index.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
SpanType,
|
|
5
5
|
Trace,
|
|
6
6
|
TraceStatus
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-53PSEBAD.mjs";
|
|
8
8
|
|
|
9
9
|
// src/client.ts
|
|
10
10
|
import debug3 from "debug";
|
|
@@ -12,8 +12,11 @@ import debug3 from "debug";
|
|
|
12
12
|
// src/instrumentation/openai.ts
|
|
13
13
|
import debug from "debug";
|
|
14
14
|
var log = debug("observyze:sdk");
|
|
15
|
+
var WRAPPED = /* @__PURE__ */ Symbol.for("observyze.openai.wrapped");
|
|
16
|
+
var MAX_CAPTURED_STREAM_CHARS = 1e6;
|
|
15
17
|
function wrapOpenAI(client, nwClient) {
|
|
16
18
|
const anyClient = client;
|
|
19
|
+
if (anyClient[WRAPPED]) return client;
|
|
17
20
|
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
18
21
|
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/openai");
|
|
19
22
|
if (!isAlreadyRedirected) {
|
|
@@ -39,7 +42,7 @@ function wrapOpenAI(client, nwClient) {
|
|
|
39
42
|
provider: "openai",
|
|
40
43
|
model: params.model
|
|
41
44
|
});
|
|
42
|
-
const span = trace.startSpan("chat.completions.create",
|
|
45
|
+
const span = trace.startSpan("chat.completions.create", "llm" /* LLM */);
|
|
43
46
|
span.setMetadata("model", params.model);
|
|
44
47
|
span.setMetadata("provider", "openai");
|
|
45
48
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -69,38 +72,64 @@ function wrapOpenAI(client, nwClient) {
|
|
|
69
72
|
output: completionResponse.usage.completion_tokens,
|
|
70
73
|
total: completionResponse.usage.total_tokens
|
|
71
74
|
});
|
|
75
|
+
span.setMetadata("token_usage_source", "provider");
|
|
76
|
+
} else {
|
|
77
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
72
78
|
}
|
|
73
79
|
span.setMetadata("latency_ms", latency);
|
|
74
80
|
span.end();
|
|
75
|
-
trace.end();
|
|
81
|
+
trace.end("success" /* SUCCESS */);
|
|
76
82
|
return response;
|
|
77
83
|
} catch (error) {
|
|
78
84
|
const latency = Date.now() - startTime;
|
|
79
85
|
span.setMetadata("latency_ms", latency);
|
|
80
86
|
span.setError(error);
|
|
81
87
|
span.end();
|
|
82
|
-
trace.end();
|
|
88
|
+
trace.end("error" /* ERROR */);
|
|
83
89
|
throw error;
|
|
84
90
|
}
|
|
85
91
|
};
|
|
92
|
+
Object.defineProperty(anyClient, WRAPPED, { value: true, enumerable: false });
|
|
86
93
|
return client;
|
|
87
94
|
}
|
|
88
95
|
function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
89
96
|
const bufferedChunks = [];
|
|
90
97
|
let streamId = "";
|
|
91
98
|
let streamModel = "";
|
|
99
|
+
let inputTokens = 0;
|
|
100
|
+
let outputTokens = 0;
|
|
101
|
+
let outputTruncated = false;
|
|
102
|
+
let capturedChars = 0;
|
|
92
103
|
return {
|
|
93
104
|
[Symbol.asyncIterator]: async function* () {
|
|
105
|
+
let completed = false;
|
|
106
|
+
let failure;
|
|
94
107
|
try {
|
|
95
108
|
for await (const chunk of stream) {
|
|
96
109
|
if (chunk.id) streamId = chunk.id;
|
|
97
110
|
if (chunk.model) streamModel = chunk.model;
|
|
98
111
|
const delta = chunk.choices[0]?.delta;
|
|
99
112
|
if (delta?.content) {
|
|
100
|
-
|
|
113
|
+
if (capturedChars < MAX_CAPTURED_STREAM_CHARS) {
|
|
114
|
+
const captured = delta.content.slice(0, MAX_CAPTURED_STREAM_CHARS - capturedChars);
|
|
115
|
+
bufferedChunks.push(captured);
|
|
116
|
+
capturedChars += captured.length;
|
|
117
|
+
if (captured.length < delta.content.length) outputTruncated = true;
|
|
118
|
+
} else {
|
|
119
|
+
outputTruncated = true;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (chunk.usage) {
|
|
123
|
+
inputTokens = Math.max(inputTokens, chunk.usage.prompt_tokens || 0);
|
|
124
|
+
outputTokens = Math.max(outputTokens, chunk.usage.completion_tokens || 0);
|
|
101
125
|
}
|
|
102
126
|
yield chunk;
|
|
103
127
|
}
|
|
128
|
+
completed = true;
|
|
129
|
+
} catch (error) {
|
|
130
|
+
failure = error;
|
|
131
|
+
throw error;
|
|
132
|
+
} finally {
|
|
104
133
|
const latency = Date.now() - startTime;
|
|
105
134
|
const completeOutput = bufferedChunks.join("");
|
|
106
135
|
span.setOutput({
|
|
@@ -110,15 +139,21 @@ function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
|
110
139
|
});
|
|
111
140
|
span.setMetadata("latency_ms", latency);
|
|
112
141
|
span.setMetadata("streaming", true);
|
|
142
|
+
span.setMetadata("stream_completed", completed);
|
|
143
|
+
span.setMetadata("output_truncated", outputTruncated);
|
|
144
|
+
if (inputTokens > 0 || outputTokens > 0) {
|
|
145
|
+
span.setTokens({ input: inputTokens, output: outputTokens, total: inputTokens + outputTokens });
|
|
146
|
+
span.setMetadata("token_usage_source", "provider");
|
|
147
|
+
} else {
|
|
148
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
149
|
+
}
|
|
150
|
+
if (failure) {
|
|
151
|
+
span.setError(failure);
|
|
152
|
+
} else if (!completed) {
|
|
153
|
+
span.setError(new Error("Stream consumption ended before provider completion"));
|
|
154
|
+
}
|
|
113
155
|
span.end();
|
|
114
|
-
trace.end();
|
|
115
|
-
} catch (error) {
|
|
116
|
-
const latency = Date.now() - startTime;
|
|
117
|
-
span.setMetadata("latency_ms", latency);
|
|
118
|
-
span.setError(error);
|
|
119
|
-
span.end();
|
|
120
|
-
trace.end();
|
|
121
|
-
throw error;
|
|
156
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
122
157
|
}
|
|
123
158
|
}
|
|
124
159
|
};
|
|
@@ -127,8 +162,11 @@ function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
|
127
162
|
// src/instrumentation/anthropic.ts
|
|
128
163
|
import debug2 from "debug";
|
|
129
164
|
var log2 = debug2("observyze:sdk");
|
|
165
|
+
var WRAPPED2 = /* @__PURE__ */ Symbol.for("observyze.anthropic.wrapped");
|
|
166
|
+
var MAX_CAPTURED_STREAM_CHARS2 = 1e6;
|
|
130
167
|
function wrapAnthropic(client, nwClient) {
|
|
131
168
|
const anyClient = client;
|
|
169
|
+
if (anyClient[WRAPPED2]) return client;
|
|
132
170
|
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
133
171
|
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/anthropic");
|
|
134
172
|
if (!isAlreadyRedirected) {
|
|
@@ -154,7 +192,7 @@ function wrapAnthropic(client, nwClient) {
|
|
|
154
192
|
provider: "anthropic",
|
|
155
193
|
model: params.model
|
|
156
194
|
});
|
|
157
|
-
const span = trace.startSpan("messages.create",
|
|
195
|
+
const span = trace.startSpan("messages.create", "llm" /* LLM */);
|
|
158
196
|
span.setMetadata("model", params.model);
|
|
159
197
|
span.setMetadata("provider", "anthropic");
|
|
160
198
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -188,20 +226,24 @@ function wrapAnthropic(client, nwClient) {
|
|
|
188
226
|
output: messageResponse.usage.output_tokens,
|
|
189
227
|
total: messageResponse.usage.input_tokens + messageResponse.usage.output_tokens
|
|
190
228
|
});
|
|
229
|
+
span.setMetadata("token_usage_source", "provider");
|
|
230
|
+
} else {
|
|
231
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
191
232
|
}
|
|
192
233
|
span.setMetadata("latency_ms", latency);
|
|
193
234
|
span.end();
|
|
194
|
-
trace.end();
|
|
235
|
+
trace.end("success" /* SUCCESS */);
|
|
195
236
|
return response;
|
|
196
237
|
} catch (error) {
|
|
197
238
|
const latency = Date.now() - startTime;
|
|
198
239
|
span.setMetadata("latency_ms", latency);
|
|
199
240
|
span.setError(error);
|
|
200
241
|
span.end();
|
|
201
|
-
trace.end();
|
|
242
|
+
trace.end("error" /* ERROR */);
|
|
202
243
|
throw error;
|
|
203
244
|
}
|
|
204
245
|
};
|
|
246
|
+
Object.defineProperty(anyClient, WRAPPED2, { value: true, enumerable: false });
|
|
205
247
|
return client;
|
|
206
248
|
}
|
|
207
249
|
function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
@@ -211,8 +253,12 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
211
253
|
let stopReason = null;
|
|
212
254
|
let inputTokens = 0;
|
|
213
255
|
let outputTokens = 0;
|
|
256
|
+
let capturedChars = 0;
|
|
257
|
+
let outputTruncated = false;
|
|
214
258
|
return {
|
|
215
259
|
[Symbol.asyncIterator]: async function* () {
|
|
260
|
+
let completed = false;
|
|
261
|
+
let failure;
|
|
216
262
|
try {
|
|
217
263
|
for await (const event of stream) {
|
|
218
264
|
if (event.type === "message_start" && event.message) {
|
|
@@ -223,7 +269,14 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
223
269
|
}
|
|
224
270
|
}
|
|
225
271
|
if (event.type === "content_block_delta" && event.delta?.text) {
|
|
226
|
-
|
|
272
|
+
if (capturedChars < MAX_CAPTURED_STREAM_CHARS2) {
|
|
273
|
+
const captured = event.delta.text.slice(0, MAX_CAPTURED_STREAM_CHARS2 - capturedChars);
|
|
274
|
+
bufferedChunks.push(captured);
|
|
275
|
+
capturedChars += captured.length;
|
|
276
|
+
if (captured.length < event.delta.text.length) outputTruncated = true;
|
|
277
|
+
} else {
|
|
278
|
+
outputTruncated = true;
|
|
279
|
+
}
|
|
227
280
|
}
|
|
228
281
|
if (event.type === "message_delta" && event.delta) {
|
|
229
282
|
if (event.delta.stop_reason) {
|
|
@@ -235,6 +288,11 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
235
288
|
}
|
|
236
289
|
yield event;
|
|
237
290
|
}
|
|
291
|
+
completed = true;
|
|
292
|
+
} catch (error) {
|
|
293
|
+
failure = error;
|
|
294
|
+
throw error;
|
|
295
|
+
} finally {
|
|
238
296
|
const latency = Date.now() - startTime;
|
|
239
297
|
const completeOutput = bufferedChunks.join("");
|
|
240
298
|
span.setOutput({
|
|
@@ -249,52 +307,747 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
249
307
|
output: outputTokens,
|
|
250
308
|
total: inputTokens + outputTokens
|
|
251
309
|
});
|
|
310
|
+
span.setMetadata("token_usage_source", "provider");
|
|
311
|
+
} else {
|
|
312
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
252
313
|
}
|
|
253
314
|
span.setMetadata("latency_ms", latency);
|
|
254
315
|
span.setMetadata("streaming", true);
|
|
316
|
+
span.setMetadata("stream_completed", completed);
|
|
317
|
+
span.setMetadata("output_truncated", outputTruncated);
|
|
318
|
+
if (failure) {
|
|
319
|
+
span.setError(failure);
|
|
320
|
+
} else if (!completed) {
|
|
321
|
+
span.setError(new Error("Stream consumption ended before provider completion"));
|
|
322
|
+
}
|
|
255
323
|
span.end();
|
|
256
|
-
trace.end();
|
|
324
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// src/instrumentation/langchain.ts
|
|
331
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
332
|
+
var WRAPPED3 = /* @__PURE__ */ Symbol.for("observyze.langchain.wrapped");
|
|
333
|
+
var nestedInstrumentation = new AsyncLocalStorage();
|
|
334
|
+
var MAX_CAPTURED_STREAM_CHARS3 = 1e6;
|
|
335
|
+
function runnableName(runnable) {
|
|
336
|
+
return String(
|
|
337
|
+
runnable.name || runnable.lc_namespace?.join(".") || runnable.constructor?.name || "Runnable"
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
function extractUsage(output) {
|
|
341
|
+
const usage2 = output?.usage_metadata || output?.response_metadata?.tokenUsage || output?.usage;
|
|
342
|
+
if (!usage2) return null;
|
|
343
|
+
const input = Number(usage2.input_tokens ?? usage2.promptTokens ?? usage2.prompt_tokens ?? 0);
|
|
344
|
+
const outputTokens = Number(usage2.output_tokens ?? usage2.completionTokens ?? usage2.completion_tokens ?? 0);
|
|
345
|
+
const total = Number(usage2.total_tokens ?? usage2.totalTokens ?? input + outputTokens);
|
|
346
|
+
return input > 0 || outputTokens > 0 || total > 0 ? { input, output: outputTokens, total } : null;
|
|
347
|
+
}
|
|
348
|
+
function chunkText(chunk) {
|
|
349
|
+
if (typeof chunk === "string") return chunk;
|
|
350
|
+
if (typeof chunk?.content === "string") return chunk.content;
|
|
351
|
+
if (typeof chunk?.text === "string") return chunk.text;
|
|
352
|
+
return "";
|
|
353
|
+
}
|
|
354
|
+
function wrapLangChain(runnable, nwClient) {
|
|
355
|
+
const target = runnable;
|
|
356
|
+
if (target[WRAPPED3]) return runnable;
|
|
357
|
+
const name = runnableName(runnable);
|
|
358
|
+
const originalInvoke = runnable.invoke.bind(runnable);
|
|
359
|
+
target.invoke = async (input, config) => {
|
|
360
|
+
if (nestedInstrumentation.getStore()) return originalInvoke(input, config);
|
|
361
|
+
const trace = nwClient.startTrace(`langchain.${name}.invoke`, { provider: "langchain", runnable: name });
|
|
362
|
+
const span = trace.startSpan(`${name}.invoke`, "chain" /* CHAIN */);
|
|
363
|
+
span.setInput(input);
|
|
364
|
+
try {
|
|
365
|
+
const output = await nestedInstrumentation.run(true, () => originalInvoke(input, config));
|
|
366
|
+
span.setOutput(output);
|
|
367
|
+
const usage2 = extractUsage(output);
|
|
368
|
+
if (usage2) {
|
|
369
|
+
span.setTokens(usage2);
|
|
370
|
+
span.setMetadata("token_usage_source", "provider");
|
|
371
|
+
} else {
|
|
372
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
373
|
+
}
|
|
374
|
+
span.end();
|
|
375
|
+
trace.end("success" /* SUCCESS */);
|
|
376
|
+
return output;
|
|
377
|
+
} catch (error) {
|
|
378
|
+
span.setError(error);
|
|
379
|
+
span.end();
|
|
380
|
+
trace.end("error" /* ERROR */);
|
|
381
|
+
throw error;
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
if (typeof runnable.stream === "function") {
|
|
385
|
+
const originalStream = runnable.stream.bind(runnable);
|
|
386
|
+
target.stream = async (input, config) => {
|
|
387
|
+
if (nestedInstrumentation.getStore()) return originalStream(input, config);
|
|
388
|
+
const trace = nwClient.startTrace(`langchain.${name}.stream`, { provider: "langchain", runnable: name });
|
|
389
|
+
const span = trace.startSpan(`${name}.stream`, "chain" /* CHAIN */);
|
|
390
|
+
span.setInput(input);
|
|
391
|
+
let source;
|
|
392
|
+
try {
|
|
393
|
+
source = await nestedInstrumentation.run(true, () => originalStream(input, config));
|
|
257
394
|
} catch (error) {
|
|
258
|
-
const latency = Date.now() - startTime;
|
|
259
|
-
span.setMetadata("latency_ms", latency);
|
|
260
395
|
span.setError(error);
|
|
261
396
|
span.end();
|
|
262
|
-
trace.end();
|
|
397
|
+
trace.end("error" /* ERROR */);
|
|
263
398
|
throw error;
|
|
264
399
|
}
|
|
400
|
+
return {
|
|
401
|
+
[Symbol.asyncIterator]: async function* () {
|
|
402
|
+
let output = "";
|
|
403
|
+
let completed = false;
|
|
404
|
+
let truncated = false;
|
|
405
|
+
let failure;
|
|
406
|
+
try {
|
|
407
|
+
for await (const chunk of source) {
|
|
408
|
+
const text = chunkText(chunk);
|
|
409
|
+
if (output.length < MAX_CAPTURED_STREAM_CHARS3) {
|
|
410
|
+
const captured = text.slice(0, MAX_CAPTURED_STREAM_CHARS3 - output.length);
|
|
411
|
+
output += captured;
|
|
412
|
+
if (captured.length < text.length) truncated = true;
|
|
413
|
+
} else if (text) {
|
|
414
|
+
truncated = true;
|
|
415
|
+
}
|
|
416
|
+
yield chunk;
|
|
417
|
+
}
|
|
418
|
+
completed = true;
|
|
419
|
+
} catch (error) {
|
|
420
|
+
failure = error;
|
|
421
|
+
throw error;
|
|
422
|
+
} finally {
|
|
423
|
+
span.setOutput({ content: output });
|
|
424
|
+
span.setMetadata("streaming", true);
|
|
425
|
+
span.setMetadata("stream_completed", completed);
|
|
426
|
+
span.setMetadata("output_truncated", truncated);
|
|
427
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
428
|
+
if (failure) span.setError(failure);
|
|
429
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before runnable completion"));
|
|
430
|
+
span.end();
|
|
431
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
Object.defineProperty(target, WRAPPED3, { value: true, enumerable: false });
|
|
438
|
+
return runnable;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// src/instrumentation/gemini.ts
|
|
442
|
+
var WRAPPED4 = /* @__PURE__ */ Symbol.for("observyze.gemini.wrapped");
|
|
443
|
+
var MAX_CAPTURED_STREAM_CHARS4 = 1e6;
|
|
444
|
+
function modelName(target, request) {
|
|
445
|
+
return String(request?.model || target?.model || target?.modelName || "unknown");
|
|
446
|
+
}
|
|
447
|
+
function responseText(response) {
|
|
448
|
+
try {
|
|
449
|
+
if (typeof response?.text === "function") return String(response.text());
|
|
450
|
+
if (typeof response?.text === "string") return response.text;
|
|
451
|
+
const parts = response?.candidates?.[0]?.content?.parts;
|
|
452
|
+
return Array.isArray(parts) ? parts.map((part) => part?.text || "").join("") : "";
|
|
453
|
+
} catch {
|
|
454
|
+
return "";
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
function usage(response) {
|
|
458
|
+
const value = response?.usageMetadata || response?.usage_metadata || response?.usage;
|
|
459
|
+
if (!value) return null;
|
|
460
|
+
const input = Number(value.promptTokenCount ?? value.inputTokens ?? value.input_tokens ?? 0);
|
|
461
|
+
const output = Number(value.candidatesTokenCount ?? value.outputTokens ?? value.output_tokens ?? 0);
|
|
462
|
+
const total = Number(value.totalTokenCount ?? value.totalTokens ?? value.total_tokens ?? input + output);
|
|
463
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
464
|
+
}
|
|
465
|
+
function instrumentMethod(target, method, nwClient) {
|
|
466
|
+
if (typeof target?.[method] !== "function") return;
|
|
467
|
+
const original = target[method].bind(target);
|
|
468
|
+
target[method] = async (...args) => {
|
|
469
|
+
const request = args[0];
|
|
470
|
+
const model = modelName(target, request);
|
|
471
|
+
const trace = nwClient.startTrace(`gemini.${method}`, { provider: "google", model });
|
|
472
|
+
const span = trace.startSpan(method, "llm" /* LLM */);
|
|
473
|
+
span.setMetadata("provider", "google");
|
|
474
|
+
span.setMetadata("model", model);
|
|
475
|
+
span.setInput(request);
|
|
476
|
+
try {
|
|
477
|
+
const result = await original(...args);
|
|
478
|
+
const stream = result?.stream || (result?.[Symbol.asyncIterator] ? result : null);
|
|
479
|
+
if (stream?.[Symbol.asyncIterator]) {
|
|
480
|
+
const wrappedStream = wrapGeminiStream(stream, result?.response, span, trace, model);
|
|
481
|
+
if (result?.stream) {
|
|
482
|
+
return new Proxy(result, {
|
|
483
|
+
get(target2, property, receiver) {
|
|
484
|
+
return property === "stream" ? wrappedStream : Reflect.get(target2, property, receiver);
|
|
485
|
+
}
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
return wrappedStream;
|
|
489
|
+
}
|
|
490
|
+
const resolved = result?.response ? await result.response : result;
|
|
491
|
+
span.setOutput({
|
|
492
|
+
text: responseText(resolved),
|
|
493
|
+
finish_reason: resolved?.candidates?.[0]?.finishReason
|
|
494
|
+
});
|
|
495
|
+
const tokenUsage = usage(resolved);
|
|
496
|
+
if (tokenUsage) {
|
|
497
|
+
span.setTokens(tokenUsage);
|
|
498
|
+
span.setMetadata("token_usage_source", "provider");
|
|
499
|
+
} else {
|
|
500
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
501
|
+
}
|
|
502
|
+
span.end();
|
|
503
|
+
trace.end("success" /* SUCCESS */);
|
|
504
|
+
return result;
|
|
505
|
+
} catch (error) {
|
|
506
|
+
span.setError(error);
|
|
507
|
+
span.end();
|
|
508
|
+
trace.end("error" /* ERROR */);
|
|
509
|
+
throw error;
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
function wrapGeminiStream(stream, finalResponse, span, trace, model) {
|
|
514
|
+
return {
|
|
515
|
+
[Symbol.asyncIterator]: async function* () {
|
|
516
|
+
let text = "";
|
|
517
|
+
let completed = false;
|
|
518
|
+
let truncated = false;
|
|
519
|
+
let failure;
|
|
520
|
+
let latestUsage = null;
|
|
521
|
+
try {
|
|
522
|
+
for await (const chunk of stream) {
|
|
523
|
+
const chunkText2 = responseText(chunk);
|
|
524
|
+
if (text.length < MAX_CAPTURED_STREAM_CHARS4) {
|
|
525
|
+
const captured = chunkText2.slice(0, MAX_CAPTURED_STREAM_CHARS4 - text.length);
|
|
526
|
+
text += captured;
|
|
527
|
+
if (captured.length < chunkText2.length) truncated = true;
|
|
528
|
+
} else if (chunkText2) {
|
|
529
|
+
truncated = true;
|
|
530
|
+
}
|
|
531
|
+
latestUsage = usage(chunk) || latestUsage;
|
|
532
|
+
yield chunk;
|
|
533
|
+
}
|
|
534
|
+
completed = true;
|
|
535
|
+
} catch (error) {
|
|
536
|
+
failure = error;
|
|
537
|
+
throw error;
|
|
538
|
+
} finally {
|
|
539
|
+
if (completed && finalResponse) {
|
|
540
|
+
try {
|
|
541
|
+
const resolved = await finalResponse;
|
|
542
|
+
latestUsage = usage(resolved) || latestUsage;
|
|
543
|
+
if (!text) text = responseText(resolved).slice(0, MAX_CAPTURED_STREAM_CHARS4);
|
|
544
|
+
} catch (error) {
|
|
545
|
+
failure = error;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
span.setOutput({ model, text });
|
|
549
|
+
span.setMetadata("streaming", true);
|
|
550
|
+
span.setMetadata("stream_completed", completed);
|
|
551
|
+
span.setMetadata("output_truncated", truncated);
|
|
552
|
+
if (latestUsage) {
|
|
553
|
+
span.setTokens(latestUsage);
|
|
554
|
+
span.setMetadata("token_usage_source", "provider");
|
|
555
|
+
} else {
|
|
556
|
+
span.setMetadata("token_usage_source", "unavailable");
|
|
557
|
+
}
|
|
558
|
+
if (failure) span.setError(failure);
|
|
559
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before Gemini completed"));
|
|
560
|
+
span.end();
|
|
561
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
function wrapGemini(client, nwClient) {
|
|
567
|
+
const target = client.models || client;
|
|
568
|
+
if (target[WRAPPED4]) return client;
|
|
569
|
+
instrumentMethod(target, "generateContent", nwClient);
|
|
570
|
+
instrumentMethod(target, "generateContentStream", nwClient);
|
|
571
|
+
Object.defineProperty(target, WRAPPED4, { value: true, enumerable: false });
|
|
572
|
+
return client;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// src/instrumentation/vercel-ai.ts
|
|
576
|
+
var MAX_CAPTURED_STREAM_CHARS5 = 1e6;
|
|
577
|
+
function modelName2(request, response) {
|
|
578
|
+
let requestModelId;
|
|
579
|
+
try {
|
|
580
|
+
requestModelId = typeof request?.model?.modelId === "function" ? request.model.modelId() : request?.model?.modelId;
|
|
581
|
+
} catch {
|
|
582
|
+
requestModelId = void 0;
|
|
583
|
+
}
|
|
584
|
+
return String(
|
|
585
|
+
response?.response?.modelId || requestModelId || request?.model || "unknown"
|
|
586
|
+
);
|
|
587
|
+
}
|
|
588
|
+
async function resolveUsage(value) {
|
|
589
|
+
const usage2 = await Promise.resolve(value?.usage).catch(() => null);
|
|
590
|
+
if (!usage2) return null;
|
|
591
|
+
const input = Number(usage2.inputTokens ?? usage2.promptTokens ?? usage2.input_tokens ?? 0);
|
|
592
|
+
const output = Number(usage2.outputTokens ?? usage2.completionTokens ?? usage2.output_tokens ?? 0);
|
|
593
|
+
const total = Number(usage2.totalTokens ?? usage2.total_tokens ?? input + output);
|
|
594
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
595
|
+
}
|
|
596
|
+
function wrapVercelAI(sdk, nwClient) {
|
|
597
|
+
const wrapped = { ...sdk };
|
|
598
|
+
const generateText = sdk.generateText;
|
|
599
|
+
const streamText = sdk.streamText;
|
|
600
|
+
if (typeof generateText === "function") {
|
|
601
|
+
wrapped.generateText = async (...args) => {
|
|
602
|
+
const request = args[0];
|
|
603
|
+
const trace = nwClient.startTrace("vercel-ai.generateText", { provider: "vercel-ai", model: modelName2(request) });
|
|
604
|
+
const span = trace.startSpan("generateText", "llm" /* LLM */);
|
|
605
|
+
span.setMetadata("provider", "vercel-ai");
|
|
606
|
+
span.setMetadata("model", modelName2(request));
|
|
607
|
+
span.setInput(request);
|
|
608
|
+
try {
|
|
609
|
+
const result = await generateText.apply(sdk, args);
|
|
610
|
+
span.setOutput({ text: result?.text, finish_reason: result?.finishReason });
|
|
611
|
+
const tokens = await resolveUsage(result);
|
|
612
|
+
if (tokens) {
|
|
613
|
+
span.setTokens(tokens);
|
|
614
|
+
span.setMetadata("token_usage_source", "provider");
|
|
615
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
616
|
+
span.end();
|
|
617
|
+
trace.end("success" /* SUCCESS */);
|
|
618
|
+
return result;
|
|
619
|
+
} catch (error) {
|
|
620
|
+
span.setError(error);
|
|
621
|
+
span.end();
|
|
622
|
+
trace.end("error" /* ERROR */);
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
if (typeof streamText === "function") {
|
|
628
|
+
wrapped.streamText = (...args) => {
|
|
629
|
+
const request = args[0];
|
|
630
|
+
const trace = nwClient.startTrace("vercel-ai.streamText", { provider: "vercel-ai", model: modelName2(request) });
|
|
631
|
+
const span = trace.startSpan("streamText", "llm" /* LLM */);
|
|
632
|
+
span.setMetadata("provider", "vercel-ai");
|
|
633
|
+
span.setMetadata("model", modelName2(request));
|
|
634
|
+
span.setInput(request);
|
|
635
|
+
let result;
|
|
636
|
+
try {
|
|
637
|
+
result = streamText.apply(sdk, args);
|
|
638
|
+
} catch (error) {
|
|
639
|
+
span.setError(error);
|
|
640
|
+
span.end();
|
|
641
|
+
trace.end("error" /* ERROR */);
|
|
642
|
+
throw error;
|
|
643
|
+
}
|
|
644
|
+
if (!result?.textStream?.[Symbol.asyncIterator]) {
|
|
645
|
+
span.setError(new Error("Vercel AI streamText returned no textStream"));
|
|
646
|
+
span.end();
|
|
647
|
+
trace.end("error" /* ERROR */);
|
|
648
|
+
return result;
|
|
649
|
+
}
|
|
650
|
+
const source = result.textStream;
|
|
651
|
+
const textStream = {
|
|
652
|
+
[Symbol.asyncIterator]: async function* () {
|
|
653
|
+
let text = "";
|
|
654
|
+
let completed = false;
|
|
655
|
+
let truncated = false;
|
|
656
|
+
let failure;
|
|
657
|
+
try {
|
|
658
|
+
for await (const chunk of source) {
|
|
659
|
+
if (text.length < MAX_CAPTURED_STREAM_CHARS5) {
|
|
660
|
+
const captured = String(chunk).slice(0, MAX_CAPTURED_STREAM_CHARS5 - text.length);
|
|
661
|
+
text += captured;
|
|
662
|
+
if (captured.length < String(chunk).length) truncated = true;
|
|
663
|
+
} else if (chunk) truncated = true;
|
|
664
|
+
yield chunk;
|
|
665
|
+
}
|
|
666
|
+
completed = true;
|
|
667
|
+
} catch (error) {
|
|
668
|
+
failure = error;
|
|
669
|
+
throw error;
|
|
670
|
+
} finally {
|
|
671
|
+
span.setOutput({ text });
|
|
672
|
+
span.setMetadata("streaming", true);
|
|
673
|
+
span.setMetadata("stream_completed", completed);
|
|
674
|
+
span.setMetadata("output_truncated", truncated);
|
|
675
|
+
const tokens = completed ? await resolveUsage(result) : null;
|
|
676
|
+
if (tokens) {
|
|
677
|
+
span.setTokens(tokens);
|
|
678
|
+
span.setMetadata("token_usage_source", "provider");
|
|
679
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
680
|
+
if (failure) span.setError(failure);
|
|
681
|
+
else if (!completed) span.setError(new Error("Stream consumption ended before Vercel AI completed"));
|
|
682
|
+
span.end();
|
|
683
|
+
trace.end(completed && !failure ? "success" /* SUCCESS */ : "error" /* ERROR */);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
};
|
|
687
|
+
return new Proxy(result, {
|
|
688
|
+
get(target, property, receiver) {
|
|
689
|
+
return property === "textStream" ? textStream : Reflect.get(target, property, receiver);
|
|
690
|
+
}
|
|
691
|
+
});
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
return wrapped;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
// src/instrumentation/llamaindex.ts
|
|
698
|
+
var WRAPPED5 = /* @__PURE__ */ Symbol.for("observyze.llamaindex.wrapped");
|
|
699
|
+
function outputValue(response) {
|
|
700
|
+
if (typeof response?.response === "string") return { text: response.response };
|
|
701
|
+
if (typeof response?.message?.content === "string") return { text: response.message.content };
|
|
702
|
+
if (typeof response === "string") return { text: response };
|
|
703
|
+
if (typeof response?.toString === "function" && response.toString !== Object.prototype.toString) {
|
|
704
|
+
try {
|
|
705
|
+
return { text: String(response.toString()) };
|
|
706
|
+
} catch {
|
|
707
|
+
return { response_type: response?.constructor?.name || typeof response };
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
return { response_type: response?.constructor?.name || typeof response };
|
|
711
|
+
}
|
|
712
|
+
function extractUsage2(response) {
|
|
713
|
+
const usage2 = response?.usage || response?.raw?.usage || response?.message?.additionalKwargs?.usage;
|
|
714
|
+
if (!usage2) return null;
|
|
715
|
+
const input = Number(usage2.prompt_tokens ?? usage2.input_tokens ?? usage2.inputTokens ?? 0);
|
|
716
|
+
const output = Number(usage2.completion_tokens ?? usage2.output_tokens ?? usage2.outputTokens ?? 0);
|
|
717
|
+
const total = Number(usage2.total_tokens ?? usage2.totalTokens ?? input + output);
|
|
718
|
+
return input > 0 || output > 0 || total > 0 ? { input, output, total } : null;
|
|
719
|
+
}
|
|
720
|
+
function instrument(target, method, nwClient) {
|
|
721
|
+
if (typeof target?.[method] !== "function") return;
|
|
722
|
+
const original = target[method].bind(target);
|
|
723
|
+
target[method] = async (...args) => {
|
|
724
|
+
const trace = nwClient.startTrace(`llamaindex.${method}`, { provider: "llamaindex" });
|
|
725
|
+
const span = trace.startSpan(method, method === "query" ? "retrieval" /* RETRIEVAL */ : "llm" /* LLM */);
|
|
726
|
+
span.setMetadata("provider", "llamaindex");
|
|
727
|
+
span.setInput(args[0]);
|
|
728
|
+
try {
|
|
729
|
+
const result = await original(...args);
|
|
730
|
+
span.setOutput(outputValue(result));
|
|
731
|
+
const tokens = extractUsage2(result);
|
|
732
|
+
if (tokens) {
|
|
733
|
+
span.setTokens(tokens);
|
|
734
|
+
span.setMetadata("token_usage_source", "provider");
|
|
735
|
+
} else span.setMetadata("token_usage_source", "unavailable");
|
|
736
|
+
span.end();
|
|
737
|
+
trace.end("success" /* SUCCESS */);
|
|
738
|
+
return result;
|
|
739
|
+
} catch (error) {
|
|
740
|
+
span.setError(error);
|
|
741
|
+
span.end();
|
|
742
|
+
trace.end("error" /* ERROR */);
|
|
743
|
+
throw error;
|
|
265
744
|
}
|
|
266
745
|
};
|
|
267
746
|
}
|
|
747
|
+
function wrapLlamaIndex(engine, nwClient) {
|
|
748
|
+
const target = engine;
|
|
749
|
+
if (target[WRAPPED5]) return engine;
|
|
750
|
+
instrument(target, "query", nwClient);
|
|
751
|
+
instrument(target, "chat", nwClient);
|
|
752
|
+
Object.defineProperty(target, WRAPPED5, { value: true, enumerable: false });
|
|
753
|
+
return engine;
|
|
754
|
+
}
|
|
268
755
|
|
|
269
756
|
// src/instrumentation/index.ts
|
|
270
757
|
function wrap(client, nwClient) {
|
|
271
|
-
|
|
758
|
+
const candidate = client;
|
|
759
|
+
if (candidate?.chat?.completions && typeof candidate.chat.completions.create === "function") {
|
|
272
760
|
return wrapOpenAI(client, nwClient);
|
|
273
761
|
}
|
|
274
|
-
if (
|
|
762
|
+
if (candidate?.messages && typeof candidate.messages.create === "function") {
|
|
275
763
|
return wrapAnthropic(client, nwClient);
|
|
276
764
|
}
|
|
765
|
+
if (typeof candidate?.generateContent === "function" || typeof candidate?.generateContentStream === "function" || typeof candidate?.models?.generateContent === "function" || typeof candidate?.models?.generateContentStream === "function") {
|
|
766
|
+
return wrapGemini(client, nwClient);
|
|
767
|
+
}
|
|
768
|
+
if (typeof candidate?.generateText === "function" || typeof candidate?.streamText === "function") {
|
|
769
|
+
return wrapVercelAI(client, nwClient);
|
|
770
|
+
}
|
|
771
|
+
if (typeof candidate?.invoke === "function") {
|
|
772
|
+
return wrapLangChain(client, nwClient);
|
|
773
|
+
}
|
|
774
|
+
if (typeof candidate?.query === "function" || typeof candidate?.chat === "function") {
|
|
775
|
+
return wrapLlamaIndex(client, nwClient);
|
|
776
|
+
}
|
|
277
777
|
throw new Error(
|
|
278
|
-
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic"
|
|
778
|
+
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic, Gemini, Vercel AI SDK, LangChain, and LlamaIndex"
|
|
279
779
|
);
|
|
280
780
|
}
|
|
281
781
|
|
|
782
|
+
// src/execution-budget.ts
|
|
783
|
+
var ExecutionBudgetExceededError = class extends Error {
|
|
784
|
+
constructor(dimension, message) {
|
|
785
|
+
super(message);
|
|
786
|
+
this.dimension = dimension;
|
|
787
|
+
this.name = "ExecutionBudgetExceededError";
|
|
788
|
+
}
|
|
789
|
+
dimension;
|
|
790
|
+
code = "OBSERVYZE_EXECUTION_BUDGET_EXCEEDED";
|
|
791
|
+
};
|
|
792
|
+
var ExecutionBudget = class {
|
|
793
|
+
constructor(options) {
|
|
794
|
+
this.options = options;
|
|
795
|
+
if (!Number.isInteger(options.maxCalls) || options.maxCalls < 1 || options.maxCalls > 1e5) {
|
|
796
|
+
throw new Error("ExecutionBudget maxCalls must be an integer between 1 and 100000");
|
|
797
|
+
}
|
|
798
|
+
if (!Number.isInteger(options.timeoutMs) || options.timeoutMs < 100 || options.timeoutMs > 864e5) {
|
|
799
|
+
throw new Error("ExecutionBudget timeoutMs must be an integer between 100 and 86400000");
|
|
800
|
+
}
|
|
801
|
+
if (options.maxTokens !== void 0 && (!Number.isInteger(options.maxTokens) || options.maxTokens < 1 || options.maxTokens > 1e10)) {
|
|
802
|
+
throw new Error("ExecutionBudget maxTokens must be an integer between 1 and 10000000000");
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
options;
|
|
806
|
+
startedAt = Date.now();
|
|
807
|
+
controller = new AbortController();
|
|
808
|
+
callsUsed = 0;
|
|
809
|
+
tokensUsed = 0;
|
|
810
|
+
get signal() {
|
|
811
|
+
return this.controller.signal;
|
|
812
|
+
}
|
|
813
|
+
get stats() {
|
|
814
|
+
const elapsedMs = Date.now() - this.startedAt;
|
|
815
|
+
return {
|
|
816
|
+
callsUsed: this.callsUsed,
|
|
817
|
+
tokensUsed: this.tokensUsed,
|
|
818
|
+
elapsedMs,
|
|
819
|
+
remainingCalls: Math.max(0, this.options.maxCalls - this.callsUsed),
|
|
820
|
+
remainingTokens: this.options.maxTokens === void 0 ? null : Math.max(0, this.options.maxTokens - this.tokensUsed),
|
|
821
|
+
remainingMs: Math.max(0, this.options.timeoutMs - elapsedMs),
|
|
822
|
+
aborted: this.controller.signal.aborted
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
abort(reason = "Execution budget aborted by the application") {
|
|
826
|
+
if (!this.controller.signal.aborted) this.controller.abort(new Error(reason));
|
|
827
|
+
}
|
|
828
|
+
consumeTokens(tokens) {
|
|
829
|
+
if (!Number.isInteger(tokens) || tokens < 0) {
|
|
830
|
+
throw new Error("ExecutionBudget token usage must be a non-negative integer");
|
|
831
|
+
}
|
|
832
|
+
this.assertTime();
|
|
833
|
+
if (this.options.maxTokens !== void 0 && this.tokensUsed + tokens > this.options.maxTokens) {
|
|
834
|
+
this.abort("Execution token budget exceeded");
|
|
835
|
+
throw new ExecutionBudgetExceededError(
|
|
836
|
+
"tokens",
|
|
837
|
+
`Execution token budget exceeded (${this.tokensUsed + tokens}/${this.options.maxTokens})`
|
|
838
|
+
);
|
|
839
|
+
}
|
|
840
|
+
this.tokensUsed += tokens;
|
|
841
|
+
}
|
|
842
|
+
async run(operation, reservedTokens = 0) {
|
|
843
|
+
this.assertTime();
|
|
844
|
+
if (this.controller.signal.aborted) {
|
|
845
|
+
throw new ExecutionBudgetExceededError("time", "Execution budget is already aborted");
|
|
846
|
+
}
|
|
847
|
+
if (this.callsUsed >= this.options.maxCalls) {
|
|
848
|
+
this.abort("Execution call budget exceeded");
|
|
849
|
+
throw new ExecutionBudgetExceededError(
|
|
850
|
+
"calls",
|
|
851
|
+
`Execution call budget exceeded (${this.callsUsed}/${this.options.maxCalls})`
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
this.consumeTokens(reservedTokens);
|
|
855
|
+
this.callsUsed += 1;
|
|
856
|
+
const remainingMs = this.options.timeoutMs - (Date.now() - this.startedAt);
|
|
857
|
+
if (remainingMs <= 0) this.assertTime();
|
|
858
|
+
let timeout;
|
|
859
|
+
const deadline = new Promise((_resolve, reject) => {
|
|
860
|
+
timeout = setTimeout(() => {
|
|
861
|
+
this.abort("Execution time budget exceeded");
|
|
862
|
+
reject(new ExecutionBudgetExceededError(
|
|
863
|
+
"time",
|
|
864
|
+
`Execution time budget exceeded (${this.options.timeoutMs}ms)`
|
|
865
|
+
));
|
|
866
|
+
}, remainingMs);
|
|
867
|
+
});
|
|
868
|
+
try {
|
|
869
|
+
return await Promise.race([operation(this.controller.signal), deadline]);
|
|
870
|
+
} finally {
|
|
871
|
+
if (timeout) clearTimeout(timeout);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
assertTime() {
|
|
875
|
+
if (Date.now() - this.startedAt >= this.options.timeoutMs) {
|
|
876
|
+
this.abort("Execution time budget exceeded");
|
|
877
|
+
throw new ExecutionBudgetExceededError(
|
|
878
|
+
"time",
|
|
879
|
+
`Execution time budget exceeded (${this.options.timeoutMs}ms)`
|
|
880
|
+
);
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
};
|
|
884
|
+
|
|
282
885
|
// src/client.ts
|
|
283
886
|
import fs from "fs";
|
|
284
887
|
import path from "path";
|
|
285
888
|
var log3 = debug3("observyze:sdk");
|
|
889
|
+
var MAX_HISTORY_FILE_BYTES = 50 * 1024 * 1024;
|
|
890
|
+
var MAX_HISTORY_TRACES = 1e4;
|
|
891
|
+
var MAX_GUARDRAIL_REQUEST_BYTES = 1024 * 1024;
|
|
892
|
+
var ALLOWED_EVALUATION_SOURCES = /* @__PURE__ */ new Set(["live", "consensus", "nli_fast_path", "error", "disabled", "fallback"]);
|
|
893
|
+
var SAFE_IMPORTED_METADATA = /* @__PURE__ */ new Set([
|
|
894
|
+
"provider",
|
|
895
|
+
"model",
|
|
896
|
+
"temperature",
|
|
897
|
+
"max_tokens",
|
|
898
|
+
"max_completion_tokens",
|
|
899
|
+
"max_output_tokens",
|
|
900
|
+
"latency_ms",
|
|
901
|
+
"streaming",
|
|
902
|
+
"stream_completed",
|
|
903
|
+
"output_truncated",
|
|
904
|
+
"token_usage_source",
|
|
905
|
+
"cost_source",
|
|
906
|
+
"known_pricing",
|
|
907
|
+
"source",
|
|
908
|
+
"lifecycle",
|
|
909
|
+
"invocation_type"
|
|
910
|
+
]);
|
|
911
|
+
function luhnValid(candidate) {
|
|
912
|
+
const digits = candidate.replace(/[^0-9]/g, "");
|
|
913
|
+
if (digits.length < 13 || digits.length > 19 || /^(\d)\1+$/.test(digits)) return false;
|
|
914
|
+
let sum = 0;
|
|
915
|
+
let shouldDouble = false;
|
|
916
|
+
for (let index = digits.length - 1; index >= 0; index -= 1) {
|
|
917
|
+
let digit = Number(digits[index]);
|
|
918
|
+
if (shouldDouble) {
|
|
919
|
+
digit *= 2;
|
|
920
|
+
if (digit > 9) digit -= 9;
|
|
921
|
+
}
|
|
922
|
+
sum += digit;
|
|
923
|
+
shouldDouble = !shouldDouble;
|
|
924
|
+
}
|
|
925
|
+
return sum % 10 === 0;
|
|
926
|
+
}
|
|
927
|
+
var PII_PATTERNS = [
|
|
928
|
+
{ name: "email", pattern: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/g, replacement: "[EMAIL_REDACTED]" },
|
|
929
|
+
{ name: "jwt", pattern: /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, replacement: "[JWT_REDACTED]" },
|
|
930
|
+
{ name: "authorization", pattern: /\b(?:Bearer|Token|Basic)\s+[A-Za-z0-9\-._~+/]+=*/gi, replacement: "[AUTH_TOKEN_REDACTED]" },
|
|
931
|
+
{ name: "aws_access_key", pattern: /\b(?:AKIA|ASIA|AROA|ANPA|ANVA|AIDA)[A-Z0-9]{16}\b/g, replacement: "[AWS_KEY_REDACTED]" },
|
|
932
|
+
{ name: "api_key", pattern: /(?<![A-Za-z0-9_])(?:(?:sk|pk)-[A-Za-z0-9-]{20,}|ob_[A-Za-z0-9]{20,}|claude-[A-Za-z0-9-]{20,})(?![A-Za-z0-9_])/g, replacement: "[API_KEY_REDACTED]" },
|
|
933
|
+
{ name: "github_token", pattern: /\b(?:ghp|gho|ghu|ghs|github_pat)_[A-Za-z0-9_]{20,}\b/g, replacement: "[API_KEY_REDACTED]" },
|
|
934
|
+
{ name: "ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/g, replacement: "[SSN_REDACTED]" },
|
|
935
|
+
{ name: "credit_card_luhn", pattern: /\b(?:\d[ -]?){12,18}\d\b/g, replacement: (match) => luhnValid(match) ? "[CC_REDACTED]" : match },
|
|
936
|
+
{ name: "phone_us", pattern: /(?<!\d)(?:\+?1[-.\\s]?)?\(?\d{3}\)?[-.\\s]?\d{3}[-.\\s]?\d{4}(?!\d)/g, replacement: "[PHONE_REDACTED]" },
|
|
937
|
+
{ name: "phone_international", pattern: /(?<!\d)\+(?:[0-9][().\s-]?){7,15}[0-9](?!\d)/g, replacement: "[PHONE_REDACTED]" },
|
|
938
|
+
{ name: "zip_context", pattern: /(?:zip\s*(?:code)?[\s:]*)\b\d{5}(?:-\d{4})?\b/gi, replacement: "zip: [ZIP_REDACTED]" },
|
|
939
|
+
{ name: "ipv4", pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, replacement: "[IP_REDACTED]" },
|
|
940
|
+
{ name: "ipv6", pattern: /\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, replacement: "[IP_REDACTED]" }
|
|
941
|
+
];
|
|
942
|
+
var SENSITIVE_KEY_SEGMENTS = /* @__PURE__ */ new Set([
|
|
943
|
+
"password",
|
|
944
|
+
"passwd",
|
|
945
|
+
"secret",
|
|
946
|
+
"token",
|
|
947
|
+
"apikey",
|
|
948
|
+
"key",
|
|
949
|
+
"authorization",
|
|
950
|
+
"credential",
|
|
951
|
+
"private",
|
|
952
|
+
"ssn",
|
|
953
|
+
"social",
|
|
954
|
+
"dob",
|
|
955
|
+
"birth",
|
|
956
|
+
"passport",
|
|
957
|
+
"credit",
|
|
958
|
+
"card",
|
|
959
|
+
"cvv",
|
|
960
|
+
"cvc",
|
|
961
|
+
"pin",
|
|
962
|
+
"bank",
|
|
963
|
+
"routing",
|
|
964
|
+
"account"
|
|
965
|
+
]);
|
|
966
|
+
function isSensitiveKey(key) {
|
|
967
|
+
const normalized = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/-/g, "_");
|
|
968
|
+
if (SENSITIVE_KEY_SEGMENTS.has(normalized)) return true;
|
|
969
|
+
if (/^(?:first|last|given|family|full|person)_?name$/.test(normalized)) return true;
|
|
970
|
+
return normalized.split("_").some((segment) => SENSITIVE_KEY_SEGMENTS.has(segment)) && /(?:password|passwd|secret|token|api_?key|authorization|credential|private_?key|ssn|social_?security|credit_?card|card_?number|cvv|cvc|bank_?account|routing_?number|passport|date_?of_?birth)/.test(normalized);
|
|
971
|
+
}
|
|
972
|
+
function redactValue(value, depth = 0) {
|
|
973
|
+
if (depth > 16) return "[MAX_DEPTH_EXCEEDED]";
|
|
974
|
+
if (typeof value === "string") {
|
|
975
|
+
let redacted = value;
|
|
976
|
+
for (const { pattern, replacement } of PII_PATTERNS) {
|
|
977
|
+
pattern.lastIndex = 0;
|
|
978
|
+
redacted = typeof replacement === "string" ? redacted.replace(pattern, replacement) : redacted.replace(pattern, replacement);
|
|
979
|
+
}
|
|
980
|
+
return redacted;
|
|
981
|
+
}
|
|
982
|
+
if (Array.isArray(value)) return value.map((item) => redactValue(item, depth + 1));
|
|
983
|
+
if (value instanceof Date) return value;
|
|
984
|
+
if (value instanceof Map) return redactValue(Object.fromEntries(value.entries()), depth + 1);
|
|
985
|
+
if (value !== null && typeof value === "object") {
|
|
986
|
+
const redacted = {};
|
|
987
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
988
|
+
redacted[key] = isSensitiveKey(key) ? "[REDACTED]" : redactValue(nested, depth + 1);
|
|
989
|
+
}
|
|
990
|
+
return redacted;
|
|
991
|
+
}
|
|
992
|
+
return value;
|
|
993
|
+
}
|
|
994
|
+
function operationalMetadata(value) {
|
|
995
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
996
|
+
return Object.fromEntries(Object.entries(value).filter(
|
|
997
|
+
([key, item]) => SAFE_IMPORTED_METADATA.has(key) && (item === null || ["string", "number", "boolean"].includes(typeof item))
|
|
998
|
+
));
|
|
999
|
+
}
|
|
1000
|
+
function metadataOnlyImportedTrace(trace) {
|
|
1001
|
+
return {
|
|
1002
|
+
trace_id: trace?.trace_id,
|
|
1003
|
+
organization_id: trace?.organization_id,
|
|
1004
|
+
project_id: trace?.project_id,
|
|
1005
|
+
name: "[CONTENT_CAPTURE_DISABLED]",
|
|
1006
|
+
status: trace?.status,
|
|
1007
|
+
start_time: trace?.start_time,
|
|
1008
|
+
end_time: trace?.end_time,
|
|
1009
|
+
duration_ms: trace?.duration_ms,
|
|
1010
|
+
metadata: operationalMetadata(trace?.metadata),
|
|
1011
|
+
spans: Array.isArray(trace?.spans) ? trace.spans.map((span) => ({
|
|
1012
|
+
span_id: span?.span_id,
|
|
1013
|
+
parent_span_id: span?.parent_span_id,
|
|
1014
|
+
name: "[CONTENT_CAPTURE_DISABLED]",
|
|
1015
|
+
type: span?.type,
|
|
1016
|
+
start_time: span?.start_time,
|
|
1017
|
+
end_time: span?.end_time,
|
|
1018
|
+
duration_ms: span?.duration_ms,
|
|
1019
|
+
input: null,
|
|
1020
|
+
output: null,
|
|
1021
|
+
...span?.error ? { error: { message: "Error details omitted because content capture is disabled" } } : {},
|
|
1022
|
+
metadata: operationalMetadata(span?.metadata),
|
|
1023
|
+
tokens: span?.tokens || span?.tokens_used
|
|
1024
|
+
})) : [],
|
|
1025
|
+
tags: [],
|
|
1026
|
+
cost: trace?.cost
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
var IngestionRequestError = class extends Error {
|
|
1030
|
+
constructor(message, retryable, retryAfterMs) {
|
|
1031
|
+
super(message);
|
|
1032
|
+
this.retryable = retryable;
|
|
1033
|
+
this.retryAfterMs = retryAfterMs;
|
|
1034
|
+
this.name = "IngestionRequestError";
|
|
1035
|
+
}
|
|
1036
|
+
retryable;
|
|
1037
|
+
retryAfterMs;
|
|
1038
|
+
};
|
|
286
1039
|
var DEFAULT_CONFIG = {
|
|
287
|
-
endpoint: "
|
|
1040
|
+
endpoint: "https://api.observyze.com",
|
|
288
1041
|
batchSize: 100,
|
|
289
1042
|
flushInterval: 5e3,
|
|
290
|
-
|
|
1043
|
+
requestTimeoutMs: 1e4,
|
|
291
1044
|
debug: false,
|
|
292
1045
|
dryRun: false,
|
|
293
1046
|
enablePiiRedaction: true,
|
|
1047
|
+
captureContent: true,
|
|
294
1048
|
hallucinationThreshold: 0.8,
|
|
295
1049
|
safetyThreshold: 0.9,
|
|
296
1050
|
confidenceThreshold: 0.4,
|
|
297
|
-
evalEndpoint: process.env.EVAL_ENDPOINT || (process.env.NODE_ENV === "production" ? "https://api.observyze.com" : "http://localhost:3001"),
|
|
298
1051
|
enableCircuitBreaker: true,
|
|
299
1052
|
failClosed: true,
|
|
300
1053
|
enableProxyRedirect: true
|
|
@@ -306,10 +1059,46 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
306
1059
|
isShuttingDown = false;
|
|
307
1060
|
MAX_QUEUE_SIZE = 1e3;
|
|
308
1061
|
RETRY_DELAYS = [1e3, 2e3, 4e3, 8e3, 16e3, 3e4];
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
1062
|
+
static async readBoundedResponse(response, maxBytes = 64 * 1024) {
|
|
1063
|
+
if (!response.body) {
|
|
1064
|
+
if (typeof response.text === "function") return String(await response.text()).slice(0, maxBytes);
|
|
1065
|
+
if (typeof response.json === "function") return JSON.stringify(await response.json()).slice(0, maxBytes);
|
|
1066
|
+
return "";
|
|
1067
|
+
}
|
|
1068
|
+
const reader = response.body.getReader();
|
|
1069
|
+
const chunks = [];
|
|
1070
|
+
let size = 0;
|
|
1071
|
+
while (true) {
|
|
1072
|
+
const { done, value } = await reader.read();
|
|
1073
|
+
if (done) break;
|
|
1074
|
+
const remaining = maxBytes - size;
|
|
1075
|
+
if (remaining <= 0) {
|
|
1076
|
+
await reader.cancel();
|
|
1077
|
+
break;
|
|
1078
|
+
}
|
|
1079
|
+
chunks.push(value.byteLength > remaining ? value.slice(0, remaining) : value);
|
|
1080
|
+
size += Math.min(value.byteLength, remaining);
|
|
1081
|
+
if (value.byteLength > remaining) {
|
|
1082
|
+
await reader.cancel();
|
|
1083
|
+
break;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
const merged = new Uint8Array(size);
|
|
1087
|
+
let offset = 0;
|
|
1088
|
+
for (const chunk of chunks) {
|
|
1089
|
+
merged.set(chunk, offset);
|
|
1090
|
+
offset += chunk.byteLength;
|
|
1091
|
+
}
|
|
1092
|
+
return new TextDecoder().decode(merged);
|
|
1093
|
+
}
|
|
1094
|
+
static retryAfterMs(response) {
|
|
1095
|
+
const value = response.headers?.get?.("retry-after");
|
|
1096
|
+
if (!value) return void 0;
|
|
1097
|
+
const seconds = Number(value);
|
|
1098
|
+
if (Number.isFinite(seconds)) return Math.min(6e4, Math.max(0, seconds * 1e3));
|
|
1099
|
+
const date = Date.parse(value);
|
|
1100
|
+
return Number.isFinite(date) ? Math.min(6e4, Math.max(0, date - Date.now())) : void 0;
|
|
1101
|
+
}
|
|
313
1102
|
static parseApiError(_response, body) {
|
|
314
1103
|
try {
|
|
315
1104
|
const parsed = JSON.parse(body);
|
|
@@ -320,18 +1109,9 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
320
1109
|
message: error.message || body.slice(0, 200)
|
|
321
1110
|
};
|
|
322
1111
|
} catch {
|
|
323
|
-
return {
|
|
324
|
-
traceId: "unknown",
|
|
325
|
-
code: "UNKNOWN_ERROR",
|
|
326
|
-
message: body.slice(0, 200)
|
|
327
|
-
};
|
|
1112
|
+
return { traceId: "unknown", code: "UNKNOWN_ERROR", message: body.slice(0, 200) };
|
|
328
1113
|
}
|
|
329
1114
|
}
|
|
330
|
-
/**
|
|
331
|
-
* Format an API error into a user-friendly message with trace_id for correlation.
|
|
332
|
-
* Example output:
|
|
333
|
-
* "Observyze API error (401 [ref: err_a1b2c3d4]): MISSING_PROVIDER_KEY — No API key configured..."
|
|
334
|
-
*/
|
|
335
1115
|
static formatApiError(response, body) {
|
|
336
1116
|
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, body);
|
|
337
1117
|
const prefix = traceId !== "unknown" ? ` [ref: ${traceId}]` : "";
|
|
@@ -341,11 +1121,27 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
341
1121
|
if (!config.apiKey) {
|
|
342
1122
|
throw new Error("Observyze SDK: apiKey is required");
|
|
343
1123
|
}
|
|
1124
|
+
if (config.batchSize !== void 0 && (!Number.isInteger(config.batchSize) || config.batchSize < 1 || config.batchSize > 100)) {
|
|
1125
|
+
throw new Error("Observyze SDK: batchSize must be an integer between 1 and 100");
|
|
1126
|
+
}
|
|
1127
|
+
if (config.flushInterval !== void 0 && (!Number.isFinite(config.flushInterval) || config.flushInterval < 100 || config.flushInterval > 36e5)) {
|
|
1128
|
+
throw new Error("Observyze SDK: flushInterval must be between 100 and 3600000 milliseconds");
|
|
1129
|
+
}
|
|
1130
|
+
if (config.requestTimeoutMs !== void 0 && (!Number.isFinite(config.requestTimeoutMs) || config.requestTimeoutMs < 100 || config.requestTimeoutMs > 12e4)) {
|
|
1131
|
+
throw new Error("Observyze SDK: requestTimeoutMs must be between 100 and 120000 milliseconds");
|
|
1132
|
+
}
|
|
1133
|
+
try {
|
|
1134
|
+
const endpoint = new URL(config.endpoint || String(DEFAULT_CONFIG.endpoint));
|
|
1135
|
+
if (!["http:", "https:"].includes(endpoint.protocol)) throw new Error("invalid protocol");
|
|
1136
|
+
} catch {
|
|
1137
|
+
throw new Error("Observyze SDK: endpoint must be a valid HTTP(S) URL");
|
|
1138
|
+
}
|
|
344
1139
|
this.config = {
|
|
345
1140
|
...DEFAULT_CONFIG,
|
|
346
1141
|
...config,
|
|
347
1142
|
organizationId: config.organizationId || "",
|
|
348
|
-
projectId: config.projectId || ""
|
|
1143
|
+
projectId: config.projectId || "",
|
|
1144
|
+
evalEndpoint: config.evalEndpoint || config.endpoint || String(DEFAULT_CONFIG.endpoint)
|
|
349
1145
|
};
|
|
350
1146
|
this.startFlushTimer();
|
|
351
1147
|
if (this.config.debug) {
|
|
@@ -364,13 +1160,14 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
364
1160
|
const trace = new Trace(
|
|
365
1161
|
name,
|
|
366
1162
|
this.config.organizationId,
|
|
367
|
-
this.config.projectId
|
|
1163
|
+
this.config.projectId,
|
|
1164
|
+
this.config.captureContent
|
|
368
1165
|
);
|
|
369
1166
|
if (metadata) {
|
|
370
1167
|
trace.setMetadataAll(metadata);
|
|
371
1168
|
}
|
|
372
1169
|
const originalEnd = trace.end.bind(trace);
|
|
373
|
-
trace.end = (status =
|
|
1170
|
+
trace.end = (status = "success" /* SUCCESS */) => {
|
|
374
1171
|
originalEnd(status);
|
|
375
1172
|
this.bufferTrace(trace);
|
|
376
1173
|
};
|
|
@@ -421,7 +1218,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
421
1218
|
}
|
|
422
1219
|
}
|
|
423
1220
|
/**
|
|
424
|
-
* Flush all buffered traces to
|
|
1221
|
+
* Flush all buffered traces to Observyze
|
|
425
1222
|
*/
|
|
426
1223
|
async flush() {
|
|
427
1224
|
if (this.traceBuffer.length === 0) {
|
|
@@ -464,37 +1261,52 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
464
1261
|
let lastError = null;
|
|
465
1262
|
for (let attempt = 0; attempt < this.RETRY_DELAYS.length + 1; attempt++) {
|
|
466
1263
|
try {
|
|
467
|
-
const
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
1264
|
+
const controller = new AbortController();
|
|
1265
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1266
|
+
let response;
|
|
1267
|
+
try {
|
|
1268
|
+
response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
1269
|
+
method: "POST",
|
|
1270
|
+
headers: {
|
|
1271
|
+
"Content-Type": "application/json",
|
|
1272
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1273
|
+
"x-api-key": this.config.apiKey
|
|
1274
|
+
},
|
|
1275
|
+
body: JSON.stringify({
|
|
1276
|
+
traces: traces.map((trace) => {
|
|
1277
|
+
const json = trace.toJSON();
|
|
1278
|
+
return this.config.enablePiiRedaction ? this.sanitizePII(json) : json;
|
|
1279
|
+
})
|
|
1280
|
+
}),
|
|
1281
|
+
signal: controller.signal
|
|
1282
|
+
});
|
|
1283
|
+
} finally {
|
|
1284
|
+
clearTimeout(timeout);
|
|
1285
|
+
}
|
|
483
1286
|
if (!response.ok) {
|
|
484
|
-
const errorBody = await
|
|
1287
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
485
1288
|
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
486
|
-
|
|
1289
|
+
const retryable = [408, 425, 429].includes(response.status) || response.status >= 500;
|
|
1290
|
+
throw new IngestionRequestError(
|
|
1291
|
+
`[Observyze SDK] Trace ingestion failed. ${formatted}`,
|
|
1292
|
+
retryable,
|
|
1293
|
+
_ObservyzeClient.retryAfterMs(response)
|
|
1294
|
+
);
|
|
487
1295
|
}
|
|
488
1296
|
if (this.config.debug) {
|
|
489
1297
|
log3(`[Observyze SDK] Successfully sent ${traces.length} traces${attempt > 0 ? ` (after ${attempt} retries)` : ""}`);
|
|
490
1298
|
}
|
|
1299
|
+
if (response.body) await response.body.cancel().catch(() => void 0);
|
|
491
1300
|
return;
|
|
492
1301
|
} catch (error) {
|
|
493
1302
|
lastError = error;
|
|
494
|
-
|
|
1303
|
+
const retryable = !(error instanceof IngestionRequestError) || error.retryable;
|
|
1304
|
+
if (!retryable || attempt >= this.RETRY_DELAYS.length) {
|
|
495
1305
|
break;
|
|
496
1306
|
}
|
|
497
|
-
const
|
|
1307
|
+
const configuredDelay = error instanceof IngestionRequestError ? error.retryAfterMs : void 0;
|
|
1308
|
+
const baseDelay = configuredDelay ?? this.RETRY_DELAYS[attempt];
|
|
1309
|
+
const delay = Math.min(6e4, Math.round(baseDelay * (0.8 + Math.random() * 0.4)));
|
|
498
1310
|
if (this.config.debug) {
|
|
499
1311
|
log3.extend("warn")(`[Observyze SDK] Attempt ${attempt + 1} failed, retrying in ${delay}ms...`, error);
|
|
500
1312
|
}
|
|
@@ -540,22 +1352,21 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
540
1352
|
return { ...this.config };
|
|
541
1353
|
}
|
|
542
1354
|
/**
|
|
543
|
-
* Wrap
|
|
544
|
-
*
|
|
1355
|
+
* Wrap a supported LLM client or framework boundary to enable auto-instrumentation.
|
|
1356
|
+
*
|
|
1357
|
+
* Supports: OpenAI, Anthropic, Google Gemini, Vercel AI SDK, LangChain, LlamaIndex.
|
|
1358
|
+
*
|
|
545
1359
|
* @example
|
|
546
1360
|
* ```typescript
|
|
547
1361
|
* import OpenAI from 'openai'
|
|
548
1362
|
* import { ObservyzeClient } from '@observyze/sdk'
|
|
549
|
-
*
|
|
1363
|
+
*
|
|
550
1364
|
* const nw = new ObservyzeClient({ apiKey: 'your-api-key' })
|
|
551
|
-
* const openai = new OpenAI({ apiKey: 'openai-key' })
|
|
552
|
-
*
|
|
553
|
-
* // Wrap the client to enable auto-instrumentation
|
|
554
|
-
* nw.wrap(openai)
|
|
555
|
-
*
|
|
1365
|
+
* const openai = nw.wrap(new OpenAI({ apiKey: 'openai-key' }))
|
|
1366
|
+
*
|
|
556
1367
|
* // All calls are now automatically traced
|
|
557
1368
|
* const response = await openai.chat.completions.create({
|
|
558
|
-
* model: 'gpt-
|
|
1369
|
+
* model: 'gpt-4o',
|
|
559
1370
|
* messages: [{ role: 'user', content: 'Hello!' }]
|
|
560
1371
|
* })
|
|
561
1372
|
* ```
|
|
@@ -563,9 +1374,64 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
563
1374
|
wrap(client) {
|
|
564
1375
|
return wrap(client, this);
|
|
565
1376
|
}
|
|
1377
|
+
/** Create an active call/token/time budget for one agent execution. */
|
|
1378
|
+
createExecutionBudget(options) {
|
|
1379
|
+
return new ExecutionBudget(options);
|
|
1380
|
+
}
|
|
566
1381
|
/**
|
|
567
|
-
*
|
|
568
|
-
*
|
|
1382
|
+
* Verify that the SDK can reach Observyze and send traces end-to-end.
|
|
1383
|
+
*
|
|
1384
|
+
* @example
|
|
1385
|
+
* ```typescript
|
|
1386
|
+
* const nw = new ObservyzeClient({ apiKey: process.env.OBSERVYZE_API_KEY })
|
|
1387
|
+
* const result = await nw.testConnection()
|
|
1388
|
+
* // { ok: true, traceId: '...', message: 'Connection successful...' }
|
|
1389
|
+
* ```
|
|
1390
|
+
*/
|
|
1391
|
+
async testConnection() {
|
|
1392
|
+
if (this.config.dryRun) {
|
|
1393
|
+
return {
|
|
1394
|
+
ok: false,
|
|
1395
|
+
message: "Dry-run mode is enabled, so no trace was actually sent. Set dryRun: false to run a real connection test."
|
|
1396
|
+
};
|
|
1397
|
+
}
|
|
1398
|
+
const trace = new Trace(
|
|
1399
|
+
"Observyze Connection Test",
|
|
1400
|
+
this.config.organizationId,
|
|
1401
|
+
this.config.projectId,
|
|
1402
|
+
this.config.captureContent
|
|
1403
|
+
);
|
|
1404
|
+
const span = trace.startSpan("connection-test", "llm" /* LLM */);
|
|
1405
|
+
span.setInput({ prompt: "Observyze SDK connection test" });
|
|
1406
|
+
span.setOutput({ response: "Connection successful" });
|
|
1407
|
+
span.setTokens({ input: 5, output: 4, total: 9 });
|
|
1408
|
+
span.setMetadata("source", "sdk-test-connection");
|
|
1409
|
+
span.end();
|
|
1410
|
+
trace.setMetadata("source", "sdk-test-connection");
|
|
1411
|
+
trace.addTag("setup-test");
|
|
1412
|
+
trace.end("success" /* SUCCESS */);
|
|
1413
|
+
try {
|
|
1414
|
+
await this.sendWithRetry([trace]);
|
|
1415
|
+
return {
|
|
1416
|
+
ok: true,
|
|
1417
|
+
traceId: trace.id,
|
|
1418
|
+
message: `Connection successful. Test trace ${trace.id} was sent to Observyze. Search for "Observyze Connection Test" in Dashboard \u2192 Traces to confirm it landed.`
|
|
1419
|
+
};
|
|
1420
|
+
} catch (error) {
|
|
1421
|
+
const rawMessage = error?.message || String(error);
|
|
1422
|
+
const statusMatch = rawMessage.match(/\((\d{3})/);
|
|
1423
|
+
const isNetworkFailure = !statusMatch && /fetch|network|ENOTFOUND|ECONNREFUSED|ETIMEDOUT/i.test(rawMessage);
|
|
1424
|
+
const hint = isNetworkFailure ? " Check that your endpoint is reachable from this environment (firewalls, proxies, DNS)." : "";
|
|
1425
|
+
return {
|
|
1426
|
+
ok: false,
|
|
1427
|
+
...statusMatch ? { status: parseInt(statusMatch[1], 10) } : {},
|
|
1428
|
+
message: rawMessage + hint
|
|
1429
|
+
};
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* Sync a local agent .history file to Observyze cloud.
|
|
1434
|
+
* Parses JSON/NDJSON agent history and sends to the ingestion endpoint.
|
|
569
1435
|
*/
|
|
570
1436
|
async syncLocalHistory(filePath) {
|
|
571
1437
|
try {
|
|
@@ -576,6 +1442,11 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
576
1442
|
if (!fs.existsSync(fullPath)) {
|
|
577
1443
|
throw new Error(`History file not found: ${fullPath}`);
|
|
578
1444
|
}
|
|
1445
|
+
const fileStats = fs.statSync(fullPath);
|
|
1446
|
+
if (!fileStats.isFile()) throw new Error("History path must reference a regular file");
|
|
1447
|
+
if (fileStats.size > MAX_HISTORY_FILE_BYTES) {
|
|
1448
|
+
throw new Error(`History file exceeds the ${MAX_HISTORY_FILE_BYTES} byte limit`);
|
|
1449
|
+
}
|
|
579
1450
|
const content = fs.readFileSync(fullPath, "utf-8");
|
|
580
1451
|
let items = [];
|
|
581
1452
|
try {
|
|
@@ -586,21 +1457,36 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
586
1457
|
if (!Array.isArray(items)) {
|
|
587
1458
|
items = [items];
|
|
588
1459
|
}
|
|
1460
|
+
if (items.length > MAX_HISTORY_TRACES) {
|
|
1461
|
+
throw new Error(`History import exceeds the ${MAX_HISTORY_TRACES} trace limit`);
|
|
1462
|
+
}
|
|
589
1463
|
if (this.config.debug) {
|
|
590
1464
|
log3(`[Observyze SDK] Syncing ${items.length} traces from ${filePath}`);
|
|
591
1465
|
}
|
|
592
1466
|
for (let i = 0; i < items.length; i += this.config.batchSize) {
|
|
593
1467
|
const batch = items.slice(i, i + this.config.batchSize);
|
|
594
|
-
const
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
|
|
1468
|
+
const privacyBoundBatch = this.config.captureContent ? batch : batch.map(metadataOnlyImportedTrace);
|
|
1469
|
+
const controller = new AbortController();
|
|
1470
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1471
|
+
let response;
|
|
1472
|
+
try {
|
|
1473
|
+
response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
1474
|
+
method: "POST",
|
|
1475
|
+
headers: {
|
|
1476
|
+
"Content-Type": "application/json",
|
|
1477
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1478
|
+
"x-api-key": this.config.apiKey
|
|
1479
|
+
},
|
|
1480
|
+
body: JSON.stringify({
|
|
1481
|
+
traces: this.config.enablePiiRedaction ? this.sanitizePII(privacyBoundBatch) : privacyBoundBatch
|
|
1482
|
+
}),
|
|
1483
|
+
signal: controller.signal
|
|
1484
|
+
});
|
|
1485
|
+
} finally {
|
|
1486
|
+
clearTimeout(timeout);
|
|
1487
|
+
}
|
|
602
1488
|
if (!response.ok) {
|
|
603
|
-
const errorBody = await
|
|
1489
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
604
1490
|
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
605
1491
|
throw new Error(`[Observyze SDK] Local history sync failed. ${formatted}`);
|
|
606
1492
|
}
|
|
@@ -614,219 +1500,106 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
614
1500
|
}
|
|
615
1501
|
}
|
|
616
1502
|
/**
|
|
617
|
-
* Industry-grade PII
|
|
618
|
-
*
|
|
619
|
-
* Recursively scrubs PII from trace data before transmission to the cloud.
|
|
620
|
-
* Coverage: emails, JWTs, bearer tokens, AWS/API keys, SSNs, credit cards,
|
|
621
|
-
* phone numbers (US + E.164), IPv4/IPv6, passport numbers, ZIP codes, plus
|
|
622
|
-
* key-based redaction for sensitive JSON fields (password, token, api_key, etc.).
|
|
623
|
-
*
|
|
624
|
-
* Design:
|
|
625
|
-
* - Pure function, never mutates the original object
|
|
626
|
-
* - Depth-limited to 16 levels to prevent stack overflow on deep agent outputs
|
|
627
|
-
* - Key-aware: sensitive key names are fully redacted regardless of value format
|
|
1503
|
+
* Industry-grade PII redaction.
|
|
1504
|
+
* Recursively scrubs PII from trace data before transmission.
|
|
628
1505
|
*/
|
|
629
|
-
static PII_PATTERNS = [
|
|
630
|
-
{ pattern: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/gi, label: "[EMAIL_REDACTED]" },
|
|
631
|
-
{ pattern: /\beyJ[A-Za-z0-9_\-]+\.eyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\b/g, label: "[JWT_REDACTED]" },
|
|
632
|
-
{ pattern: /\b(Bearer|Token|Basic)\s+[A-Za-z0-9\-.~+\/]+=*\b/gi, label: "[AUTH_TOKEN_REDACTED]" },
|
|
633
|
-
{ pattern: /\b(AKIA|ASIA|AROA|ANPA|ANVA|AIDA)[A-Z0-9]{16}\b/g, label: "[AWS_KEY_REDACTED]" },
|
|
634
|
-
{ pattern: /\b(sk-[A-Za-z0-9\-]{20,}|pk-[A-Za-z0-9\-]{20,}|ob_[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{36,}|gho_[A-Za-z0-9]{36,})\b/g, label: "[API_KEY_REDACTED]" },
|
|
635
|
-
{ pattern: /\b\d{3}-\d{2}-\d{4}\b/g, label: "[SSN_REDACTED]" },
|
|
636
|
-
{ pattern: /\b(?:\d[ \-]?){13,18}\d\b/g, label: "[CC_REDACTED]" },
|
|
637
|
-
{ pattern: /(?:\+1[\s.\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b/g, label: "[PHONE_REDACTED]" },
|
|
638
|
-
{ pattern: /\+\d{1,3}[\s.\-]?\(?\d{1,4}\)?[\s.\-]?\d{1,4}[\s.\-]?\d{1,9}/g, label: "[PHONE_REDACTED]" },
|
|
639
|
-
{ pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, label: "[IP_REDACTED]" },
|
|
640
|
-
{ pattern: /\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, label: "[IP_REDACTED]" }
|
|
641
|
-
];
|
|
642
|
-
static SENSITIVE_KEYS = /* @__PURE__ */ new Set([
|
|
643
|
-
"password",
|
|
644
|
-
"passwd",
|
|
645
|
-
"secret",
|
|
646
|
-
"token",
|
|
647
|
-
"apikey",
|
|
648
|
-
"api_key",
|
|
649
|
-
"accesstoken",
|
|
650
|
-
"access_token",
|
|
651
|
-
"refreshtoken",
|
|
652
|
-
"refresh_token",
|
|
653
|
-
"authorization",
|
|
654
|
-
"auth",
|
|
655
|
-
"credential",
|
|
656
|
-
"credentials",
|
|
657
|
-
"private_key",
|
|
658
|
-
"privatekey",
|
|
659
|
-
"client_secret",
|
|
660
|
-
"clientsecret",
|
|
661
|
-
"ssn",
|
|
662
|
-
"social_security",
|
|
663
|
-
"dob",
|
|
664
|
-
"date_of_birth",
|
|
665
|
-
"dateofbirth",
|
|
666
|
-
"passport",
|
|
667
|
-
"passport_number",
|
|
668
|
-
"credit_card",
|
|
669
|
-
"creditcard",
|
|
670
|
-
"card_number",
|
|
671
|
-
"cardnumber",
|
|
672
|
-
"cvv",
|
|
673
|
-
"cvc",
|
|
674
|
-
"pin",
|
|
675
|
-
"bank_account",
|
|
676
|
-
"routing_number"
|
|
677
|
-
]);
|
|
678
|
-
static isSensitiveKey(key) {
|
|
679
|
-
const normalized = key.toLowerCase().replace(/-/g, "_");
|
|
680
|
-
if (_ObservyzeClient.SENSITIVE_KEYS.has(normalized)) return true;
|
|
681
|
-
const segments = normalized.split("_");
|
|
682
|
-
for (const segment of segments) {
|
|
683
|
-
if (_ObservyzeClient.SENSITIVE_KEYS.has(segment)) return true;
|
|
684
|
-
}
|
|
685
|
-
return false;
|
|
686
|
-
}
|
|
687
1506
|
sanitizePII(data, depth = 0) {
|
|
688
|
-
|
|
689
|
-
if (data === null || data === void 0) return data;
|
|
690
|
-
if (typeof data === "string") {
|
|
691
|
-
let result = data;
|
|
692
|
-
for (const { pattern, label } of _ObservyzeClient.PII_PATTERNS) {
|
|
693
|
-
pattern.lastIndex = 0;
|
|
694
|
-
result = result.replace(pattern, label);
|
|
695
|
-
}
|
|
696
|
-
return result;
|
|
697
|
-
}
|
|
698
|
-
if (typeof data === "number" || typeof data === "boolean") return data;
|
|
699
|
-
if (Array.isArray(data)) return data.map((item) => this.sanitizePII(item, depth + 1));
|
|
700
|
-
if (typeof data === "object") {
|
|
701
|
-
const sanitized = {};
|
|
702
|
-
for (const [k, v] of Object.entries(data)) {
|
|
703
|
-
sanitized[k] = _ObservyzeClient.isSensitiveKey(k) ? "[REDACTED]" : this.sanitizePII(v, depth + 1);
|
|
704
|
-
}
|
|
705
|
-
return sanitized;
|
|
706
|
-
}
|
|
707
|
-
return data;
|
|
1507
|
+
return redactValue(data, depth);
|
|
708
1508
|
}
|
|
709
1509
|
/**
|
|
710
|
-
*
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
1510
|
+
* Explicitly evaluate content before an application action.
|
|
1511
|
+
* Requires an active Observyze subscription with guardrails enabled.
|
|
1512
|
+
*
|
|
714
1513
|
* Returns GuardrailResult with score: null when evaluation couldn't be performed.
|
|
715
|
-
* In failClosed mode, null scores
|
|
716
|
-
*
|
|
1514
|
+
* In failClosed mode (default), null scores block execution.
|
|
1515
|
+
*
|
|
1516
|
+
* @example
|
|
1517
|
+
* ```typescript
|
|
1518
|
+
* const result = await nw.checkGuardrails(llmOutput)
|
|
1519
|
+
* if (!result.pass) {
|
|
1520
|
+
* throw new Error('Guardrail blocked: ' + result.reason)
|
|
1521
|
+
* }
|
|
1522
|
+
* ```
|
|
717
1523
|
*/
|
|
718
1524
|
async checkGuardrails(content) {
|
|
719
1525
|
if (!this.config.enableCircuitBreaker) {
|
|
720
1526
|
return { pass: true, score: 0, safetyScore: 0, evaluationSource: "disabled" };
|
|
721
1527
|
}
|
|
1528
|
+
if (!this.config.captureContent) {
|
|
1529
|
+
const reason = "Guardrail content dispatch is disabled because captureContent is false";
|
|
1530
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "disabled", fallbackReason: reason, reason } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "disabled", fallbackReason: reason };
|
|
1531
|
+
}
|
|
722
1532
|
try {
|
|
723
1533
|
if (this.config.debug) {
|
|
724
|
-
log3(`[Observyze Guardrail] Analyzing payload
|
|
1534
|
+
log3(`[Observyze Guardrail] Analyzing payload...`);
|
|
725
1535
|
}
|
|
726
1536
|
const evalEndpoint = this.config.evalEndpoint;
|
|
727
|
-
const
|
|
728
|
-
const
|
|
729
|
-
const
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
1537
|
+
const protectedContent = this.config.enablePiiRedaction ? this.sanitizePII(content) : content;
|
|
1538
|
+
const payload = typeof protectedContent === "string" ? { text: protectedContent, organization_id: this.config.organizationId } : { trace: protectedContent, organization_id: this.config.organizationId };
|
|
1539
|
+
const serializedPayload = JSON.stringify(payload);
|
|
1540
|
+
if (Buffer.byteLength(serializedPayload, "utf8") > MAX_GUARDRAIL_REQUEST_BYTES) {
|
|
1541
|
+
throw new Error(`Guardrail payload exceeds the ${MAX_GUARDRAIL_REQUEST_BYTES} byte limit`);
|
|
1542
|
+
}
|
|
1543
|
+
const evaluate = async (type) => {
|
|
1544
|
+
const controller = new AbortController();
|
|
1545
|
+
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
|
1546
|
+
try {
|
|
1547
|
+
const response = await fetch(`${evalEndpoint}/api/v1/evaluate/${type}`, {
|
|
1548
|
+
method: "POST",
|
|
1549
|
+
headers: {
|
|
1550
|
+
"Content-Type": "application/json",
|
|
1551
|
+
"Authorization": `Bearer ${this.config.apiKey}`,
|
|
1552
|
+
"x-api-key": this.config.apiKey
|
|
1553
|
+
},
|
|
1554
|
+
body: serializedPayload,
|
|
1555
|
+
signal: controller.signal
|
|
1556
|
+
});
|
|
1557
|
+
if (response.ok) {
|
|
1558
|
+
const text = await _ObservyzeClient.readBoundedResponse(response);
|
|
1559
|
+
const parsed = JSON.parse(text);
|
|
1560
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
1561
|
+
}
|
|
1562
|
+
const errorBody = await _ObservyzeClient.readBoundedResponse(response);
|
|
746
1563
|
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, errorBody);
|
|
747
1564
|
if (this.config.debug) {
|
|
748
|
-
log3.extend("warn")(`[Observyze Guardrail]
|
|
1565
|
+
log3.extend("warn")(`[Observyze Guardrail] ${type} eval returned ${response.status} [${code}] (ref: ${traceId}): ${message}`);
|
|
749
1566
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
clearTimeout(timeout);
|
|
753
|
-
if (fetchError.name === "AbortError") {
|
|
1567
|
+
return null;
|
|
1568
|
+
} catch (fetchError) {
|
|
754
1569
|
if (this.config.debug) {
|
|
755
|
-
|
|
1570
|
+
const reason = fetchError?.name === "AbortError" ? `timed out after ${this.config.requestTimeoutMs}ms` : `failed: ${fetchError?.message || "unknown error"}`;
|
|
1571
|
+
log3.extend("warn")(`[Observyze Guardrail] ${type} evaluation ${reason}`);
|
|
756
1572
|
}
|
|
757
|
-
|
|
758
|
-
|
|
1573
|
+
return null;
|
|
1574
|
+
} finally {
|
|
1575
|
+
clearTimeout(timeout);
|
|
759
1576
|
}
|
|
1577
|
+
};
|
|
1578
|
+
const [hallucinationResult, safetyResult] = await Promise.all([
|
|
1579
|
+
evaluate("hallucination"),
|
|
1580
|
+
evaluate("safety")
|
|
1581
|
+
]);
|
|
1582
|
+
const unavailableReason = "One or more required guardrail evaluations were unavailable";
|
|
1583
|
+
if (!hallucinationResult || !safetyResult) {
|
|
1584
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: unavailableReason, reason: `${unavailableReason}. Fail-closed: execution blocked.` } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: unavailableReason };
|
|
760
1585
|
}
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
confidence
|
|
770
|
-
if (hallucinationScore === null) {
|
|
771
|
-
if (this.config.failClosed) {
|
|
772
|
-
if (this.config.debug) {
|
|
773
|
-
log3.extend("warn")("[Observyze Guardrail] Eval returned null score \u2014 failing closed (blocking)");
|
|
774
|
-
}
|
|
775
|
-
return {
|
|
776
|
-
pass: false,
|
|
777
|
-
score: null,
|
|
778
|
-
confidence: null,
|
|
779
|
-
safetyScore: null,
|
|
780
|
-
evaluationSource: "error",
|
|
781
|
-
fallbackReason: evalResult.message || "Evaluation failed to produce a score",
|
|
782
|
-
reason: "Evaluation service failed to produce a score. Fail-closed: execution blocked."
|
|
783
|
-
};
|
|
784
|
-
}
|
|
785
|
-
if (this.config.debug) {
|
|
786
|
-
log3("[Observyze Guardrail] Eval returned null score \u2014 allowing (fail-open)");
|
|
787
|
-
}
|
|
788
|
-
return {
|
|
789
|
-
pass: true,
|
|
790
|
-
score: null,
|
|
791
|
-
confidence: null,
|
|
792
|
-
safetyScore: null,
|
|
793
|
-
evaluationSource: "error",
|
|
794
|
-
fallbackReason: evalResult.message || "Evaluation failed to produce a score"
|
|
795
|
-
};
|
|
796
|
-
}
|
|
797
|
-
} else if (this.config.failClosed) {
|
|
798
|
-
if (this.config.debug) {
|
|
799
|
-
log3.extend("warn")("[Observyze Guardrail] Eval unavailable \u2014 failing closed (blocking)");
|
|
800
|
-
}
|
|
801
|
-
return {
|
|
802
|
-
pass: false,
|
|
803
|
-
score: null,
|
|
804
|
-
confidence: null,
|
|
805
|
-
safetyScore: null,
|
|
806
|
-
evaluationSource: "error",
|
|
807
|
-
fallbackReason: "Evaluation service unreachable",
|
|
808
|
-
reason: "Evaluation service unreachable. Fail-closed: execution blocked."
|
|
809
|
-
};
|
|
810
|
-
} else {
|
|
811
|
-
if (this.config.debug) {
|
|
812
|
-
log3("[Observyze Guardrail] Eval unavailable \u2014 allowing (fail-open)");
|
|
813
|
-
}
|
|
814
|
-
return {
|
|
815
|
-
pass: true,
|
|
816
|
-
score: null,
|
|
817
|
-
confidence: null,
|
|
818
|
-
safetyScore: null,
|
|
819
|
-
evaluationSource: "error",
|
|
820
|
-
fallbackReason: "Evaluation service unreachable"
|
|
821
|
-
};
|
|
1586
|
+
const validScore = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1 ? value : null;
|
|
1587
|
+
const hallucinationScore = validScore(hallucinationResult.score ?? hallucinationResult.hallucination_score);
|
|
1588
|
+
const safetyScore = validScore(safetyResult.score ?? safetyResult.safety_score);
|
|
1589
|
+
const rawSource = hallucinationResult.evaluation_source ?? safetyResult.evaluation_source ?? "live";
|
|
1590
|
+
const evaluationSource = ALLOWED_EVALUATION_SOURCES.has(rawSource) ? rawSource : "live";
|
|
1591
|
+
const confidence = validScore(hallucinationResult.confidence ?? safetyResult.confidence);
|
|
1592
|
+
if (hallucinationScore === null || safetyScore === null) {
|
|
1593
|
+
const reason = hallucinationResult.message || safetyResult.message || "Evaluation failed to produce a score";
|
|
1594
|
+
return this.config.failClosed ? { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: reason, reason: `${reason}. Fail-closed: execution blocked.` } : { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: reason };
|
|
822
1595
|
}
|
|
823
1596
|
const hallThreshold = this.config.hallucinationThreshold;
|
|
824
1597
|
const safeThreshold = this.config.safetyThreshold;
|
|
825
1598
|
const confThreshold = this.config.confidenceThreshold;
|
|
826
1599
|
if (confidence !== null && confidence < confThreshold) {
|
|
827
|
-
if (hallucinationScore >= hallThreshold) {
|
|
1600
|
+
if (hallucinationScore >= hallThreshold || safetyScore >= safeThreshold) {
|
|
828
1601
|
if (this.config.debug) {
|
|
829
|
-
log3.extend("warn")(`[Observyze Guardrail] High score
|
|
1602
|
+
log3.extend("warn")(`[Observyze Guardrail] High score but low confidence (${confidence.toFixed(2)}). Alerting only.`);
|
|
830
1603
|
}
|
|
831
1604
|
return {
|
|
832
1605
|
pass: true,
|
|
@@ -834,7 +1607,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
834
1607
|
confidence,
|
|
835
1608
|
safetyScore,
|
|
836
1609
|
evaluationSource,
|
|
837
|
-
reason: `
|
|
1610
|
+
reason: `Risk score exceeded a threshold but confidence ${confidence.toFixed(2)} is below ${confThreshold.toFixed(2)}. Execution allowed with alert.`
|
|
838
1611
|
};
|
|
839
1612
|
}
|
|
840
1613
|
}
|
|
@@ -874,9 +1647,8 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
874
1647
|
}
|
|
875
1648
|
}
|
|
876
1649
|
/**
|
|
877
|
-
*
|
|
878
|
-
*
|
|
879
|
-
* Pauses execution if hallucination score >= hallucinationThreshold and requests human review.
|
|
1650
|
+
* Execute an application action only after an explicit pre-execution
|
|
1651
|
+
* guardrail check passes.
|
|
880
1652
|
* @throws Error when execution is blocked by circuit breaker
|
|
881
1653
|
*/
|
|
882
1654
|
async executeWithCircuitBreaker(agentExecution, traceContext) {
|
|
@@ -885,7 +1657,9 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
885
1657
|
}
|
|
886
1658
|
const guardResult = await this.checkGuardrails(traceContext || "execution context");
|
|
887
1659
|
if (!guardResult.pass) {
|
|
888
|
-
const error = new Error(
|
|
1660
|
+
const error = new Error(
|
|
1661
|
+
`[Observyze] Execution Blocked by Autonomous Circuit Breaker. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}. Reason: ${guardResult.reason}. Human approval required before agent can continue.`
|
|
1662
|
+
);
|
|
889
1663
|
if (this.config.debug) {
|
|
890
1664
|
log3.extend("error")("[Observyze CircuitBreaker] Execution blocked:", error.message);
|
|
891
1665
|
}
|
|
@@ -896,34 +1670,10 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
896
1670
|
}
|
|
897
1671
|
return await agentExecution();
|
|
898
1672
|
}
|
|
899
|
-
/**
|
|
900
|
-
* Phase 4: Bug Bounty Protocol (Automated) (Requirement 4.2)
|
|
901
|
-
* Automatically shard persistent failure cases to external security researcher endpoints (e.g. HackerOne wrapper)
|
|
902
|
-
*/
|
|
903
|
-
async reportBugBounty(traceId, securityEndpoint, failureContext) {
|
|
904
|
-
try {
|
|
905
|
-
if (this.config.debug) {
|
|
906
|
-
log3(`[Observyze SDK] Sharding persistent failure case ${traceId} to Bug Bounty Protocol endpoint...`);
|
|
907
|
-
}
|
|
908
|
-
await fetch(securityEndpoint, {
|
|
909
|
-
method: "POST",
|
|
910
|
-
headers: { "Content-Type": "application/json" },
|
|
911
|
-
body: JSON.stringify({
|
|
912
|
-
alert: "persistent_failure_sharded",
|
|
913
|
-
trace_id: traceId,
|
|
914
|
-
context: failureContext,
|
|
915
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
916
|
-
})
|
|
917
|
-
});
|
|
918
|
-
if (this.config.debug) {
|
|
919
|
-
log3(`[Observyze SDK] Bug Bounty payload successfully transmitted.`);
|
|
920
|
-
}
|
|
921
|
-
} catch (err) {
|
|
922
|
-
log3.extend("error")("[Observyze Bug Bounty] Failed to shard failure case:", err);
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
1673
|
};
|
|
926
1674
|
export {
|
|
1675
|
+
ExecutionBudget,
|
|
1676
|
+
ExecutionBudgetExceededError,
|
|
927
1677
|
ObservyzeClient,
|
|
928
1678
|
ObservyzeSpanExporter,
|
|
929
1679
|
Span,
|
|
@@ -932,5 +1682,9 @@ export {
|
|
|
932
1682
|
TraceStatus,
|
|
933
1683
|
wrap,
|
|
934
1684
|
wrapAnthropic,
|
|
935
|
-
|
|
1685
|
+
wrapGemini,
|
|
1686
|
+
wrapLangChain,
|
|
1687
|
+
wrapLlamaIndex,
|
|
1688
|
+
wrapOpenAI,
|
|
1689
|
+
wrapVercelAI
|
|
936
1690
|
};
|