@observyze/sdk 0.1.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-FQBYUOJB.mjs +316 -0
- package/dist/chunk-YSYKPKKX.mjs +357 -0
- package/dist/index--b41_E-1.d.mts +440 -0
- package/dist/index--b41_E-1.d.ts +440 -0
- package/dist/index-Cr-FN-y5.d.mts +405 -0
- package/dist/index-Cr-FN-y5.d.ts +405 -0
- package/dist/index-D4UXMom5.d.mts +429 -0
- package/dist/index-D4UXMom5.d.ts +429 -0
- package/dist/index.d.mts +27 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +1282 -0
- package/dist/index.mjs +936 -0
- package/dist/opentelemetry/index.d.mts +2 -0
- package/dist/opentelemetry/index.d.ts +2 -0
- package/dist/opentelemetry/index.js +338 -0
- package/dist/opentelemetry/index.mjs +6 -0
- package/package.json +17 -3
- package/INSTRUMENTATION_SUMMARY.md +0 -184
- package/examples/auto-instrumentation.ts +0 -210
- package/src/client.ts +0 -578
- package/src/index.ts +0 -21
- package/src/instrumentation/README.md +0 -227
- package/src/instrumentation/anthropic.ts +0 -233
- package/src/instrumentation/index.ts +0 -43
- package/src/instrumentation/openai.ts +0 -193
- package/src/trace.ts +0 -242
- package/src/types.ts +0 -102
- package/tsconfig.json +0 -14
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,936 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ObservyzeSpanExporter,
|
|
3
|
+
Span,
|
|
4
|
+
SpanType,
|
|
5
|
+
Trace,
|
|
6
|
+
TraceStatus
|
|
7
|
+
} from "./chunk-FQBYUOJB.mjs";
|
|
8
|
+
|
|
9
|
+
// src/client.ts
|
|
10
|
+
import debug3 from "debug";
|
|
11
|
+
|
|
12
|
+
// src/instrumentation/openai.ts
|
|
13
|
+
import debug from "debug";
|
|
14
|
+
var log = debug("observyze:sdk");
|
|
15
|
+
function wrapOpenAI(client, nwClient) {
|
|
16
|
+
const anyClient = client;
|
|
17
|
+
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
18
|
+
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/openai");
|
|
19
|
+
if (!isAlreadyRedirected) {
|
|
20
|
+
const originalApiKey = anyClient.apiKey;
|
|
21
|
+
anyClient.baseURL = `${nwClient.getConfig().endpoint}/api/v1/proxy/openai/v1`;
|
|
22
|
+
anyClient.apiKey = nwClient.getConfig().apiKey;
|
|
23
|
+
anyClient.defaultHeaders = {
|
|
24
|
+
...anyClient.defaultHeaders,
|
|
25
|
+
"x-provider-key": originalApiKey
|
|
26
|
+
};
|
|
27
|
+
if (nwClient.getConfig().debug) {
|
|
28
|
+
log("[Observyze SDK] Transparently redirected OpenAI client to proxy gateway:", anyClient.baseURL);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const originalCreate = client.chat.completions.create.bind(client.chat.completions);
|
|
33
|
+
client.chat.completions.create = async function(params, options) {
|
|
34
|
+
const isProxyRedirected = nwClient.getConfig().enableProxyRedirect && anyClient.baseURL?.includes("/api/v1/proxy/openai");
|
|
35
|
+
if (isProxyRedirected) {
|
|
36
|
+
return originalCreate(params, options);
|
|
37
|
+
}
|
|
38
|
+
const trace = nwClient.startTrace(`openai.chat.completions.create`, {
|
|
39
|
+
provider: "openai",
|
|
40
|
+
model: params.model
|
|
41
|
+
});
|
|
42
|
+
const span = trace.startSpan("chat.completions.create", SpanType.LLM);
|
|
43
|
+
span.setMetadata("model", params.model);
|
|
44
|
+
span.setMetadata("provider", "openai");
|
|
45
|
+
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
46
|
+
if (params.max_tokens !== void 0) span.setMetadata("max_tokens", params.max_tokens);
|
|
47
|
+
span.setInput({
|
|
48
|
+
model: params.model,
|
|
49
|
+
messages: params.messages,
|
|
50
|
+
temperature: params.temperature,
|
|
51
|
+
max_tokens: params.max_tokens
|
|
52
|
+
});
|
|
53
|
+
const startTime = Date.now();
|
|
54
|
+
try {
|
|
55
|
+
const response = await originalCreate(params, options);
|
|
56
|
+
if (params.stream) {
|
|
57
|
+
return wrapOpenAIStream(response, span, trace, startTime);
|
|
58
|
+
}
|
|
59
|
+
const completionResponse = response;
|
|
60
|
+
const latency = Date.now() - startTime;
|
|
61
|
+
span.setOutput({
|
|
62
|
+
id: completionResponse.id,
|
|
63
|
+
model: completionResponse.model,
|
|
64
|
+
choices: completionResponse.choices
|
|
65
|
+
});
|
|
66
|
+
if (completionResponse.usage) {
|
|
67
|
+
span.setTokens({
|
|
68
|
+
input: completionResponse.usage.prompt_tokens,
|
|
69
|
+
output: completionResponse.usage.completion_tokens,
|
|
70
|
+
total: completionResponse.usage.total_tokens
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
span.setMetadata("latency_ms", latency);
|
|
74
|
+
span.end();
|
|
75
|
+
trace.end();
|
|
76
|
+
return response;
|
|
77
|
+
} catch (error) {
|
|
78
|
+
const latency = Date.now() - startTime;
|
|
79
|
+
span.setMetadata("latency_ms", latency);
|
|
80
|
+
span.setError(error);
|
|
81
|
+
span.end();
|
|
82
|
+
trace.end();
|
|
83
|
+
throw error;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
return client;
|
|
87
|
+
}
|
|
88
|
+
function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
89
|
+
const bufferedChunks = [];
|
|
90
|
+
let streamId = "";
|
|
91
|
+
let streamModel = "";
|
|
92
|
+
return {
|
|
93
|
+
[Symbol.asyncIterator]: async function* () {
|
|
94
|
+
try {
|
|
95
|
+
for await (const chunk of stream) {
|
|
96
|
+
if (chunk.id) streamId = chunk.id;
|
|
97
|
+
if (chunk.model) streamModel = chunk.model;
|
|
98
|
+
const delta = chunk.choices[0]?.delta;
|
|
99
|
+
if (delta?.content) {
|
|
100
|
+
bufferedChunks.push(delta.content);
|
|
101
|
+
}
|
|
102
|
+
yield chunk;
|
|
103
|
+
}
|
|
104
|
+
const latency = Date.now() - startTime;
|
|
105
|
+
const completeOutput = bufferedChunks.join("");
|
|
106
|
+
span.setOutput({
|
|
107
|
+
id: streamId,
|
|
108
|
+
model: streamModel,
|
|
109
|
+
content: completeOutput
|
|
110
|
+
});
|
|
111
|
+
span.setMetadata("latency_ms", latency);
|
|
112
|
+
span.setMetadata("streaming", true);
|
|
113
|
+
span.end();
|
|
114
|
+
trace.end();
|
|
115
|
+
} catch (error) {
|
|
116
|
+
const latency = Date.now() - startTime;
|
|
117
|
+
span.setMetadata("latency_ms", latency);
|
|
118
|
+
span.setError(error);
|
|
119
|
+
span.end();
|
|
120
|
+
trace.end();
|
|
121
|
+
throw error;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// src/instrumentation/anthropic.ts
|
|
128
|
+
import debug2 from "debug";
|
|
129
|
+
var log2 = debug2("observyze:sdk");
|
|
130
|
+
function wrapAnthropic(client, nwClient) {
|
|
131
|
+
const anyClient = client;
|
|
132
|
+
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
133
|
+
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/anthropic");
|
|
134
|
+
if (!isAlreadyRedirected) {
|
|
135
|
+
const originalApiKey = anyClient.apiKey;
|
|
136
|
+
anyClient.baseURL = `${nwClient.getConfig().endpoint}/api/v1/proxy/anthropic/v1`;
|
|
137
|
+
anyClient.apiKey = nwClient.getConfig().apiKey;
|
|
138
|
+
anyClient.defaultHeaders = {
|
|
139
|
+
...anyClient.defaultHeaders,
|
|
140
|
+
"x-provider-key": originalApiKey
|
|
141
|
+
};
|
|
142
|
+
if (nwClient.getConfig().debug) {
|
|
143
|
+
log2("[Observyze SDK] Transparently redirected Anthropic client to proxy gateway:", anyClient.baseURL);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
const originalCreate = client.messages.create.bind(client.messages);
|
|
148
|
+
client.messages.create = async function(params, options) {
|
|
149
|
+
const isProxyRedirected = nwClient.getConfig().enableProxyRedirect && anyClient.baseURL?.includes("/api/v1/proxy/anthropic");
|
|
150
|
+
if (isProxyRedirected) {
|
|
151
|
+
return originalCreate(params, options);
|
|
152
|
+
}
|
|
153
|
+
const trace = nwClient.startTrace(`anthropic.messages.create`, {
|
|
154
|
+
provider: "anthropic",
|
|
155
|
+
model: params.model
|
|
156
|
+
});
|
|
157
|
+
const span = trace.startSpan("messages.create", SpanType.LLM);
|
|
158
|
+
span.setMetadata("model", params.model);
|
|
159
|
+
span.setMetadata("provider", "anthropic");
|
|
160
|
+
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
161
|
+
if (params.max_tokens !== void 0) span.setMetadata("max_tokens", params.max_tokens);
|
|
162
|
+
if (params.system !== void 0) span.setMetadata("system", params.system);
|
|
163
|
+
span.setInput({
|
|
164
|
+
model: params.model,
|
|
165
|
+
messages: params.messages,
|
|
166
|
+
max_tokens: params.max_tokens,
|
|
167
|
+
temperature: params.temperature,
|
|
168
|
+
system: params.system
|
|
169
|
+
});
|
|
170
|
+
const startTime = Date.now();
|
|
171
|
+
try {
|
|
172
|
+
const response = await originalCreate(params, options);
|
|
173
|
+
if (params.stream) {
|
|
174
|
+
return wrapAnthropicStream(response, span, trace, startTime);
|
|
175
|
+
}
|
|
176
|
+
const messageResponse = response;
|
|
177
|
+
const latency = Date.now() - startTime;
|
|
178
|
+
span.setOutput({
|
|
179
|
+
id: messageResponse.id,
|
|
180
|
+
model: messageResponse.model,
|
|
181
|
+
role: messageResponse.role,
|
|
182
|
+
content: messageResponse.content,
|
|
183
|
+
stop_reason: messageResponse.stop_reason
|
|
184
|
+
});
|
|
185
|
+
if (messageResponse.usage) {
|
|
186
|
+
span.setTokens({
|
|
187
|
+
input: messageResponse.usage.input_tokens,
|
|
188
|
+
output: messageResponse.usage.output_tokens,
|
|
189
|
+
total: messageResponse.usage.input_tokens + messageResponse.usage.output_tokens
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
span.setMetadata("latency_ms", latency);
|
|
193
|
+
span.end();
|
|
194
|
+
trace.end();
|
|
195
|
+
return response;
|
|
196
|
+
} catch (error) {
|
|
197
|
+
const latency = Date.now() - startTime;
|
|
198
|
+
span.setMetadata("latency_ms", latency);
|
|
199
|
+
span.setError(error);
|
|
200
|
+
span.end();
|
|
201
|
+
trace.end();
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
return client;
|
|
206
|
+
}
|
|
207
|
+
function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
208
|
+
const bufferedChunks = [];
|
|
209
|
+
let messageId = "";
|
|
210
|
+
let messageModel = "";
|
|
211
|
+
let stopReason = null;
|
|
212
|
+
let inputTokens = 0;
|
|
213
|
+
let outputTokens = 0;
|
|
214
|
+
return {
|
|
215
|
+
[Symbol.asyncIterator]: async function* () {
|
|
216
|
+
try {
|
|
217
|
+
for await (const event of stream) {
|
|
218
|
+
if (event.type === "message_start" && event.message) {
|
|
219
|
+
messageId = event.message.id;
|
|
220
|
+
messageModel = event.message.model;
|
|
221
|
+
if (event.message.usage) {
|
|
222
|
+
inputTokens = event.message.usage.input_tokens;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (event.type === "content_block_delta" && event.delta?.text) {
|
|
226
|
+
bufferedChunks.push(event.delta.text);
|
|
227
|
+
}
|
|
228
|
+
if (event.type === "message_delta" && event.delta) {
|
|
229
|
+
if (event.delta.stop_reason) {
|
|
230
|
+
stopReason = event.delta.stop_reason;
|
|
231
|
+
}
|
|
232
|
+
if (event.usage?.output_tokens) {
|
|
233
|
+
outputTokens = event.usage.output_tokens;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
yield event;
|
|
237
|
+
}
|
|
238
|
+
const latency = Date.now() - startTime;
|
|
239
|
+
const completeOutput = bufferedChunks.join("");
|
|
240
|
+
span.setOutput({
|
|
241
|
+
id: messageId,
|
|
242
|
+
model: messageModel,
|
|
243
|
+
content: completeOutput,
|
|
244
|
+
stop_reason: stopReason
|
|
245
|
+
});
|
|
246
|
+
if (inputTokens > 0 || outputTokens > 0) {
|
|
247
|
+
span.setTokens({
|
|
248
|
+
input: inputTokens,
|
|
249
|
+
output: outputTokens,
|
|
250
|
+
total: inputTokens + outputTokens
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
span.setMetadata("latency_ms", latency);
|
|
254
|
+
span.setMetadata("streaming", true);
|
|
255
|
+
span.end();
|
|
256
|
+
trace.end();
|
|
257
|
+
} catch (error) {
|
|
258
|
+
const latency = Date.now() - startTime;
|
|
259
|
+
span.setMetadata("latency_ms", latency);
|
|
260
|
+
span.setError(error);
|
|
261
|
+
span.end();
|
|
262
|
+
trace.end();
|
|
263
|
+
throw error;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// src/instrumentation/index.ts
|
|
270
|
+
function wrap(client, nwClient) {
|
|
271
|
+
if ("chat" in client && client.chat && "completions" in client.chat) {
|
|
272
|
+
return wrapOpenAI(client, nwClient);
|
|
273
|
+
}
|
|
274
|
+
if ("messages" in client && client.messages && "create" in client.messages) {
|
|
275
|
+
return wrapAnthropic(client, nwClient);
|
|
276
|
+
}
|
|
277
|
+
throw new Error(
|
|
278
|
+
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic"
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// src/client.ts
|
|
283
|
+
import fs from "fs";
|
|
284
|
+
import path from "path";
|
|
285
|
+
var log3 = debug3("observyze:sdk");
|
|
286
|
+
var DEFAULT_CONFIG = {
|
|
287
|
+
endpoint: "http://localhost:3001",
|
|
288
|
+
batchSize: 100,
|
|
289
|
+
flushInterval: 5e3,
|
|
290
|
+
enableAutoInstrumentation: true,
|
|
291
|
+
debug: false,
|
|
292
|
+
dryRun: false,
|
|
293
|
+
enablePiiRedaction: true,
|
|
294
|
+
hallucinationThreshold: 0.8,
|
|
295
|
+
safetyThreshold: 0.9,
|
|
296
|
+
confidenceThreshold: 0.4,
|
|
297
|
+
evalEndpoint: process.env.EVAL_ENDPOINT || (process.env.NODE_ENV === "production" ? "https://api.observyze.com" : "http://localhost:3001"),
|
|
298
|
+
enableCircuitBreaker: true,
|
|
299
|
+
failClosed: true,
|
|
300
|
+
enableProxyRedirect: true
|
|
301
|
+
};
|
|
302
|
+
var ObservyzeClient = class _ObservyzeClient {
|
|
303
|
+
config;
|
|
304
|
+
traceBuffer = [];
|
|
305
|
+
flushTimer = null;
|
|
306
|
+
isShuttingDown = false;
|
|
307
|
+
MAX_QUEUE_SIZE = 1e3;
|
|
308
|
+
RETRY_DELAYS = [1e3, 2e3, 4e3, 8e3, 16e3, 3e4];
|
|
309
|
+
/**
|
|
310
|
+
* Parse a JSON API error response and extract trace_id, error code, and message.
|
|
311
|
+
* The api-gateway error handler includes these fields in every error response.
|
|
312
|
+
*/
|
|
313
|
+
static parseApiError(_response, body) {
|
|
314
|
+
try {
|
|
315
|
+
const parsed = JSON.parse(body);
|
|
316
|
+
const error = parsed.error || parsed;
|
|
317
|
+
return {
|
|
318
|
+
traceId: error.trace_id || "unknown",
|
|
319
|
+
code: error.code || "UNKNOWN_ERROR",
|
|
320
|
+
message: error.message || body.slice(0, 200)
|
|
321
|
+
};
|
|
322
|
+
} catch {
|
|
323
|
+
return {
|
|
324
|
+
traceId: "unknown",
|
|
325
|
+
code: "UNKNOWN_ERROR",
|
|
326
|
+
message: body.slice(0, 200)
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Format an API error into a user-friendly message with trace_id for correlation.
|
|
332
|
+
* Example output:
|
|
333
|
+
* "Observyze API error (401 [ref: err_a1b2c3d4]): MISSING_PROVIDER_KEY — No API key configured..."
|
|
334
|
+
*/
|
|
335
|
+
static formatApiError(response, body) {
|
|
336
|
+
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, body);
|
|
337
|
+
const prefix = traceId !== "unknown" ? ` [ref: ${traceId}]` : "";
|
|
338
|
+
return `Observyze API error (${response.status}${prefix}): ${code} \u2014 ${message}`;
|
|
339
|
+
}
|
|
340
|
+
constructor(config) {
|
|
341
|
+
if (!config.apiKey) {
|
|
342
|
+
throw new Error("Observyze SDK: apiKey is required");
|
|
343
|
+
}
|
|
344
|
+
this.config = {
|
|
345
|
+
...DEFAULT_CONFIG,
|
|
346
|
+
...config,
|
|
347
|
+
organizationId: config.organizationId || "",
|
|
348
|
+
projectId: config.projectId || ""
|
|
349
|
+
};
|
|
350
|
+
this.startFlushTimer();
|
|
351
|
+
if (this.config.debug) {
|
|
352
|
+
log3("[Observyze SDK] Initialized with config:", {
|
|
353
|
+
endpoint: this.config.endpoint,
|
|
354
|
+
batchSize: this.config.batchSize,
|
|
355
|
+
flushInterval: this.config.flushInterval,
|
|
356
|
+
dryRun: this.config.dryRun
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Start a new trace
|
|
362
|
+
*/
|
|
363
|
+
startTrace(name, metadata) {
|
|
364
|
+
const trace = new Trace(
|
|
365
|
+
name,
|
|
366
|
+
this.config.organizationId,
|
|
367
|
+
this.config.projectId
|
|
368
|
+
);
|
|
369
|
+
if (metadata) {
|
|
370
|
+
trace.setMetadataAll(metadata);
|
|
371
|
+
}
|
|
372
|
+
const originalEnd = trace.end.bind(trace);
|
|
373
|
+
trace.end = (status = TraceStatus.SUCCESS) => {
|
|
374
|
+
originalEnd(status);
|
|
375
|
+
this.bufferTrace(trace);
|
|
376
|
+
};
|
|
377
|
+
return trace;
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Buffer a completed trace for batch sending
|
|
381
|
+
*/
|
|
382
|
+
bufferTrace(trace) {
|
|
383
|
+
if (!trace.isEnded) {
|
|
384
|
+
if (this.config.debug) {
|
|
385
|
+
log3.extend("warn")("[Observyze SDK] Attempted to buffer a trace that has not ended");
|
|
386
|
+
}
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (this.traceBuffer.length >= this.MAX_QUEUE_SIZE) {
|
|
390
|
+
if (this.config.debug) {
|
|
391
|
+
log3.extend("warn")(`[Observyze SDK] Queue at max capacity (${this.MAX_QUEUE_SIZE}), dropping oldest trace`);
|
|
392
|
+
}
|
|
393
|
+
this.traceBuffer.shift();
|
|
394
|
+
}
|
|
395
|
+
this.traceBuffer.push(trace);
|
|
396
|
+
if (this.config.debug) {
|
|
397
|
+
log3(`[Observyze SDK] Buffered trace ${trace.id} (${this.traceBuffer.length}/${this.config.batchSize})`);
|
|
398
|
+
}
|
|
399
|
+
if (this.traceBuffer.length >= this.config.batchSize) {
|
|
400
|
+
this.flush().catch((err) => {
|
|
401
|
+
log3.extend("error")("[Observyze SDK] Error flushing buffer:", err);
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Start the auto-flush timer
|
|
407
|
+
*/
|
|
408
|
+
startFlushTimer() {
|
|
409
|
+
if (this.flushTimer) {
|
|
410
|
+
clearInterval(this.flushTimer);
|
|
411
|
+
}
|
|
412
|
+
this.flushTimer = setInterval(() => {
|
|
413
|
+
if (this.traceBuffer.length > 0) {
|
|
414
|
+
this.flush().catch((err) => {
|
|
415
|
+
log3.extend("error")("[Observyze SDK] Error in auto-flush:", err);
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
}, this.config.flushInterval);
|
|
419
|
+
if (this.flushTimer.unref) {
|
|
420
|
+
this.flushTimer.unref();
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Flush all buffered traces to the Ingestion Service
|
|
425
|
+
*/
|
|
426
|
+
async flush() {
|
|
427
|
+
if (this.traceBuffer.length === 0) {
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
const tracesToSend = this.traceBuffer.splice(0, this.config.batchSize);
|
|
431
|
+
if (this.config.debug) {
|
|
432
|
+
log3(`[Observyze SDK] Flushing ${tracesToSend.length} traces`);
|
|
433
|
+
}
|
|
434
|
+
if (this.config.dryRun) {
|
|
435
|
+
if (this.config.debug) {
|
|
436
|
+
log3("[Observyze SDK] Dry-run mode: traces not sent");
|
|
437
|
+
}
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
try {
|
|
441
|
+
await this.sendWithRetry(tracesToSend);
|
|
442
|
+
} catch (error) {
|
|
443
|
+
const remainingSpace = this.MAX_QUEUE_SIZE - this.traceBuffer.length;
|
|
444
|
+
if (remainingSpace > 0) {
|
|
445
|
+
this.traceBuffer.unshift(...tracesToSend.slice(0, remainingSpace));
|
|
446
|
+
if (this.config.debug) {
|
|
447
|
+
log3(`[Observyze SDK] Re-queued ${Math.min(tracesToSend.length, remainingSpace)} traces after failure`);
|
|
448
|
+
}
|
|
449
|
+
} else {
|
|
450
|
+
if (this.config.debug) {
|
|
451
|
+
log3.extend("warn")(`[Observyze SDK] Queue full, dropped ${tracesToSend.length} traces`);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (this.config.debug) {
|
|
455
|
+
log3.extend("error")("[Observyze SDK] Failed to send traces after retries:", error);
|
|
456
|
+
}
|
|
457
|
+
throw error;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Send traces with exponential backoff retry
|
|
462
|
+
*/
|
|
463
|
+
async sendWithRetry(traces) {
|
|
464
|
+
let lastError = null;
|
|
465
|
+
for (let attempt = 0; attempt < this.RETRY_DELAYS.length + 1; attempt++) {
|
|
466
|
+
try {
|
|
467
|
+
const response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
468
|
+
method: "POST",
|
|
469
|
+
headers: {
|
|
470
|
+
"Content-Type": "application/json",
|
|
471
|
+
"Authorization": `Bearer ${this.config.apiKey}`
|
|
472
|
+
},
|
|
473
|
+
body: JSON.stringify({
|
|
474
|
+
traces: traces.map((trace) => {
|
|
475
|
+
const json = trace.toJSON();
|
|
476
|
+
if (this.config.enablePiiRedaction) {
|
|
477
|
+
json.spans = this.sanitizePII(json.spans);
|
|
478
|
+
}
|
|
479
|
+
return json;
|
|
480
|
+
})
|
|
481
|
+
})
|
|
482
|
+
});
|
|
483
|
+
if (!response.ok) {
|
|
484
|
+
const errorBody = await response.text();
|
|
485
|
+
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
486
|
+
throw new Error(`[Observyze SDK] Trace ingestion failed. ${formatted}`);
|
|
487
|
+
}
|
|
488
|
+
if (this.config.debug) {
|
|
489
|
+
log3(`[Observyze SDK] Successfully sent ${traces.length} traces${attempt > 0 ? ` (after ${attempt} retries)` : ""}`);
|
|
490
|
+
}
|
|
491
|
+
return;
|
|
492
|
+
} catch (error) {
|
|
493
|
+
lastError = error;
|
|
494
|
+
if (attempt >= this.RETRY_DELAYS.length) {
|
|
495
|
+
break;
|
|
496
|
+
}
|
|
497
|
+
const delay = this.RETRY_DELAYS[attempt];
|
|
498
|
+
if (this.config.debug) {
|
|
499
|
+
log3.extend("warn")(`[Observyze SDK] Attempt ${attempt + 1} failed, retrying in ${delay}ms...`, error);
|
|
500
|
+
}
|
|
501
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
throw lastError || new Error("Failed to send traces after all retries");
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Shutdown the SDK and flush remaining traces
|
|
508
|
+
*/
|
|
509
|
+
async shutdown() {
|
|
510
|
+
if (this.isShuttingDown) {
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
this.isShuttingDown = true;
|
|
514
|
+
if (this.config.debug) {
|
|
515
|
+
log3("[Observyze SDK] Shutting down...");
|
|
516
|
+
}
|
|
517
|
+
if (this.flushTimer) {
|
|
518
|
+
clearInterval(this.flushTimer);
|
|
519
|
+
this.flushTimer = null;
|
|
520
|
+
}
|
|
521
|
+
try {
|
|
522
|
+
await this.flush();
|
|
523
|
+
} catch (error) {
|
|
524
|
+
log3.extend("error")("[Observyze SDK] Error during shutdown flush:", error);
|
|
525
|
+
}
|
|
526
|
+
if (this.config.debug) {
|
|
527
|
+
log3("[Observyze SDK] Shutdown complete");
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Get current buffer size
|
|
532
|
+
*/
|
|
533
|
+
get bufferSize() {
|
|
534
|
+
return this.traceBuffer.length;
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Get SDK configuration
|
|
538
|
+
*/
|
|
539
|
+
getConfig() {
|
|
540
|
+
return { ...this.config };
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Wrap an LLM client (OpenAI, Anthropic) to enable auto-instrumentation
|
|
544
|
+
*
|
|
545
|
+
* @example
|
|
546
|
+
* ```typescript
|
|
547
|
+
* import OpenAI from 'openai'
|
|
548
|
+
* import { ObservyzeClient } from '@observyze/sdk'
|
|
549
|
+
*
|
|
550
|
+
* const nw = new ObservyzeClient({ apiKey: 'your-api-key' })
|
|
551
|
+
* const openai = new OpenAI({ apiKey: 'openai-key' })
|
|
552
|
+
*
|
|
553
|
+
* // Wrap the client to enable auto-instrumentation
|
|
554
|
+
* nw.wrap(openai)
|
|
555
|
+
*
|
|
556
|
+
* // All calls are now automatically traced
|
|
557
|
+
* const response = await openai.chat.completions.create({
|
|
558
|
+
* model: 'gpt-4',
|
|
559
|
+
* messages: [{ role: 'user', content: 'Hello!' }]
|
|
560
|
+
* })
|
|
561
|
+
* ```
|
|
562
|
+
*/
|
|
563
|
+
wrap(client) {
|
|
564
|
+
return wrap(client, this);
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Sync local agent .history file to Observyze cloud
|
|
568
|
+
* Parses JSON/NDJSON agent history and sends to ingestion endpoint.
|
|
569
|
+
*/
|
|
570
|
+
async syncLocalHistory(filePath) {
|
|
571
|
+
try {
|
|
572
|
+
if (typeof process === "undefined" || !process.versions?.node) {
|
|
573
|
+
throw new Error("syncLocalHistory is only available in Node.js environments");
|
|
574
|
+
}
|
|
575
|
+
const fullPath = path.resolve(process.cwd(), filePath);
|
|
576
|
+
if (!fs.existsSync(fullPath)) {
|
|
577
|
+
throw new Error(`History file not found: ${fullPath}`);
|
|
578
|
+
}
|
|
579
|
+
const content = fs.readFileSync(fullPath, "utf-8");
|
|
580
|
+
let items = [];
|
|
581
|
+
try {
|
|
582
|
+
items = JSON.parse(content);
|
|
583
|
+
} catch (e) {
|
|
584
|
+
items = content.split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l));
|
|
585
|
+
}
|
|
586
|
+
if (!Array.isArray(items)) {
|
|
587
|
+
items = [items];
|
|
588
|
+
}
|
|
589
|
+
if (this.config.debug) {
|
|
590
|
+
log3(`[Observyze SDK] Syncing ${items.length} traces from ${filePath}`);
|
|
591
|
+
}
|
|
592
|
+
for (let i = 0; i < items.length; i += this.config.batchSize) {
|
|
593
|
+
const batch = items.slice(i, i + this.config.batchSize);
|
|
594
|
+
const response = await fetch(`${this.config.endpoint}/api/v1/ingest/batch`, {
|
|
595
|
+
method: "POST",
|
|
596
|
+
headers: {
|
|
597
|
+
"Content-Type": "application/json",
|
|
598
|
+
"Authorization": `Bearer ${this.config.apiKey}`
|
|
599
|
+
},
|
|
600
|
+
body: JSON.stringify({ traces: batch })
|
|
601
|
+
});
|
|
602
|
+
if (!response.ok) {
|
|
603
|
+
const errorBody = await response.text();
|
|
604
|
+
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
605
|
+
throw new Error(`[Observyze SDK] Local history sync failed. ${formatted}`);
|
|
606
|
+
}
|
|
607
|
+
if (this.config.debug) {
|
|
608
|
+
log3(`[Observyze SDK] Synced batch of ${batch.length} traces from local history`);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
} catch (err) {
|
|
612
|
+
log3.extend("error")("[Observyze SDK] Failed to sync local history:", err);
|
|
613
|
+
throw err;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Industry-grade PII Redaction (Compliance & RBAC)
|
|
618
|
+
*
|
|
619
|
+
* Recursively scrubs PII from trace data before transmission to the cloud.
|
|
620
|
+
* Coverage: emails, JWTs, bearer tokens, AWS/API keys, SSNs, credit cards,
|
|
621
|
+
* phone numbers (US + E.164), IPv4/IPv6, passport numbers, ZIP codes, plus
|
|
622
|
+
* key-based redaction for sensitive JSON fields (password, token, api_key, etc.).
|
|
623
|
+
*
|
|
624
|
+
* Design:
|
|
625
|
+
* - Pure function, never mutates the original object
|
|
626
|
+
* - Depth-limited to 16 levels to prevent stack overflow on deep agent outputs
|
|
627
|
+
* - Key-aware: sensitive key names are fully redacted regardless of value format
|
|
628
|
+
*/
|
|
629
|
+
static PII_PATTERNS = [
|
|
630
|
+
{ pattern: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/gi, label: "[EMAIL_REDACTED]" },
|
|
631
|
+
{ pattern: /\beyJ[A-Za-z0-9_\-]+\.eyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\b/g, label: "[JWT_REDACTED]" },
|
|
632
|
+
{ pattern: /\b(Bearer|Token|Basic)\s+[A-Za-z0-9\-.~+\/]+=*\b/gi, label: "[AUTH_TOKEN_REDACTED]" },
|
|
633
|
+
{ pattern: /\b(AKIA|ASIA|AROA|ANPA|ANVA|AIDA)[A-Z0-9]{16}\b/g, label: "[AWS_KEY_REDACTED]" },
|
|
634
|
+
{ pattern: /\b(sk-[A-Za-z0-9\-]{20,}|pk-[A-Za-z0-9\-]{20,}|ob_[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{36,}|gho_[A-Za-z0-9]{36,})\b/g, label: "[API_KEY_REDACTED]" },
|
|
635
|
+
{ pattern: /\b\d{3}-\d{2}-\d{4}\b/g, label: "[SSN_REDACTED]" },
|
|
636
|
+
{ pattern: /\b(?:\d[ \-]?){13,18}\d\b/g, label: "[CC_REDACTED]" },
|
|
637
|
+
{ pattern: /(?:\+1[\s.\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b/g, label: "[PHONE_REDACTED]" },
|
|
638
|
+
{ pattern: /\+\d{1,3}[\s.\-]?\(?\d{1,4}\)?[\s.\-]?\d{1,4}[\s.\-]?\d{1,9}/g, label: "[PHONE_REDACTED]" },
|
|
639
|
+
{ pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, label: "[IP_REDACTED]" },
|
|
640
|
+
{ pattern: /\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, label: "[IP_REDACTED]" }
|
|
641
|
+
];
|
|
642
|
+
static SENSITIVE_KEYS = /* @__PURE__ */ new Set([
|
|
643
|
+
"password",
|
|
644
|
+
"passwd",
|
|
645
|
+
"secret",
|
|
646
|
+
"token",
|
|
647
|
+
"apikey",
|
|
648
|
+
"api_key",
|
|
649
|
+
"accesstoken",
|
|
650
|
+
"access_token",
|
|
651
|
+
"refreshtoken",
|
|
652
|
+
"refresh_token",
|
|
653
|
+
"authorization",
|
|
654
|
+
"auth",
|
|
655
|
+
"credential",
|
|
656
|
+
"credentials",
|
|
657
|
+
"private_key",
|
|
658
|
+
"privatekey",
|
|
659
|
+
"client_secret",
|
|
660
|
+
"clientsecret",
|
|
661
|
+
"ssn",
|
|
662
|
+
"social_security",
|
|
663
|
+
"dob",
|
|
664
|
+
"date_of_birth",
|
|
665
|
+
"dateofbirth",
|
|
666
|
+
"passport",
|
|
667
|
+
"passport_number",
|
|
668
|
+
"credit_card",
|
|
669
|
+
"creditcard",
|
|
670
|
+
"card_number",
|
|
671
|
+
"cardnumber",
|
|
672
|
+
"cvv",
|
|
673
|
+
"cvc",
|
|
674
|
+
"pin",
|
|
675
|
+
"bank_account",
|
|
676
|
+
"routing_number"
|
|
677
|
+
]);
|
|
678
|
+
static isSensitiveKey(key) {
|
|
679
|
+
const normalized = key.toLowerCase().replace(/-/g, "_");
|
|
680
|
+
if (_ObservyzeClient.SENSITIVE_KEYS.has(normalized)) return true;
|
|
681
|
+
const segments = normalized.split("_");
|
|
682
|
+
for (const segment of segments) {
|
|
683
|
+
if (_ObservyzeClient.SENSITIVE_KEYS.has(segment)) return true;
|
|
684
|
+
}
|
|
685
|
+
return false;
|
|
686
|
+
}
|
|
687
|
+
sanitizePII(data, depth = 0) {
|
|
688
|
+
if (depth > 16) return "[MAX_DEPTH_EXCEEDED]";
|
|
689
|
+
if (data === null || data === void 0) return data;
|
|
690
|
+
if (typeof data === "string") {
|
|
691
|
+
let result = data;
|
|
692
|
+
for (const { pattern, label } of _ObservyzeClient.PII_PATTERNS) {
|
|
693
|
+
pattern.lastIndex = 0;
|
|
694
|
+
result = result.replace(pattern, label);
|
|
695
|
+
}
|
|
696
|
+
return result;
|
|
697
|
+
}
|
|
698
|
+
if (typeof data === "number" || typeof data === "boolean") return data;
|
|
699
|
+
if (Array.isArray(data)) return data.map((item) => this.sanitizePII(item, depth + 1));
|
|
700
|
+
if (typeof data === "object") {
|
|
701
|
+
const sanitized = {};
|
|
702
|
+
for (const [k, v] of Object.entries(data)) {
|
|
703
|
+
sanitized[k] = _ObservyzeClient.isSensitiveKey(k) ? "[REDACTED]" : this.sanitizePII(v, depth + 1);
|
|
704
|
+
}
|
|
705
|
+
return sanitized;
|
|
706
|
+
}
|
|
707
|
+
return data;
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Phase 4: Autonomous Circuit Breakers (Requirement 4.1)
|
|
711
|
+
* Evaluate a trace or text for hallucination in real-time.
|
|
712
|
+
* If hallucination score > hallucinationThreshold, the SDK blocks execution.
|
|
713
|
+
*
|
|
714
|
+
* Returns GuardrailResult with score: null when evaluation couldn't be performed.
|
|
715
|
+
* In failClosed mode, null scores result in blocked execution.
|
|
716
|
+
* In failOpen mode, null scores allow execution through.
|
|
717
|
+
*/
|
|
718
|
+
async checkGuardrails(content) {
|
|
719
|
+
if (!this.config.enableCircuitBreaker) {
|
|
720
|
+
return { pass: true, score: 0, safetyScore: 0, evaluationSource: "disabled" };
|
|
721
|
+
}
|
|
722
|
+
try {
|
|
723
|
+
if (this.config.debug) {
|
|
724
|
+
log3(`[Observyze Guardrail] Analyzing payload for hallucination anomalies...`);
|
|
725
|
+
}
|
|
726
|
+
const evalEndpoint = this.config.evalEndpoint;
|
|
727
|
+
const payload = typeof content === "string" ? { text: content, organization_id: this.config.organizationId } : { trace: content, organization_id: this.config.organizationId };
|
|
728
|
+
const controller = new AbortController();
|
|
729
|
+
const timeout = setTimeout(() => controller.abort(), 5e3);
|
|
730
|
+
let evalResult = null;
|
|
731
|
+
try {
|
|
732
|
+
const response = await fetch(`${evalEndpoint}/api/v1/evaluate/hallucination`, {
|
|
733
|
+
method: "POST",
|
|
734
|
+
headers: {
|
|
735
|
+
"Content-Type": "application/json",
|
|
736
|
+
"Authorization": `Bearer ${this.config.apiKey}`
|
|
737
|
+
},
|
|
738
|
+
body: JSON.stringify(payload),
|
|
739
|
+
signal: controller.signal
|
|
740
|
+
});
|
|
741
|
+
clearTimeout(timeout);
|
|
742
|
+
if (response.ok) {
|
|
743
|
+
evalResult = await response.json();
|
|
744
|
+
} else {
|
|
745
|
+
const errorBody = await response.text();
|
|
746
|
+
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, errorBody);
|
|
747
|
+
if (this.config.debug) {
|
|
748
|
+
log3.extend("warn")(`[Observyze Guardrail] Eval returned ${response.status} [${code}] (ref: ${traceId}): ${message}`);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
} catch (fetchError) {
|
|
752
|
+
clearTimeout(timeout);
|
|
753
|
+
if (fetchError.name === "AbortError") {
|
|
754
|
+
if (this.config.debug) {
|
|
755
|
+
log3.extend("warn")("[Observyze Guardrail] Evaluation timed out after 5s");
|
|
756
|
+
}
|
|
757
|
+
} else if (this.config.debug) {
|
|
758
|
+
log3.extend("warn")("[Observyze Guardrail] Evaluation request failed:", fetchError.message);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
let hallucinationScore = 0;
|
|
762
|
+
let safetyScore = 0;
|
|
763
|
+
let evaluationSource = "live";
|
|
764
|
+
let confidence = null;
|
|
765
|
+
if (evalResult) {
|
|
766
|
+
hallucinationScore = evalResult.score ?? evalResult.hallucination_score ?? null;
|
|
767
|
+
safetyScore = evalResult.safety_score ?? 0;
|
|
768
|
+
evaluationSource = evalResult.evaluation_source ?? "live";
|
|
769
|
+
confidence = evalResult.confidence ?? null;
|
|
770
|
+
if (hallucinationScore === null) {
|
|
771
|
+
if (this.config.failClosed) {
|
|
772
|
+
if (this.config.debug) {
|
|
773
|
+
log3.extend("warn")("[Observyze Guardrail] Eval returned null score \u2014 failing closed (blocking)");
|
|
774
|
+
}
|
|
775
|
+
return {
|
|
776
|
+
pass: false,
|
|
777
|
+
score: null,
|
|
778
|
+
confidence: null,
|
|
779
|
+
safetyScore: null,
|
|
780
|
+
evaluationSource: "error",
|
|
781
|
+
fallbackReason: evalResult.message || "Evaluation failed to produce a score",
|
|
782
|
+
reason: "Evaluation service failed to produce a score. Fail-closed: execution blocked."
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
if (this.config.debug) {
|
|
786
|
+
log3("[Observyze Guardrail] Eval returned null score \u2014 allowing (fail-open)");
|
|
787
|
+
}
|
|
788
|
+
return {
|
|
789
|
+
pass: true,
|
|
790
|
+
score: null,
|
|
791
|
+
confidence: null,
|
|
792
|
+
safetyScore: null,
|
|
793
|
+
evaluationSource: "error",
|
|
794
|
+
fallbackReason: evalResult.message || "Evaluation failed to produce a score"
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
} else if (this.config.failClosed) {
|
|
798
|
+
if (this.config.debug) {
|
|
799
|
+
log3.extend("warn")("[Observyze Guardrail] Eval unavailable \u2014 failing closed (blocking)");
|
|
800
|
+
}
|
|
801
|
+
return {
|
|
802
|
+
pass: false,
|
|
803
|
+
score: null,
|
|
804
|
+
confidence: null,
|
|
805
|
+
safetyScore: null,
|
|
806
|
+
evaluationSource: "error",
|
|
807
|
+
fallbackReason: "Evaluation service unreachable",
|
|
808
|
+
reason: "Evaluation service unreachable. Fail-closed: execution blocked."
|
|
809
|
+
};
|
|
810
|
+
} else {
|
|
811
|
+
if (this.config.debug) {
|
|
812
|
+
log3("[Observyze Guardrail] Eval unavailable \u2014 allowing (fail-open)");
|
|
813
|
+
}
|
|
814
|
+
return {
|
|
815
|
+
pass: true,
|
|
816
|
+
score: null,
|
|
817
|
+
confidence: null,
|
|
818
|
+
safetyScore: null,
|
|
819
|
+
evaluationSource: "error",
|
|
820
|
+
fallbackReason: "Evaluation service unreachable"
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
const hallThreshold = this.config.hallucinationThreshold;
|
|
824
|
+
const safeThreshold = this.config.safetyThreshold;
|
|
825
|
+
const confThreshold = this.config.confidenceThreshold;
|
|
826
|
+
if (confidence !== null && confidence < confThreshold) {
|
|
827
|
+
if (hallucinationScore >= hallThreshold) {
|
|
828
|
+
if (this.config.debug) {
|
|
829
|
+
log3.extend("warn")(`[Observyze Guardrail] High score (${hallucinationScore.toFixed(2)}) but low confidence (${confidence.toFixed(2)}). Alerting only.`);
|
|
830
|
+
}
|
|
831
|
+
return {
|
|
832
|
+
pass: true,
|
|
833
|
+
score: hallucinationScore,
|
|
834
|
+
confidence,
|
|
835
|
+
safetyScore,
|
|
836
|
+
evaluationSource,
|
|
837
|
+
reason: `Score ${hallucinationScore.toFixed(2)} but confidence ${confidence.toFixed(2)} is low. Execution allowed with alert.`
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
if (hallucinationScore >= hallThreshold) {
|
|
842
|
+
if (this.config.debug) {
|
|
843
|
+
log3.extend("warn")(`[Observyze Guardrail] Hallucination circuit breached! Score: ${hallucinationScore.toFixed(2)} >= ${hallThreshold}`);
|
|
844
|
+
}
|
|
845
|
+
return {
|
|
846
|
+
pass: false,
|
|
847
|
+
score: hallucinationScore,
|
|
848
|
+
confidence,
|
|
849
|
+
safetyScore,
|
|
850
|
+
evaluationSource,
|
|
851
|
+
reason: `Hallucination score ${hallucinationScore.toFixed(2)} exceeds threshold ${hallThreshold}. Execution blocked for human review.`
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
if (safetyScore !== null && safetyScore >= safeThreshold) {
|
|
855
|
+
if (this.config.debug) {
|
|
856
|
+
log3.extend("warn")(`[Observyze Guardrail] Safety circuit breached! Score: ${safetyScore.toFixed(2)} >= ${safeThreshold}`);
|
|
857
|
+
}
|
|
858
|
+
return {
|
|
859
|
+
pass: false,
|
|
860
|
+
score: hallucinationScore,
|
|
861
|
+
confidence,
|
|
862
|
+
safetyScore,
|
|
863
|
+
evaluationSource,
|
|
864
|
+
reason: `Safety score ${safetyScore.toFixed(2)} exceeds threshold ${safeThreshold}. Execution blocked for safety review.`
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
return { pass: true, score: hallucinationScore, confidence, safetyScore, evaluationSource };
|
|
868
|
+
} catch (err) {
|
|
869
|
+
log3.extend("error")("[Observyze Guardrail] Failed to evaluate:", err);
|
|
870
|
+
if (this.config.failClosed) {
|
|
871
|
+
return { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: "Guardrail exception", reason: "Guardrail error \u2014 fail-closed: execution blocked." };
|
|
872
|
+
}
|
|
873
|
+
return { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: "Guardrail exception" };
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* Phase 4: Autonomous Circuit Breakers
|
|
878
|
+
* Execute an agent action wrapped with the Circuit Breaker.
|
|
879
|
+
* Pauses execution if hallucination score >= hallucinationThreshold and requests human review.
|
|
880
|
+
* @throws Error when execution is blocked by circuit breaker
|
|
881
|
+
*/
|
|
882
|
+
async executeWithCircuitBreaker(agentExecution, traceContext) {
|
|
883
|
+
if (!this.config.enableCircuitBreaker) {
|
|
884
|
+
return await agentExecution();
|
|
885
|
+
}
|
|
886
|
+
const guardResult = await this.checkGuardrails(traceContext || "execution context");
|
|
887
|
+
if (!guardResult.pass) {
|
|
888
|
+
const error = new Error(`[Observyze] Execution Blocked by Autonomous Circuit Breaker. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}. Reason: ${guardResult.reason}. Human approval required before agent can continue.`);
|
|
889
|
+
if (this.config.debug) {
|
|
890
|
+
log3.extend("error")("[Observyze CircuitBreaker] Execution blocked:", error.message);
|
|
891
|
+
}
|
|
892
|
+
throw error;
|
|
893
|
+
}
|
|
894
|
+
if (this.config.debug) {
|
|
895
|
+
log3(`[Observyze CircuitBreaker] Execution allowed. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}`);
|
|
896
|
+
}
|
|
897
|
+
return await agentExecution();
|
|
898
|
+
}
|
|
899
|
+
/**
|
|
900
|
+
* Phase 4: Bug Bounty Protocol (Automated) (Requirement 4.2)
|
|
901
|
+
* Automatically shard persistent failure cases to external security researcher endpoints (e.g. HackerOne wrapper)
|
|
902
|
+
*/
|
|
903
|
+
async reportBugBounty(traceId, securityEndpoint, failureContext) {
|
|
904
|
+
try {
|
|
905
|
+
if (this.config.debug) {
|
|
906
|
+
log3(`[Observyze SDK] Sharding persistent failure case ${traceId} to Bug Bounty Protocol endpoint...`);
|
|
907
|
+
}
|
|
908
|
+
await fetch(securityEndpoint, {
|
|
909
|
+
method: "POST",
|
|
910
|
+
headers: { "Content-Type": "application/json" },
|
|
911
|
+
body: JSON.stringify({
|
|
912
|
+
alert: "persistent_failure_sharded",
|
|
913
|
+
trace_id: traceId,
|
|
914
|
+
context: failureContext,
|
|
915
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
916
|
+
})
|
|
917
|
+
});
|
|
918
|
+
if (this.config.debug) {
|
|
919
|
+
log3(`[Observyze SDK] Bug Bounty payload successfully transmitted.`);
|
|
920
|
+
}
|
|
921
|
+
} catch (err) {
|
|
922
|
+
log3.extend("error")("[Observyze Bug Bounty] Failed to shard failure case:", err);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
};
|
|
926
|
+
export {
|
|
927
|
+
ObservyzeClient,
|
|
928
|
+
ObservyzeSpanExporter,
|
|
929
|
+
Span,
|
|
930
|
+
SpanType,
|
|
931
|
+
Trace,
|
|
932
|
+
TraceStatus,
|
|
933
|
+
wrap,
|
|
934
|
+
wrapAnthropic,
|
|
935
|
+
wrapOpenAI
|
|
936
|
+
};
|