@observyze/sdk 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -10
- package/dist/chunk-FQBYUOJB.mjs +316 -0
- package/dist/{chunk-YRMQCX2P.mjs → chunk-YSYKPKKX.mjs} +5 -20
- package/dist/index--b41_E-1.d.mts +440 -0
- package/dist/index--b41_E-1.d.ts +440 -0
- package/dist/{index-DEorAmFu.d.mts → index-Cr-FN-y5.d.mts} +89 -89
- package/dist/{index-DEorAmFu.d.ts → index-Cr-FN-y5.d.ts} +89 -89
- package/dist/index-D4UXMom5.d.mts +429 -0
- package/dist/index-D4UXMom5.d.ts +429 -0
- package/dist/index.d.mts +4 -37
- package/dist/index.d.ts +4 -37
- package/dist/index.js +630 -377
- package/dist/index.mjs +333 -70
- package/dist/opentelemetry/index.d.mts +2 -1
- package/dist/opentelemetry/index.d.ts +2 -1
- package/dist/opentelemetry/index.js +7 -4
- package/dist/opentelemetry/index.mjs +1 -1
- package/package.json +13 -6
- package/LICENSE +0 -21
package/dist/index.mjs
CHANGED
|
@@ -3,24 +3,43 @@ import {
|
|
|
3
3
|
Span,
|
|
4
4
|
SpanType,
|
|
5
5
|
Trace,
|
|
6
|
-
TraceStatus
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
init_types
|
|
12
|
-
} from "./chunk-YRMQCX2P.mjs";
|
|
6
|
+
TraceStatus
|
|
7
|
+
} from "./chunk-FQBYUOJB.mjs";
|
|
8
|
+
|
|
9
|
+
// src/client.ts
|
|
10
|
+
import debug3 from "debug";
|
|
13
11
|
|
|
14
12
|
// src/instrumentation/openai.ts
|
|
13
|
+
import debug from "debug";
|
|
14
|
+
var log = debug("observyze:sdk");
|
|
15
15
|
function wrapOpenAI(client, nwClient) {
|
|
16
16
|
const anyClient = client;
|
|
17
|
+
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
18
|
+
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/openai");
|
|
19
|
+
if (!isAlreadyRedirected) {
|
|
20
|
+
const originalApiKey = anyClient.apiKey;
|
|
21
|
+
anyClient.baseURL = `${nwClient.getConfig().endpoint}/api/v1/proxy/openai/v1`;
|
|
22
|
+
anyClient.apiKey = nwClient.getConfig().apiKey;
|
|
23
|
+
anyClient.defaultHeaders = {
|
|
24
|
+
...anyClient.defaultHeaders,
|
|
25
|
+
"x-provider-key": originalApiKey
|
|
26
|
+
};
|
|
27
|
+
if (nwClient.getConfig().debug) {
|
|
28
|
+
log("[Observyze SDK] Transparently redirected OpenAI client to proxy gateway:", anyClient.baseURL);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
17
32
|
const originalCreate = client.chat.completions.create.bind(client.chat.completions);
|
|
18
33
|
client.chat.completions.create = async function(params, options) {
|
|
34
|
+
const isProxyRedirected = nwClient.getConfig().enableProxyRedirect && anyClient.baseURL?.includes("/api/v1/proxy/openai");
|
|
35
|
+
if (isProxyRedirected) {
|
|
36
|
+
return originalCreate(params, options);
|
|
37
|
+
}
|
|
19
38
|
const trace = nwClient.startTrace(`openai.chat.completions.create`, {
|
|
20
39
|
provider: "openai",
|
|
21
40
|
model: params.model
|
|
22
41
|
});
|
|
23
|
-
const span = trace.startSpan("chat.completions.create",
|
|
42
|
+
const span = trace.startSpan("chat.completions.create", SpanType.LLM);
|
|
24
43
|
span.setMetadata("model", params.model);
|
|
25
44
|
span.setMetadata("provider", "openai");
|
|
26
45
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -104,23 +123,38 @@ function wrapOpenAIStream(stream, span, trace, startTime) {
|
|
|
104
123
|
}
|
|
105
124
|
};
|
|
106
125
|
}
|
|
107
|
-
var init_openai = __esm({
|
|
108
|
-
"src/instrumentation/openai.ts"() {
|
|
109
|
-
"use strict";
|
|
110
|
-
init_types();
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
126
|
|
|
114
127
|
// src/instrumentation/anthropic.ts
|
|
128
|
+
import debug2 from "debug";
|
|
129
|
+
var log2 = debug2("observyze:sdk");
|
|
115
130
|
function wrapAnthropic(client, nwClient) {
|
|
116
131
|
const anyClient = client;
|
|
132
|
+
if (nwClient.getConfig().enableProxyRedirect && anyClient.baseURL && anyClient.apiKey) {
|
|
133
|
+
const isAlreadyRedirected = anyClient.baseURL.includes("/api/v1/proxy/anthropic");
|
|
134
|
+
if (!isAlreadyRedirected) {
|
|
135
|
+
const originalApiKey = anyClient.apiKey;
|
|
136
|
+
anyClient.baseURL = `${nwClient.getConfig().endpoint}/api/v1/proxy/anthropic/v1`;
|
|
137
|
+
anyClient.apiKey = nwClient.getConfig().apiKey;
|
|
138
|
+
anyClient.defaultHeaders = {
|
|
139
|
+
...anyClient.defaultHeaders,
|
|
140
|
+
"x-provider-key": originalApiKey
|
|
141
|
+
};
|
|
142
|
+
if (nwClient.getConfig().debug) {
|
|
143
|
+
log2("[Observyze SDK] Transparently redirected Anthropic client to proxy gateway:", anyClient.baseURL);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
117
147
|
const originalCreate = client.messages.create.bind(client.messages);
|
|
118
148
|
client.messages.create = async function(params, options) {
|
|
149
|
+
const isProxyRedirected = nwClient.getConfig().enableProxyRedirect && anyClient.baseURL?.includes("/api/v1/proxy/anthropic");
|
|
150
|
+
if (isProxyRedirected) {
|
|
151
|
+
return originalCreate(params, options);
|
|
152
|
+
}
|
|
119
153
|
const trace = nwClient.startTrace(`anthropic.messages.create`, {
|
|
120
154
|
provider: "anthropic",
|
|
121
155
|
model: params.model
|
|
122
156
|
});
|
|
123
|
-
const span = trace.startSpan("messages.create",
|
|
157
|
+
const span = trace.startSpan("messages.create", SpanType.LLM);
|
|
124
158
|
span.setMetadata("model", params.model);
|
|
125
159
|
span.setMetadata("provider", "anthropic");
|
|
126
160
|
if (params.temperature !== void 0) span.setMetadata("temperature", params.temperature);
|
|
@@ -231,20 +265,8 @@ function wrapAnthropicStream(stream, span, trace, startTime) {
|
|
|
231
265
|
}
|
|
232
266
|
};
|
|
233
267
|
}
|
|
234
|
-
var init_anthropic = __esm({
|
|
235
|
-
"src/instrumentation/anthropic.ts"() {
|
|
236
|
-
"use strict";
|
|
237
|
-
init_types();
|
|
238
|
-
}
|
|
239
|
-
});
|
|
240
268
|
|
|
241
269
|
// src/instrumentation/index.ts
|
|
242
|
-
var instrumentation_exports = {};
|
|
243
|
-
__export(instrumentation_exports, {
|
|
244
|
-
wrap: () => wrap,
|
|
245
|
-
wrapAnthropic: () => wrapAnthropic,
|
|
246
|
-
wrapOpenAI: () => wrapOpenAI
|
|
247
|
-
});
|
|
248
270
|
function wrap(client, nwClient) {
|
|
249
271
|
if ("chat" in client && client.chat && "completions" in client.chat) {
|
|
250
272
|
return wrapOpenAI(client, nwClient);
|
|
@@ -256,26 +278,26 @@ function wrap(client, nwClient) {
|
|
|
256
278
|
"Observyze SDK: Unsupported client type. Supported clients: OpenAI, Anthropic"
|
|
257
279
|
);
|
|
258
280
|
}
|
|
259
|
-
var init_instrumentation = __esm({
|
|
260
|
-
"src/instrumentation/index.ts"() {
|
|
261
|
-
"use strict";
|
|
262
|
-
init_openai();
|
|
263
|
-
init_anthropic();
|
|
264
|
-
init_openai();
|
|
265
|
-
init_anthropic();
|
|
266
|
-
}
|
|
267
|
-
});
|
|
268
281
|
|
|
269
282
|
// src/client.ts
|
|
270
|
-
|
|
283
|
+
import fs from "fs";
|
|
284
|
+
import path from "path";
|
|
285
|
+
var log3 = debug3("observyze:sdk");
|
|
271
286
|
var DEFAULT_CONFIG = {
|
|
272
|
-
endpoint: "
|
|
287
|
+
endpoint: "http://localhost:3001",
|
|
273
288
|
batchSize: 100,
|
|
274
289
|
flushInterval: 5e3,
|
|
275
290
|
enableAutoInstrumentation: true,
|
|
276
291
|
debug: false,
|
|
277
292
|
dryRun: false,
|
|
278
|
-
enablePiiRedaction: true
|
|
293
|
+
enablePiiRedaction: true,
|
|
294
|
+
hallucinationThreshold: 0.8,
|
|
295
|
+
safetyThreshold: 0.9,
|
|
296
|
+
confidenceThreshold: 0.4,
|
|
297
|
+
evalEndpoint: process.env.EVAL_ENDPOINT || (process.env.NODE_ENV === "production" ? "https://api.observyze.com" : "http://localhost:3001"),
|
|
298
|
+
enableCircuitBreaker: true,
|
|
299
|
+
failClosed: true,
|
|
300
|
+
enableProxyRedirect: true
|
|
279
301
|
};
|
|
280
302
|
var ObservyzeClient = class _ObservyzeClient {
|
|
281
303
|
config;
|
|
@@ -284,7 +306,37 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
284
306
|
isShuttingDown = false;
|
|
285
307
|
MAX_QUEUE_SIZE = 1e3;
|
|
286
308
|
RETRY_DELAYS = [1e3, 2e3, 4e3, 8e3, 16e3, 3e4];
|
|
287
|
-
|
|
309
|
+
/**
|
|
310
|
+
* Parse a JSON API error response and extract trace_id, error code, and message.
|
|
311
|
+
* The api-gateway error handler includes these fields in every error response.
|
|
312
|
+
*/
|
|
313
|
+
static parseApiError(_response, body) {
|
|
314
|
+
try {
|
|
315
|
+
const parsed = JSON.parse(body);
|
|
316
|
+
const error = parsed.error || parsed;
|
|
317
|
+
return {
|
|
318
|
+
traceId: error.trace_id || "unknown",
|
|
319
|
+
code: error.code || "UNKNOWN_ERROR",
|
|
320
|
+
message: error.message || body.slice(0, 200)
|
|
321
|
+
};
|
|
322
|
+
} catch {
|
|
323
|
+
return {
|
|
324
|
+
traceId: "unknown",
|
|
325
|
+
code: "UNKNOWN_ERROR",
|
|
326
|
+
message: body.slice(0, 200)
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Format an API error into a user-friendly message with trace_id for correlation.
|
|
332
|
+
* Example output:
|
|
333
|
+
* "Observyze API error (401 [ref: err_a1b2c3d4]): MISSING_PROVIDER_KEY — No API key configured..."
|
|
334
|
+
*/
|
|
335
|
+
static formatApiError(response, body) {
|
|
336
|
+
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, body);
|
|
337
|
+
const prefix = traceId !== "unknown" ? ` [ref: ${traceId}]` : "";
|
|
338
|
+
return `Observyze API error (${response.status}${prefix}): ${code} \u2014 ${message}`;
|
|
339
|
+
}
|
|
288
340
|
constructor(config) {
|
|
289
341
|
if (!config.apiKey) {
|
|
290
342
|
throw new Error("Observyze SDK: apiKey is required");
|
|
@@ -297,7 +349,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
297
349
|
};
|
|
298
350
|
this.startFlushTimer();
|
|
299
351
|
if (this.config.debug) {
|
|
300
|
-
|
|
352
|
+
log3("[Observyze SDK] Initialized with config:", {
|
|
301
353
|
endpoint: this.config.endpoint,
|
|
302
354
|
batchSize: this.config.batchSize,
|
|
303
355
|
flushInterval: this.config.flushInterval,
|
|
@@ -318,7 +370,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
318
370
|
trace.setMetadataAll(metadata);
|
|
319
371
|
}
|
|
320
372
|
const originalEnd = trace.end.bind(trace);
|
|
321
|
-
trace.end = (status =
|
|
373
|
+
trace.end = (status = TraceStatus.SUCCESS) => {
|
|
322
374
|
originalEnd(status);
|
|
323
375
|
this.bufferTrace(trace);
|
|
324
376
|
};
|
|
@@ -330,23 +382,23 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
330
382
|
bufferTrace(trace) {
|
|
331
383
|
if (!trace.isEnded) {
|
|
332
384
|
if (this.config.debug) {
|
|
333
|
-
|
|
385
|
+
log3.extend("warn")("[Observyze SDK] Attempted to buffer a trace that has not ended");
|
|
334
386
|
}
|
|
335
387
|
return;
|
|
336
388
|
}
|
|
337
389
|
if (this.traceBuffer.length >= this.MAX_QUEUE_SIZE) {
|
|
338
390
|
if (this.config.debug) {
|
|
339
|
-
|
|
391
|
+
log3.extend("warn")(`[Observyze SDK] Queue at max capacity (${this.MAX_QUEUE_SIZE}), dropping oldest trace`);
|
|
340
392
|
}
|
|
341
393
|
this.traceBuffer.shift();
|
|
342
394
|
}
|
|
343
395
|
this.traceBuffer.push(trace);
|
|
344
396
|
if (this.config.debug) {
|
|
345
|
-
|
|
397
|
+
log3(`[Observyze SDK] Buffered trace ${trace.id} (${this.traceBuffer.length}/${this.config.batchSize})`);
|
|
346
398
|
}
|
|
347
399
|
if (this.traceBuffer.length >= this.config.batchSize) {
|
|
348
400
|
this.flush().catch((err) => {
|
|
349
|
-
|
|
401
|
+
log3.extend("error")("[Observyze SDK] Error flushing buffer:", err);
|
|
350
402
|
});
|
|
351
403
|
}
|
|
352
404
|
}
|
|
@@ -360,7 +412,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
360
412
|
this.flushTimer = setInterval(() => {
|
|
361
413
|
if (this.traceBuffer.length > 0) {
|
|
362
414
|
this.flush().catch((err) => {
|
|
363
|
-
|
|
415
|
+
log3.extend("error")("[Observyze SDK] Error in auto-flush:", err);
|
|
364
416
|
});
|
|
365
417
|
}
|
|
366
418
|
}, this.config.flushInterval);
|
|
@@ -377,11 +429,11 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
377
429
|
}
|
|
378
430
|
const tracesToSend = this.traceBuffer.splice(0, this.config.batchSize);
|
|
379
431
|
if (this.config.debug) {
|
|
380
|
-
|
|
432
|
+
log3(`[Observyze SDK] Flushing ${tracesToSend.length} traces`);
|
|
381
433
|
}
|
|
382
434
|
if (this.config.dryRun) {
|
|
383
435
|
if (this.config.debug) {
|
|
384
|
-
|
|
436
|
+
log3("[Observyze SDK] Dry-run mode: traces not sent");
|
|
385
437
|
}
|
|
386
438
|
return;
|
|
387
439
|
}
|
|
@@ -392,15 +444,15 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
392
444
|
if (remainingSpace > 0) {
|
|
393
445
|
this.traceBuffer.unshift(...tracesToSend.slice(0, remainingSpace));
|
|
394
446
|
if (this.config.debug) {
|
|
395
|
-
|
|
447
|
+
log3(`[Observyze SDK] Re-queued ${Math.min(tracesToSend.length, remainingSpace)} traces after failure`);
|
|
396
448
|
}
|
|
397
449
|
} else {
|
|
398
450
|
if (this.config.debug) {
|
|
399
|
-
|
|
451
|
+
log3.extend("warn")(`[Observyze SDK] Queue full, dropped ${tracesToSend.length} traces`);
|
|
400
452
|
}
|
|
401
453
|
}
|
|
402
454
|
if (this.config.debug) {
|
|
403
|
-
|
|
455
|
+
log3.extend("error")("[Observyze SDK] Failed to send traces after retries:", error);
|
|
404
456
|
}
|
|
405
457
|
throw error;
|
|
406
458
|
}
|
|
@@ -430,10 +482,11 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
430
482
|
});
|
|
431
483
|
if (!response.ok) {
|
|
432
484
|
const errorBody = await response.text();
|
|
433
|
-
|
|
485
|
+
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
486
|
+
throw new Error(`[Observyze SDK] Trace ingestion failed. ${formatted}`);
|
|
434
487
|
}
|
|
435
488
|
if (this.config.debug) {
|
|
436
|
-
|
|
489
|
+
log3(`[Observyze SDK] Successfully sent ${traces.length} traces${attempt > 0 ? ` (after ${attempt} retries)` : ""}`);
|
|
437
490
|
}
|
|
438
491
|
return;
|
|
439
492
|
} catch (error) {
|
|
@@ -443,7 +496,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
443
496
|
}
|
|
444
497
|
const delay = this.RETRY_DELAYS[attempt];
|
|
445
498
|
if (this.config.debug) {
|
|
446
|
-
|
|
499
|
+
log3.extend("warn")(`[Observyze SDK] Attempt ${attempt + 1} failed, retrying in ${delay}ms...`, error);
|
|
447
500
|
}
|
|
448
501
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
449
502
|
}
|
|
@@ -459,7 +512,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
459
512
|
}
|
|
460
513
|
this.isShuttingDown = true;
|
|
461
514
|
if (this.config.debug) {
|
|
462
|
-
|
|
515
|
+
log3("[Observyze SDK] Shutting down...");
|
|
463
516
|
}
|
|
464
517
|
if (this.flushTimer) {
|
|
465
518
|
clearInterval(this.flushTimer);
|
|
@@ -468,10 +521,10 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
468
521
|
try {
|
|
469
522
|
await this.flush();
|
|
470
523
|
} catch (error) {
|
|
471
|
-
|
|
524
|
+
log3.extend("error")("[Observyze SDK] Error during shutdown flush:", error);
|
|
472
525
|
}
|
|
473
526
|
if (this.config.debug) {
|
|
474
|
-
|
|
527
|
+
log3("[Observyze SDK] Shutdown complete");
|
|
475
528
|
}
|
|
476
529
|
}
|
|
477
530
|
/**
|
|
@@ -508,8 +561,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
508
561
|
* ```
|
|
509
562
|
*/
|
|
510
563
|
wrap(client) {
|
|
511
|
-
|
|
512
|
-
return wrapClient(client, this);
|
|
564
|
+
return wrap(client, this);
|
|
513
565
|
}
|
|
514
566
|
/**
|
|
515
567
|
* Sync local agent .history file to Observyze cloud
|
|
@@ -520,8 +572,6 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
520
572
|
if (typeof process === "undefined" || !process.versions?.node) {
|
|
521
573
|
throw new Error("syncLocalHistory is only available in Node.js environments");
|
|
522
574
|
}
|
|
523
|
-
const fs = __require("fs");
|
|
524
|
-
const path = __require("path");
|
|
525
575
|
const fullPath = path.resolve(process.cwd(), filePath);
|
|
526
576
|
if (!fs.existsSync(fullPath)) {
|
|
527
577
|
throw new Error(`History file not found: ${fullPath}`);
|
|
@@ -537,7 +587,7 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
537
587
|
items = [items];
|
|
538
588
|
}
|
|
539
589
|
if (this.config.debug) {
|
|
540
|
-
|
|
590
|
+
log3(`[Observyze SDK] Syncing ${items.length} traces from ${filePath}`);
|
|
541
591
|
}
|
|
542
592
|
for (let i = 0; i < items.length; i += this.config.batchSize) {
|
|
543
593
|
const batch = items.slice(i, i + this.config.batchSize);
|
|
@@ -551,14 +601,15 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
551
601
|
});
|
|
552
602
|
if (!response.ok) {
|
|
553
603
|
const errorBody = await response.text();
|
|
554
|
-
|
|
604
|
+
const formatted = _ObservyzeClient.formatApiError(response, errorBody);
|
|
605
|
+
throw new Error(`[Observyze SDK] Local history sync failed. ${formatted}`);
|
|
555
606
|
}
|
|
556
607
|
if (this.config.debug) {
|
|
557
|
-
|
|
608
|
+
log3(`[Observyze SDK] Synced batch of ${batch.length} traces from local history`);
|
|
558
609
|
}
|
|
559
610
|
}
|
|
560
611
|
} catch (err) {
|
|
561
|
-
|
|
612
|
+
log3.extend("error")("[Observyze SDK] Failed to sync local history:", err);
|
|
562
613
|
throw err;
|
|
563
614
|
}
|
|
564
615
|
}
|
|
@@ -655,11 +706,223 @@ var ObservyzeClient = class _ObservyzeClient {
|
|
|
655
706
|
}
|
|
656
707
|
return data;
|
|
657
708
|
}
|
|
709
|
+
/**
|
|
710
|
+
* Phase 4: Autonomous Circuit Breakers (Requirement 4.1)
|
|
711
|
+
* Evaluate a trace or text for hallucination in real-time.
|
|
712
|
+
* If hallucination score > hallucinationThreshold, the SDK blocks execution.
|
|
713
|
+
*
|
|
714
|
+
* Returns GuardrailResult with score: null when evaluation couldn't be performed.
|
|
715
|
+
* In failClosed mode, null scores result in blocked execution.
|
|
716
|
+
* In failOpen mode, null scores allow execution through.
|
|
717
|
+
*/
|
|
718
|
+
async checkGuardrails(content) {
|
|
719
|
+
if (!this.config.enableCircuitBreaker) {
|
|
720
|
+
return { pass: true, score: 0, safetyScore: 0, evaluationSource: "disabled" };
|
|
721
|
+
}
|
|
722
|
+
try {
|
|
723
|
+
if (this.config.debug) {
|
|
724
|
+
log3(`[Observyze Guardrail] Analyzing payload for hallucination anomalies...`);
|
|
725
|
+
}
|
|
726
|
+
const evalEndpoint = this.config.evalEndpoint;
|
|
727
|
+
const payload = typeof content === "string" ? { text: content, organization_id: this.config.organizationId } : { trace: content, organization_id: this.config.organizationId };
|
|
728
|
+
const controller = new AbortController();
|
|
729
|
+
const timeout = setTimeout(() => controller.abort(), 5e3);
|
|
730
|
+
let evalResult = null;
|
|
731
|
+
try {
|
|
732
|
+
const response = await fetch(`${evalEndpoint}/api/v1/evaluate/hallucination`, {
|
|
733
|
+
method: "POST",
|
|
734
|
+
headers: {
|
|
735
|
+
"Content-Type": "application/json",
|
|
736
|
+
"Authorization": `Bearer ${this.config.apiKey}`
|
|
737
|
+
},
|
|
738
|
+
body: JSON.stringify(payload),
|
|
739
|
+
signal: controller.signal
|
|
740
|
+
});
|
|
741
|
+
clearTimeout(timeout);
|
|
742
|
+
if (response.ok) {
|
|
743
|
+
evalResult = await response.json();
|
|
744
|
+
} else {
|
|
745
|
+
const errorBody = await response.text();
|
|
746
|
+
const { traceId, code, message } = _ObservyzeClient.parseApiError(response, errorBody);
|
|
747
|
+
if (this.config.debug) {
|
|
748
|
+
log3.extend("warn")(`[Observyze Guardrail] Eval returned ${response.status} [${code}] (ref: ${traceId}): ${message}`);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
} catch (fetchError) {
|
|
752
|
+
clearTimeout(timeout);
|
|
753
|
+
if (fetchError.name === "AbortError") {
|
|
754
|
+
if (this.config.debug) {
|
|
755
|
+
log3.extend("warn")("[Observyze Guardrail] Evaluation timed out after 5s");
|
|
756
|
+
}
|
|
757
|
+
} else if (this.config.debug) {
|
|
758
|
+
log3.extend("warn")("[Observyze Guardrail] Evaluation request failed:", fetchError.message);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
let hallucinationScore = 0;
|
|
762
|
+
let safetyScore = 0;
|
|
763
|
+
let evaluationSource = "live";
|
|
764
|
+
let confidence = null;
|
|
765
|
+
if (evalResult) {
|
|
766
|
+
hallucinationScore = evalResult.score ?? evalResult.hallucination_score ?? null;
|
|
767
|
+
safetyScore = evalResult.safety_score ?? 0;
|
|
768
|
+
evaluationSource = evalResult.evaluation_source ?? "live";
|
|
769
|
+
confidence = evalResult.confidence ?? null;
|
|
770
|
+
if (hallucinationScore === null) {
|
|
771
|
+
if (this.config.failClosed) {
|
|
772
|
+
if (this.config.debug) {
|
|
773
|
+
log3.extend("warn")("[Observyze Guardrail] Eval returned null score \u2014 failing closed (blocking)");
|
|
774
|
+
}
|
|
775
|
+
return {
|
|
776
|
+
pass: false,
|
|
777
|
+
score: null,
|
|
778
|
+
confidence: null,
|
|
779
|
+
safetyScore: null,
|
|
780
|
+
evaluationSource: "error",
|
|
781
|
+
fallbackReason: evalResult.message || "Evaluation failed to produce a score",
|
|
782
|
+
reason: "Evaluation service failed to produce a score. Fail-closed: execution blocked."
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
if (this.config.debug) {
|
|
786
|
+
log3("[Observyze Guardrail] Eval returned null score \u2014 allowing (fail-open)");
|
|
787
|
+
}
|
|
788
|
+
return {
|
|
789
|
+
pass: true,
|
|
790
|
+
score: null,
|
|
791
|
+
confidence: null,
|
|
792
|
+
safetyScore: null,
|
|
793
|
+
evaluationSource: "error",
|
|
794
|
+
fallbackReason: evalResult.message || "Evaluation failed to produce a score"
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
} else if (this.config.failClosed) {
|
|
798
|
+
if (this.config.debug) {
|
|
799
|
+
log3.extend("warn")("[Observyze Guardrail] Eval unavailable \u2014 failing closed (blocking)");
|
|
800
|
+
}
|
|
801
|
+
return {
|
|
802
|
+
pass: false,
|
|
803
|
+
score: null,
|
|
804
|
+
confidence: null,
|
|
805
|
+
safetyScore: null,
|
|
806
|
+
evaluationSource: "error",
|
|
807
|
+
fallbackReason: "Evaluation service unreachable",
|
|
808
|
+
reason: "Evaluation service unreachable. Fail-closed: execution blocked."
|
|
809
|
+
};
|
|
810
|
+
} else {
|
|
811
|
+
if (this.config.debug) {
|
|
812
|
+
log3("[Observyze Guardrail] Eval unavailable \u2014 allowing (fail-open)");
|
|
813
|
+
}
|
|
814
|
+
return {
|
|
815
|
+
pass: true,
|
|
816
|
+
score: null,
|
|
817
|
+
confidence: null,
|
|
818
|
+
safetyScore: null,
|
|
819
|
+
evaluationSource: "error",
|
|
820
|
+
fallbackReason: "Evaluation service unreachable"
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
const hallThreshold = this.config.hallucinationThreshold;
|
|
824
|
+
const safeThreshold = this.config.safetyThreshold;
|
|
825
|
+
const confThreshold = this.config.confidenceThreshold;
|
|
826
|
+
if (confidence !== null && confidence < confThreshold) {
|
|
827
|
+
if (hallucinationScore >= hallThreshold) {
|
|
828
|
+
if (this.config.debug) {
|
|
829
|
+
log3.extend("warn")(`[Observyze Guardrail] High score (${hallucinationScore.toFixed(2)}) but low confidence (${confidence.toFixed(2)}). Alerting only.`);
|
|
830
|
+
}
|
|
831
|
+
return {
|
|
832
|
+
pass: true,
|
|
833
|
+
score: hallucinationScore,
|
|
834
|
+
confidence,
|
|
835
|
+
safetyScore,
|
|
836
|
+
evaluationSource,
|
|
837
|
+
reason: `Score ${hallucinationScore.toFixed(2)} but confidence ${confidence.toFixed(2)} is low. Execution allowed with alert.`
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
if (hallucinationScore >= hallThreshold) {
|
|
842
|
+
if (this.config.debug) {
|
|
843
|
+
log3.extend("warn")(`[Observyze Guardrail] Hallucination circuit breached! Score: ${hallucinationScore.toFixed(2)} >= ${hallThreshold}`);
|
|
844
|
+
}
|
|
845
|
+
return {
|
|
846
|
+
pass: false,
|
|
847
|
+
score: hallucinationScore,
|
|
848
|
+
confidence,
|
|
849
|
+
safetyScore,
|
|
850
|
+
evaluationSource,
|
|
851
|
+
reason: `Hallucination score ${hallucinationScore.toFixed(2)} exceeds threshold ${hallThreshold}. Execution blocked for human review.`
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
if (safetyScore !== null && safetyScore >= safeThreshold) {
|
|
855
|
+
if (this.config.debug) {
|
|
856
|
+
log3.extend("warn")(`[Observyze Guardrail] Safety circuit breached! Score: ${safetyScore.toFixed(2)} >= ${safeThreshold}`);
|
|
857
|
+
}
|
|
858
|
+
return {
|
|
859
|
+
pass: false,
|
|
860
|
+
score: hallucinationScore,
|
|
861
|
+
confidence,
|
|
862
|
+
safetyScore,
|
|
863
|
+
evaluationSource,
|
|
864
|
+
reason: `Safety score ${safetyScore.toFixed(2)} exceeds threshold ${safeThreshold}. Execution blocked for safety review.`
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
return { pass: true, score: hallucinationScore, confidence, safetyScore, evaluationSource };
|
|
868
|
+
} catch (err) {
|
|
869
|
+
log3.extend("error")("[Observyze Guardrail] Failed to evaluate:", err);
|
|
870
|
+
if (this.config.failClosed) {
|
|
871
|
+
return { pass: false, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: "Guardrail exception", reason: "Guardrail error \u2014 fail-closed: execution blocked." };
|
|
872
|
+
}
|
|
873
|
+
return { pass: true, score: null, confidence: null, safetyScore: null, evaluationSource: "error", fallbackReason: "Guardrail exception" };
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* Phase 4: Autonomous Circuit Breakers
|
|
878
|
+
* Execute an agent action wrapped with the Circuit Breaker.
|
|
879
|
+
* Pauses execution if hallucination score >= hallucinationThreshold and requests human review.
|
|
880
|
+
* @throws Error when execution is blocked by circuit breaker
|
|
881
|
+
*/
|
|
882
|
+
async executeWithCircuitBreaker(agentExecution, traceContext) {
|
|
883
|
+
if (!this.config.enableCircuitBreaker) {
|
|
884
|
+
return await agentExecution();
|
|
885
|
+
}
|
|
886
|
+
const guardResult = await this.checkGuardrails(traceContext || "execution context");
|
|
887
|
+
if (!guardResult.pass) {
|
|
888
|
+
const error = new Error(`[Observyze] Execution Blocked by Autonomous Circuit Breaker. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}. Reason: ${guardResult.reason}. Human approval required before agent can continue.`);
|
|
889
|
+
if (this.config.debug) {
|
|
890
|
+
log3.extend("error")("[Observyze CircuitBreaker] Execution blocked:", error.message);
|
|
891
|
+
}
|
|
892
|
+
throw error;
|
|
893
|
+
}
|
|
894
|
+
if (this.config.debug) {
|
|
895
|
+
log3(`[Observyze CircuitBreaker] Execution allowed. Hallucination: ${guardResult.score?.toFixed(2) ?? "N/A"}, Safety: ${(guardResult.safetyScore ?? 0)?.toFixed(2) ?? "N/A"}`);
|
|
896
|
+
}
|
|
897
|
+
return await agentExecution();
|
|
898
|
+
}
|
|
899
|
+
/**
|
|
900
|
+
* Phase 4: Bug Bounty Protocol (Automated) (Requirement 4.2)
|
|
901
|
+
* Automatically shard persistent failure cases to external security researcher endpoints (e.g. HackerOne wrapper)
|
|
902
|
+
*/
|
|
903
|
+
async reportBugBounty(traceId, securityEndpoint, failureContext) {
|
|
904
|
+
try {
|
|
905
|
+
if (this.config.debug) {
|
|
906
|
+
log3(`[Observyze SDK] Sharding persistent failure case ${traceId} to Bug Bounty Protocol endpoint...`);
|
|
907
|
+
}
|
|
908
|
+
await fetch(securityEndpoint, {
|
|
909
|
+
method: "POST",
|
|
910
|
+
headers: { "Content-Type": "application/json" },
|
|
911
|
+
body: JSON.stringify({
|
|
912
|
+
alert: "persistent_failure_sharded",
|
|
913
|
+
trace_id: traceId,
|
|
914
|
+
context: failureContext,
|
|
915
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
916
|
+
})
|
|
917
|
+
});
|
|
918
|
+
if (this.config.debug) {
|
|
919
|
+
log3(`[Observyze SDK] Bug Bounty payload successfully transmitted.`);
|
|
920
|
+
}
|
|
921
|
+
} catch (err) {
|
|
922
|
+
log3.extend("error")("[Observyze Bug Bounty] Failed to shard failure case:", err);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
658
925
|
};
|
|
659
|
-
|
|
660
|
-
// src/index.ts
|
|
661
|
-
init_types();
|
|
662
|
-
init_instrumentation();
|
|
663
926
|
export {
|
|
664
927
|
ObservyzeClient,
|
|
665
928
|
ObservyzeSpanExporter,
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
export { a as ObservyzeExporterConfig, b as ObservyzeSpanExporter } from '../index-
|
|
1
|
+
export { a as ObservyzeExporterConfig, b as ObservyzeSpanExporter } from '../index--b41_E-1.mjs';
|
|
2
|
+
import '@observyze/types';
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
export { a as ObservyzeExporterConfig, b as ObservyzeSpanExporter } from '../index-
|
|
1
|
+
export { a as ObservyzeExporterConfig, b as ObservyzeSpanExporter } from '../index--b41_E-1.js';
|
|
2
|
+
import '@observyze/types';
|
|
@@ -24,6 +24,9 @@ __export(opentelemetry_exports, {
|
|
|
24
24
|
});
|
|
25
25
|
module.exports = __toCommonJS(opentelemetry_exports);
|
|
26
26
|
|
|
27
|
+
// src/types.ts
|
|
28
|
+
var import_types = require("@observyze/types");
|
|
29
|
+
|
|
27
30
|
// src/trace.ts
|
|
28
31
|
var import_crypto = require("crypto");
|
|
29
32
|
function generateId() {
|
|
@@ -127,7 +130,7 @@ var Trace = class {
|
|
|
127
130
|
organization_id: organizationId,
|
|
128
131
|
project_id: projectId,
|
|
129
132
|
name,
|
|
130
|
-
status:
|
|
133
|
+
status: import_types.TraceStatus.RUNNING,
|
|
131
134
|
start_time: new Date(this.startTime),
|
|
132
135
|
end_time: new Date(this.startTime),
|
|
133
136
|
// Will be updated on end()
|
|
@@ -195,7 +198,7 @@ var Trace = class {
|
|
|
195
198
|
/**
|
|
196
199
|
* End the trace with a final status
|
|
197
200
|
*/
|
|
198
|
-
end(status =
|
|
201
|
+
end(status = import_types.TraceStatus.SUCCESS) {
|
|
199
202
|
if (this.ended) {
|
|
200
203
|
return;
|
|
201
204
|
}
|
|
@@ -296,9 +299,9 @@ var ObservyzeSpanExporter = class {
|
|
|
296
299
|
const statusCode = status?.code;
|
|
297
300
|
if (statusCode === 2) {
|
|
298
301
|
oSpan.setError(new Error(status?.message || "OTel span error"));
|
|
299
|
-
trace.end(
|
|
302
|
+
trace.end(import_types.TraceStatus.ERROR);
|
|
300
303
|
} else {
|
|
301
|
-
trace.end(
|
|
304
|
+
trace.end(import_types.TraceStatus.SUCCESS);
|
|
302
305
|
}
|
|
303
306
|
}
|
|
304
307
|
resultCallback({ code: 0 });
|