@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +1002 -120
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +315 -6
- package/dist/cjs/proto/index.cjs +7 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +31 -2
- package/dist/esm/cli.d.ts +8 -1
- package/dist/esm/cli.js +536 -47
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +315 -6
- package/dist/esm/index.js +978 -120
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +31 -2
- package/dist/esm/proto/index.js +5 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +3 -3
package/dist/esm/index.js
CHANGED
|
@@ -14,12 +14,14 @@ var REASON_MISSING_EVIDENCE = "missing_evidence";
|
|
|
14
14
|
var REASON_SHORT_CIRCUITED = "short_circuited";
|
|
15
15
|
var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
|
|
16
16
|
var REASON_JUDGE_ABSTAINED = "judge_abstained";
|
|
17
|
+
var REASON_REPLAY_MISS = "replay_miss";
|
|
17
18
|
var OUTCOME_PASS = "pass";
|
|
18
19
|
var OUTCOME_FAIL = "fail";
|
|
19
20
|
var OUTCOME_SKIP = "skip";
|
|
20
21
|
var TYPE_SCHEMA = "schema";
|
|
21
22
|
var TYPE_CONSTRAINT = "constraint";
|
|
22
23
|
var TYPE_TRACE = "trace";
|
|
24
|
+
var TYPE_EFFECT = "effect";
|
|
23
25
|
var TYPE_CONTENT = "content";
|
|
24
26
|
var TYPE_EMBEDDING = "embedding";
|
|
25
27
|
var TYPE_LLM_JUDGE = "llm_judge";
|
|
@@ -57,6 +59,7 @@ function layerForType(assertionType) {
|
|
|
57
59
|
return 2;
|
|
58
60
|
case "trace":
|
|
59
61
|
case "trace_tree":
|
|
62
|
+
case "effect":
|
|
60
63
|
return 3;
|
|
61
64
|
case "content":
|
|
62
65
|
return 4;
|
|
@@ -216,8 +219,8 @@ function extractId(response) {
|
|
|
216
219
|
}
|
|
217
220
|
|
|
218
221
|
// src/version.ts
|
|
219
|
-
var VERSION = "1.0.0-rc.
|
|
220
|
-
var ENGINE_VERSION = "1.0.0-rc.
|
|
222
|
+
var VERSION = "1.0.0-rc.4";
|
|
223
|
+
var ENGINE_VERSION = "1.0.0-rc.4";
|
|
221
224
|
|
|
222
225
|
// src/engine-manager.ts
|
|
223
226
|
import { spawn, spawnSync } from "child_process";
|
|
@@ -285,7 +288,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
285
288
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
286
289
|
);
|
|
287
290
|
}
|
|
288
|
-
return new Promise((
|
|
291
|
+
return new Promise((resolve4, reject) => {
|
|
289
292
|
const client = url.startsWith("https:") ? https : http;
|
|
290
293
|
client.get(url, (res) => {
|
|
291
294
|
const status = res.statusCode ?? 0;
|
|
@@ -296,7 +299,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
296
299
|
return;
|
|
297
300
|
}
|
|
298
301
|
res.resume();
|
|
299
|
-
|
|
302
|
+
resolve4(httpsGet(location, redirectCount + 1));
|
|
300
303
|
return;
|
|
301
304
|
}
|
|
302
305
|
if (status < 200 || status >= 300) {
|
|
@@ -306,7 +309,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
306
309
|
}
|
|
307
310
|
const chunks = [];
|
|
308
311
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
309
|
-
res.on("end", () =>
|
|
312
|
+
res.on("end", () => resolve4(Buffer.concat(chunks)));
|
|
310
313
|
res.on("error", reject);
|
|
311
314
|
}).on("error", reject);
|
|
312
315
|
});
|
|
@@ -437,8 +440,8 @@ Install options:
|
|
|
437
440
|
}
|
|
438
441
|
return [await downloadEngine(), "download"];
|
|
439
442
|
}
|
|
440
|
-
async function resolveEngine(explicitPath,
|
|
441
|
-
const [enginePath, source] = await locateEngine(explicitPath,
|
|
443
|
+
async function resolveEngine(explicitPath, options2 = {}) {
|
|
444
|
+
const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
|
|
442
445
|
const found = engineVersion(enginePath);
|
|
443
446
|
if (found !== ENGINE_VERSION) {
|
|
444
447
|
throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
|
|
@@ -453,9 +456,9 @@ var EngineManager = class {
|
|
|
453
456
|
initialized = false;
|
|
454
457
|
requestId = 0;
|
|
455
458
|
initResult = null;
|
|
456
|
-
constructor(
|
|
457
|
-
this.enginePath =
|
|
458
|
-
this.logLevel =
|
|
459
|
+
constructor(options2) {
|
|
460
|
+
this.enginePath = options2?.enginePath ?? "";
|
|
461
|
+
this.logLevel = options2?.logLevel ?? "warn";
|
|
459
462
|
}
|
|
460
463
|
async start() {
|
|
461
464
|
this.enginePath = await resolveEngine(this.enginePath || void 0);
|
|
@@ -533,11 +536,11 @@ var EngineManager = class {
|
|
|
533
536
|
}
|
|
534
537
|
this.requestId += 1;
|
|
535
538
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
536
|
-
return new Promise((
|
|
539
|
+
return new Promise((resolve4, reject) => {
|
|
537
540
|
const onLine = (line) => {
|
|
538
541
|
try {
|
|
539
542
|
const response = decodeResponse(line);
|
|
540
|
-
|
|
543
|
+
resolve4(extractResult(response));
|
|
541
544
|
} catch (err) {
|
|
542
545
|
reject(err);
|
|
543
546
|
}
|
|
@@ -571,17 +574,17 @@ var EngineManager = class {
|
|
|
571
574
|
// src/config.ts
|
|
572
575
|
var _simulationMode = false;
|
|
573
576
|
var _sampleRate = 0;
|
|
574
|
-
function config(
|
|
575
|
-
if (
|
|
576
|
-
_simulationMode =
|
|
577
|
+
function config(options2) {
|
|
578
|
+
if (options2.simulation !== void 0) {
|
|
579
|
+
_simulationMode = options2.simulation;
|
|
577
580
|
}
|
|
578
|
-
if (
|
|
579
|
-
if (
|
|
581
|
+
if (options2.sampleRate !== void 0) {
|
|
582
|
+
if (options2.sampleRate < 0 || options2.sampleRate > 1) {
|
|
580
583
|
throw new RangeError(
|
|
581
|
-
`sampleRate must be between 0.0 and 1.0, got ${
|
|
584
|
+
`sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
|
|
582
585
|
);
|
|
583
586
|
}
|
|
584
|
-
_sampleRate =
|
|
587
|
+
_sampleRate = options2.sampleRate;
|
|
585
588
|
}
|
|
586
589
|
}
|
|
587
590
|
function isSimulationMode() {
|
|
@@ -737,16 +740,16 @@ var AttestClient = class {
|
|
|
737
740
|
desyncThreshold;
|
|
738
741
|
desyncWindowMs;
|
|
739
742
|
desynced = false;
|
|
740
|
-
constructor(engine,
|
|
743
|
+
constructor(engine, options2) {
|
|
741
744
|
this.engine = engine;
|
|
742
|
-
this.logger =
|
|
745
|
+
this.logger = options2?.logger ?? envLoggerDefault();
|
|
743
746
|
this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
|
|
744
|
-
|
|
747
|
+
options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
|
|
745
748
|
);
|
|
746
|
-
this.desyncThreshold =
|
|
747
|
-
this.desyncWindowMs =
|
|
748
|
-
if (
|
|
749
|
-
this.diagnosticListeners.add(
|
|
749
|
+
this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
|
|
750
|
+
this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
|
|
751
|
+
if (options2?.onDiagnostic !== void 0) {
|
|
752
|
+
this.diagnosticListeners.add(options2.onDiagnostic);
|
|
750
753
|
}
|
|
751
754
|
}
|
|
752
755
|
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
@@ -904,12 +907,12 @@ var AttestClient = class {
|
|
|
904
907
|
throw new Error("Engine process not started.");
|
|
905
908
|
}
|
|
906
909
|
let capturedReqId;
|
|
907
|
-
const requestPromise = new Promise((
|
|
910
|
+
const requestPromise = new Promise((resolve4, reject) => {
|
|
908
911
|
this.writeChain = this.writeChain.then(() => {
|
|
909
912
|
this.requestId += 1;
|
|
910
913
|
const reqId = this.requestId;
|
|
911
914
|
capturedReqId = reqId;
|
|
912
|
-
this.pending.set(reqId, { resolve, reject });
|
|
915
|
+
this.pending.set(reqId, { resolve: resolve4, reject });
|
|
913
916
|
const requestStr = encodeRequest(reqId, method, params);
|
|
914
917
|
cp.stdin.write(requestStr, (err) => {
|
|
915
918
|
if (err) {
|
|
@@ -927,23 +930,24 @@ var AttestClient = class {
|
|
|
927
930
|
throw err;
|
|
928
931
|
});
|
|
929
932
|
}
|
|
930
|
-
async evaluateBatch(trace, assertions,
|
|
933
|
+
async evaluateBatch(trace, assertions, options2) {
|
|
931
934
|
if (isSimulationMode()) {
|
|
932
935
|
requireSimulationCapability(this.engine["initResult"]);
|
|
933
936
|
}
|
|
934
937
|
const params = {
|
|
935
938
|
trace,
|
|
936
939
|
assertions: [...assertions],
|
|
937
|
-
...
|
|
940
|
+
...options2?.budget !== void 0 ? { budget: options2.budget } : {},
|
|
941
|
+
...options2?.context !== void 0 ? { context: options2.context } : {}
|
|
938
942
|
};
|
|
939
943
|
const raw = await this.sendRequest(
|
|
940
944
|
"evaluate_batch",
|
|
941
945
|
params,
|
|
942
|
-
|
|
946
|
+
options2?.timeout
|
|
943
947
|
);
|
|
944
948
|
return { ...raw, usage: parseUsage(raw["usage"]) };
|
|
945
949
|
}
|
|
946
|
-
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation,
|
|
950
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
|
|
947
951
|
const result = {
|
|
948
952
|
assertion_id: assertionId2,
|
|
949
953
|
status,
|
|
@@ -961,11 +965,11 @@ var AttestClient = class {
|
|
|
961
965
|
const raw = await this.sendRequest(
|
|
962
966
|
"submit_plugin_result",
|
|
963
967
|
params,
|
|
964
|
-
|
|
968
|
+
options2?.timeout
|
|
965
969
|
);
|
|
966
970
|
return Boolean(raw?.accepted ?? false);
|
|
967
971
|
}
|
|
968
|
-
async queryDrift(assertionId2, windowSize = 50,
|
|
972
|
+
async queryDrift(assertionId2, windowSize = 50, options2) {
|
|
969
973
|
if (isSimulationMode()) {
|
|
970
974
|
return {
|
|
971
975
|
assertion_id: assertionId2,
|
|
@@ -980,11 +984,11 @@ var AttestClient = class {
|
|
|
980
984
|
const raw = await this.sendRequest(
|
|
981
985
|
"query_drift",
|
|
982
986
|
{ assertion_id: assertionId2, window_size: windowSize },
|
|
983
|
-
|
|
987
|
+
options2?.timeout
|
|
984
988
|
);
|
|
985
989
|
return raw.report;
|
|
986
990
|
}
|
|
987
|
-
async generateUserMessage(persona, conversationHistory, faultConfig,
|
|
991
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
|
|
988
992
|
if (isSimulationMode()) {
|
|
989
993
|
return `[simulation] Hello from ${persona.name}`;
|
|
990
994
|
}
|
|
@@ -998,7 +1002,7 @@ var AttestClient = class {
|
|
|
998
1002
|
const raw = await this.sendRequest(
|
|
999
1003
|
"generate_user_message",
|
|
1000
1004
|
params,
|
|
1001
|
-
|
|
1005
|
+
options2?.timeout
|
|
1002
1006
|
);
|
|
1003
1007
|
return String(raw.message);
|
|
1004
1008
|
}
|
|
@@ -1043,33 +1047,33 @@ var TraceBuilder = class {
|
|
|
1043
1047
|
this.input = input;
|
|
1044
1048
|
return this;
|
|
1045
1049
|
}
|
|
1046
|
-
addLlmCall(name,
|
|
1050
|
+
addLlmCall(name, options2) {
|
|
1047
1051
|
this.steps.push({
|
|
1048
1052
|
type: STEP_LLM_CALL,
|
|
1049
1053
|
name,
|
|
1050
|
-
args:
|
|
1051
|
-
result:
|
|
1052
|
-
metadata:
|
|
1054
|
+
args: options2?.args,
|
|
1055
|
+
result: options2?.result,
|
|
1056
|
+
metadata: options2?.metadata
|
|
1053
1057
|
});
|
|
1054
1058
|
return this;
|
|
1055
1059
|
}
|
|
1056
|
-
addToolCall(name,
|
|
1060
|
+
addToolCall(name, options2) {
|
|
1057
1061
|
this.steps.push({
|
|
1058
1062
|
type: STEP_TOOL_CALL,
|
|
1059
1063
|
name,
|
|
1060
|
-
args:
|
|
1061
|
-
result:
|
|
1062
|
-
metadata:
|
|
1064
|
+
args: options2?.args,
|
|
1065
|
+
result: options2?.result,
|
|
1066
|
+
metadata: options2?.metadata
|
|
1063
1067
|
});
|
|
1064
1068
|
return this;
|
|
1065
1069
|
}
|
|
1066
|
-
addRetrieval(name,
|
|
1070
|
+
addRetrieval(name, options2) {
|
|
1067
1071
|
this.steps.push({
|
|
1068
1072
|
type: STEP_RETRIEVAL,
|
|
1069
1073
|
name,
|
|
1070
|
-
args:
|
|
1071
|
-
result:
|
|
1072
|
-
metadata:
|
|
1074
|
+
args: options2?.args,
|
|
1075
|
+
result: options2?.result,
|
|
1076
|
+
metadata: options2?.metadata
|
|
1073
1077
|
});
|
|
1074
1078
|
return this;
|
|
1075
1079
|
}
|
|
@@ -1147,10 +1151,10 @@ var AgentResult = class {
|
|
|
1147
1151
|
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
1148
1152
|
* when only soft_fail rows are present (see `softFailures`).
|
|
1149
1153
|
*/
|
|
1150
|
-
outcome(
|
|
1154
|
+
outcome(options2) {
|
|
1151
1155
|
if (this.hardFailures.length > 0) return OUTCOME_FAIL;
|
|
1152
1156
|
if (this.notEvaluated.length > 0) {
|
|
1153
|
-
return
|
|
1157
|
+
return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
|
|
1154
1158
|
}
|
|
1155
1159
|
return OUTCOME_PASS;
|
|
1156
1160
|
}
|
|
@@ -1260,9 +1264,9 @@ function truncate(value, limit = 280) {
|
|
|
1260
1264
|
}
|
|
1261
1265
|
return value.slice(0, limit - 3) + "...";
|
|
1262
1266
|
}
|
|
1263
|
-
function renderAssertionFailure(result,
|
|
1264
|
-
const color =
|
|
1265
|
-
const testFile =
|
|
1267
|
+
function renderAssertionFailure(result, options2 = {}) {
|
|
1268
|
+
const color = options2.color ?? false;
|
|
1269
|
+
const testFile = options2.testFile;
|
|
1266
1270
|
const layer = result.layer ?? 0;
|
|
1267
1271
|
const label = layerLabel(layer, result.type);
|
|
1268
1272
|
const lines = [];
|
|
@@ -1343,8 +1347,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1343
1347
|
);
|
|
1344
1348
|
}
|
|
1345
1349
|
}
|
|
1346
|
-
function renderSummary(result,
|
|
1347
|
-
const color =
|
|
1350
|
+
function renderSummary(result, options2 = {}) {
|
|
1351
|
+
const color = options2.color ?? false;
|
|
1348
1352
|
const passN = result.passCount;
|
|
1349
1353
|
const soft = result.assertionResults.filter(
|
|
1350
1354
|
(r) => r.status === STATUS_SOFT_FAIL
|
|
@@ -1362,9 +1366,9 @@ function renderSummary(result, options = {}) {
|
|
|
1362
1366
|
];
|
|
1363
1367
|
return parts.join(" | ");
|
|
1364
1368
|
}
|
|
1365
|
-
function renderDiagnostics(result,
|
|
1366
|
-
const color =
|
|
1367
|
-
const includePassing =
|
|
1369
|
+
function renderDiagnostics(result, options2 = {}) {
|
|
1370
|
+
const color = options2.color ?? false;
|
|
1371
|
+
const includePassing = options2.includePassing ?? false;
|
|
1368
1372
|
const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
1369
1373
|
if (rows.length === 0) {
|
|
1370
1374
|
return dim(" (no failures)\n", color);
|
|
@@ -1372,7 +1376,7 @@ function renderDiagnostics(result, options = {}) {
|
|
|
1372
1376
|
const out = [""];
|
|
1373
1377
|
out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
|
|
1374
1378
|
for (const r of rows) {
|
|
1375
|
-
out.push(renderAssertionFailure(r,
|
|
1379
|
+
out.push(renderAssertionFailure(r, options2));
|
|
1376
1380
|
}
|
|
1377
1381
|
out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
|
|
1378
1382
|
return out.join("\n");
|
|
@@ -1489,9 +1493,64 @@ var TraceTree = class {
|
|
|
1489
1493
|
|
|
1490
1494
|
// src/expect.ts
|
|
1491
1495
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
1496
|
+
var MAX_WORLD_BYTES = 9 * 1024 * 1024;
|
|
1497
|
+
var WorldChecks = class {
|
|
1498
|
+
_snapshot;
|
|
1499
|
+
_add;
|
|
1500
|
+
constructor(snapshot, add) {
|
|
1501
|
+
this._snapshot = snapshot;
|
|
1502
|
+
this._add = add;
|
|
1503
|
+
}
|
|
1504
|
+
addCheck(spec, where) {
|
|
1505
|
+
const full = { ...spec, world: this._snapshot };
|
|
1506
|
+
if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
|
|
1507
|
+
return this._add(full);
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
|
|
1511
|
+
* `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
|
|
1512
|
+
*
|
|
1513
|
+
* @throws Error when no bound is given, or `equals` is combined with another bound.
|
|
1514
|
+
*
|
|
1515
|
+
* @example
|
|
1516
|
+
* ```ts
|
|
1517
|
+
* expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
|
|
1518
|
+
* ```
|
|
1519
|
+
*/
|
|
1520
|
+
count(resource, opts) {
|
|
1521
|
+
if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
1522
|
+
throw new Error("count needs equals, atMost or atLeast");
|
|
1523
|
+
}
|
|
1524
|
+
if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
|
|
1525
|
+
throw new Error("count: equals excludes atMost and atLeast");
|
|
1526
|
+
}
|
|
1527
|
+
const spec = { check: "count", resource };
|
|
1528
|
+
if (opts.equals !== void 0) spec.equals = opts.equals;
|
|
1529
|
+
if (opts.atMost !== void 0) spec.at_most = opts.atMost;
|
|
1530
|
+
if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
|
|
1531
|
+
return this.addCheck(spec, opts.where);
|
|
1532
|
+
}
|
|
1533
|
+
/** Assert no two matching records share the same values for field(s) `by`. */
|
|
1534
|
+
unique(resource, opts) {
|
|
1535
|
+
const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
|
|
1536
|
+
if (fields.length === 0) throw new Error("unique needs at least one field in by");
|
|
1537
|
+
return this.addCheck({ check: "unique", resource, fields }, opts.where);
|
|
1538
|
+
}
|
|
1539
|
+
/**
|
|
1540
|
+
* Assert every matching record carries a distinct value in field `key` (default
|
|
1541
|
+
* `idempotency_key`), the record field that stores the idempotency key.
|
|
1542
|
+
*/
|
|
1543
|
+
idempotent(resource, opts = {}) {
|
|
1544
|
+
return this.addCheck(
|
|
1545
|
+
{ check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
|
|
1546
|
+
opts.where
|
|
1547
|
+
);
|
|
1548
|
+
}
|
|
1549
|
+
};
|
|
1492
1550
|
var ExpectChain = class {
|
|
1493
1551
|
_result;
|
|
1494
1552
|
_assertions = [];
|
|
1553
|
+
_worldBytes = 0;
|
|
1495
1554
|
constructor(result) {
|
|
1496
1555
|
this._result = result;
|
|
1497
1556
|
}
|
|
@@ -1920,6 +1979,24 @@ var ExpectChain = class {
|
|
|
1920
1979
|
soft: opts?.soft ?? false
|
|
1921
1980
|
});
|
|
1922
1981
|
}
|
|
1982
|
+
// -- Layer 3: Effect checks (the external system the agent acted on) --
|
|
1983
|
+
/**
|
|
1984
|
+
* Check effects in a world snapshot:
|
|
1985
|
+
* `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
|
|
1986
|
+
* a world helper: the checks are `not_evaluated` (`missing_evidence`).
|
|
1987
|
+
*/
|
|
1988
|
+
world(snapshot) {
|
|
1989
|
+
const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
|
|
1990
|
+
return new WorldChecks(snapshot, (spec) => {
|
|
1991
|
+
this._worldBytes += size;
|
|
1992
|
+
if (this._worldBytes > MAX_WORLD_BYTES) {
|
|
1993
|
+
throw new Error(
|
|
1994
|
+
`effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
|
|
1995
|
+
);
|
|
1996
|
+
}
|
|
1997
|
+
return this.add(TYPE_EFFECT, spec);
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
1923
2000
|
// -- Layer 4: Content --
|
|
1924
2001
|
/**
|
|
1925
2002
|
* Assert the output message contains the given string.
|
|
@@ -2413,6 +2490,728 @@ function attestExpect(result) {
|
|
|
2413
2490
|
return new ExpectChain(new AgentResult(result));
|
|
2414
2491
|
}
|
|
2415
2492
|
|
|
2493
|
+
// src/capture.ts
|
|
2494
|
+
import { spawn as spawn2 } from "child_process";
|
|
2495
|
+
import * as fs2 from "fs";
|
|
2496
|
+
import * as path2 from "path";
|
|
2497
|
+
import { fileURLToPath } from "url";
|
|
2498
|
+
var FIXTURE_VERSION = 1;
|
|
2499
|
+
var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
|
|
2500
|
+
var CaptureError = class extends Error {
|
|
2501
|
+
/** The fixture written, when the engine wrote one. */
|
|
2502
|
+
fixture;
|
|
2503
|
+
constructor(message, fixture = null) {
|
|
2504
|
+
super(message);
|
|
2505
|
+
this.name = "CaptureError";
|
|
2506
|
+
this.fixture = fixture;
|
|
2507
|
+
}
|
|
2508
|
+
};
|
|
2509
|
+
function toPath(p) {
|
|
2510
|
+
return p instanceof URL ? fileURLToPath(p) : p;
|
|
2511
|
+
}
|
|
2512
|
+
function loadFixture(file) {
|
|
2513
|
+
const p = toPath(file);
|
|
2514
|
+
const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
|
|
2515
|
+
if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
|
|
2516
|
+
throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
|
|
2517
|
+
}
|
|
2518
|
+
validate(p, doc);
|
|
2519
|
+
return doc;
|
|
2520
|
+
}
|
|
2521
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
2522
|
+
function isObject(v) {
|
|
2523
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2524
|
+
}
|
|
2525
|
+
function kindOf(v) {
|
|
2526
|
+
if (Array.isArray(v)) return "array";
|
|
2527
|
+
if (isObject(v)) return "object";
|
|
2528
|
+
if (typeof v === "string" || typeof v === "boolean") return typeof v;
|
|
2529
|
+
return "other";
|
|
2530
|
+
}
|
|
2531
|
+
function validate(p, doc) {
|
|
2532
|
+
const need = (where, obj, fields) => {
|
|
2533
|
+
if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
|
|
2534
|
+
for (const [key, kind] of Object.entries(fields)) {
|
|
2535
|
+
if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
|
|
2536
|
+
}
|
|
2537
|
+
return obj;
|
|
2538
|
+
};
|
|
2539
|
+
need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
|
|
2540
|
+
if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
|
|
2541
|
+
if (!NAME_PATTERN.test(doc["name"])) {
|
|
2542
|
+
throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
|
|
2543
|
+
}
|
|
2544
|
+
const v = need("fixture.verification", doc["verification"], {
|
|
2545
|
+
verified: "boolean",
|
|
2546
|
+
expect_fail: "boolean",
|
|
2547
|
+
results: "array",
|
|
2548
|
+
expected_failures: "array"
|
|
2549
|
+
});
|
|
2550
|
+
doc["contracts"].forEach(
|
|
2551
|
+
(c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
|
|
2552
|
+
);
|
|
2553
|
+
v["results"].forEach(
|
|
2554
|
+
(r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
|
|
2555
|
+
);
|
|
2556
|
+
}
|
|
2557
|
+
function objectKey(k) {
|
|
2558
|
+
if (k === "__proto__") return `[${JSON.stringify(k)}]`;
|
|
2559
|
+
return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
|
|
2560
|
+
}
|
|
2561
|
+
function comment(text) {
|
|
2562
|
+
return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
|
|
2563
|
+
}
|
|
2564
|
+
function ts(value) {
|
|
2565
|
+
if (value === null || typeof value === "boolean" || typeof value === "number") {
|
|
2566
|
+
return JSON.stringify(value);
|
|
2567
|
+
}
|
|
2568
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
2569
|
+
if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
|
|
2570
|
+
if (typeof value === "object") {
|
|
2571
|
+
const entries = Object.entries(value).map(
|
|
2572
|
+
([k, v]) => `${objectKey(k)}: ${ts(v)}`
|
|
2573
|
+
);
|
|
2574
|
+
return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
|
|
2575
|
+
}
|
|
2576
|
+
throw new Error(`value ${String(value)} has no literal`);
|
|
2577
|
+
}
|
|
2578
|
+
function str(spec, key) {
|
|
2579
|
+
const v = spec[key];
|
|
2580
|
+
if (typeof v !== "string") throw new Error(`missing ${key}`);
|
|
2581
|
+
return v;
|
|
2582
|
+
}
|
|
2583
|
+
function options(entries) {
|
|
2584
|
+
const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
|
|
2585
|
+
return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
|
|
2586
|
+
}
|
|
2587
|
+
function call(method, args, opts) {
|
|
2588
|
+
return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
|
|
2589
|
+
}
|
|
2590
|
+
function dslCall(contract) {
|
|
2591
|
+
const s = contract.spec;
|
|
2592
|
+
const soft = s["soft"] === true ? true : void 0;
|
|
2593
|
+
try {
|
|
2594
|
+
if (contract.type === "trace") {
|
|
2595
|
+
const check = s["check"];
|
|
2596
|
+
if (check === "trajectory" && s["mode"] !== "partial_order") {
|
|
2597
|
+
const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
|
|
2598
|
+
if (!Array.isArray(s["tools"])) throw new Error("missing tools");
|
|
2599
|
+
return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
|
|
2600
|
+
}
|
|
2601
|
+
if (check === "tool_order") {
|
|
2602
|
+
return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
|
|
2603
|
+
}
|
|
2604
|
+
if (check === "tool_cardinality") {
|
|
2605
|
+
return call(
|
|
2606
|
+
"toolCardinality",
|
|
2607
|
+
[ts(str(s, "tool"))],
|
|
2608
|
+
options([
|
|
2609
|
+
["per", s["per"]],
|
|
2610
|
+
["atLeast", s["min"]],
|
|
2611
|
+
["atMost", s["max"]],
|
|
2612
|
+
["soft", soft]
|
|
2613
|
+
])
|
|
2614
|
+
);
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
if (contract.type === "schema" && s["target"] === "output.structured") {
|
|
2618
|
+
if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
|
|
2619
|
+
return call("outputMatchesSchema", [ts(s["schema"])], "");
|
|
2620
|
+
}
|
|
2621
|
+
if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
|
|
2622
|
+
if (typeof s["value"] !== "number") throw new Error("missing value");
|
|
2623
|
+
return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
|
|
2624
|
+
}
|
|
2625
|
+
} catch (err) {
|
|
2626
|
+
throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
|
|
2627
|
+
}
|
|
2628
|
+
throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
|
|
2629
|
+
}
|
|
2630
|
+
function testPathFor(fixturePath) {
|
|
2631
|
+
const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
|
|
2632
|
+
return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
|
|
2633
|
+
}
|
|
2634
|
+
function renderVitest(fixture, fixturePath, testPath) {
|
|
2635
|
+
const calls = [];
|
|
2636
|
+
const skipped = [];
|
|
2637
|
+
for (const c of fixture.contracts) {
|
|
2638
|
+
try {
|
|
2639
|
+
calls.push([c, dslCall(c)]);
|
|
2640
|
+
} catch (err) {
|
|
2641
|
+
skipped.push(err.message);
|
|
2642
|
+
}
|
|
2643
|
+
}
|
|
2644
|
+
const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
|
|
2645
|
+
const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
|
|
2646
|
+
const expectFail = fixture.verification.expect_fail;
|
|
2647
|
+
const lines = [
|
|
2648
|
+
`// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
|
|
2649
|
+
"//",
|
|
2650
|
+
"// Review and edit before committing: they check the captured trace without running",
|
|
2651
|
+
"// the agent. Re-capturing does not overwrite this file.",
|
|
2652
|
+
`import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
|
|
2653
|
+
`import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
|
|
2654
|
+
`import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
|
|
2655
|
+
"",
|
|
2656
|
+
`const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
|
|
2657
|
+
];
|
|
2658
|
+
if (skipped.length > 0) {
|
|
2659
|
+
lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
|
|
2660
|
+
for (const s of skipped) lines.push(`// - ${comment(s)}`);
|
|
2661
|
+
}
|
|
2662
|
+
lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
|
|
2663
|
+
for (const [c, dsl] of calls) {
|
|
2664
|
+
lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
|
|
2665
|
+
}
|
|
2666
|
+
lines[lines.length - 1] += ";";
|
|
2667
|
+
lines.push("}");
|
|
2668
|
+
if (expectFail) {
|
|
2669
|
+
lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
|
|
2670
|
+
lines.push("const EXPECTED: Array<[string, string]> = [");
|
|
2671
|
+
for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
|
|
2672
|
+
lines.push(
|
|
2673
|
+
"];",
|
|
2674
|
+
"",
|
|
2675
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2676
|
+
` it("reproduces the incident", async () => {`,
|
|
2677
|
+
" const result = await inspect(contracts());",
|
|
2678
|
+
"",
|
|
2679
|
+
" expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
|
|
2680
|
+
" });",
|
|
2681
|
+
"});"
|
|
2682
|
+
);
|
|
2683
|
+
} else {
|
|
2684
|
+
lines.push(
|
|
2685
|
+
"",
|
|
2686
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2687
|
+
` it("keeps its drafted contracts", async () => {`,
|
|
2688
|
+
" await evaluate(contracts());",
|
|
2689
|
+
" });",
|
|
2690
|
+
"});"
|
|
2691
|
+
);
|
|
2692
|
+
}
|
|
2693
|
+
return `${lines.join("\n")}
|
|
2694
|
+
`;
|
|
2695
|
+
}
|
|
2696
|
+
function emitVitest(fixturePath) {
|
|
2697
|
+
const testPath = testPathFor(fixturePath);
|
|
2698
|
+
const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
|
|
2699
|
+
if (existing?.isSymbolicLink()) {
|
|
2700
|
+
throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
|
|
2701
|
+
}
|
|
2702
|
+
if (existing !== void 0) return null;
|
|
2703
|
+
const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
|
|
2704
|
+
fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
|
|
2705
|
+
try {
|
|
2706
|
+
fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
|
|
2707
|
+
} catch (err) {
|
|
2708
|
+
if (err.code === "EEXIST") {
|
|
2709
|
+
throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
|
|
2710
|
+
}
|
|
2711
|
+
throw err;
|
|
2712
|
+
}
|
|
2713
|
+
return testPath;
|
|
2714
|
+
}
|
|
2715
|
+
function serve(engine, args) {
|
|
2716
|
+
const child = spawn2(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2717
|
+
let stdout = "";
|
|
2718
|
+
let stderr = "";
|
|
2719
|
+
let resolveReady = () => void 0;
|
|
2720
|
+
const ready = new Promise((resolve4) => {
|
|
2721
|
+
resolveReady = resolve4;
|
|
2722
|
+
});
|
|
2723
|
+
const exited = new Promise((resolve4) => {
|
|
2724
|
+
child.on("close", (code, signal) => {
|
|
2725
|
+
resolveReady(null);
|
|
2726
|
+
resolve4({ code, signal });
|
|
2727
|
+
});
|
|
2728
|
+
child.on("error", (err) => {
|
|
2729
|
+
stderr += err.message;
|
|
2730
|
+
resolveReady(null);
|
|
2731
|
+
resolve4({ code: null, signal: null });
|
|
2732
|
+
});
|
|
2733
|
+
});
|
|
2734
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
2735
|
+
stdout += chunk;
|
|
2736
|
+
const nl = stdout.indexOf("\n");
|
|
2737
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
2738
|
+
});
|
|
2739
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
2740
|
+
stderr += chunk;
|
|
2741
|
+
});
|
|
2742
|
+
child.stdin.on("error", () => void 0);
|
|
2743
|
+
return {
|
|
2744
|
+
ready,
|
|
2745
|
+
async finish(finalize) {
|
|
2746
|
+
if (finalize) {
|
|
2747
|
+
const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
|
|
2748
|
+
child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
|
|
2749
|
+
`);
|
|
2750
|
+
}
|
|
2751
|
+
child.stdin.end();
|
|
2752
|
+
const { code, signal } = await exited;
|
|
2753
|
+
return { code, signal, stdout, stderr };
|
|
2754
|
+
}
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
function readyEnv(line) {
|
|
2758
|
+
if (line === null) return null;
|
|
2759
|
+
let parsed;
|
|
2760
|
+
try {
|
|
2761
|
+
parsed = JSON.parse(line);
|
|
2762
|
+
} catch {
|
|
2763
|
+
return null;
|
|
2764
|
+
}
|
|
2765
|
+
const env = parsed?.env;
|
|
2766
|
+
if (typeof env !== "object" || env === null) return null;
|
|
2767
|
+
const entries = Object.entries(env);
|
|
2768
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
2769
|
+
return Object.fromEntries(entries);
|
|
2770
|
+
}
|
|
2771
|
+
function exitDetail(exit) {
|
|
2772
|
+
const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
|
|
2773
|
+
const err = exit.stderr.trim();
|
|
2774
|
+
return err ? `${how}: ${err}` : how;
|
|
2775
|
+
}
|
|
2776
|
+
function restore(saved) {
|
|
2777
|
+
for (const [k, v] of saved) {
|
|
2778
|
+
if (v === void 0) delete process.env[k];
|
|
2779
|
+
else process.env[k] = v;
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
var recording = false;
|
|
2783
|
+
async function record(name, fn, opts = {}) {
|
|
2784
|
+
if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
|
|
2785
|
+
recording = true;
|
|
2786
|
+
try {
|
|
2787
|
+
return await recordExclusive(name, fn, opts);
|
|
2788
|
+
} finally {
|
|
2789
|
+
recording = false;
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
async function recordExclusive(name, fn, opts) {
|
|
2793
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2794
|
+
const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
|
|
2795
|
+
if (opts.expectFail) args.push("--expect-fail");
|
|
2796
|
+
if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
|
|
2797
|
+
if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
|
|
2798
|
+
const proc = serve(engine, args);
|
|
2799
|
+
const env = readyEnv(await proc.ready);
|
|
2800
|
+
if (env === null) {
|
|
2801
|
+
const exit2 = await proc.finish(false);
|
|
2802
|
+
throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
|
|
2803
|
+
}
|
|
2804
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
2805
|
+
Object.assign(process.env, env);
|
|
2806
|
+
let value;
|
|
2807
|
+
try {
|
|
2808
|
+
value = await fn({ name, env });
|
|
2809
|
+
if (opts.flush) await opts.flush();
|
|
2810
|
+
} catch (err) {
|
|
2811
|
+
restore(saved);
|
|
2812
|
+
await proc.finish(false);
|
|
2813
|
+
throw err;
|
|
2814
|
+
}
|
|
2815
|
+
restore(saved);
|
|
2816
|
+
const exit = await proc.finish(true);
|
|
2817
|
+
process.stderr.write(exit.stderr);
|
|
2818
|
+
const lines = exit.stdout.trim().split("\n");
|
|
2819
|
+
if (lines.length < 2) {
|
|
2820
|
+
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2821
|
+
}
|
|
2822
|
+
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2823
|
+
if (exit.code !== 0 || !outcome.verified) {
|
|
2824
|
+
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2825
|
+
}
|
|
2826
|
+
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2827
|
+
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
2828
|
+
}
|
|
2829
|
+
|
|
2830
|
+
// src/replay.ts
|
|
2831
|
+
import { spawn as spawn3 } from "child_process";
|
|
2832
|
+
import * as path3 from "path";
|
|
2833
|
+
|
|
2834
|
+
// src/egress.ts
|
|
2835
|
+
import * as net from "net";
|
|
2836
|
+
var PROVIDER_HOSTS = [
|
|
2837
|
+
"api.openai.com",
|
|
2838
|
+
"api.anthropic.com",
|
|
2839
|
+
"generativelanguage.googleapis.com",
|
|
2840
|
+
/\.openai\.azure\.com$/
|
|
2841
|
+
];
|
|
2842
|
+
var EgressBlockedError = class extends Error {
|
|
2843
|
+
constructor(host) {
|
|
2844
|
+
super(
|
|
2845
|
+
`attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
|
|
2846
|
+
);
|
|
2847
|
+
this.host = host;
|
|
2848
|
+
this.name = "EgressBlockedError";
|
|
2849
|
+
}
|
|
2850
|
+
host;
|
|
2851
|
+
};
|
|
2852
|
+
function isProviderHost(host) {
|
|
2853
|
+
const h = host.toLowerCase().replace(/\.$/, "");
|
|
2854
|
+
return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
|
|
2855
|
+
}
|
|
2856
|
+
function egressBlockRequested(env = process.env) {
|
|
2857
|
+
const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
|
|
2858
|
+
if (v !== "" && v !== "block" && v !== "allow") {
|
|
2859
|
+
throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
|
|
2860
|
+
}
|
|
2861
|
+
return v === "block";
|
|
2862
|
+
}
|
|
2863
|
+
var original = null;
|
|
2864
|
+
var installs = 0;
|
|
2865
|
+
function hostOf(args) {
|
|
2866
|
+
const first = Array.isArray(args[0]) ? args[0][0] : args[0];
|
|
2867
|
+
if (typeof first === "object" && first !== null) {
|
|
2868
|
+
const o = first;
|
|
2869
|
+
return typeof o.host === "string" ? o.host : null;
|
|
2870
|
+
}
|
|
2871
|
+
if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
|
|
2872
|
+
return typeof args[1] === "string" ? args[1] : null;
|
|
2873
|
+
}
|
|
2874
|
+
return null;
|
|
2875
|
+
}
|
|
2876
|
+
function installEgressGuard() {
|
|
2877
|
+
installs++;
|
|
2878
|
+
if (original === null) {
|
|
2879
|
+
const proto = net.Socket.prototype;
|
|
2880
|
+
const orig = proto.connect;
|
|
2881
|
+
original = orig;
|
|
2882
|
+
proto.connect = function(...args) {
|
|
2883
|
+
const host = hostOf(args);
|
|
2884
|
+
if (host !== null && isProviderHost(host)) {
|
|
2885
|
+
const err = new EgressBlockedError(host);
|
|
2886
|
+
process.nextTick(() => this.destroy(err));
|
|
2887
|
+
return this;
|
|
2888
|
+
}
|
|
2889
|
+
return orig.apply(this, args);
|
|
2890
|
+
};
|
|
2891
|
+
}
|
|
2892
|
+
let released = false;
|
|
2893
|
+
return () => {
|
|
2894
|
+
if (released) return;
|
|
2895
|
+
released = true;
|
|
2896
|
+
installs--;
|
|
2897
|
+
if (installs === 0) uninstallEgressGuard();
|
|
2898
|
+
};
|
|
2899
|
+
}
|
|
2900
|
+
function egressGuardActive() {
|
|
2901
|
+
return installs > 0;
|
|
2902
|
+
}
|
|
2903
|
+
function uninstallEgressGuard() {
|
|
2904
|
+
installs = 0;
|
|
2905
|
+
if (original === null) return;
|
|
2906
|
+
net.Socket.prototype.connect = original;
|
|
2907
|
+
original = null;
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
// src/replay.ts
|
|
2911
|
+
var ReplayMissError = class extends Error {
|
|
2912
|
+
constructor(message, summary) {
|
|
2913
|
+
super(message);
|
|
2914
|
+
this.summary = summary;
|
|
2915
|
+
this.name = "ReplayMissError";
|
|
2916
|
+
}
|
|
2917
|
+
summary;
|
|
2918
|
+
};
|
|
2919
|
+
var ReplayNotEvaluatedError = class extends Error {
|
|
2920
|
+
constructor(message, summary) {
|
|
2921
|
+
super(message);
|
|
2922
|
+
this.summary = summary;
|
|
2923
|
+
this.name = "ReplayNotEvaluatedError";
|
|
2924
|
+
}
|
|
2925
|
+
summary;
|
|
2926
|
+
};
|
|
2927
|
+
function escapeControl(text) {
|
|
2928
|
+
return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
|
|
2929
|
+
}
|
|
2930
|
+
var ReplayError = class extends Error {
|
|
2931
|
+
constructor(message) {
|
|
2932
|
+
super(escapeControl(message));
|
|
2933
|
+
this.name = "ReplayError";
|
|
2934
|
+
}
|
|
2935
|
+
};
|
|
2936
|
+
function resolveMode(explicit, env = process.env) {
|
|
2937
|
+
if (explicit !== void 0) return checkMode(explicit, "mode");
|
|
2938
|
+
const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
|
|
2939
|
+
if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
|
|
2940
|
+
const ci = (env.CI ?? "").trim().toLowerCase();
|
|
2941
|
+
return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
|
|
2942
|
+
}
|
|
2943
|
+
function checkMode(value, where) {
|
|
2944
|
+
if (value === "strict" || value === "auto" || value === "off") return value;
|
|
2945
|
+
throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
|
|
2946
|
+
}
|
|
2947
|
+
var active = null;
|
|
2948
|
+
async function startReplay(name, opts = {}) {
|
|
2949
|
+
if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
|
|
2950
|
+
const resolved = resolveMode(opts.mode);
|
|
2951
|
+
if (resolved === "off") return offSession(name);
|
|
2952
|
+
const mode = resolved;
|
|
2953
|
+
if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
|
|
2954
|
+
active = name;
|
|
2955
|
+
try {
|
|
2956
|
+
return await start(name, mode, opts);
|
|
2957
|
+
} catch (err) {
|
|
2958
|
+
active = null;
|
|
2959
|
+
throw err;
|
|
2960
|
+
}
|
|
2961
|
+
}
|
|
2962
|
+
function offSession(name) {
|
|
2963
|
+
return {
|
|
2964
|
+
name,
|
|
2965
|
+
mode: "off",
|
|
2966
|
+
env: {},
|
|
2967
|
+
summary: null,
|
|
2968
|
+
finish: () => Promise.resolve(null),
|
|
2969
|
+
error: () => null,
|
|
2970
|
+
check: () => Promise.resolve()
|
|
2971
|
+
};
|
|
2972
|
+
}
|
|
2973
|
+
function missError(mode, summary) {
|
|
2974
|
+
if (summary.misses === 0) return null;
|
|
2975
|
+
const message = escapeControl(firstMiss(summary) ?? "replay miss");
|
|
2976
|
+
return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
|
|
2977
|
+
}
|
|
2978
|
+
async function start(name, mode, opts) {
|
|
2979
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2980
|
+
const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
|
|
2981
|
+
const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
|
|
2982
|
+
const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
|
|
2983
|
+
if (blocking) args.push("--no-live");
|
|
2984
|
+
const child = spawn3(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2985
|
+
let stdout = "";
|
|
2986
|
+
let stderr = "";
|
|
2987
|
+
let resolveReady = () => void 0;
|
|
2988
|
+
const ready = new Promise((resolve4) => {
|
|
2989
|
+
resolveReady = resolve4;
|
|
2990
|
+
});
|
|
2991
|
+
const exited = new Promise((resolve4) => {
|
|
2992
|
+
child.on("close", (code) => {
|
|
2993
|
+
resolveReady(null);
|
|
2994
|
+
resolve4(code);
|
|
2995
|
+
});
|
|
2996
|
+
child.on("error", (err) => {
|
|
2997
|
+
stderr += err.message;
|
|
2998
|
+
resolveReady(null);
|
|
2999
|
+
resolve4(null);
|
|
3000
|
+
});
|
|
3001
|
+
});
|
|
3002
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3003
|
+
stdout += chunk;
|
|
3004
|
+
const nl = stdout.indexOf("\n");
|
|
3005
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3006
|
+
});
|
|
3007
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3008
|
+
stderr += chunk;
|
|
3009
|
+
});
|
|
3010
|
+
child.stdin.on("error", () => void 0);
|
|
3011
|
+
const stop = async () => {
|
|
3012
|
+
child.stdin.write("stop\n");
|
|
3013
|
+
child.stdin.end();
|
|
3014
|
+
return { code: await exited, stdout, stderr };
|
|
3015
|
+
};
|
|
3016
|
+
const env = parseEnv(await ready);
|
|
3017
|
+
if (env === null) {
|
|
3018
|
+
const out = await stop();
|
|
3019
|
+
active = null;
|
|
3020
|
+
throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3021
|
+
}
|
|
3022
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3023
|
+
Object.assign(process.env, env);
|
|
3024
|
+
let summary = null;
|
|
3025
|
+
let finishing = null;
|
|
3026
|
+
const finish = () => {
|
|
3027
|
+
finishing ??= (async () => {
|
|
3028
|
+
for (const [k, v] of saved) {
|
|
3029
|
+
if (v === void 0) delete process.env[k];
|
|
3030
|
+
else process.env[k] = v;
|
|
3031
|
+
}
|
|
3032
|
+
const out = await stop();
|
|
3033
|
+
active = null;
|
|
3034
|
+
const parsed = parseSummary(out.stdout);
|
|
3035
|
+
if (parsed === null) {
|
|
3036
|
+
throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3037
|
+
}
|
|
3038
|
+
if (blocking && parsed.live > 0) {
|
|
3039
|
+
throw new ReplayError(
|
|
3040
|
+
`replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
|
|
3041
|
+
);
|
|
3042
|
+
}
|
|
3043
|
+
summary = parsed;
|
|
3044
|
+
return parsed;
|
|
3045
|
+
})();
|
|
3046
|
+
return finishing;
|
|
3047
|
+
};
|
|
3048
|
+
return {
|
|
3049
|
+
name,
|
|
3050
|
+
mode,
|
|
3051
|
+
env,
|
|
3052
|
+
get summary() {
|
|
3053
|
+
return summary;
|
|
3054
|
+
},
|
|
3055
|
+
finish,
|
|
3056
|
+
error: () => summary === null ? null : missError(mode, summary),
|
|
3057
|
+
check: async () => {
|
|
3058
|
+
await finish();
|
|
3059
|
+
const err = summary === null ? null : missError(mode, summary);
|
|
3060
|
+
if (err !== null) throw err;
|
|
3061
|
+
}
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
function parseEnv(line) {
|
|
3065
|
+
if (line === null) return null;
|
|
3066
|
+
try {
|
|
3067
|
+
const env = JSON.parse(line).env;
|
|
3068
|
+
if (typeof env !== "object" || env === null) return null;
|
|
3069
|
+
const entries = Object.entries(env);
|
|
3070
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
3071
|
+
return Object.fromEntries(entries);
|
|
3072
|
+
} catch {
|
|
3073
|
+
return null;
|
|
3074
|
+
}
|
|
3075
|
+
}
|
|
3076
|
+
function parseSummary(out) {
|
|
3077
|
+
const lines = out.trim().split("\n");
|
|
3078
|
+
try {
|
|
3079
|
+
const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
|
|
3080
|
+
if (s === void 0 || !Array.isArray(s.records)) return null;
|
|
3081
|
+
for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
|
|
3082
|
+
return s;
|
|
3083
|
+
} catch {
|
|
3084
|
+
return null;
|
|
3085
|
+
}
|
|
3086
|
+
}
|
|
3087
|
+
function firstMiss(summary) {
|
|
3088
|
+
return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
|
|
3089
|
+
}
|
|
3090
|
+
function liveLine(live, costUsd, unpriced = 0) {
|
|
3091
|
+
const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
|
|
3092
|
+
return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
// src/world.ts
|
|
3096
|
+
import { spawn as spawn4 } from "child_process";
|
|
3097
|
+
import * as path4 from "path";
|
|
3098
|
+
var STATE_PATH = "/_attest/state";
|
|
3099
|
+
var STATE_TIMEOUT_MS = 1e4;
|
|
3100
|
+
var WorldError = class extends Error {
|
|
3101
|
+
constructor(message) {
|
|
3102
|
+
super(escapeControl(message));
|
|
3103
|
+
this.name = "WorldError";
|
|
3104
|
+
}
|
|
3105
|
+
};
|
|
3106
|
+
var active2 = null;
|
|
3107
|
+
async function startWorld(worldFile, opts = {}) {
|
|
3108
|
+
if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
|
|
3109
|
+
const file = path4.resolve(worldFile);
|
|
3110
|
+
active2 = file;
|
|
3111
|
+
try {
|
|
3112
|
+
return await start2(file, opts);
|
|
3113
|
+
} catch (err) {
|
|
3114
|
+
active2 = null;
|
|
3115
|
+
throw err;
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
async function start2(file, opts) {
|
|
3119
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3120
|
+
const child = spawn4(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
3121
|
+
let stdout = "";
|
|
3122
|
+
let stderr = "";
|
|
3123
|
+
let resolveReady = () => void 0;
|
|
3124
|
+
const ready = new Promise((resolve4) => {
|
|
3125
|
+
resolveReady = resolve4;
|
|
3126
|
+
});
|
|
3127
|
+
const exited = new Promise((resolve4) => {
|
|
3128
|
+
child.on("close", (code) => {
|
|
3129
|
+
resolveReady(null);
|
|
3130
|
+
resolve4(code);
|
|
3131
|
+
});
|
|
3132
|
+
child.on("error", (err) => {
|
|
3133
|
+
stderr += err.message;
|
|
3134
|
+
resolveReady(null);
|
|
3135
|
+
resolve4(null);
|
|
3136
|
+
});
|
|
3137
|
+
});
|
|
3138
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3139
|
+
stdout += chunk;
|
|
3140
|
+
const nl = stdout.indexOf("\n");
|
|
3141
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3142
|
+
});
|
|
3143
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3144
|
+
stderr += chunk;
|
|
3145
|
+
});
|
|
3146
|
+
child.stdin.on("error", () => void 0);
|
|
3147
|
+
const halt = async () => {
|
|
3148
|
+
child.stdin.write("stop\n");
|
|
3149
|
+
child.stdin.end();
|
|
3150
|
+
return { code: await exited, stdout, stderr };
|
|
3151
|
+
};
|
|
3152
|
+
const env = parseEnv(await ready);
|
|
3153
|
+
if (env === null) {
|
|
3154
|
+
const out = await halt();
|
|
3155
|
+
throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3156
|
+
}
|
|
3157
|
+
const urls = Object.values(env);
|
|
3158
|
+
if (urls.length !== 1) {
|
|
3159
|
+
await halt();
|
|
3160
|
+
throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
|
|
3161
|
+
}
|
|
3162
|
+
const baseUrl = urls[0];
|
|
3163
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3164
|
+
Object.assign(process.env, env);
|
|
3165
|
+
let stopped = false;
|
|
3166
|
+
let stopping = null;
|
|
3167
|
+
const stop = () => {
|
|
3168
|
+
stopped = true;
|
|
3169
|
+
stopping ??= (async () => {
|
|
3170
|
+
for (const [k, v] of saved) {
|
|
3171
|
+
if (v === void 0) delete process.env[k];
|
|
3172
|
+
else process.env[k] = v;
|
|
3173
|
+
}
|
|
3174
|
+
const out = await halt();
|
|
3175
|
+
active2 = null;
|
|
3176
|
+
const final = parseFinal(out.stdout);
|
|
3177
|
+
if (out.code !== 0 || final === null) {
|
|
3178
|
+
throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3179
|
+
}
|
|
3180
|
+
return final;
|
|
3181
|
+
})();
|
|
3182
|
+
return stopping;
|
|
3183
|
+
};
|
|
3184
|
+
const state = async () => {
|
|
3185
|
+
if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
|
|
3186
|
+
const url = baseUrl + STATE_PATH;
|
|
3187
|
+
let res;
|
|
3188
|
+
try {
|
|
3189
|
+
res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
|
|
3190
|
+
} catch (err) {
|
|
3191
|
+
throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3192
|
+
}
|
|
3193
|
+
if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
|
|
3194
|
+
const body = await res.json().catch(() => {
|
|
3195
|
+
throw new WorldError(`GET ${url}: the body is not JSON`);
|
|
3196
|
+
});
|
|
3197
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
3198
|
+
throw new WorldError(`GET ${url}: the body is not a snapshot object`);
|
|
3199
|
+
}
|
|
3200
|
+
return body;
|
|
3201
|
+
};
|
|
3202
|
+
return { path: file, env, baseUrl, state, stop };
|
|
3203
|
+
}
|
|
3204
|
+
function parseFinal(out) {
|
|
3205
|
+
const lines = out.trim().split("\n");
|
|
3206
|
+
try {
|
|
3207
|
+
const world = JSON.parse(lines[lines.length - 1] ?? "").world;
|
|
3208
|
+
if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
|
|
3209
|
+
return world;
|
|
3210
|
+
} catch {
|
|
3211
|
+
return null;
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3214
|
+
|
|
2416
3215
|
// src/context.ts
|
|
2417
3216
|
import { AsyncLocalStorage } from "async_hooks";
|
|
2418
3217
|
var activeBuilder = new AsyncLocalStorage();
|
|
@@ -2616,12 +3415,12 @@ var ContinuousEvalRunner = class {
|
|
|
2616
3415
|
maxSize;
|
|
2617
3416
|
running = false;
|
|
2618
3417
|
loopTimer = null;
|
|
2619
|
-
constructor(client, assertions,
|
|
3418
|
+
constructor(client, assertions, options2) {
|
|
2620
3419
|
this.client = client;
|
|
2621
3420
|
this.assertions = assertions;
|
|
2622
|
-
this.sampler = new Sampler(
|
|
2623
|
-
this.dispatcher = new AlertDispatcher(
|
|
2624
|
-
this.maxSize =
|
|
3421
|
+
this.sampler = new Sampler(options2?.sampleRate ?? 1);
|
|
3422
|
+
this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
|
|
3423
|
+
this.maxSize = options2?.maxSize ?? resolveQueueSize();
|
|
2625
3424
|
}
|
|
2626
3425
|
get alertEmitter() {
|
|
2627
3426
|
return this.dispatcher;
|
|
@@ -2732,13 +3531,13 @@ function loadLabelsCSV(text) {
|
|
|
2732
3531
|
if (rows.length === 0) {
|
|
2733
3532
|
throw new Error("empty CSV");
|
|
2734
3533
|
}
|
|
2735
|
-
let
|
|
3534
|
+
let start3 = 0;
|
|
2736
3535
|
if (rows[0].length >= 2) {
|
|
2737
3536
|
const firstHuman = parseFloatStrict(rows[0][1]);
|
|
2738
|
-
if (firstHuman === null)
|
|
3537
|
+
if (firstHuman === null) start3 = 1;
|
|
2739
3538
|
}
|
|
2740
3539
|
const out = [];
|
|
2741
|
-
for (let idx =
|
|
3540
|
+
for (let idx = start3; idx < rows.length; idx++) {
|
|
2742
3541
|
const row = rows[idx];
|
|
2743
3542
|
const lineNum = idx + 1;
|
|
2744
3543
|
if (row.length === 0) continue;
|
|
@@ -2947,10 +3746,10 @@ var OpenAIAdapter = class {
|
|
|
2947
3746
|
constructor(agentId2) {
|
|
2948
3747
|
this.agentId = agentId2;
|
|
2949
3748
|
}
|
|
2950
|
-
traceFromResponse(response,
|
|
3749
|
+
traceFromResponse(response, options2) {
|
|
2951
3750
|
const builder = new TraceBuilder(this.agentId);
|
|
2952
|
-
if (
|
|
2953
|
-
builder.setInput({ messages:
|
|
3751
|
+
if (options2?.inputMessages) {
|
|
3752
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2954
3753
|
}
|
|
2955
3754
|
const message = response.choices[0].message;
|
|
2956
3755
|
const completionText = message.content ?? "";
|
|
@@ -2970,12 +3769,12 @@ var OpenAIAdapter = class {
|
|
|
2970
3769
|
}
|
|
2971
3770
|
builder.setOutput({
|
|
2972
3771
|
message: completionText,
|
|
2973
|
-
structured:
|
|
3772
|
+
structured: options2?.structuredOutput ?? {}
|
|
2974
3773
|
});
|
|
2975
3774
|
builder.setMetadata({
|
|
2976
3775
|
total_tokens: response.usage?.total_tokens,
|
|
2977
|
-
cost_usd:
|
|
2978
|
-
latency_ms:
|
|
3776
|
+
cost_usd: options2?.costUsd,
|
|
3777
|
+
latency_ms: options2?.latencyMs,
|
|
2979
3778
|
model: response.model
|
|
2980
3779
|
});
|
|
2981
3780
|
return builder.build();
|
|
@@ -2988,10 +3787,10 @@ var AnthropicAdapter = class {
|
|
|
2988
3787
|
constructor(agentId2) {
|
|
2989
3788
|
this.agentId = agentId2;
|
|
2990
3789
|
}
|
|
2991
|
-
traceFromResponse(response,
|
|
3790
|
+
traceFromResponse(response, options2) {
|
|
2992
3791
|
const builder = new TraceBuilder(this.agentId);
|
|
2993
|
-
if (
|
|
2994
|
-
builder.setInput({ messages:
|
|
3792
|
+
if (options2?.inputMessages) {
|
|
3793
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2995
3794
|
}
|
|
2996
3795
|
const completionParts = [];
|
|
2997
3796
|
for (const block of response.content) {
|
|
@@ -3014,8 +3813,8 @@ var AnthropicAdapter = class {
|
|
|
3014
3813
|
}
|
|
3015
3814
|
builder.setMetadata({
|
|
3016
3815
|
total_tokens: totalTokens,
|
|
3017
|
-
cost_usd:
|
|
3018
|
-
latency_ms:
|
|
3816
|
+
cost_usd: options2?.costUsd,
|
|
3817
|
+
latency_ms: options2?.latencyMs,
|
|
3019
3818
|
model: response.model
|
|
3020
3819
|
});
|
|
3021
3820
|
return builder.build();
|
|
@@ -3028,10 +3827,10 @@ var GeminiAdapter = class {
|
|
|
3028
3827
|
constructor(agentId2) {
|
|
3029
3828
|
this.agentId = agentId2;
|
|
3030
3829
|
}
|
|
3031
|
-
traceFromResponse(response,
|
|
3830
|
+
traceFromResponse(response, options2) {
|
|
3032
3831
|
const builder = new TraceBuilder(this.agentId);
|
|
3033
|
-
if (
|
|
3034
|
-
builder.setInput({ text:
|
|
3832
|
+
if (options2?.inputText) {
|
|
3833
|
+
builder.setInput({ text: options2.inputText });
|
|
3035
3834
|
}
|
|
3036
3835
|
let completionText = "";
|
|
3037
3836
|
if (response.text != null) {
|
|
@@ -3051,9 +3850,9 @@ var GeminiAdapter = class {
|
|
|
3051
3850
|
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
3052
3851
|
builder.setOutput({ message: completionText });
|
|
3053
3852
|
builder.setMetadata({
|
|
3054
|
-
cost_usd:
|
|
3055
|
-
latency_ms:
|
|
3056
|
-
model:
|
|
3853
|
+
cost_usd: options2?.costUsd,
|
|
3854
|
+
latency_ms: options2?.latencyMs,
|
|
3855
|
+
model: options2?.model
|
|
3057
3856
|
});
|
|
3058
3857
|
return builder.build();
|
|
3059
3858
|
}
|
|
@@ -3065,10 +3864,10 @@ var OllamaAdapter = class {
|
|
|
3065
3864
|
constructor(agentId2) {
|
|
3066
3865
|
this.agentId = agentId2;
|
|
3067
3866
|
}
|
|
3068
|
-
traceFromResponse(response,
|
|
3867
|
+
traceFromResponse(response, options2) {
|
|
3069
3868
|
const builder = new TraceBuilder(this.agentId);
|
|
3070
|
-
if (
|
|
3071
|
-
builder.setInput({ messages:
|
|
3869
|
+
if (options2?.inputMessages) {
|
|
3870
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3072
3871
|
}
|
|
3073
3872
|
const completionText = response.message?.content ?? "";
|
|
3074
3873
|
builder.addLlmCall("completion", {
|
|
@@ -3082,7 +3881,7 @@ var OllamaAdapter = class {
|
|
|
3082
3881
|
}
|
|
3083
3882
|
builder.setMetadata({
|
|
3084
3883
|
total_tokens: totalTokens,
|
|
3085
|
-
latency_ms:
|
|
3884
|
+
latency_ms: options2?.latencyMs,
|
|
3086
3885
|
model: response.model
|
|
3087
3886
|
});
|
|
3088
3887
|
return builder.build();
|
|
@@ -3090,7 +3889,7 @@ var OllamaAdapter = class {
|
|
|
3090
3889
|
};
|
|
3091
3890
|
|
|
3092
3891
|
// src/adapters/otel.ts
|
|
3093
|
-
import {
|
|
3892
|
+
import { spawn as spawn5 } from "child_process";
|
|
3094
3893
|
var CONVERT_TIMEOUT_MS = 6e4;
|
|
3095
3894
|
var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
3096
3895
|
var STATUS_CODE_ERROR = 2;
|
|
@@ -3113,7 +3912,7 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3113
3912
|
* @throws RangeError when a parsed export holds a non-finite number.
|
|
3114
3913
|
* @throws OTLPConversionError when the engine times out, overflows the output buffer or
|
|
3115
3914
|
* prints no trace.
|
|
3116
|
-
* @throws Error
|
|
3915
|
+
* @throws Error when the engine cannot start.
|
|
3117
3916
|
*/
|
|
3118
3917
|
static async fromOtlp(otlp, agentId2) {
|
|
3119
3918
|
const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
|
|
@@ -3125,33 +3924,9 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3125
3924
|
const args = ["trace", "convert"];
|
|
3126
3925
|
if (agentId2 !== void 0) args.push("--agent-id", agentId2);
|
|
3127
3926
|
args.push("-");
|
|
3128
|
-
const
|
|
3129
|
-
input: payload,
|
|
3130
|
-
timeout: CONVERT_TIMEOUT_MS,
|
|
3131
|
-
maxBuffer: MAX_TRACE_OUTPUT_BYTES
|
|
3132
|
-
});
|
|
3133
|
-
if (proc.error) {
|
|
3134
|
-
const code = proc.error.code;
|
|
3135
|
-
if (code === "ETIMEDOUT") {
|
|
3136
|
-
throw new OTLPConversionError(
|
|
3137
|
-
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3138
|
-
);
|
|
3139
|
-
}
|
|
3140
|
-
if (code === "ENOBUFS") {
|
|
3141
|
-
throw new OTLPConversionError(
|
|
3142
|
-
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3143
|
-
);
|
|
3144
|
-
}
|
|
3145
|
-
throw proc.error;
|
|
3146
|
-
}
|
|
3147
|
-
if (proc.status !== 0) {
|
|
3148
|
-
const message = proc.stderr.toString("utf8").trim();
|
|
3149
|
-
throw new OTLPConversionError(
|
|
3150
|
-
message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
|
|
3151
|
-
);
|
|
3152
|
-
}
|
|
3927
|
+
const stdout = await runConvert(await resolveEngine(), args, payload);
|
|
3153
3928
|
try {
|
|
3154
|
-
return JSON.parse(
|
|
3929
|
+
return JSON.parse(stdout);
|
|
3155
3930
|
} catch (err) {
|
|
3156
3931
|
throw new OTLPConversionError(
|
|
3157
3932
|
`attest-engine trace convert printed output that is not JSON: ${String(err)}`
|
|
@@ -3159,6 +3934,65 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3159
3934
|
}
|
|
3160
3935
|
}
|
|
3161
3936
|
};
|
|
3937
|
+
function runConvert(engine, args, payload) {
|
|
3938
|
+
return new Promise((resolve4, reject) => {
|
|
3939
|
+
const child = spawn5(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3940
|
+
let timer;
|
|
3941
|
+
child.on("error", (e) => {
|
|
3942
|
+
clearTimeout(timer);
|
|
3943
|
+
reject(e);
|
|
3944
|
+
});
|
|
3945
|
+
const { stdin, stdout, stderr } = child;
|
|
3946
|
+
if (stdin === null || stdout === null || stderr === null) return;
|
|
3947
|
+
const out = [];
|
|
3948
|
+
const err = [];
|
|
3949
|
+
let outBytes = 0;
|
|
3950
|
+
let failure = null;
|
|
3951
|
+
const fail = (e) => {
|
|
3952
|
+
failure ??= e;
|
|
3953
|
+
child.kill("SIGKILL");
|
|
3954
|
+
};
|
|
3955
|
+
timer = setTimeout(() => {
|
|
3956
|
+
fail(
|
|
3957
|
+
new OTLPConversionError(
|
|
3958
|
+
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3959
|
+
)
|
|
3960
|
+
);
|
|
3961
|
+
}, CONVERT_TIMEOUT_MS);
|
|
3962
|
+
stdout.on("data", (chunk) => {
|
|
3963
|
+
outBytes += chunk.length;
|
|
3964
|
+
if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
|
|
3965
|
+
fail(
|
|
3966
|
+
new OTLPConversionError(
|
|
3967
|
+
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3968
|
+
)
|
|
3969
|
+
);
|
|
3970
|
+
return;
|
|
3971
|
+
}
|
|
3972
|
+
out.push(chunk);
|
|
3973
|
+
});
|
|
3974
|
+
stderr.on("data", (chunk) => err.push(chunk));
|
|
3975
|
+
stdin.on("error", () => void 0);
|
|
3976
|
+
child.on("close", (code, signal) => {
|
|
3977
|
+
clearTimeout(timer);
|
|
3978
|
+
if (failure !== null) {
|
|
3979
|
+
reject(failure);
|
|
3980
|
+
return;
|
|
3981
|
+
}
|
|
3982
|
+
if (code !== 0) {
|
|
3983
|
+
const message = Buffer.concat(err).toString("utf8").trim();
|
|
3984
|
+
reject(
|
|
3985
|
+
new OTLPConversionError(
|
|
3986
|
+
message || `attest-engine trace convert exited ${String(code ?? signal)}`
|
|
3987
|
+
)
|
|
3988
|
+
);
|
|
3989
|
+
return;
|
|
3990
|
+
}
|
|
3991
|
+
resolve4(Buffer.concat(out).toString("utf8"));
|
|
3992
|
+
});
|
|
3993
|
+
stdin.end(payload);
|
|
3994
|
+
});
|
|
3995
|
+
}
|
|
3162
3996
|
function hrTimeToNanos(time, spanName) {
|
|
3163
3997
|
if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
|
|
3164
3998
|
throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
|
|
@@ -3264,13 +4098,13 @@ var LangChainAdapter = class {
|
|
|
3264
4098
|
* @param options.model - Model name override.
|
|
3265
4099
|
* @returns A finalized Trace object.
|
|
3266
4100
|
*/
|
|
3267
|
-
buildTrace(
|
|
4101
|
+
buildTrace(options2) {
|
|
3268
4102
|
const builder = new TraceBuilder(this.agentId);
|
|
3269
4103
|
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
3270
4104
|
let totalTokens;
|
|
3271
4105
|
for (const llm of this.llmCalls) {
|
|
3272
4106
|
const args = {};
|
|
3273
|
-
if (llm.model ??
|
|
4107
|
+
if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
|
|
3274
4108
|
const result = { completion: llm.text };
|
|
3275
4109
|
if (llm.tokens !== void 0) {
|
|
3276
4110
|
result.tokens = llm.tokens;
|
|
@@ -3287,9 +4121,9 @@ var LangChainAdapter = class {
|
|
|
3287
4121
|
builder.setOutput({ message: outputMessage });
|
|
3288
4122
|
builder.setMetadata({
|
|
3289
4123
|
total_tokens: totalTokens,
|
|
3290
|
-
cost_usd:
|
|
4124
|
+
cost_usd: options2?.costUsd,
|
|
3291
4125
|
latency_ms: latencyMs,
|
|
3292
|
-
model:
|
|
4126
|
+
model: options2?.model ?? this.llmCalls[0]?.model
|
|
3293
4127
|
});
|
|
3294
4128
|
return builder.build();
|
|
3295
4129
|
}
|
|
@@ -3303,6 +4137,7 @@ export {
|
|
|
3303
4137
|
AttestClient,
|
|
3304
4138
|
CONFUSED_USER,
|
|
3305
4139
|
COOPERATIVE_USER,
|
|
4140
|
+
CaptureError,
|
|
3306
4141
|
ContinuousEvalRunner,
|
|
3307
4142
|
ENGINE_VERSION,
|
|
3308
4143
|
ERR_ASSERTION_ERROR,
|
|
@@ -3311,6 +4146,7 @@ export {
|
|
|
3311
4146
|
ERR_PROVIDER_ERROR,
|
|
3312
4147
|
ERR_SESSION_ERROR,
|
|
3313
4148
|
ERR_TIMEOUT,
|
|
4149
|
+
EgressBlockedError,
|
|
3314
4150
|
EngineManager,
|
|
3315
4151
|
EngineTimeoutError,
|
|
3316
4152
|
ExpectChain,
|
|
@@ -3335,8 +4171,12 @@ export {
|
|
|
3335
4171
|
REASON_MISSING_EVIDENCE,
|
|
3336
4172
|
REASON_PROVIDER_NOT_CONFIGURED,
|
|
3337
4173
|
REASON_PROVIDER_UNAVAILABLE,
|
|
4174
|
+
REASON_REPLAY_MISS,
|
|
3338
4175
|
REASON_SHORT_CIRCUITED,
|
|
3339
4176
|
REASON_SIMULATION,
|
|
4177
|
+
ReplayError,
|
|
4178
|
+
ReplayMissError,
|
|
4179
|
+
ReplayNotEvaluatedError,
|
|
3340
4180
|
STATUS_HARD_FAIL,
|
|
3341
4181
|
STATUS_NOT_EVALUATED,
|
|
3342
4182
|
STATUS_PASS,
|
|
@@ -3351,6 +4191,7 @@ export {
|
|
|
3351
4191
|
TIER_3,
|
|
3352
4192
|
TYPE_CONSTRAINT,
|
|
3353
4193
|
TYPE_CONTENT,
|
|
4194
|
+
TYPE_EFFECT,
|
|
3354
4195
|
TYPE_EMBEDDING,
|
|
3355
4196
|
TYPE_LLM_JUDGE,
|
|
3356
4197
|
TYPE_PLUGIN,
|
|
@@ -3360,6 +4201,8 @@ export {
|
|
|
3360
4201
|
TraceBuilder,
|
|
3361
4202
|
TraceTree,
|
|
3362
4203
|
VERSION,
|
|
4204
|
+
WorldChecks,
|
|
4205
|
+
WorldError,
|
|
3363
4206
|
activeBuilder,
|
|
3364
4207
|
addUsage,
|
|
3365
4208
|
agent,
|
|
@@ -3370,28 +4213,43 @@ export {
|
|
|
3370
4213
|
config,
|
|
3371
4214
|
decodeResponse,
|
|
3372
4215
|
delegate,
|
|
4216
|
+
dslCall,
|
|
4217
|
+
egressBlockRequested,
|
|
4218
|
+
emitVitest,
|
|
3373
4219
|
encodeRequest,
|
|
3374
4220
|
executePluginAssertion,
|
|
3375
4221
|
extractId,
|
|
3376
4222
|
extractResult,
|
|
4223
|
+
firstMiss,
|
|
3377
4224
|
formatUsage,
|
|
3378
4225
|
getSampleRate,
|
|
4226
|
+
installEgressGuard,
|
|
4227
|
+
isProviderHost,
|
|
3379
4228
|
isSimulationMode,
|
|
3380
4229
|
layerForType,
|
|
4230
|
+
liveLine,
|
|
4231
|
+
loadFixture,
|
|
3381
4232
|
loadLabelsCSV,
|
|
3382
4233
|
loadLabelsJSONL,
|
|
3383
4234
|
parseUsage,
|
|
3384
4235
|
previewLine,
|
|
3385
4236
|
promptHash,
|
|
4237
|
+
record,
|
|
3386
4238
|
renderAssertionFailure,
|
|
3387
4239
|
renderDiagnostics,
|
|
3388
4240
|
renderSummary,
|
|
4241
|
+
renderVitest,
|
|
3389
4242
|
requireSimulationCapability,
|
|
3390
4243
|
resetConfig,
|
|
4244
|
+
resolveMode,
|
|
3391
4245
|
simulationEnvEnabled,
|
|
4246
|
+
startReplay,
|
|
4247
|
+
startWorld,
|
|
3392
4248
|
sumUsage,
|
|
4249
|
+
testPathFor,
|
|
3393
4250
|
tier,
|
|
3394
4251
|
traceId,
|
|
4252
|
+
uninstallEgressGuard,
|
|
3395
4253
|
usageSummaryLine,
|
|
3396
4254
|
zeroUsage
|
|
3397
4255
|
};
|