@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +1001 -120
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +313 -5
- package/dist/cjs/proto/index.cjs +7 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +18 -2
- package/dist/esm/cli.d.ts +6 -1
- package/dist/esm/cli.js +310 -20
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +313 -5
- package/dist/esm/index.js +977 -120
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +18 -2
- package/dist/esm/proto/index.js +5 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +3 -3
package/dist/esm/index.js
CHANGED
|
@@ -14,12 +14,14 @@ var REASON_MISSING_EVIDENCE = "missing_evidence";
|
|
|
14
14
|
var REASON_SHORT_CIRCUITED = "short_circuited";
|
|
15
15
|
var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
|
|
16
16
|
var REASON_JUDGE_ABSTAINED = "judge_abstained";
|
|
17
|
+
var REASON_REPLAY_MISS = "replay_miss";
|
|
17
18
|
var OUTCOME_PASS = "pass";
|
|
18
19
|
var OUTCOME_FAIL = "fail";
|
|
19
20
|
var OUTCOME_SKIP = "skip";
|
|
20
21
|
var TYPE_SCHEMA = "schema";
|
|
21
22
|
var TYPE_CONSTRAINT = "constraint";
|
|
22
23
|
var TYPE_TRACE = "trace";
|
|
24
|
+
var TYPE_EFFECT = "effect";
|
|
23
25
|
var TYPE_CONTENT = "content";
|
|
24
26
|
var TYPE_EMBEDDING = "embedding";
|
|
25
27
|
var TYPE_LLM_JUDGE = "llm_judge";
|
|
@@ -57,6 +59,7 @@ function layerForType(assertionType) {
|
|
|
57
59
|
return 2;
|
|
58
60
|
case "trace":
|
|
59
61
|
case "trace_tree":
|
|
62
|
+
case "effect":
|
|
60
63
|
return 3;
|
|
61
64
|
case "content":
|
|
62
65
|
return 4;
|
|
@@ -216,8 +219,8 @@ function extractId(response) {
|
|
|
216
219
|
}
|
|
217
220
|
|
|
218
221
|
// src/version.ts
|
|
219
|
-
var VERSION = "1.0.0-rc.
|
|
220
|
-
var ENGINE_VERSION = "1.0.0-rc.
|
|
222
|
+
var VERSION = "1.0.0-rc.3";
|
|
223
|
+
var ENGINE_VERSION = "1.0.0-rc.3";
|
|
221
224
|
|
|
222
225
|
// src/engine-manager.ts
|
|
223
226
|
import { spawn, spawnSync } from "child_process";
|
|
@@ -285,7 +288,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
285
288
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
286
289
|
);
|
|
287
290
|
}
|
|
288
|
-
return new Promise((
|
|
291
|
+
return new Promise((resolve4, reject) => {
|
|
289
292
|
const client = url.startsWith("https:") ? https : http;
|
|
290
293
|
client.get(url, (res) => {
|
|
291
294
|
const status = res.statusCode ?? 0;
|
|
@@ -296,7 +299,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
296
299
|
return;
|
|
297
300
|
}
|
|
298
301
|
res.resume();
|
|
299
|
-
|
|
302
|
+
resolve4(httpsGet(location, redirectCount + 1));
|
|
300
303
|
return;
|
|
301
304
|
}
|
|
302
305
|
if (status < 200 || status >= 300) {
|
|
@@ -306,7 +309,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
306
309
|
}
|
|
307
310
|
const chunks = [];
|
|
308
311
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
309
|
-
res.on("end", () =>
|
|
312
|
+
res.on("end", () => resolve4(Buffer.concat(chunks)));
|
|
310
313
|
res.on("error", reject);
|
|
311
314
|
}).on("error", reject);
|
|
312
315
|
});
|
|
@@ -437,8 +440,8 @@ Install options:
|
|
|
437
440
|
}
|
|
438
441
|
return [await downloadEngine(), "download"];
|
|
439
442
|
}
|
|
440
|
-
async function resolveEngine(explicitPath,
|
|
441
|
-
const [enginePath, source] = await locateEngine(explicitPath,
|
|
443
|
+
async function resolveEngine(explicitPath, options2 = {}) {
|
|
444
|
+
const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
|
|
442
445
|
const found = engineVersion(enginePath);
|
|
443
446
|
if (found !== ENGINE_VERSION) {
|
|
444
447
|
throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
|
|
@@ -453,9 +456,9 @@ var EngineManager = class {
|
|
|
453
456
|
initialized = false;
|
|
454
457
|
requestId = 0;
|
|
455
458
|
initResult = null;
|
|
456
|
-
constructor(
|
|
457
|
-
this.enginePath =
|
|
458
|
-
this.logLevel =
|
|
459
|
+
constructor(options2) {
|
|
460
|
+
this.enginePath = options2?.enginePath ?? "";
|
|
461
|
+
this.logLevel = options2?.logLevel ?? "warn";
|
|
459
462
|
}
|
|
460
463
|
async start() {
|
|
461
464
|
this.enginePath = await resolveEngine(this.enginePath || void 0);
|
|
@@ -533,11 +536,11 @@ var EngineManager = class {
|
|
|
533
536
|
}
|
|
534
537
|
this.requestId += 1;
|
|
535
538
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
536
|
-
return new Promise((
|
|
539
|
+
return new Promise((resolve4, reject) => {
|
|
537
540
|
const onLine = (line) => {
|
|
538
541
|
try {
|
|
539
542
|
const response = decodeResponse(line);
|
|
540
|
-
|
|
543
|
+
resolve4(extractResult(response));
|
|
541
544
|
} catch (err) {
|
|
542
545
|
reject(err);
|
|
543
546
|
}
|
|
@@ -571,17 +574,17 @@ var EngineManager = class {
|
|
|
571
574
|
// src/config.ts
|
|
572
575
|
var _simulationMode = false;
|
|
573
576
|
var _sampleRate = 0;
|
|
574
|
-
function config(
|
|
575
|
-
if (
|
|
576
|
-
_simulationMode =
|
|
577
|
+
function config(options2) {
|
|
578
|
+
if (options2.simulation !== void 0) {
|
|
579
|
+
_simulationMode = options2.simulation;
|
|
577
580
|
}
|
|
578
|
-
if (
|
|
579
|
-
if (
|
|
581
|
+
if (options2.sampleRate !== void 0) {
|
|
582
|
+
if (options2.sampleRate < 0 || options2.sampleRate > 1) {
|
|
580
583
|
throw new RangeError(
|
|
581
|
-
`sampleRate must be between 0.0 and 1.0, got ${
|
|
584
|
+
`sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
|
|
582
585
|
);
|
|
583
586
|
}
|
|
584
|
-
_sampleRate =
|
|
587
|
+
_sampleRate = options2.sampleRate;
|
|
585
588
|
}
|
|
586
589
|
}
|
|
587
590
|
function isSimulationMode() {
|
|
@@ -737,16 +740,16 @@ var AttestClient = class {
|
|
|
737
740
|
desyncThreshold;
|
|
738
741
|
desyncWindowMs;
|
|
739
742
|
desynced = false;
|
|
740
|
-
constructor(engine,
|
|
743
|
+
constructor(engine, options2) {
|
|
741
744
|
this.engine = engine;
|
|
742
|
-
this.logger =
|
|
745
|
+
this.logger = options2?.logger ?? envLoggerDefault();
|
|
743
746
|
this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
|
|
744
|
-
|
|
747
|
+
options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
|
|
745
748
|
);
|
|
746
|
-
this.desyncThreshold =
|
|
747
|
-
this.desyncWindowMs =
|
|
748
|
-
if (
|
|
749
|
-
this.diagnosticListeners.add(
|
|
749
|
+
this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
|
|
750
|
+
this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
|
|
751
|
+
if (options2?.onDiagnostic !== void 0) {
|
|
752
|
+
this.diagnosticListeners.add(options2.onDiagnostic);
|
|
750
753
|
}
|
|
751
754
|
}
|
|
752
755
|
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
@@ -904,12 +907,12 @@ var AttestClient = class {
|
|
|
904
907
|
throw new Error("Engine process not started.");
|
|
905
908
|
}
|
|
906
909
|
let capturedReqId;
|
|
907
|
-
const requestPromise = new Promise((
|
|
910
|
+
const requestPromise = new Promise((resolve4, reject) => {
|
|
908
911
|
this.writeChain = this.writeChain.then(() => {
|
|
909
912
|
this.requestId += 1;
|
|
910
913
|
const reqId = this.requestId;
|
|
911
914
|
capturedReqId = reqId;
|
|
912
|
-
this.pending.set(reqId, { resolve, reject });
|
|
915
|
+
this.pending.set(reqId, { resolve: resolve4, reject });
|
|
913
916
|
const requestStr = encodeRequest(reqId, method, params);
|
|
914
917
|
cp.stdin.write(requestStr, (err) => {
|
|
915
918
|
if (err) {
|
|
@@ -927,23 +930,23 @@ var AttestClient = class {
|
|
|
927
930
|
throw err;
|
|
928
931
|
});
|
|
929
932
|
}
|
|
930
|
-
async evaluateBatch(trace, assertions,
|
|
933
|
+
async evaluateBatch(trace, assertions, options2) {
|
|
931
934
|
if (isSimulationMode()) {
|
|
932
935
|
requireSimulationCapability(this.engine["initResult"]);
|
|
933
936
|
}
|
|
934
937
|
const params = {
|
|
935
938
|
trace,
|
|
936
939
|
assertions: [...assertions],
|
|
937
|
-
...
|
|
940
|
+
...options2?.budget !== void 0 ? { budget: options2.budget } : {}
|
|
938
941
|
};
|
|
939
942
|
const raw = await this.sendRequest(
|
|
940
943
|
"evaluate_batch",
|
|
941
944
|
params,
|
|
942
|
-
|
|
945
|
+
options2?.timeout
|
|
943
946
|
);
|
|
944
947
|
return { ...raw, usage: parseUsage(raw["usage"]) };
|
|
945
948
|
}
|
|
946
|
-
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation,
|
|
949
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
|
|
947
950
|
const result = {
|
|
948
951
|
assertion_id: assertionId2,
|
|
949
952
|
status,
|
|
@@ -961,11 +964,11 @@ var AttestClient = class {
|
|
|
961
964
|
const raw = await this.sendRequest(
|
|
962
965
|
"submit_plugin_result",
|
|
963
966
|
params,
|
|
964
|
-
|
|
967
|
+
options2?.timeout
|
|
965
968
|
);
|
|
966
969
|
return Boolean(raw?.accepted ?? false);
|
|
967
970
|
}
|
|
968
|
-
async queryDrift(assertionId2, windowSize = 50,
|
|
971
|
+
async queryDrift(assertionId2, windowSize = 50, options2) {
|
|
969
972
|
if (isSimulationMode()) {
|
|
970
973
|
return {
|
|
971
974
|
assertion_id: assertionId2,
|
|
@@ -980,11 +983,11 @@ var AttestClient = class {
|
|
|
980
983
|
const raw = await this.sendRequest(
|
|
981
984
|
"query_drift",
|
|
982
985
|
{ assertion_id: assertionId2, window_size: windowSize },
|
|
983
|
-
|
|
986
|
+
options2?.timeout
|
|
984
987
|
);
|
|
985
988
|
return raw.report;
|
|
986
989
|
}
|
|
987
|
-
async generateUserMessage(persona, conversationHistory, faultConfig,
|
|
990
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
|
|
988
991
|
if (isSimulationMode()) {
|
|
989
992
|
return `[simulation] Hello from ${persona.name}`;
|
|
990
993
|
}
|
|
@@ -998,7 +1001,7 @@ var AttestClient = class {
|
|
|
998
1001
|
const raw = await this.sendRequest(
|
|
999
1002
|
"generate_user_message",
|
|
1000
1003
|
params,
|
|
1001
|
-
|
|
1004
|
+
options2?.timeout
|
|
1002
1005
|
);
|
|
1003
1006
|
return String(raw.message);
|
|
1004
1007
|
}
|
|
@@ -1043,33 +1046,33 @@ var TraceBuilder = class {
|
|
|
1043
1046
|
this.input = input;
|
|
1044
1047
|
return this;
|
|
1045
1048
|
}
|
|
1046
|
-
addLlmCall(name,
|
|
1049
|
+
addLlmCall(name, options2) {
|
|
1047
1050
|
this.steps.push({
|
|
1048
1051
|
type: STEP_LLM_CALL,
|
|
1049
1052
|
name,
|
|
1050
|
-
args:
|
|
1051
|
-
result:
|
|
1052
|
-
metadata:
|
|
1053
|
+
args: options2?.args,
|
|
1054
|
+
result: options2?.result,
|
|
1055
|
+
metadata: options2?.metadata
|
|
1053
1056
|
});
|
|
1054
1057
|
return this;
|
|
1055
1058
|
}
|
|
1056
|
-
addToolCall(name,
|
|
1059
|
+
addToolCall(name, options2) {
|
|
1057
1060
|
this.steps.push({
|
|
1058
1061
|
type: STEP_TOOL_CALL,
|
|
1059
1062
|
name,
|
|
1060
|
-
args:
|
|
1061
|
-
result:
|
|
1062
|
-
metadata:
|
|
1063
|
+
args: options2?.args,
|
|
1064
|
+
result: options2?.result,
|
|
1065
|
+
metadata: options2?.metadata
|
|
1063
1066
|
});
|
|
1064
1067
|
return this;
|
|
1065
1068
|
}
|
|
1066
|
-
addRetrieval(name,
|
|
1069
|
+
addRetrieval(name, options2) {
|
|
1067
1070
|
this.steps.push({
|
|
1068
1071
|
type: STEP_RETRIEVAL,
|
|
1069
1072
|
name,
|
|
1070
|
-
args:
|
|
1071
|
-
result:
|
|
1072
|
-
metadata:
|
|
1073
|
+
args: options2?.args,
|
|
1074
|
+
result: options2?.result,
|
|
1075
|
+
metadata: options2?.metadata
|
|
1073
1076
|
});
|
|
1074
1077
|
return this;
|
|
1075
1078
|
}
|
|
@@ -1147,10 +1150,10 @@ var AgentResult = class {
|
|
|
1147
1150
|
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
1148
1151
|
* when only soft_fail rows are present (see `softFailures`).
|
|
1149
1152
|
*/
|
|
1150
|
-
outcome(
|
|
1153
|
+
outcome(options2) {
|
|
1151
1154
|
if (this.hardFailures.length > 0) return OUTCOME_FAIL;
|
|
1152
1155
|
if (this.notEvaluated.length > 0) {
|
|
1153
|
-
return
|
|
1156
|
+
return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
|
|
1154
1157
|
}
|
|
1155
1158
|
return OUTCOME_PASS;
|
|
1156
1159
|
}
|
|
@@ -1260,9 +1263,9 @@ function truncate(value, limit = 280) {
|
|
|
1260
1263
|
}
|
|
1261
1264
|
return value.slice(0, limit - 3) + "...";
|
|
1262
1265
|
}
|
|
1263
|
-
function renderAssertionFailure(result,
|
|
1264
|
-
const color =
|
|
1265
|
-
const testFile =
|
|
1266
|
+
function renderAssertionFailure(result, options2 = {}) {
|
|
1267
|
+
const color = options2.color ?? false;
|
|
1268
|
+
const testFile = options2.testFile;
|
|
1266
1269
|
const layer = result.layer ?? 0;
|
|
1267
1270
|
const label = layerLabel(layer, result.type);
|
|
1268
1271
|
const lines = [];
|
|
@@ -1343,8 +1346,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1343
1346
|
);
|
|
1344
1347
|
}
|
|
1345
1348
|
}
|
|
1346
|
-
function renderSummary(result,
|
|
1347
|
-
const color =
|
|
1349
|
+
function renderSummary(result, options2 = {}) {
|
|
1350
|
+
const color = options2.color ?? false;
|
|
1348
1351
|
const passN = result.passCount;
|
|
1349
1352
|
const soft = result.assertionResults.filter(
|
|
1350
1353
|
(r) => r.status === STATUS_SOFT_FAIL
|
|
@@ -1362,9 +1365,9 @@ function renderSummary(result, options = {}) {
|
|
|
1362
1365
|
];
|
|
1363
1366
|
return parts.join(" | ");
|
|
1364
1367
|
}
|
|
1365
|
-
function renderDiagnostics(result,
|
|
1366
|
-
const color =
|
|
1367
|
-
const includePassing =
|
|
1368
|
+
function renderDiagnostics(result, options2 = {}) {
|
|
1369
|
+
const color = options2.color ?? false;
|
|
1370
|
+
const includePassing = options2.includePassing ?? false;
|
|
1368
1371
|
const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
1369
1372
|
if (rows.length === 0) {
|
|
1370
1373
|
return dim(" (no failures)\n", color);
|
|
@@ -1372,7 +1375,7 @@ function renderDiagnostics(result, options = {}) {
|
|
|
1372
1375
|
const out = [""];
|
|
1373
1376
|
out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
|
|
1374
1377
|
for (const r of rows) {
|
|
1375
|
-
out.push(renderAssertionFailure(r,
|
|
1378
|
+
out.push(renderAssertionFailure(r, options2));
|
|
1376
1379
|
}
|
|
1377
1380
|
out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
|
|
1378
1381
|
return out.join("\n");
|
|
@@ -1489,9 +1492,64 @@ var TraceTree = class {
|
|
|
1489
1492
|
|
|
1490
1493
|
// src/expect.ts
|
|
1491
1494
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
1495
|
+
var MAX_WORLD_BYTES = 9 * 1024 * 1024;
|
|
1496
|
+
var WorldChecks = class {
|
|
1497
|
+
_snapshot;
|
|
1498
|
+
_add;
|
|
1499
|
+
constructor(snapshot, add) {
|
|
1500
|
+
this._snapshot = snapshot;
|
|
1501
|
+
this._add = add;
|
|
1502
|
+
}
|
|
1503
|
+
addCheck(spec, where) {
|
|
1504
|
+
const full = { ...spec, world: this._snapshot };
|
|
1505
|
+
if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
|
|
1506
|
+
return this._add(full);
|
|
1507
|
+
}
|
|
1508
|
+
/**
|
|
1509
|
+
* Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
|
|
1510
|
+
* `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
|
|
1511
|
+
*
|
|
1512
|
+
* @throws Error when no bound is given, or `equals` is combined with another bound.
|
|
1513
|
+
*
|
|
1514
|
+
* @example
|
|
1515
|
+
* ```ts
|
|
1516
|
+
* expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
|
|
1517
|
+
* ```
|
|
1518
|
+
*/
|
|
1519
|
+
count(resource, opts) {
|
|
1520
|
+
if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
1521
|
+
throw new Error("count needs equals, atMost or atLeast");
|
|
1522
|
+
}
|
|
1523
|
+
if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
|
|
1524
|
+
throw new Error("count: equals excludes atMost and atLeast");
|
|
1525
|
+
}
|
|
1526
|
+
const spec = { check: "count", resource };
|
|
1527
|
+
if (opts.equals !== void 0) spec.equals = opts.equals;
|
|
1528
|
+
if (opts.atMost !== void 0) spec.at_most = opts.atMost;
|
|
1529
|
+
if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
|
|
1530
|
+
return this.addCheck(spec, opts.where);
|
|
1531
|
+
}
|
|
1532
|
+
/** Assert no two matching records share the same values for field(s) `by`. */
|
|
1533
|
+
unique(resource, opts) {
|
|
1534
|
+
const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
|
|
1535
|
+
if (fields.length === 0) throw new Error("unique needs at least one field in by");
|
|
1536
|
+
return this.addCheck({ check: "unique", resource, fields }, opts.where);
|
|
1537
|
+
}
|
|
1538
|
+
/**
|
|
1539
|
+
* Assert every matching record carries a distinct value in field `key` (default
|
|
1540
|
+
* `idempotency_key`), the record field that stores the idempotency key.
|
|
1541
|
+
*/
|
|
1542
|
+
idempotent(resource, opts = {}) {
|
|
1543
|
+
return this.addCheck(
|
|
1544
|
+
{ check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
|
|
1545
|
+
opts.where
|
|
1546
|
+
);
|
|
1547
|
+
}
|
|
1548
|
+
};
|
|
1492
1549
|
var ExpectChain = class {
|
|
1493
1550
|
_result;
|
|
1494
1551
|
_assertions = [];
|
|
1552
|
+
_worldBytes = 0;
|
|
1495
1553
|
constructor(result) {
|
|
1496
1554
|
this._result = result;
|
|
1497
1555
|
}
|
|
@@ -1920,6 +1978,24 @@ var ExpectChain = class {
|
|
|
1920
1978
|
soft: opts?.soft ?? false
|
|
1921
1979
|
});
|
|
1922
1980
|
}
|
|
1981
|
+
// -- Layer 3: Effect checks (the external system the agent acted on) --
|
|
1982
|
+
/**
|
|
1983
|
+
* Check effects in a world snapshot:
|
|
1984
|
+
* `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
|
|
1985
|
+
* a world helper: the checks are `not_evaluated` (`missing_evidence`).
|
|
1986
|
+
*/
|
|
1987
|
+
world(snapshot) {
|
|
1988
|
+
const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
|
|
1989
|
+
return new WorldChecks(snapshot, (spec) => {
|
|
1990
|
+
this._worldBytes += size;
|
|
1991
|
+
if (this._worldBytes > MAX_WORLD_BYTES) {
|
|
1992
|
+
throw new Error(
|
|
1993
|
+
`effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
|
|
1994
|
+
);
|
|
1995
|
+
}
|
|
1996
|
+
return this.add(TYPE_EFFECT, spec);
|
|
1997
|
+
});
|
|
1998
|
+
}
|
|
1923
1999
|
// -- Layer 4: Content --
|
|
1924
2000
|
/**
|
|
1925
2001
|
* Assert the output message contains the given string.
|
|
@@ -2413,6 +2489,728 @@ function attestExpect(result) {
|
|
|
2413
2489
|
return new ExpectChain(new AgentResult(result));
|
|
2414
2490
|
}
|
|
2415
2491
|
|
|
2492
|
+
// src/capture.ts
|
|
2493
|
+
import { spawn as spawn2 } from "child_process";
|
|
2494
|
+
import * as fs2 from "fs";
|
|
2495
|
+
import * as path2 from "path";
|
|
2496
|
+
import { fileURLToPath } from "url";
|
|
2497
|
+
var FIXTURE_VERSION = 1;
|
|
2498
|
+
var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
|
|
2499
|
+
var CaptureError = class extends Error {
|
|
2500
|
+
/** The fixture written, when the engine wrote one. */
|
|
2501
|
+
fixture;
|
|
2502
|
+
constructor(message, fixture = null) {
|
|
2503
|
+
super(message);
|
|
2504
|
+
this.name = "CaptureError";
|
|
2505
|
+
this.fixture = fixture;
|
|
2506
|
+
}
|
|
2507
|
+
};
|
|
2508
|
+
function toPath(p) {
|
|
2509
|
+
return p instanceof URL ? fileURLToPath(p) : p;
|
|
2510
|
+
}
|
|
2511
|
+
function loadFixture(file) {
|
|
2512
|
+
const p = toPath(file);
|
|
2513
|
+
const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
|
|
2514
|
+
if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
|
|
2515
|
+
throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
|
|
2516
|
+
}
|
|
2517
|
+
validate(p, doc);
|
|
2518
|
+
return doc;
|
|
2519
|
+
}
|
|
2520
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
2521
|
+
function isObject(v) {
|
|
2522
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2523
|
+
}
|
|
2524
|
+
function kindOf(v) {
|
|
2525
|
+
if (Array.isArray(v)) return "array";
|
|
2526
|
+
if (isObject(v)) return "object";
|
|
2527
|
+
if (typeof v === "string" || typeof v === "boolean") return typeof v;
|
|
2528
|
+
return "other";
|
|
2529
|
+
}
|
|
2530
|
+
function validate(p, doc) {
|
|
2531
|
+
const need = (where, obj, fields) => {
|
|
2532
|
+
if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
|
|
2533
|
+
for (const [key, kind] of Object.entries(fields)) {
|
|
2534
|
+
if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
|
|
2535
|
+
}
|
|
2536
|
+
return obj;
|
|
2537
|
+
};
|
|
2538
|
+
need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
|
|
2539
|
+
if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
|
|
2540
|
+
if (!NAME_PATTERN.test(doc["name"])) {
|
|
2541
|
+
throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
|
|
2542
|
+
}
|
|
2543
|
+
const v = need("fixture.verification", doc["verification"], {
|
|
2544
|
+
verified: "boolean",
|
|
2545
|
+
expect_fail: "boolean",
|
|
2546
|
+
results: "array",
|
|
2547
|
+
expected_failures: "array"
|
|
2548
|
+
});
|
|
2549
|
+
doc["contracts"].forEach(
|
|
2550
|
+
(c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
|
|
2551
|
+
);
|
|
2552
|
+
v["results"].forEach(
|
|
2553
|
+
(r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
|
|
2554
|
+
);
|
|
2555
|
+
}
|
|
2556
|
+
function objectKey(k) {
|
|
2557
|
+
if (k === "__proto__") return `[${JSON.stringify(k)}]`;
|
|
2558
|
+
return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
|
|
2559
|
+
}
|
|
2560
|
+
function comment(text) {
|
|
2561
|
+
return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
|
|
2562
|
+
}
|
|
2563
|
+
function ts(value) {
|
|
2564
|
+
if (value === null || typeof value === "boolean" || typeof value === "number") {
|
|
2565
|
+
return JSON.stringify(value);
|
|
2566
|
+
}
|
|
2567
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
2568
|
+
if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
|
|
2569
|
+
if (typeof value === "object") {
|
|
2570
|
+
const entries = Object.entries(value).map(
|
|
2571
|
+
([k, v]) => `${objectKey(k)}: ${ts(v)}`
|
|
2572
|
+
);
|
|
2573
|
+
return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
|
|
2574
|
+
}
|
|
2575
|
+
throw new Error(`value ${String(value)} has no literal`);
|
|
2576
|
+
}
|
|
2577
|
+
function str(spec, key) {
|
|
2578
|
+
const v = spec[key];
|
|
2579
|
+
if (typeof v !== "string") throw new Error(`missing ${key}`);
|
|
2580
|
+
return v;
|
|
2581
|
+
}
|
|
2582
|
+
function options(entries) {
|
|
2583
|
+
const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
|
|
2584
|
+
return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
|
|
2585
|
+
}
|
|
2586
|
+
function call(method, args, opts) {
|
|
2587
|
+
return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
|
|
2588
|
+
}
|
|
2589
|
+
function dslCall(contract) {
|
|
2590
|
+
const s = contract.spec;
|
|
2591
|
+
const soft = s["soft"] === true ? true : void 0;
|
|
2592
|
+
try {
|
|
2593
|
+
if (contract.type === "trace") {
|
|
2594
|
+
const check = s["check"];
|
|
2595
|
+
if (check === "trajectory" && s["mode"] !== "partial_order") {
|
|
2596
|
+
const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
|
|
2597
|
+
if (!Array.isArray(s["tools"])) throw new Error("missing tools");
|
|
2598
|
+
return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
|
|
2599
|
+
}
|
|
2600
|
+
if (check === "tool_order") {
|
|
2601
|
+
return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
|
|
2602
|
+
}
|
|
2603
|
+
if (check === "tool_cardinality") {
|
|
2604
|
+
return call(
|
|
2605
|
+
"toolCardinality",
|
|
2606
|
+
[ts(str(s, "tool"))],
|
|
2607
|
+
options([
|
|
2608
|
+
["per", s["per"]],
|
|
2609
|
+
["atLeast", s["min"]],
|
|
2610
|
+
["atMost", s["max"]],
|
|
2611
|
+
["soft", soft]
|
|
2612
|
+
])
|
|
2613
|
+
);
|
|
2614
|
+
}
|
|
2615
|
+
}
|
|
2616
|
+
if (contract.type === "schema" && s["target"] === "output.structured") {
|
|
2617
|
+
if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
|
|
2618
|
+
return call("outputMatchesSchema", [ts(s["schema"])], "");
|
|
2619
|
+
}
|
|
2620
|
+
if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
|
|
2621
|
+
if (typeof s["value"] !== "number") throw new Error("missing value");
|
|
2622
|
+
return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
|
|
2623
|
+
}
|
|
2624
|
+
} catch (err) {
|
|
2625
|
+
throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
|
|
2626
|
+
}
|
|
2627
|
+
throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
|
|
2628
|
+
}
|
|
2629
|
+
function testPathFor(fixturePath) {
|
|
2630
|
+
const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
|
|
2631
|
+
return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
|
|
2632
|
+
}
|
|
2633
|
+
function renderVitest(fixture, fixturePath, testPath) {
|
|
2634
|
+
const calls = [];
|
|
2635
|
+
const skipped = [];
|
|
2636
|
+
for (const c of fixture.contracts) {
|
|
2637
|
+
try {
|
|
2638
|
+
calls.push([c, dslCall(c)]);
|
|
2639
|
+
} catch (err) {
|
|
2640
|
+
skipped.push(err.message);
|
|
2641
|
+
}
|
|
2642
|
+
}
|
|
2643
|
+
const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
|
|
2644
|
+
const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
|
|
2645
|
+
const expectFail = fixture.verification.expect_fail;
|
|
2646
|
+
const lines = [
|
|
2647
|
+
`// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
|
|
2648
|
+
"//",
|
|
2649
|
+
"// Review and edit before committing: they check the captured trace without running",
|
|
2650
|
+
"// the agent. Re-capturing does not overwrite this file.",
|
|
2651
|
+
`import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
|
|
2652
|
+
`import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
|
|
2653
|
+
`import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
|
|
2654
|
+
"",
|
|
2655
|
+
`const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
|
|
2656
|
+
];
|
|
2657
|
+
if (skipped.length > 0) {
|
|
2658
|
+
lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
|
|
2659
|
+
for (const s of skipped) lines.push(`// - ${comment(s)}`);
|
|
2660
|
+
}
|
|
2661
|
+
lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
|
|
2662
|
+
for (const [c, dsl] of calls) {
|
|
2663
|
+
lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
|
|
2664
|
+
}
|
|
2665
|
+
lines[lines.length - 1] += ";";
|
|
2666
|
+
lines.push("}");
|
|
2667
|
+
if (expectFail) {
|
|
2668
|
+
lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
|
|
2669
|
+
lines.push("const EXPECTED: Array<[string, string]> = [");
|
|
2670
|
+
for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
|
|
2671
|
+
lines.push(
|
|
2672
|
+
"];",
|
|
2673
|
+
"",
|
|
2674
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2675
|
+
` it("reproduces the incident", async () => {`,
|
|
2676
|
+
" const result = await inspect(contracts());",
|
|
2677
|
+
"",
|
|
2678
|
+
" expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
|
|
2679
|
+
" });",
|
|
2680
|
+
"});"
|
|
2681
|
+
);
|
|
2682
|
+
} else {
|
|
2683
|
+
lines.push(
|
|
2684
|
+
"",
|
|
2685
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2686
|
+
` it("keeps its drafted contracts", async () => {`,
|
|
2687
|
+
" await evaluate(contracts());",
|
|
2688
|
+
" });",
|
|
2689
|
+
"});"
|
|
2690
|
+
);
|
|
2691
|
+
}
|
|
2692
|
+
return `${lines.join("\n")}
|
|
2693
|
+
`;
|
|
2694
|
+
}
|
|
2695
|
+
function emitVitest(fixturePath) {
|
|
2696
|
+
const testPath = testPathFor(fixturePath);
|
|
2697
|
+
const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
|
|
2698
|
+
if (existing?.isSymbolicLink()) {
|
|
2699
|
+
throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
|
|
2700
|
+
}
|
|
2701
|
+
if (existing !== void 0) return null;
|
|
2702
|
+
const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
|
|
2703
|
+
fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
|
|
2704
|
+
try {
|
|
2705
|
+
fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
|
|
2706
|
+
} catch (err) {
|
|
2707
|
+
if (err.code === "EEXIST") {
|
|
2708
|
+
throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
|
|
2709
|
+
}
|
|
2710
|
+
throw err;
|
|
2711
|
+
}
|
|
2712
|
+
return testPath;
|
|
2713
|
+
}
|
|
2714
|
+
function serve(engine, args) {
|
|
2715
|
+
const child = spawn2(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2716
|
+
let stdout = "";
|
|
2717
|
+
let stderr = "";
|
|
2718
|
+
let resolveReady = () => void 0;
|
|
2719
|
+
const ready = new Promise((resolve4) => {
|
|
2720
|
+
resolveReady = resolve4;
|
|
2721
|
+
});
|
|
2722
|
+
const exited = new Promise((resolve4) => {
|
|
2723
|
+
child.on("close", (code, signal) => {
|
|
2724
|
+
resolveReady(null);
|
|
2725
|
+
resolve4({ code, signal });
|
|
2726
|
+
});
|
|
2727
|
+
child.on("error", (err) => {
|
|
2728
|
+
stderr += err.message;
|
|
2729
|
+
resolveReady(null);
|
|
2730
|
+
resolve4({ code: null, signal: null });
|
|
2731
|
+
});
|
|
2732
|
+
});
|
|
2733
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
2734
|
+
stdout += chunk;
|
|
2735
|
+
const nl = stdout.indexOf("\n");
|
|
2736
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
2737
|
+
});
|
|
2738
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
2739
|
+
stderr += chunk;
|
|
2740
|
+
});
|
|
2741
|
+
child.stdin.on("error", () => void 0);
|
|
2742
|
+
return {
|
|
2743
|
+
ready,
|
|
2744
|
+
async finish(finalize) {
|
|
2745
|
+
if (finalize) {
|
|
2746
|
+
const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
|
|
2747
|
+
child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
|
|
2748
|
+
`);
|
|
2749
|
+
}
|
|
2750
|
+
child.stdin.end();
|
|
2751
|
+
const { code, signal } = await exited;
|
|
2752
|
+
return { code, signal, stdout, stderr };
|
|
2753
|
+
}
|
|
2754
|
+
};
|
|
2755
|
+
}
|
|
2756
|
+
function readyEnv(line) {
|
|
2757
|
+
if (line === null) return null;
|
|
2758
|
+
let parsed;
|
|
2759
|
+
try {
|
|
2760
|
+
parsed = JSON.parse(line);
|
|
2761
|
+
} catch {
|
|
2762
|
+
return null;
|
|
2763
|
+
}
|
|
2764
|
+
const env = parsed?.env;
|
|
2765
|
+
if (typeof env !== "object" || env === null) return null;
|
|
2766
|
+
const entries = Object.entries(env);
|
|
2767
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
2768
|
+
return Object.fromEntries(entries);
|
|
2769
|
+
}
|
|
2770
|
+
function exitDetail(exit) {
|
|
2771
|
+
const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
|
|
2772
|
+
const err = exit.stderr.trim();
|
|
2773
|
+
return err ? `${how}: ${err}` : how;
|
|
2774
|
+
}
|
|
2775
|
+
function restore(saved) {
|
|
2776
|
+
for (const [k, v] of saved) {
|
|
2777
|
+
if (v === void 0) delete process.env[k];
|
|
2778
|
+
else process.env[k] = v;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
var recording = false;
|
|
2782
|
+
async function record(name, fn, opts = {}) {
|
|
2783
|
+
if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
|
|
2784
|
+
recording = true;
|
|
2785
|
+
try {
|
|
2786
|
+
return await recordExclusive(name, fn, opts);
|
|
2787
|
+
} finally {
|
|
2788
|
+
recording = false;
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
async function recordExclusive(name, fn, opts) {
|
|
2792
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2793
|
+
const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
|
|
2794
|
+
if (opts.expectFail) args.push("--expect-fail");
|
|
2795
|
+
if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
|
|
2796
|
+
if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
|
|
2797
|
+
const proc = serve(engine, args);
|
|
2798
|
+
const env = readyEnv(await proc.ready);
|
|
2799
|
+
if (env === null) {
|
|
2800
|
+
const exit2 = await proc.finish(false);
|
|
2801
|
+
throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
|
|
2802
|
+
}
|
|
2803
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
2804
|
+
Object.assign(process.env, env);
|
|
2805
|
+
let value;
|
|
2806
|
+
try {
|
|
2807
|
+
value = await fn({ name, env });
|
|
2808
|
+
if (opts.flush) await opts.flush();
|
|
2809
|
+
} catch (err) {
|
|
2810
|
+
restore(saved);
|
|
2811
|
+
await proc.finish(false);
|
|
2812
|
+
throw err;
|
|
2813
|
+
}
|
|
2814
|
+
restore(saved);
|
|
2815
|
+
const exit = await proc.finish(true);
|
|
2816
|
+
process.stderr.write(exit.stderr);
|
|
2817
|
+
const lines = exit.stdout.trim().split("\n");
|
|
2818
|
+
if (lines.length < 2) {
|
|
2819
|
+
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2820
|
+
}
|
|
2821
|
+
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2822
|
+
if (exit.code !== 0 || !outcome.verified) {
|
|
2823
|
+
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2824
|
+
}
|
|
2825
|
+
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2826
|
+
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
// src/replay.ts
|
|
2830
|
+
import { spawn as spawn3 } from "child_process";
|
|
2831
|
+
import * as path3 from "path";
|
|
2832
|
+
|
|
2833
|
+
// src/egress.ts
|
|
2834
|
+
import * as net from "net";
|
|
2835
|
+
var PROVIDER_HOSTS = [
|
|
2836
|
+
"api.openai.com",
|
|
2837
|
+
"api.anthropic.com",
|
|
2838
|
+
"generativelanguage.googleapis.com",
|
|
2839
|
+
/\.openai\.azure\.com$/
|
|
2840
|
+
];
|
|
2841
|
+
var EgressBlockedError = class extends Error {
|
|
2842
|
+
constructor(host) {
|
|
2843
|
+
super(
|
|
2844
|
+
`attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
|
|
2845
|
+
);
|
|
2846
|
+
this.host = host;
|
|
2847
|
+
this.name = "EgressBlockedError";
|
|
2848
|
+
}
|
|
2849
|
+
host;
|
|
2850
|
+
};
|
|
2851
|
+
function isProviderHost(host) {
|
|
2852
|
+
const h = host.toLowerCase().replace(/\.$/, "");
|
|
2853
|
+
return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
|
|
2854
|
+
}
|
|
2855
|
+
function egressBlockRequested(env = process.env) {
|
|
2856
|
+
const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
|
|
2857
|
+
if (v !== "" && v !== "block" && v !== "allow") {
|
|
2858
|
+
throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
|
|
2859
|
+
}
|
|
2860
|
+
return v === "block";
|
|
2861
|
+
}
|
|
2862
|
+
var original = null;
|
|
2863
|
+
var installs = 0;
|
|
2864
|
+
function hostOf(args) {
|
|
2865
|
+
const first = Array.isArray(args[0]) ? args[0][0] : args[0];
|
|
2866
|
+
if (typeof first === "object" && first !== null) {
|
|
2867
|
+
const o = first;
|
|
2868
|
+
return typeof o.host === "string" ? o.host : null;
|
|
2869
|
+
}
|
|
2870
|
+
if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
|
|
2871
|
+
return typeof args[1] === "string" ? args[1] : null;
|
|
2872
|
+
}
|
|
2873
|
+
return null;
|
|
2874
|
+
}
|
|
2875
|
+
function installEgressGuard() {
|
|
2876
|
+
installs++;
|
|
2877
|
+
if (original === null) {
|
|
2878
|
+
const proto = net.Socket.prototype;
|
|
2879
|
+
const orig = proto.connect;
|
|
2880
|
+
original = orig;
|
|
2881
|
+
proto.connect = function(...args) {
|
|
2882
|
+
const host = hostOf(args);
|
|
2883
|
+
if (host !== null && isProviderHost(host)) {
|
|
2884
|
+
const err = new EgressBlockedError(host);
|
|
2885
|
+
process.nextTick(() => this.destroy(err));
|
|
2886
|
+
return this;
|
|
2887
|
+
}
|
|
2888
|
+
return orig.apply(this, args);
|
|
2889
|
+
};
|
|
2890
|
+
}
|
|
2891
|
+
let released = false;
|
|
2892
|
+
return () => {
|
|
2893
|
+
if (released) return;
|
|
2894
|
+
released = true;
|
|
2895
|
+
installs--;
|
|
2896
|
+
if (installs === 0) uninstallEgressGuard();
|
|
2897
|
+
};
|
|
2898
|
+
}
|
|
2899
|
+
function egressGuardActive() {
|
|
2900
|
+
return installs > 0;
|
|
2901
|
+
}
|
|
2902
|
+
function uninstallEgressGuard() {
|
|
2903
|
+
installs = 0;
|
|
2904
|
+
if (original === null) return;
|
|
2905
|
+
net.Socket.prototype.connect = original;
|
|
2906
|
+
original = null;
|
|
2907
|
+
}
|
|
2908
|
+
|
|
2909
|
+
// src/replay.ts
|
|
2910
|
+
var ReplayMissError = class extends Error {
|
|
2911
|
+
constructor(message, summary) {
|
|
2912
|
+
super(message);
|
|
2913
|
+
this.summary = summary;
|
|
2914
|
+
this.name = "ReplayMissError";
|
|
2915
|
+
}
|
|
2916
|
+
summary;
|
|
2917
|
+
};
|
|
2918
|
+
var ReplayNotEvaluatedError = class extends Error {
|
|
2919
|
+
constructor(message, summary) {
|
|
2920
|
+
super(message);
|
|
2921
|
+
this.summary = summary;
|
|
2922
|
+
this.name = "ReplayNotEvaluatedError";
|
|
2923
|
+
}
|
|
2924
|
+
summary;
|
|
2925
|
+
};
|
|
2926
|
+
function escapeControl(text) {
|
|
2927
|
+
return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
|
|
2928
|
+
}
|
|
2929
|
+
var ReplayError = class extends Error {
|
|
2930
|
+
constructor(message) {
|
|
2931
|
+
super(escapeControl(message));
|
|
2932
|
+
this.name = "ReplayError";
|
|
2933
|
+
}
|
|
2934
|
+
};
|
|
2935
|
+
function resolveMode(explicit, env = process.env) {
|
|
2936
|
+
if (explicit !== void 0) return checkMode(explicit, "mode");
|
|
2937
|
+
const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
|
|
2938
|
+
if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
|
|
2939
|
+
const ci = (env.CI ?? "").trim().toLowerCase();
|
|
2940
|
+
return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
|
|
2941
|
+
}
|
|
2942
|
+
function checkMode(value, where) {
|
|
2943
|
+
if (value === "strict" || value === "auto" || value === "off") return value;
|
|
2944
|
+
throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
|
|
2945
|
+
}
|
|
2946
|
+
var active = null;
|
|
2947
|
+
async function startReplay(name, opts = {}) {
|
|
2948
|
+
if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
|
|
2949
|
+
const resolved = resolveMode(opts.mode);
|
|
2950
|
+
if (resolved === "off") return offSession(name);
|
|
2951
|
+
const mode = resolved;
|
|
2952
|
+
if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
|
|
2953
|
+
active = name;
|
|
2954
|
+
try {
|
|
2955
|
+
return await start(name, mode, opts);
|
|
2956
|
+
} catch (err) {
|
|
2957
|
+
active = null;
|
|
2958
|
+
throw err;
|
|
2959
|
+
}
|
|
2960
|
+
}
|
|
2961
|
+
function offSession(name) {
|
|
2962
|
+
return {
|
|
2963
|
+
name,
|
|
2964
|
+
mode: "off",
|
|
2965
|
+
env: {},
|
|
2966
|
+
summary: null,
|
|
2967
|
+
finish: () => Promise.resolve(null),
|
|
2968
|
+
error: () => null,
|
|
2969
|
+
check: () => Promise.resolve()
|
|
2970
|
+
};
|
|
2971
|
+
}
|
|
2972
|
+
function missError(mode, summary) {
|
|
2973
|
+
if (summary.misses === 0) return null;
|
|
2974
|
+
const message = escapeControl(firstMiss(summary) ?? "replay miss");
|
|
2975
|
+
return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
|
|
2976
|
+
}
|
|
2977
|
+
async function start(name, mode, opts) {
|
|
2978
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2979
|
+
const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
|
|
2980
|
+
const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
|
|
2981
|
+
const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
|
|
2982
|
+
if (blocking) args.push("--no-live");
|
|
2983
|
+
const child = spawn3(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2984
|
+
let stdout = "";
|
|
2985
|
+
let stderr = "";
|
|
2986
|
+
let resolveReady = () => void 0;
|
|
2987
|
+
const ready = new Promise((resolve4) => {
|
|
2988
|
+
resolveReady = resolve4;
|
|
2989
|
+
});
|
|
2990
|
+
const exited = new Promise((resolve4) => {
|
|
2991
|
+
child.on("close", (code) => {
|
|
2992
|
+
resolveReady(null);
|
|
2993
|
+
resolve4(code);
|
|
2994
|
+
});
|
|
2995
|
+
child.on("error", (err) => {
|
|
2996
|
+
stderr += err.message;
|
|
2997
|
+
resolveReady(null);
|
|
2998
|
+
resolve4(null);
|
|
2999
|
+
});
|
|
3000
|
+
});
|
|
3001
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3002
|
+
stdout += chunk;
|
|
3003
|
+
const nl = stdout.indexOf("\n");
|
|
3004
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3005
|
+
});
|
|
3006
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3007
|
+
stderr += chunk;
|
|
3008
|
+
});
|
|
3009
|
+
child.stdin.on("error", () => void 0);
|
|
3010
|
+
const stop = async () => {
|
|
3011
|
+
child.stdin.write("stop\n");
|
|
3012
|
+
child.stdin.end();
|
|
3013
|
+
return { code: await exited, stdout, stderr };
|
|
3014
|
+
};
|
|
3015
|
+
const env = parseEnv(await ready);
|
|
3016
|
+
if (env === null) {
|
|
3017
|
+
const out = await stop();
|
|
3018
|
+
active = null;
|
|
3019
|
+
throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3020
|
+
}
|
|
3021
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3022
|
+
Object.assign(process.env, env);
|
|
3023
|
+
let summary = null;
|
|
3024
|
+
let finishing = null;
|
|
3025
|
+
const finish = () => {
|
|
3026
|
+
finishing ??= (async () => {
|
|
3027
|
+
for (const [k, v] of saved) {
|
|
3028
|
+
if (v === void 0) delete process.env[k];
|
|
3029
|
+
else process.env[k] = v;
|
|
3030
|
+
}
|
|
3031
|
+
const out = await stop();
|
|
3032
|
+
active = null;
|
|
3033
|
+
const parsed = parseSummary(out.stdout);
|
|
3034
|
+
if (parsed === null) {
|
|
3035
|
+
throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3036
|
+
}
|
|
3037
|
+
if (blocking && parsed.live > 0) {
|
|
3038
|
+
throw new ReplayError(
|
|
3039
|
+
`replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
|
|
3040
|
+
);
|
|
3041
|
+
}
|
|
3042
|
+
summary = parsed;
|
|
3043
|
+
return parsed;
|
|
3044
|
+
})();
|
|
3045
|
+
return finishing;
|
|
3046
|
+
};
|
|
3047
|
+
return {
|
|
3048
|
+
name,
|
|
3049
|
+
mode,
|
|
3050
|
+
env,
|
|
3051
|
+
get summary() {
|
|
3052
|
+
return summary;
|
|
3053
|
+
},
|
|
3054
|
+
finish,
|
|
3055
|
+
error: () => summary === null ? null : missError(mode, summary),
|
|
3056
|
+
check: async () => {
|
|
3057
|
+
await finish();
|
|
3058
|
+
const err = summary === null ? null : missError(mode, summary);
|
|
3059
|
+
if (err !== null) throw err;
|
|
3060
|
+
}
|
|
3061
|
+
};
|
|
3062
|
+
}
|
|
3063
|
+
function parseEnv(line) {
|
|
3064
|
+
if (line === null) return null;
|
|
3065
|
+
try {
|
|
3066
|
+
const env = JSON.parse(line).env;
|
|
3067
|
+
if (typeof env !== "object" || env === null) return null;
|
|
3068
|
+
const entries = Object.entries(env);
|
|
3069
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
3070
|
+
return Object.fromEntries(entries);
|
|
3071
|
+
} catch {
|
|
3072
|
+
return null;
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
function parseSummary(out) {
|
|
3076
|
+
const lines = out.trim().split("\n");
|
|
3077
|
+
try {
|
|
3078
|
+
const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
|
|
3079
|
+
if (s === void 0 || !Array.isArray(s.records)) return null;
|
|
3080
|
+
for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
|
|
3081
|
+
return s;
|
|
3082
|
+
} catch {
|
|
3083
|
+
return null;
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
function firstMiss(summary) {
|
|
3087
|
+
return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
|
|
3088
|
+
}
|
|
3089
|
+
function liveLine(live, costUsd, unpriced = 0) {
|
|
3090
|
+
const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
|
|
3091
|
+
return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
// src/world.ts
|
|
3095
|
+
import { spawn as spawn4 } from "child_process";
|
|
3096
|
+
import * as path4 from "path";
|
|
3097
|
+
var STATE_PATH = "/_attest/state";
|
|
3098
|
+
var STATE_TIMEOUT_MS = 1e4;
|
|
3099
|
+
var WorldError = class extends Error {
|
|
3100
|
+
constructor(message) {
|
|
3101
|
+
super(escapeControl(message));
|
|
3102
|
+
this.name = "WorldError";
|
|
3103
|
+
}
|
|
3104
|
+
};
|
|
3105
|
+
var active2 = null;
|
|
3106
|
+
async function startWorld(worldFile, opts = {}) {
|
|
3107
|
+
if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
|
|
3108
|
+
const file = path4.resolve(worldFile);
|
|
3109
|
+
active2 = file;
|
|
3110
|
+
try {
|
|
3111
|
+
return await start2(file, opts);
|
|
3112
|
+
} catch (err) {
|
|
3113
|
+
active2 = null;
|
|
3114
|
+
throw err;
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
async function start2(file, opts) {
|
|
3118
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3119
|
+
const child = spawn4(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
3120
|
+
let stdout = "";
|
|
3121
|
+
let stderr = "";
|
|
3122
|
+
let resolveReady = () => void 0;
|
|
3123
|
+
const ready = new Promise((resolve4) => {
|
|
3124
|
+
resolveReady = resolve4;
|
|
3125
|
+
});
|
|
3126
|
+
const exited = new Promise((resolve4) => {
|
|
3127
|
+
child.on("close", (code) => {
|
|
3128
|
+
resolveReady(null);
|
|
3129
|
+
resolve4(code);
|
|
3130
|
+
});
|
|
3131
|
+
child.on("error", (err) => {
|
|
3132
|
+
stderr += err.message;
|
|
3133
|
+
resolveReady(null);
|
|
3134
|
+
resolve4(null);
|
|
3135
|
+
});
|
|
3136
|
+
});
|
|
3137
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3138
|
+
stdout += chunk;
|
|
3139
|
+
const nl = stdout.indexOf("\n");
|
|
3140
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3141
|
+
});
|
|
3142
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3143
|
+
stderr += chunk;
|
|
3144
|
+
});
|
|
3145
|
+
child.stdin.on("error", () => void 0);
|
|
3146
|
+
const halt = async () => {
|
|
3147
|
+
child.stdin.write("stop\n");
|
|
3148
|
+
child.stdin.end();
|
|
3149
|
+
return { code: await exited, stdout, stderr };
|
|
3150
|
+
};
|
|
3151
|
+
const env = parseEnv(await ready);
|
|
3152
|
+
if (env === null) {
|
|
3153
|
+
const out = await halt();
|
|
3154
|
+
throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3155
|
+
}
|
|
3156
|
+
const urls = Object.values(env);
|
|
3157
|
+
if (urls.length !== 1) {
|
|
3158
|
+
await halt();
|
|
3159
|
+
throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
|
|
3160
|
+
}
|
|
3161
|
+
const baseUrl = urls[0];
|
|
3162
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3163
|
+
Object.assign(process.env, env);
|
|
3164
|
+
let stopped = false;
|
|
3165
|
+
let stopping = null;
|
|
3166
|
+
const stop = () => {
|
|
3167
|
+
stopped = true;
|
|
3168
|
+
stopping ??= (async () => {
|
|
3169
|
+
for (const [k, v] of saved) {
|
|
3170
|
+
if (v === void 0) delete process.env[k];
|
|
3171
|
+
else process.env[k] = v;
|
|
3172
|
+
}
|
|
3173
|
+
const out = await halt();
|
|
3174
|
+
active2 = null;
|
|
3175
|
+
const final = parseFinal(out.stdout);
|
|
3176
|
+
if (out.code !== 0 || final === null) {
|
|
3177
|
+
throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3178
|
+
}
|
|
3179
|
+
return final;
|
|
3180
|
+
})();
|
|
3181
|
+
return stopping;
|
|
3182
|
+
};
|
|
3183
|
+
const state = async () => {
|
|
3184
|
+
if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
|
|
3185
|
+
const url = baseUrl + STATE_PATH;
|
|
3186
|
+
let res;
|
|
3187
|
+
try {
|
|
3188
|
+
res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
|
|
3189
|
+
} catch (err) {
|
|
3190
|
+
throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3191
|
+
}
|
|
3192
|
+
if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
|
|
3193
|
+
const body = await res.json().catch(() => {
|
|
3194
|
+
throw new WorldError(`GET ${url}: the body is not JSON`);
|
|
3195
|
+
});
|
|
3196
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
3197
|
+
throw new WorldError(`GET ${url}: the body is not a snapshot object`);
|
|
3198
|
+
}
|
|
3199
|
+
return body;
|
|
3200
|
+
};
|
|
3201
|
+
return { path: file, env, baseUrl, state, stop };
|
|
3202
|
+
}
|
|
3203
|
+
function parseFinal(out) {
|
|
3204
|
+
const lines = out.trim().split("\n");
|
|
3205
|
+
try {
|
|
3206
|
+
const world = JSON.parse(lines[lines.length - 1] ?? "").world;
|
|
3207
|
+
if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
|
|
3208
|
+
return world;
|
|
3209
|
+
} catch {
|
|
3210
|
+
return null;
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
3213
|
+
|
|
2416
3214
|
// src/context.ts
|
|
2417
3215
|
import { AsyncLocalStorage } from "async_hooks";
|
|
2418
3216
|
var activeBuilder = new AsyncLocalStorage();
|
|
@@ -2616,12 +3414,12 @@ var ContinuousEvalRunner = class {
|
|
|
2616
3414
|
maxSize;
|
|
2617
3415
|
running = false;
|
|
2618
3416
|
loopTimer = null;
|
|
2619
|
-
constructor(client, assertions,
|
|
3417
|
+
constructor(client, assertions, options2) {
|
|
2620
3418
|
this.client = client;
|
|
2621
3419
|
this.assertions = assertions;
|
|
2622
|
-
this.sampler = new Sampler(
|
|
2623
|
-
this.dispatcher = new AlertDispatcher(
|
|
2624
|
-
this.maxSize =
|
|
3420
|
+
this.sampler = new Sampler(options2?.sampleRate ?? 1);
|
|
3421
|
+
this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
|
|
3422
|
+
this.maxSize = options2?.maxSize ?? resolveQueueSize();
|
|
2625
3423
|
}
|
|
2626
3424
|
get alertEmitter() {
|
|
2627
3425
|
return this.dispatcher;
|
|
@@ -2732,13 +3530,13 @@ function loadLabelsCSV(text) {
|
|
|
2732
3530
|
if (rows.length === 0) {
|
|
2733
3531
|
throw new Error("empty CSV");
|
|
2734
3532
|
}
|
|
2735
|
-
let
|
|
3533
|
+
let start3 = 0;
|
|
2736
3534
|
if (rows[0].length >= 2) {
|
|
2737
3535
|
const firstHuman = parseFloatStrict(rows[0][1]);
|
|
2738
|
-
if (firstHuman === null)
|
|
3536
|
+
if (firstHuman === null) start3 = 1;
|
|
2739
3537
|
}
|
|
2740
3538
|
const out = [];
|
|
2741
|
-
for (let idx =
|
|
3539
|
+
for (let idx = start3; idx < rows.length; idx++) {
|
|
2742
3540
|
const row = rows[idx];
|
|
2743
3541
|
const lineNum = idx + 1;
|
|
2744
3542
|
if (row.length === 0) continue;
|
|
@@ -2947,10 +3745,10 @@ var OpenAIAdapter = class {
|
|
|
2947
3745
|
constructor(agentId2) {
|
|
2948
3746
|
this.agentId = agentId2;
|
|
2949
3747
|
}
|
|
2950
|
-
traceFromResponse(response,
|
|
3748
|
+
traceFromResponse(response, options2) {
|
|
2951
3749
|
const builder = new TraceBuilder(this.agentId);
|
|
2952
|
-
if (
|
|
2953
|
-
builder.setInput({ messages:
|
|
3750
|
+
if (options2?.inputMessages) {
|
|
3751
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2954
3752
|
}
|
|
2955
3753
|
const message = response.choices[0].message;
|
|
2956
3754
|
const completionText = message.content ?? "";
|
|
@@ -2970,12 +3768,12 @@ var OpenAIAdapter = class {
|
|
|
2970
3768
|
}
|
|
2971
3769
|
builder.setOutput({
|
|
2972
3770
|
message: completionText,
|
|
2973
|
-
structured:
|
|
3771
|
+
structured: options2?.structuredOutput ?? {}
|
|
2974
3772
|
});
|
|
2975
3773
|
builder.setMetadata({
|
|
2976
3774
|
total_tokens: response.usage?.total_tokens,
|
|
2977
|
-
cost_usd:
|
|
2978
|
-
latency_ms:
|
|
3775
|
+
cost_usd: options2?.costUsd,
|
|
3776
|
+
latency_ms: options2?.latencyMs,
|
|
2979
3777
|
model: response.model
|
|
2980
3778
|
});
|
|
2981
3779
|
return builder.build();
|
|
@@ -2988,10 +3786,10 @@ var AnthropicAdapter = class {
|
|
|
2988
3786
|
constructor(agentId2) {
|
|
2989
3787
|
this.agentId = agentId2;
|
|
2990
3788
|
}
|
|
2991
|
-
traceFromResponse(response,
|
|
3789
|
+
traceFromResponse(response, options2) {
|
|
2992
3790
|
const builder = new TraceBuilder(this.agentId);
|
|
2993
|
-
if (
|
|
2994
|
-
builder.setInput({ messages:
|
|
3791
|
+
if (options2?.inputMessages) {
|
|
3792
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2995
3793
|
}
|
|
2996
3794
|
const completionParts = [];
|
|
2997
3795
|
for (const block of response.content) {
|
|
@@ -3014,8 +3812,8 @@ var AnthropicAdapter = class {
|
|
|
3014
3812
|
}
|
|
3015
3813
|
builder.setMetadata({
|
|
3016
3814
|
total_tokens: totalTokens,
|
|
3017
|
-
cost_usd:
|
|
3018
|
-
latency_ms:
|
|
3815
|
+
cost_usd: options2?.costUsd,
|
|
3816
|
+
latency_ms: options2?.latencyMs,
|
|
3019
3817
|
model: response.model
|
|
3020
3818
|
});
|
|
3021
3819
|
return builder.build();
|
|
@@ -3028,10 +3826,10 @@ var GeminiAdapter = class {
|
|
|
3028
3826
|
constructor(agentId2) {
|
|
3029
3827
|
this.agentId = agentId2;
|
|
3030
3828
|
}
|
|
3031
|
-
traceFromResponse(response,
|
|
3829
|
+
traceFromResponse(response, options2) {
|
|
3032
3830
|
const builder = new TraceBuilder(this.agentId);
|
|
3033
|
-
if (
|
|
3034
|
-
builder.setInput({ text:
|
|
3831
|
+
if (options2?.inputText) {
|
|
3832
|
+
builder.setInput({ text: options2.inputText });
|
|
3035
3833
|
}
|
|
3036
3834
|
let completionText = "";
|
|
3037
3835
|
if (response.text != null) {
|
|
@@ -3051,9 +3849,9 @@ var GeminiAdapter = class {
|
|
|
3051
3849
|
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
3052
3850
|
builder.setOutput({ message: completionText });
|
|
3053
3851
|
builder.setMetadata({
|
|
3054
|
-
cost_usd:
|
|
3055
|
-
latency_ms:
|
|
3056
|
-
model:
|
|
3852
|
+
cost_usd: options2?.costUsd,
|
|
3853
|
+
latency_ms: options2?.latencyMs,
|
|
3854
|
+
model: options2?.model
|
|
3057
3855
|
});
|
|
3058
3856
|
return builder.build();
|
|
3059
3857
|
}
|
|
@@ -3065,10 +3863,10 @@ var OllamaAdapter = class {
|
|
|
3065
3863
|
constructor(agentId2) {
|
|
3066
3864
|
this.agentId = agentId2;
|
|
3067
3865
|
}
|
|
3068
|
-
traceFromResponse(response,
|
|
3866
|
+
traceFromResponse(response, options2) {
|
|
3069
3867
|
const builder = new TraceBuilder(this.agentId);
|
|
3070
|
-
if (
|
|
3071
|
-
builder.setInput({ messages:
|
|
3868
|
+
if (options2?.inputMessages) {
|
|
3869
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3072
3870
|
}
|
|
3073
3871
|
const completionText = response.message?.content ?? "";
|
|
3074
3872
|
builder.addLlmCall("completion", {
|
|
@@ -3082,7 +3880,7 @@ var OllamaAdapter = class {
|
|
|
3082
3880
|
}
|
|
3083
3881
|
builder.setMetadata({
|
|
3084
3882
|
total_tokens: totalTokens,
|
|
3085
|
-
latency_ms:
|
|
3883
|
+
latency_ms: options2?.latencyMs,
|
|
3086
3884
|
model: response.model
|
|
3087
3885
|
});
|
|
3088
3886
|
return builder.build();
|
|
@@ -3090,7 +3888,7 @@ var OllamaAdapter = class {
|
|
|
3090
3888
|
};
|
|
3091
3889
|
|
|
3092
3890
|
// src/adapters/otel.ts
|
|
3093
|
-
import {
|
|
3891
|
+
import { spawn as spawn5 } from "child_process";
|
|
3094
3892
|
var CONVERT_TIMEOUT_MS = 6e4;
|
|
3095
3893
|
var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
3096
3894
|
var STATUS_CODE_ERROR = 2;
|
|
@@ -3113,7 +3911,7 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3113
3911
|
* @throws RangeError when a parsed export holds a non-finite number.
|
|
3114
3912
|
* @throws OTLPConversionError when the engine times out, overflows the output buffer or
|
|
3115
3913
|
* prints no trace.
|
|
3116
|
-
* @throws Error
|
|
3914
|
+
* @throws Error when the engine cannot start.
|
|
3117
3915
|
*/
|
|
3118
3916
|
static async fromOtlp(otlp, agentId2) {
|
|
3119
3917
|
const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
|
|
@@ -3125,33 +3923,9 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3125
3923
|
const args = ["trace", "convert"];
|
|
3126
3924
|
if (agentId2 !== void 0) args.push("--agent-id", agentId2);
|
|
3127
3925
|
args.push("-");
|
|
3128
|
-
const
|
|
3129
|
-
input: payload,
|
|
3130
|
-
timeout: CONVERT_TIMEOUT_MS,
|
|
3131
|
-
maxBuffer: MAX_TRACE_OUTPUT_BYTES
|
|
3132
|
-
});
|
|
3133
|
-
if (proc.error) {
|
|
3134
|
-
const code = proc.error.code;
|
|
3135
|
-
if (code === "ETIMEDOUT") {
|
|
3136
|
-
throw new OTLPConversionError(
|
|
3137
|
-
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3138
|
-
);
|
|
3139
|
-
}
|
|
3140
|
-
if (code === "ENOBUFS") {
|
|
3141
|
-
throw new OTLPConversionError(
|
|
3142
|
-
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3143
|
-
);
|
|
3144
|
-
}
|
|
3145
|
-
throw proc.error;
|
|
3146
|
-
}
|
|
3147
|
-
if (proc.status !== 0) {
|
|
3148
|
-
const message = proc.stderr.toString("utf8").trim();
|
|
3149
|
-
throw new OTLPConversionError(
|
|
3150
|
-
message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
|
|
3151
|
-
);
|
|
3152
|
-
}
|
|
3926
|
+
const stdout = await runConvert(await resolveEngine(), args, payload);
|
|
3153
3927
|
try {
|
|
3154
|
-
return JSON.parse(
|
|
3928
|
+
return JSON.parse(stdout);
|
|
3155
3929
|
} catch (err) {
|
|
3156
3930
|
throw new OTLPConversionError(
|
|
3157
3931
|
`attest-engine trace convert printed output that is not JSON: ${String(err)}`
|
|
@@ -3159,6 +3933,65 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3159
3933
|
}
|
|
3160
3934
|
}
|
|
3161
3935
|
};
|
|
3936
|
+
function runConvert(engine, args, payload) {
|
|
3937
|
+
return new Promise((resolve4, reject) => {
|
|
3938
|
+
const child = spawn5(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3939
|
+
let timer;
|
|
3940
|
+
child.on("error", (e) => {
|
|
3941
|
+
clearTimeout(timer);
|
|
3942
|
+
reject(e);
|
|
3943
|
+
});
|
|
3944
|
+
const { stdin, stdout, stderr } = child;
|
|
3945
|
+
if (stdin === null || stdout === null || stderr === null) return;
|
|
3946
|
+
const out = [];
|
|
3947
|
+
const err = [];
|
|
3948
|
+
let outBytes = 0;
|
|
3949
|
+
let failure = null;
|
|
3950
|
+
const fail = (e) => {
|
|
3951
|
+
failure ??= e;
|
|
3952
|
+
child.kill("SIGKILL");
|
|
3953
|
+
};
|
|
3954
|
+
timer = setTimeout(() => {
|
|
3955
|
+
fail(
|
|
3956
|
+
new OTLPConversionError(
|
|
3957
|
+
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3958
|
+
)
|
|
3959
|
+
);
|
|
3960
|
+
}, CONVERT_TIMEOUT_MS);
|
|
3961
|
+
stdout.on("data", (chunk) => {
|
|
3962
|
+
outBytes += chunk.length;
|
|
3963
|
+
if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
|
|
3964
|
+
fail(
|
|
3965
|
+
new OTLPConversionError(
|
|
3966
|
+
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3967
|
+
)
|
|
3968
|
+
);
|
|
3969
|
+
return;
|
|
3970
|
+
}
|
|
3971
|
+
out.push(chunk);
|
|
3972
|
+
});
|
|
3973
|
+
stderr.on("data", (chunk) => err.push(chunk));
|
|
3974
|
+
stdin.on("error", () => void 0);
|
|
3975
|
+
child.on("close", (code, signal) => {
|
|
3976
|
+
clearTimeout(timer);
|
|
3977
|
+
if (failure !== null) {
|
|
3978
|
+
reject(failure);
|
|
3979
|
+
return;
|
|
3980
|
+
}
|
|
3981
|
+
if (code !== 0) {
|
|
3982
|
+
const message = Buffer.concat(err).toString("utf8").trim();
|
|
3983
|
+
reject(
|
|
3984
|
+
new OTLPConversionError(
|
|
3985
|
+
message || `attest-engine trace convert exited ${String(code ?? signal)}`
|
|
3986
|
+
)
|
|
3987
|
+
);
|
|
3988
|
+
return;
|
|
3989
|
+
}
|
|
3990
|
+
resolve4(Buffer.concat(out).toString("utf8"));
|
|
3991
|
+
});
|
|
3992
|
+
stdin.end(payload);
|
|
3993
|
+
});
|
|
3994
|
+
}
|
|
3162
3995
|
function hrTimeToNanos(time, spanName) {
|
|
3163
3996
|
if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
|
|
3164
3997
|
throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
|
|
@@ -3264,13 +4097,13 @@ var LangChainAdapter = class {
|
|
|
3264
4097
|
* @param options.model - Model name override.
|
|
3265
4098
|
* @returns A finalized Trace object.
|
|
3266
4099
|
*/
|
|
3267
|
-
buildTrace(
|
|
4100
|
+
buildTrace(options2) {
|
|
3268
4101
|
const builder = new TraceBuilder(this.agentId);
|
|
3269
4102
|
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
3270
4103
|
let totalTokens;
|
|
3271
4104
|
for (const llm of this.llmCalls) {
|
|
3272
4105
|
const args = {};
|
|
3273
|
-
if (llm.model ??
|
|
4106
|
+
if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
|
|
3274
4107
|
const result = { completion: llm.text };
|
|
3275
4108
|
if (llm.tokens !== void 0) {
|
|
3276
4109
|
result.tokens = llm.tokens;
|
|
@@ -3287,9 +4120,9 @@ var LangChainAdapter = class {
|
|
|
3287
4120
|
builder.setOutput({ message: outputMessage });
|
|
3288
4121
|
builder.setMetadata({
|
|
3289
4122
|
total_tokens: totalTokens,
|
|
3290
|
-
cost_usd:
|
|
4123
|
+
cost_usd: options2?.costUsd,
|
|
3291
4124
|
latency_ms: latencyMs,
|
|
3292
|
-
model:
|
|
4125
|
+
model: options2?.model ?? this.llmCalls[0]?.model
|
|
3293
4126
|
});
|
|
3294
4127
|
return builder.build();
|
|
3295
4128
|
}
|
|
@@ -3303,6 +4136,7 @@ export {
|
|
|
3303
4136
|
AttestClient,
|
|
3304
4137
|
CONFUSED_USER,
|
|
3305
4138
|
COOPERATIVE_USER,
|
|
4139
|
+
CaptureError,
|
|
3306
4140
|
ContinuousEvalRunner,
|
|
3307
4141
|
ENGINE_VERSION,
|
|
3308
4142
|
ERR_ASSERTION_ERROR,
|
|
@@ -3311,6 +4145,7 @@ export {
|
|
|
3311
4145
|
ERR_PROVIDER_ERROR,
|
|
3312
4146
|
ERR_SESSION_ERROR,
|
|
3313
4147
|
ERR_TIMEOUT,
|
|
4148
|
+
EgressBlockedError,
|
|
3314
4149
|
EngineManager,
|
|
3315
4150
|
EngineTimeoutError,
|
|
3316
4151
|
ExpectChain,
|
|
@@ -3335,8 +4170,12 @@ export {
|
|
|
3335
4170
|
REASON_MISSING_EVIDENCE,
|
|
3336
4171
|
REASON_PROVIDER_NOT_CONFIGURED,
|
|
3337
4172
|
REASON_PROVIDER_UNAVAILABLE,
|
|
4173
|
+
REASON_REPLAY_MISS,
|
|
3338
4174
|
REASON_SHORT_CIRCUITED,
|
|
3339
4175
|
REASON_SIMULATION,
|
|
4176
|
+
ReplayError,
|
|
4177
|
+
ReplayMissError,
|
|
4178
|
+
ReplayNotEvaluatedError,
|
|
3340
4179
|
STATUS_HARD_FAIL,
|
|
3341
4180
|
STATUS_NOT_EVALUATED,
|
|
3342
4181
|
STATUS_PASS,
|
|
@@ -3351,6 +4190,7 @@ export {
|
|
|
3351
4190
|
TIER_3,
|
|
3352
4191
|
TYPE_CONSTRAINT,
|
|
3353
4192
|
TYPE_CONTENT,
|
|
4193
|
+
TYPE_EFFECT,
|
|
3354
4194
|
TYPE_EMBEDDING,
|
|
3355
4195
|
TYPE_LLM_JUDGE,
|
|
3356
4196
|
TYPE_PLUGIN,
|
|
@@ -3360,6 +4200,8 @@ export {
|
|
|
3360
4200
|
TraceBuilder,
|
|
3361
4201
|
TraceTree,
|
|
3362
4202
|
VERSION,
|
|
4203
|
+
WorldChecks,
|
|
4204
|
+
WorldError,
|
|
3363
4205
|
activeBuilder,
|
|
3364
4206
|
addUsage,
|
|
3365
4207
|
agent,
|
|
@@ -3370,28 +4212,43 @@ export {
|
|
|
3370
4212
|
config,
|
|
3371
4213
|
decodeResponse,
|
|
3372
4214
|
delegate,
|
|
4215
|
+
dslCall,
|
|
4216
|
+
egressBlockRequested,
|
|
4217
|
+
emitVitest,
|
|
3373
4218
|
encodeRequest,
|
|
3374
4219
|
executePluginAssertion,
|
|
3375
4220
|
extractId,
|
|
3376
4221
|
extractResult,
|
|
4222
|
+
firstMiss,
|
|
3377
4223
|
formatUsage,
|
|
3378
4224
|
getSampleRate,
|
|
4225
|
+
installEgressGuard,
|
|
4226
|
+
isProviderHost,
|
|
3379
4227
|
isSimulationMode,
|
|
3380
4228
|
layerForType,
|
|
4229
|
+
liveLine,
|
|
4230
|
+
loadFixture,
|
|
3381
4231
|
loadLabelsCSV,
|
|
3382
4232
|
loadLabelsJSONL,
|
|
3383
4233
|
parseUsage,
|
|
3384
4234
|
previewLine,
|
|
3385
4235
|
promptHash,
|
|
4236
|
+
record,
|
|
3386
4237
|
renderAssertionFailure,
|
|
3387
4238
|
renderDiagnostics,
|
|
3388
4239
|
renderSummary,
|
|
4240
|
+
renderVitest,
|
|
3389
4241
|
requireSimulationCapability,
|
|
3390
4242
|
resetConfig,
|
|
4243
|
+
resolveMode,
|
|
3391
4244
|
simulationEnvEnabled,
|
|
4245
|
+
startReplay,
|
|
4246
|
+
startWorld,
|
|
3392
4247
|
sumUsage,
|
|
4248
|
+
testPathFor,
|
|
3393
4249
|
tier,
|
|
3394
4250
|
traceId,
|
|
4251
|
+
uninstallEgressGuard,
|
|
3395
4252
|
usageSummaryLine,
|
|
3396
4253
|
zeroUsage
|
|
3397
4254
|
};
|