@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/esm/index.js CHANGED
@@ -14,12 +14,14 @@ var REASON_MISSING_EVIDENCE = "missing_evidence";
14
14
  var REASON_SHORT_CIRCUITED = "short_circuited";
15
15
  var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
16
16
  var REASON_JUDGE_ABSTAINED = "judge_abstained";
17
+ var REASON_REPLAY_MISS = "replay_miss";
17
18
  var OUTCOME_PASS = "pass";
18
19
  var OUTCOME_FAIL = "fail";
19
20
  var OUTCOME_SKIP = "skip";
20
21
  var TYPE_SCHEMA = "schema";
21
22
  var TYPE_CONSTRAINT = "constraint";
22
23
  var TYPE_TRACE = "trace";
24
+ var TYPE_EFFECT = "effect";
23
25
  var TYPE_CONTENT = "content";
24
26
  var TYPE_EMBEDDING = "embedding";
25
27
  var TYPE_LLM_JUDGE = "llm_judge";
@@ -57,6 +59,7 @@ function layerForType(assertionType) {
57
59
  return 2;
58
60
  case "trace":
59
61
  case "trace_tree":
62
+ case "effect":
60
63
  return 3;
61
64
  case "content":
62
65
  return 4;
@@ -216,8 +219,8 @@ function extractId(response) {
216
219
  }
217
220
 
218
221
  // src/version.ts
219
- var VERSION = "1.0.0-rc.2";
220
- var ENGINE_VERSION = "1.0.0-rc.2";
222
+ var VERSION = "1.0.0-rc.3";
223
+ var ENGINE_VERSION = "1.0.0-rc.3";
221
224
 
222
225
  // src/engine-manager.ts
223
226
  import { spawn, spawnSync } from "child_process";
@@ -285,7 +288,7 @@ function httpsGet(url, redirectCount = 0) {
285
288
  new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
286
289
  );
287
290
  }
288
- return new Promise((resolve, reject) => {
291
+ return new Promise((resolve4, reject) => {
289
292
  const client = url.startsWith("https:") ? https : http;
290
293
  client.get(url, (res) => {
291
294
  const status = res.statusCode ?? 0;
@@ -296,7 +299,7 @@ function httpsGet(url, redirectCount = 0) {
296
299
  return;
297
300
  }
298
301
  res.resume();
299
- resolve(httpsGet(location, redirectCount + 1));
302
+ resolve4(httpsGet(location, redirectCount + 1));
300
303
  return;
301
304
  }
302
305
  if (status < 200 || status >= 300) {
@@ -306,7 +309,7 @@ function httpsGet(url, redirectCount = 0) {
306
309
  }
307
310
  const chunks = [];
308
311
  res.on("data", (chunk) => chunks.push(chunk));
309
- res.on("end", () => resolve(Buffer.concat(chunks)));
312
+ res.on("end", () => resolve4(Buffer.concat(chunks)));
310
313
  res.on("error", reject);
311
314
  }).on("error", reject);
312
315
  });
@@ -437,8 +440,8 @@ Install options:
437
440
  }
438
441
  return [await downloadEngine(), "download"];
439
442
  }
440
- async function resolveEngine(explicitPath, options = {}) {
441
- const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);
443
+ async function resolveEngine(explicitPath, options2 = {}) {
444
+ const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
442
445
  const found = engineVersion(enginePath);
443
446
  if (found !== ENGINE_VERSION) {
444
447
  throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
@@ -453,9 +456,9 @@ var EngineManager = class {
453
456
  initialized = false;
454
457
  requestId = 0;
455
458
  initResult = null;
456
- constructor(options) {
457
- this.enginePath = options?.enginePath ?? "";
458
- this.logLevel = options?.logLevel ?? "warn";
459
+ constructor(options2) {
460
+ this.enginePath = options2?.enginePath ?? "";
461
+ this.logLevel = options2?.logLevel ?? "warn";
459
462
  }
460
463
  async start() {
461
464
  this.enginePath = await resolveEngine(this.enginePath || void 0);
@@ -533,11 +536,11 @@ var EngineManager = class {
533
536
  }
534
537
  this.requestId += 1;
535
538
  const requestStr = encodeRequest(this.requestId, method, params);
536
- return new Promise((resolve, reject) => {
539
+ return new Promise((resolve4, reject) => {
537
540
  const onLine = (line) => {
538
541
  try {
539
542
  const response = decodeResponse(line);
540
- resolve(extractResult(response));
543
+ resolve4(extractResult(response));
541
544
  } catch (err) {
542
545
  reject(err);
543
546
  }
@@ -571,17 +574,17 @@ var EngineManager = class {
571
574
  // src/config.ts
572
575
  var _simulationMode = false;
573
576
  var _sampleRate = 0;
574
- function config(options) {
575
- if (options.simulation !== void 0) {
576
- _simulationMode = options.simulation;
577
+ function config(options2) {
578
+ if (options2.simulation !== void 0) {
579
+ _simulationMode = options2.simulation;
577
580
  }
578
- if (options.sampleRate !== void 0) {
579
- if (options.sampleRate < 0 || options.sampleRate > 1) {
581
+ if (options2.sampleRate !== void 0) {
582
+ if (options2.sampleRate < 0 || options2.sampleRate > 1) {
580
583
  throw new RangeError(
581
- `sampleRate must be between 0.0 and 1.0, got ${options.sampleRate}`
584
+ `sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
582
585
  );
583
586
  }
584
- _sampleRate = options.sampleRate;
587
+ _sampleRate = options2.sampleRate;
585
588
  }
586
589
  }
587
590
  function isSimulationMode() {
@@ -737,16 +740,16 @@ var AttestClient = class {
737
740
  desyncThreshold;
738
741
  desyncWindowMs;
739
742
  desynced = false;
740
- constructor(engine, options) {
743
+ constructor(engine, options2) {
741
744
  this.engine = engine;
742
- this.logger = options?.logger ?? envLoggerDefault();
745
+ this.logger = options2?.logger ?? envLoggerDefault();
743
746
  this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
744
- options?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
747
+ options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
745
748
  );
746
- this.desyncThreshold = options?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
747
- this.desyncWindowMs = options?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
748
- if (options?.onDiagnostic !== void 0) {
749
- this.diagnosticListeners.add(options.onDiagnostic);
749
+ this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
750
+ this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
751
+ if (options2?.onDiagnostic !== void 0) {
752
+ this.diagnosticListeners.add(options2.onDiagnostic);
750
753
  }
751
754
  }
752
755
  /** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
@@ -904,12 +907,12 @@ var AttestClient = class {
904
907
  throw new Error("Engine process not started.");
905
908
  }
906
909
  let capturedReqId;
907
- const requestPromise = new Promise((resolve, reject) => {
910
+ const requestPromise = new Promise((resolve4, reject) => {
908
911
  this.writeChain = this.writeChain.then(() => {
909
912
  this.requestId += 1;
910
913
  const reqId = this.requestId;
911
914
  capturedReqId = reqId;
912
- this.pending.set(reqId, { resolve, reject });
915
+ this.pending.set(reqId, { resolve: resolve4, reject });
913
916
  const requestStr = encodeRequest(reqId, method, params);
914
917
  cp.stdin.write(requestStr, (err) => {
915
918
  if (err) {
@@ -927,23 +930,23 @@ var AttestClient = class {
927
930
  throw err;
928
931
  });
929
932
  }
930
- async evaluateBatch(trace, assertions, options) {
933
+ async evaluateBatch(trace, assertions, options2) {
931
934
  if (isSimulationMode()) {
932
935
  requireSimulationCapability(this.engine["initResult"]);
933
936
  }
934
937
  const params = {
935
938
  trace,
936
939
  assertions: [...assertions],
937
- ...options?.budget !== void 0 ? { budget: options.budget } : {}
940
+ ...options2?.budget !== void 0 ? { budget: options2.budget } : {}
938
941
  };
939
942
  const raw = await this.sendRequest(
940
943
  "evaluate_batch",
941
944
  params,
942
- options?.timeout
945
+ options2?.timeout
943
946
  );
944
947
  return { ...raw, usage: parseUsage(raw["usage"]) };
945
948
  }
946
- async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options) {
949
+ async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
947
950
  const result = {
948
951
  assertion_id: assertionId2,
949
952
  status,
@@ -961,11 +964,11 @@ var AttestClient = class {
961
964
  const raw = await this.sendRequest(
962
965
  "submit_plugin_result",
963
966
  params,
964
- options?.timeout
967
+ options2?.timeout
965
968
  );
966
969
  return Boolean(raw?.accepted ?? false);
967
970
  }
968
- async queryDrift(assertionId2, windowSize = 50, options) {
971
+ async queryDrift(assertionId2, windowSize = 50, options2) {
969
972
  if (isSimulationMode()) {
970
973
  return {
971
974
  assertion_id: assertionId2,
@@ -980,11 +983,11 @@ var AttestClient = class {
980
983
  const raw = await this.sendRequest(
981
984
  "query_drift",
982
985
  { assertion_id: assertionId2, window_size: windowSize },
983
- options?.timeout
986
+ options2?.timeout
984
987
  );
985
988
  return raw.report;
986
989
  }
987
- async generateUserMessage(persona, conversationHistory, faultConfig, options) {
990
+ async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
988
991
  if (isSimulationMode()) {
989
992
  return `[simulation] Hello from ${persona.name}`;
990
993
  }
@@ -998,7 +1001,7 @@ var AttestClient = class {
998
1001
  const raw = await this.sendRequest(
999
1002
  "generate_user_message",
1000
1003
  params,
1001
- options?.timeout
1004
+ options2?.timeout
1002
1005
  );
1003
1006
  return String(raw.message);
1004
1007
  }
@@ -1043,33 +1046,33 @@ var TraceBuilder = class {
1043
1046
  this.input = input;
1044
1047
  return this;
1045
1048
  }
1046
- addLlmCall(name, options) {
1049
+ addLlmCall(name, options2) {
1047
1050
  this.steps.push({
1048
1051
  type: STEP_LLM_CALL,
1049
1052
  name,
1050
- args: options?.args,
1051
- result: options?.result,
1052
- metadata: options?.metadata
1053
+ args: options2?.args,
1054
+ result: options2?.result,
1055
+ metadata: options2?.metadata
1053
1056
  });
1054
1057
  return this;
1055
1058
  }
1056
- addToolCall(name, options) {
1059
+ addToolCall(name, options2) {
1057
1060
  this.steps.push({
1058
1061
  type: STEP_TOOL_CALL,
1059
1062
  name,
1060
- args: options?.args,
1061
- result: options?.result,
1062
- metadata: options?.metadata
1063
+ args: options2?.args,
1064
+ result: options2?.result,
1065
+ metadata: options2?.metadata
1063
1066
  });
1064
1067
  return this;
1065
1068
  }
1066
- addRetrieval(name, options) {
1069
+ addRetrieval(name, options2) {
1067
1070
  this.steps.push({
1068
1071
  type: STEP_RETRIEVAL,
1069
1072
  name,
1070
- args: options?.args,
1071
- result: options?.result,
1072
- metadata: options?.metadata
1073
+ args: options2?.args,
1074
+ result: options2?.result,
1075
+ metadata: options2?.metadata
1073
1076
  });
1074
1077
  return this;
1075
1078
  }
@@ -1147,10 +1150,10 @@ var AgentResult = class {
1147
1150
  * not_evaluated skips (fails when `strict`); otherwise pass, including
1148
1151
  * when only soft_fail rows are present (see `softFailures`).
1149
1152
  */
1150
- outcome(options) {
1153
+ outcome(options2) {
1151
1154
  if (this.hardFailures.length > 0) return OUTCOME_FAIL;
1152
1155
  if (this.notEvaluated.length > 0) {
1153
- return options?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
1156
+ return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
1154
1157
  }
1155
1158
  return OUTCOME_PASS;
1156
1159
  }
@@ -1260,9 +1263,9 @@ function truncate(value, limit = 280) {
1260
1263
  }
1261
1264
  return value.slice(0, limit - 3) + "...";
1262
1265
  }
1263
- function renderAssertionFailure(result, options = {}) {
1264
- const color = options.color ?? false;
1265
- const testFile = options.testFile;
1266
+ function renderAssertionFailure(result, options2 = {}) {
1267
+ const color = options2.color ?? false;
1268
+ const testFile = options2.testFile;
1266
1269
  const layer = result.layer ?? 0;
1267
1270
  const label = layerLabel(layer, result.type);
1268
1271
  const lines = [];
@@ -1343,8 +1346,8 @@ function appendJudgeMetadata(lines, meta) {
1343
1346
  );
1344
1347
  }
1345
1348
  }
1346
- function renderSummary(result, options = {}) {
1347
- const color = options.color ?? false;
1349
+ function renderSummary(result, options2 = {}) {
1350
+ const color = options2.color ?? false;
1348
1351
  const passN = result.passCount;
1349
1352
  const soft = result.assertionResults.filter(
1350
1353
  (r) => r.status === STATUS_SOFT_FAIL
@@ -1362,9 +1365,9 @@ function renderSummary(result, options = {}) {
1362
1365
  ];
1363
1366
  return parts.join(" | ");
1364
1367
  }
1365
- function renderDiagnostics(result, options = {}) {
1366
- const color = options.color ?? false;
1367
- const includePassing = options.includePassing ?? false;
1368
+ function renderDiagnostics(result, options2 = {}) {
1369
+ const color = options2.color ?? false;
1370
+ const includePassing = options2.includePassing ?? false;
1368
1371
  const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
1369
1372
  if (rows.length === 0) {
1370
1373
  return dim(" (no failures)\n", color);
@@ -1372,7 +1375,7 @@ function renderDiagnostics(result, options = {}) {
1372
1375
  const out = [""];
1373
1376
  out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
1374
1377
  for (const r of rows) {
1375
- out.push(renderAssertionFailure(r, options));
1378
+ out.push(renderAssertionFailure(r, options2));
1376
1379
  }
1377
1380
  out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
1378
1381
  return out.join("\n");
@@ -1489,9 +1492,64 @@ var TraceTree = class {
1489
1492
 
1490
1493
  // src/expect.ts
1491
1494
  import { randomUUID as randomUUID2 } from "crypto";
1495
+ var MAX_WORLD_BYTES = 9 * 1024 * 1024;
1496
+ var WorldChecks = class {
1497
+ _snapshot;
1498
+ _add;
1499
+ constructor(snapshot, add) {
1500
+ this._snapshot = snapshot;
1501
+ this._add = add;
1502
+ }
1503
+ addCheck(spec, where) {
1504
+ const full = { ...spec, world: this._snapshot };
1505
+ if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
1506
+ return this._add(full);
1507
+ }
1508
+ /**
1509
+ * Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
1510
+ * `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
1511
+ *
1512
+ * @throws Error when no bound is given, or `equals` is combined with another bound.
1513
+ *
1514
+ * @example
1515
+ * ```ts
1516
+ * expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
1517
+ * ```
1518
+ */
1519
+ count(resource, opts) {
1520
+ if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
1521
+ throw new Error("count needs equals, atMost or atLeast");
1522
+ }
1523
+ if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
1524
+ throw new Error("count: equals excludes atMost and atLeast");
1525
+ }
1526
+ const spec = { check: "count", resource };
1527
+ if (opts.equals !== void 0) spec.equals = opts.equals;
1528
+ if (opts.atMost !== void 0) spec.at_most = opts.atMost;
1529
+ if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
1530
+ return this.addCheck(spec, opts.where);
1531
+ }
1532
+ /** Assert no two matching records share the same values for field(s) `by`. */
1533
+ unique(resource, opts) {
1534
+ const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
1535
+ if (fields.length === 0) throw new Error("unique needs at least one field in by");
1536
+ return this.addCheck({ check: "unique", resource, fields }, opts.where);
1537
+ }
1538
+ /**
1539
+ * Assert every matching record carries a distinct value in field `key` (default
1540
+ * `idempotency_key`), the record field that stores the idempotency key.
1541
+ */
1542
+ idempotent(resource, opts = {}) {
1543
+ return this.addCheck(
1544
+ { check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
1545
+ opts.where
1546
+ );
1547
+ }
1548
+ };
1492
1549
  var ExpectChain = class {
1493
1550
  _result;
1494
1551
  _assertions = [];
1552
+ _worldBytes = 0;
1495
1553
  constructor(result) {
1496
1554
  this._result = result;
1497
1555
  }
@@ -1920,6 +1978,24 @@ var ExpectChain = class {
1920
1978
  soft: opts?.soft ?? false
1921
1979
  });
1922
1980
  }
1981
+ // -- Layer 3: Effect checks (the external system the agent acted on) --
1982
+ /**
1983
+ * Check effects in a world snapshot:
1984
+ * `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
1985
+ * a world helper: the checks are `not_evaluated` (`missing_evidence`).
1986
+ */
1987
+ world(snapshot) {
1988
+ const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
1989
+ return new WorldChecks(snapshot, (spec) => {
1990
+ this._worldBytes += size;
1991
+ if (this._worldBytes > MAX_WORLD_BYTES) {
1992
+ throw new Error(
1993
+ `effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
1994
+ );
1995
+ }
1996
+ return this.add(TYPE_EFFECT, spec);
1997
+ });
1998
+ }
1923
1999
  // -- Layer 4: Content --
1924
2000
  /**
1925
2001
  * Assert the output message contains the given string.
@@ -2413,6 +2489,728 @@ function attestExpect(result) {
2413
2489
  return new ExpectChain(new AgentResult(result));
2414
2490
  }
2415
2491
 
2492
+ // src/capture.ts
2493
+ import { spawn as spawn2 } from "child_process";
2494
+ import * as fs2 from "fs";
2495
+ import * as path2 from "path";
2496
+ import { fileURLToPath } from "url";
2497
+ var FIXTURE_VERSION = 1;
2498
+ var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
2499
+ var CaptureError = class extends Error {
2500
+ /** The fixture written, when the engine wrote one. */
2501
+ fixture;
2502
+ constructor(message, fixture = null) {
2503
+ super(message);
2504
+ this.name = "CaptureError";
2505
+ this.fixture = fixture;
2506
+ }
2507
+ };
2508
+ function toPath(p) {
2509
+ return p instanceof URL ? fileURLToPath(p) : p;
2510
+ }
2511
+ function loadFixture(file) {
2512
+ const p = toPath(file);
2513
+ const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
2514
+ if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
2515
+ throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
2516
+ }
2517
+ validate(p, doc);
2518
+ return doc;
2519
+ }
2520
+ var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
2521
+ function isObject(v) {
2522
+ return typeof v === "object" && v !== null && !Array.isArray(v);
2523
+ }
2524
+ function kindOf(v) {
2525
+ if (Array.isArray(v)) return "array";
2526
+ if (isObject(v)) return "object";
2527
+ if (typeof v === "string" || typeof v === "boolean") return typeof v;
2528
+ return "other";
2529
+ }
2530
+ function validate(p, doc) {
2531
+ const need = (where, obj, fields) => {
2532
+ if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
2533
+ for (const [key, kind] of Object.entries(fields)) {
2534
+ if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
2535
+ }
2536
+ return obj;
2537
+ };
2538
+ need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
2539
+ if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
2540
+ if (!NAME_PATTERN.test(doc["name"])) {
2541
+ throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
2542
+ }
2543
+ const v = need("fixture.verification", doc["verification"], {
2544
+ verified: "boolean",
2545
+ expect_fail: "boolean",
2546
+ results: "array",
2547
+ expected_failures: "array"
2548
+ });
2549
+ doc["contracts"].forEach(
2550
+ (c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
2551
+ );
2552
+ v["results"].forEach(
2553
+ (r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
2554
+ );
2555
+ }
2556
+ function objectKey(k) {
2557
+ if (k === "__proto__") return `[${JSON.stringify(k)}]`;
2558
+ return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
2559
+ }
2560
+ function comment(text) {
2561
+ return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
2562
+ }
2563
+ function ts(value) {
2564
+ if (value === null || typeof value === "boolean" || typeof value === "number") {
2565
+ return JSON.stringify(value);
2566
+ }
2567
+ if (typeof value === "string") return JSON.stringify(value);
2568
+ if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
2569
+ if (typeof value === "object") {
2570
+ const entries = Object.entries(value).map(
2571
+ ([k, v]) => `${objectKey(k)}: ${ts(v)}`
2572
+ );
2573
+ return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
2574
+ }
2575
+ throw new Error(`value ${String(value)} has no literal`);
2576
+ }
2577
+ function str(spec, key) {
2578
+ const v = spec[key];
2579
+ if (typeof v !== "string") throw new Error(`missing ${key}`);
2580
+ return v;
2581
+ }
2582
+ function options(entries) {
2583
+ const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
2584
+ return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
2585
+ }
2586
+ function call(method, args, opts) {
2587
+ return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
2588
+ }
2589
+ function dslCall(contract) {
2590
+ const s = contract.spec;
2591
+ const soft = s["soft"] === true ? true : void 0;
2592
+ try {
2593
+ if (contract.type === "trace") {
2594
+ const check = s["check"];
2595
+ if (check === "trajectory" && s["mode"] !== "partial_order") {
2596
+ const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
2597
+ if (!Array.isArray(s["tools"])) throw new Error("missing tools");
2598
+ return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
2599
+ }
2600
+ if (check === "tool_order") {
2601
+ return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
2602
+ }
2603
+ if (check === "tool_cardinality") {
2604
+ return call(
2605
+ "toolCardinality",
2606
+ [ts(str(s, "tool"))],
2607
+ options([
2608
+ ["per", s["per"]],
2609
+ ["atLeast", s["min"]],
2610
+ ["atMost", s["max"]],
2611
+ ["soft", soft]
2612
+ ])
2613
+ );
2614
+ }
2615
+ }
2616
+ if (contract.type === "schema" && s["target"] === "output.structured") {
2617
+ if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
2618
+ return call("outputMatchesSchema", [ts(s["schema"])], "");
2619
+ }
2620
+ if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
2621
+ if (typeof s["value"] !== "number") throw new Error("missing value");
2622
+ return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
2623
+ }
2624
+ } catch (err) {
2625
+ throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
2626
+ }
2627
+ throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
2628
+ }
2629
+ function testPathFor(fixturePath) {
2630
+ const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
2631
+ return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
2632
+ }
2633
+ function renderVitest(fixture, fixturePath, testPath) {
2634
+ const calls = [];
2635
+ const skipped = [];
2636
+ for (const c of fixture.contracts) {
2637
+ try {
2638
+ calls.push([c, dslCall(c)]);
2639
+ } catch (err) {
2640
+ skipped.push(err.message);
2641
+ }
2642
+ }
2643
+ const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
2644
+ const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
2645
+ const expectFail = fixture.verification.expect_fail;
2646
+ const lines = [
2647
+ `// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
2648
+ "//",
2649
+ "// Review and edit before committing: they check the captured trace without running",
2650
+ "// the agent. Re-capturing does not overwrite this file.",
2651
+ `import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
2652
+ `import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
2653
+ `import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
2654
+ "",
2655
+ `const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
2656
+ ];
2657
+ if (skipped.length > 0) {
2658
+ lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
2659
+ for (const s of skipped) lines.push(`// - ${comment(s)}`);
2660
+ }
2661
+ lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
2662
+ for (const [c, dsl] of calls) {
2663
+ lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
2664
+ }
2665
+ lines[lines.length - 1] += ";";
2666
+ lines.push("}");
2667
+ if (expectFail) {
2668
+ lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
2669
+ lines.push("const EXPECTED: Array<[string, string]> = [");
2670
+ for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
2671
+ lines.push(
2672
+ "];",
2673
+ "",
2674
+ `describe(${ts(fixture.name)}, () => {`,
2675
+ ` it("reproduces the incident", async () => {`,
2676
+ " const result = await inspect(contracts());",
2677
+ "",
2678
+ " expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
2679
+ " });",
2680
+ "});"
2681
+ );
2682
+ } else {
2683
+ lines.push(
2684
+ "",
2685
+ `describe(${ts(fixture.name)}, () => {`,
2686
+ ` it("keeps its drafted contracts", async () => {`,
2687
+ " await evaluate(contracts());",
2688
+ " });",
2689
+ "});"
2690
+ );
2691
+ }
2692
+ return `${lines.join("\n")}
2693
+ `;
2694
+ }
2695
+ function emitVitest(fixturePath) {
2696
+ const testPath = testPathFor(fixturePath);
2697
+ const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
2698
+ if (existing?.isSymbolicLink()) {
2699
+ throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
2700
+ }
2701
+ if (existing !== void 0) return null;
2702
+ const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
2703
+ fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
2704
+ try {
2705
+ fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
2706
+ } catch (err) {
2707
+ if (err.code === "EEXIST") {
2708
+ throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
2709
+ }
2710
+ throw err;
2711
+ }
2712
+ return testPath;
2713
+ }
2714
+ function serve(engine, args) {
2715
+ const child = spawn2(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
2716
+ let stdout = "";
2717
+ let stderr = "";
2718
+ let resolveReady = () => void 0;
2719
+ const ready = new Promise((resolve4) => {
2720
+ resolveReady = resolve4;
2721
+ });
2722
+ const exited = new Promise((resolve4) => {
2723
+ child.on("close", (code, signal) => {
2724
+ resolveReady(null);
2725
+ resolve4({ code, signal });
2726
+ });
2727
+ child.on("error", (err) => {
2728
+ stderr += err.message;
2729
+ resolveReady(null);
2730
+ resolve4({ code: null, signal: null });
2731
+ });
2732
+ });
2733
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
2734
+ stdout += chunk;
2735
+ const nl = stdout.indexOf("\n");
2736
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
2737
+ });
2738
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
2739
+ stderr += chunk;
2740
+ });
2741
+ child.stdin.on("error", () => void 0);
2742
+ return {
2743
+ ready,
2744
+ async finish(finalize) {
2745
+ if (finalize) {
2746
+ const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
2747
+ child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
2748
+ `);
2749
+ }
2750
+ child.stdin.end();
2751
+ const { code, signal } = await exited;
2752
+ return { code, signal, stdout, stderr };
2753
+ }
2754
+ };
2755
+ }
2756
+ function readyEnv(line) {
2757
+ if (line === null) return null;
2758
+ let parsed;
2759
+ try {
2760
+ parsed = JSON.parse(line);
2761
+ } catch {
2762
+ return null;
2763
+ }
2764
+ const env = parsed?.env;
2765
+ if (typeof env !== "object" || env === null) return null;
2766
+ const entries = Object.entries(env);
2767
+ if (!entries.every(([, v]) => typeof v === "string")) return null;
2768
+ return Object.fromEntries(entries);
2769
+ }
2770
+ function exitDetail(exit) {
2771
+ const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
2772
+ const err = exit.stderr.trim();
2773
+ return err ? `${how}: ${err}` : how;
2774
+ }
2775
+ function restore(saved) {
2776
+ for (const [k, v] of saved) {
2777
+ if (v === void 0) delete process.env[k];
2778
+ else process.env[k] = v;
2779
+ }
2780
+ }
2781
+ var recording = false;
2782
+ async function record(name, fn, opts = {}) {
2783
+ if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
2784
+ recording = true;
2785
+ try {
2786
+ return await recordExclusive(name, fn, opts);
2787
+ } finally {
2788
+ recording = false;
2789
+ }
2790
+ }
2791
+ async function recordExclusive(name, fn, opts) {
2792
+ const engine = await resolveEngine(opts.enginePath);
2793
+ const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
2794
+ if (opts.expectFail) args.push("--expect-fail");
2795
+ if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
2796
+ if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
2797
+ const proc = serve(engine, args);
2798
+ const env = readyEnv(await proc.ready);
2799
+ if (env === null) {
2800
+ const exit2 = await proc.finish(false);
2801
+ throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
2802
+ }
2803
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
2804
+ Object.assign(process.env, env);
2805
+ let value;
2806
+ try {
2807
+ value = await fn({ name, env });
2808
+ if (opts.flush) await opts.flush();
2809
+ } catch (err) {
2810
+ restore(saved);
2811
+ await proc.finish(false);
2812
+ throw err;
2813
+ }
2814
+ restore(saved);
2815
+ const exit = await proc.finish(true);
2816
+ process.stderr.write(exit.stderr);
2817
+ const lines = exit.stdout.trim().split("\n");
2818
+ if (lines.length < 2) {
2819
+ throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
2820
+ }
2821
+ const outcome = JSON.parse(lines[lines.length - 1] ?? "");
2822
+ if (exit.code !== 0 || !outcome.verified) {
2823
+ throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
2824
+ }
2825
+ const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
2826
+ return { value, fixture: outcome.fixture, verified: true, testFile };
2827
+ }
2828
+
2829
+ // src/replay.ts
2830
+ import { spawn as spawn3 } from "child_process";
2831
+ import * as path3 from "path";
2832
+
2833
+ // src/egress.ts
2834
+ import * as net from "net";
2835
+ var PROVIDER_HOSTS = [
2836
+ "api.openai.com",
2837
+ "api.anthropic.com",
2838
+ "generativelanguage.googleapis.com",
2839
+ /\.openai\.azure\.com$/
2840
+ ];
2841
+ var EgressBlockedError = class extends Error {
2842
+ constructor(host) {
2843
+ super(
2844
+ `attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
2845
+ );
2846
+ this.host = host;
2847
+ this.name = "EgressBlockedError";
2848
+ }
2849
+ host;
2850
+ };
2851
+ function isProviderHost(host) {
2852
+ const h = host.toLowerCase().replace(/\.$/, "");
2853
+ return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
2854
+ }
2855
+ function egressBlockRequested(env = process.env) {
2856
+ const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
2857
+ if (v !== "" && v !== "block" && v !== "allow") {
2858
+ throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
2859
+ }
2860
+ return v === "block";
2861
+ }
2862
+ var original = null;
2863
+ var installs = 0;
2864
+ function hostOf(args) {
2865
+ const first = Array.isArray(args[0]) ? args[0][0] : args[0];
2866
+ if (typeof first === "object" && first !== null) {
2867
+ const o = first;
2868
+ return typeof o.host === "string" ? o.host : null;
2869
+ }
2870
+ if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
2871
+ return typeof args[1] === "string" ? args[1] : null;
2872
+ }
2873
+ return null;
2874
+ }
2875
+ function installEgressGuard() {
2876
+ installs++;
2877
+ if (original === null) {
2878
+ const proto = net.Socket.prototype;
2879
+ const orig = proto.connect;
2880
+ original = orig;
2881
+ proto.connect = function(...args) {
2882
+ const host = hostOf(args);
2883
+ if (host !== null && isProviderHost(host)) {
2884
+ const err = new EgressBlockedError(host);
2885
+ process.nextTick(() => this.destroy(err));
2886
+ return this;
2887
+ }
2888
+ return orig.apply(this, args);
2889
+ };
2890
+ }
2891
+ let released = false;
2892
+ return () => {
2893
+ if (released) return;
2894
+ released = true;
2895
+ installs--;
2896
+ if (installs === 0) uninstallEgressGuard();
2897
+ };
2898
+ }
2899
+ function egressGuardActive() {
2900
+ return installs > 0;
2901
+ }
2902
+ function uninstallEgressGuard() {
2903
+ installs = 0;
2904
+ if (original === null) return;
2905
+ net.Socket.prototype.connect = original;
2906
+ original = null;
2907
+ }
2908
+
2909
+ // src/replay.ts
2910
+ var ReplayMissError = class extends Error {
2911
+ constructor(message, summary) {
2912
+ super(message);
2913
+ this.summary = summary;
2914
+ this.name = "ReplayMissError";
2915
+ }
2916
+ summary;
2917
+ };
2918
+ var ReplayNotEvaluatedError = class extends Error {
2919
+ constructor(message, summary) {
2920
+ super(message);
2921
+ this.summary = summary;
2922
+ this.name = "ReplayNotEvaluatedError";
2923
+ }
2924
+ summary;
2925
+ };
2926
+ function escapeControl(text) {
2927
+ return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
2928
+ }
2929
+ var ReplayError = class extends Error {
2930
+ constructor(message) {
2931
+ super(escapeControl(message));
2932
+ this.name = "ReplayError";
2933
+ }
2934
+ };
2935
+ function resolveMode(explicit, env = process.env) {
2936
+ if (explicit !== void 0) return checkMode(explicit, "mode");
2937
+ const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
2938
+ if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
2939
+ const ci = (env.CI ?? "").trim().toLowerCase();
2940
+ return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
2941
+ }
2942
+ function checkMode(value, where) {
2943
+ if (value === "strict" || value === "auto" || value === "off") return value;
2944
+ throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
2945
+ }
2946
+ var active = null;
2947
+ async function startReplay(name, opts = {}) {
2948
+ if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
2949
+ const resolved = resolveMode(opts.mode);
2950
+ if (resolved === "off") return offSession(name);
2951
+ const mode = resolved;
2952
+ if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
2953
+ active = name;
2954
+ try {
2955
+ return await start(name, mode, opts);
2956
+ } catch (err) {
2957
+ active = null;
2958
+ throw err;
2959
+ }
2960
+ }
2961
+ function offSession(name) {
2962
+ return {
2963
+ name,
2964
+ mode: "off",
2965
+ env: {},
2966
+ summary: null,
2967
+ finish: () => Promise.resolve(null),
2968
+ error: () => null,
2969
+ check: () => Promise.resolve()
2970
+ };
2971
+ }
2972
+ function missError(mode, summary) {
2973
+ if (summary.misses === 0) return null;
2974
+ const message = escapeControl(firstMiss(summary) ?? "replay miss");
2975
+ return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
2976
+ }
2977
+ async function start(name, mode, opts) {
2978
+ const engine = await resolveEngine(opts.enginePath);
2979
+ const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
2980
+ const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
2981
+ const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
2982
+ if (blocking) args.push("--no-live");
2983
+ const child = spawn3(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
2984
+ let stdout = "";
2985
+ let stderr = "";
2986
+ let resolveReady = () => void 0;
2987
+ const ready = new Promise((resolve4) => {
2988
+ resolveReady = resolve4;
2989
+ });
2990
+ const exited = new Promise((resolve4) => {
2991
+ child.on("close", (code) => {
2992
+ resolveReady(null);
2993
+ resolve4(code);
2994
+ });
2995
+ child.on("error", (err) => {
2996
+ stderr += err.message;
2997
+ resolveReady(null);
2998
+ resolve4(null);
2999
+ });
3000
+ });
3001
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
3002
+ stdout += chunk;
3003
+ const nl = stdout.indexOf("\n");
3004
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
3005
+ });
3006
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
3007
+ stderr += chunk;
3008
+ });
3009
+ child.stdin.on("error", () => void 0);
3010
+ const stop = async () => {
3011
+ child.stdin.write("stop\n");
3012
+ child.stdin.end();
3013
+ return { code: await exited, stdout, stderr };
3014
+ };
3015
+ const env = parseEnv(await ready);
3016
+ if (env === null) {
3017
+ const out = await stop();
3018
+ active = null;
3019
+ throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
3020
+ }
3021
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
3022
+ Object.assign(process.env, env);
3023
+ let summary = null;
3024
+ let finishing = null;
3025
+ const finish = () => {
3026
+ finishing ??= (async () => {
3027
+ for (const [k, v] of saved) {
3028
+ if (v === void 0) delete process.env[k];
3029
+ else process.env[k] = v;
3030
+ }
3031
+ const out = await stop();
3032
+ active = null;
3033
+ const parsed = parseSummary(out.stdout);
3034
+ if (parsed === null) {
3035
+ throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
3036
+ }
3037
+ if (blocking && parsed.live > 0) {
3038
+ throw new ReplayError(
3039
+ `replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
3040
+ );
3041
+ }
3042
+ summary = parsed;
3043
+ return parsed;
3044
+ })();
3045
+ return finishing;
3046
+ };
3047
+ return {
3048
+ name,
3049
+ mode,
3050
+ env,
3051
+ get summary() {
3052
+ return summary;
3053
+ },
3054
+ finish,
3055
+ error: () => summary === null ? null : missError(mode, summary),
3056
+ check: async () => {
3057
+ await finish();
3058
+ const err = summary === null ? null : missError(mode, summary);
3059
+ if (err !== null) throw err;
3060
+ }
3061
+ };
3062
+ }
3063
+ function parseEnv(line) {
3064
+ if (line === null) return null;
3065
+ try {
3066
+ const env = JSON.parse(line).env;
3067
+ if (typeof env !== "object" || env === null) return null;
3068
+ const entries = Object.entries(env);
3069
+ if (!entries.every(([, v]) => typeof v === "string")) return null;
3070
+ return Object.fromEntries(entries);
3071
+ } catch {
3072
+ return null;
3073
+ }
3074
+ }
3075
+ function parseSummary(out) {
3076
+ const lines = out.trim().split("\n");
3077
+ try {
3078
+ const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
3079
+ if (s === void 0 || !Array.isArray(s.records)) return null;
3080
+ for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
3081
+ return s;
3082
+ } catch {
3083
+ return null;
3084
+ }
3085
+ }
3086
+ function firstMiss(summary) {
3087
+ return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
3088
+ }
3089
+ function liveLine(live, costUsd, unpriced = 0) {
3090
+ const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
3091
+ return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
3092
+ }
3093
+
3094
+ // src/world.ts
3095
+ import { spawn as spawn4 } from "child_process";
3096
+ import * as path4 from "path";
3097
+ var STATE_PATH = "/_attest/state";
3098
+ var STATE_TIMEOUT_MS = 1e4;
3099
+ var WorldError = class extends Error {
3100
+ constructor(message) {
3101
+ super(escapeControl(message));
3102
+ this.name = "WorldError";
3103
+ }
3104
+ };
3105
+ var active2 = null;
3106
+ async function startWorld(worldFile, opts = {}) {
3107
+ if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
3108
+ const file = path4.resolve(worldFile);
3109
+ active2 = file;
3110
+ try {
3111
+ return await start2(file, opts);
3112
+ } catch (err) {
3113
+ active2 = null;
3114
+ throw err;
3115
+ }
3116
+ }
3117
+ async function start2(file, opts) {
3118
+ const engine = await resolveEngine(opts.enginePath);
3119
+ const child = spawn4(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
3120
+ let stdout = "";
3121
+ let stderr = "";
3122
+ let resolveReady = () => void 0;
3123
+ const ready = new Promise((resolve4) => {
3124
+ resolveReady = resolve4;
3125
+ });
3126
+ const exited = new Promise((resolve4) => {
3127
+ child.on("close", (code) => {
3128
+ resolveReady(null);
3129
+ resolve4(code);
3130
+ });
3131
+ child.on("error", (err) => {
3132
+ stderr += err.message;
3133
+ resolveReady(null);
3134
+ resolve4(null);
3135
+ });
3136
+ });
3137
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
3138
+ stdout += chunk;
3139
+ const nl = stdout.indexOf("\n");
3140
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
3141
+ });
3142
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
3143
+ stderr += chunk;
3144
+ });
3145
+ child.stdin.on("error", () => void 0);
3146
+ const halt = async () => {
3147
+ child.stdin.write("stop\n");
3148
+ child.stdin.end();
3149
+ return { code: await exited, stdout, stderr };
3150
+ };
3151
+ const env = parseEnv(await ready);
3152
+ if (env === null) {
3153
+ const out = await halt();
3154
+ throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
3155
+ }
3156
+ const urls = Object.values(env);
3157
+ if (urls.length !== 1) {
3158
+ await halt();
3159
+ throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
3160
+ }
3161
+ const baseUrl = urls[0];
3162
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
3163
+ Object.assign(process.env, env);
3164
+ let stopped = false;
3165
+ let stopping = null;
3166
+ const stop = () => {
3167
+ stopped = true;
3168
+ stopping ??= (async () => {
3169
+ for (const [k, v] of saved) {
3170
+ if (v === void 0) delete process.env[k];
3171
+ else process.env[k] = v;
3172
+ }
3173
+ const out = await halt();
3174
+ active2 = null;
3175
+ const final = parseFinal(out.stdout);
3176
+ if (out.code !== 0 || final === null) {
3177
+ throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
3178
+ }
3179
+ return final;
3180
+ })();
3181
+ return stopping;
3182
+ };
3183
+ const state = async () => {
3184
+ if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
3185
+ const url = baseUrl + STATE_PATH;
3186
+ let res;
3187
+ try {
3188
+ res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
3189
+ } catch (err) {
3190
+ throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
3191
+ }
3192
+ if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
3193
+ const body = await res.json().catch(() => {
3194
+ throw new WorldError(`GET ${url}: the body is not JSON`);
3195
+ });
3196
+ if (typeof body !== "object" || body === null || Array.isArray(body)) {
3197
+ throw new WorldError(`GET ${url}: the body is not a snapshot object`);
3198
+ }
3199
+ return body;
3200
+ };
3201
+ return { path: file, env, baseUrl, state, stop };
3202
+ }
3203
+ function parseFinal(out) {
3204
+ const lines = out.trim().split("\n");
3205
+ try {
3206
+ const world = JSON.parse(lines[lines.length - 1] ?? "").world;
3207
+ if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
3208
+ return world;
3209
+ } catch {
3210
+ return null;
3211
+ }
3212
+ }
3213
+
2416
3214
  // src/context.ts
2417
3215
  import { AsyncLocalStorage } from "async_hooks";
2418
3216
  var activeBuilder = new AsyncLocalStorage();
@@ -2616,12 +3414,12 @@ var ContinuousEvalRunner = class {
2616
3414
  maxSize;
2617
3415
  running = false;
2618
3416
  loopTimer = null;
2619
- constructor(client, assertions, options) {
3417
+ constructor(client, assertions, options2) {
2620
3418
  this.client = client;
2621
3419
  this.assertions = assertions;
2622
- this.sampler = new Sampler(options?.sampleRate ?? 1);
2623
- this.dispatcher = new AlertDispatcher(options?.alertWebhook, options?.alertSlackUrl);
2624
- this.maxSize = options?.maxSize ?? resolveQueueSize();
3420
+ this.sampler = new Sampler(options2?.sampleRate ?? 1);
3421
+ this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
3422
+ this.maxSize = options2?.maxSize ?? resolveQueueSize();
2625
3423
  }
2626
3424
  get alertEmitter() {
2627
3425
  return this.dispatcher;
@@ -2732,13 +3530,13 @@ function loadLabelsCSV(text) {
2732
3530
  if (rows.length === 0) {
2733
3531
  throw new Error("empty CSV");
2734
3532
  }
2735
- let start = 0;
3533
+ let start3 = 0;
2736
3534
  if (rows[0].length >= 2) {
2737
3535
  const firstHuman = parseFloatStrict(rows[0][1]);
2738
- if (firstHuman === null) start = 1;
3536
+ if (firstHuman === null) start3 = 1;
2739
3537
  }
2740
3538
  const out = [];
2741
- for (let idx = start; idx < rows.length; idx++) {
3539
+ for (let idx = start3; idx < rows.length; idx++) {
2742
3540
  const row = rows[idx];
2743
3541
  const lineNum = idx + 1;
2744
3542
  if (row.length === 0) continue;
@@ -2947,10 +3745,10 @@ var OpenAIAdapter = class {
2947
3745
  constructor(agentId2) {
2948
3746
  this.agentId = agentId2;
2949
3747
  }
2950
- traceFromResponse(response, options) {
3748
+ traceFromResponse(response, options2) {
2951
3749
  const builder = new TraceBuilder(this.agentId);
2952
- if (options?.inputMessages) {
2953
- builder.setInput({ messages: options.inputMessages });
3750
+ if (options2?.inputMessages) {
3751
+ builder.setInput({ messages: options2.inputMessages });
2954
3752
  }
2955
3753
  const message = response.choices[0].message;
2956
3754
  const completionText = message.content ?? "";
@@ -2970,12 +3768,12 @@ var OpenAIAdapter = class {
2970
3768
  }
2971
3769
  builder.setOutput({
2972
3770
  message: completionText,
2973
- structured: options?.structuredOutput ?? {}
3771
+ structured: options2?.structuredOutput ?? {}
2974
3772
  });
2975
3773
  builder.setMetadata({
2976
3774
  total_tokens: response.usage?.total_tokens,
2977
- cost_usd: options?.costUsd,
2978
- latency_ms: options?.latencyMs,
3775
+ cost_usd: options2?.costUsd,
3776
+ latency_ms: options2?.latencyMs,
2979
3777
  model: response.model
2980
3778
  });
2981
3779
  return builder.build();
@@ -2988,10 +3786,10 @@ var AnthropicAdapter = class {
2988
3786
  constructor(agentId2) {
2989
3787
  this.agentId = agentId2;
2990
3788
  }
2991
- traceFromResponse(response, options) {
3789
+ traceFromResponse(response, options2) {
2992
3790
  const builder = new TraceBuilder(this.agentId);
2993
- if (options?.inputMessages) {
2994
- builder.setInput({ messages: options.inputMessages });
3791
+ if (options2?.inputMessages) {
3792
+ builder.setInput({ messages: options2.inputMessages });
2995
3793
  }
2996
3794
  const completionParts = [];
2997
3795
  for (const block of response.content) {
@@ -3014,8 +3812,8 @@ var AnthropicAdapter = class {
3014
3812
  }
3015
3813
  builder.setMetadata({
3016
3814
  total_tokens: totalTokens,
3017
- cost_usd: options?.costUsd,
3018
- latency_ms: options?.latencyMs,
3815
+ cost_usd: options2?.costUsd,
3816
+ latency_ms: options2?.latencyMs,
3019
3817
  model: response.model
3020
3818
  });
3021
3819
  return builder.build();
@@ -3028,10 +3826,10 @@ var GeminiAdapter = class {
3028
3826
  constructor(agentId2) {
3029
3827
  this.agentId = agentId2;
3030
3828
  }
3031
- traceFromResponse(response, options) {
3829
+ traceFromResponse(response, options2) {
3032
3830
  const builder = new TraceBuilder(this.agentId);
3033
- if (options?.inputText) {
3034
- builder.setInput({ text: options.inputText });
3831
+ if (options2?.inputText) {
3832
+ builder.setInput({ text: options2.inputText });
3035
3833
  }
3036
3834
  let completionText = "";
3037
3835
  if (response.text != null) {
@@ -3051,9 +3849,9 @@ var GeminiAdapter = class {
3051
3849
  builder.addLlmCall("completion", { result: { completion: completionText } });
3052
3850
  builder.setOutput({ message: completionText });
3053
3851
  builder.setMetadata({
3054
- cost_usd: options?.costUsd,
3055
- latency_ms: options?.latencyMs,
3056
- model: options?.model
3852
+ cost_usd: options2?.costUsd,
3853
+ latency_ms: options2?.latencyMs,
3854
+ model: options2?.model
3057
3855
  });
3058
3856
  return builder.build();
3059
3857
  }
@@ -3065,10 +3863,10 @@ var OllamaAdapter = class {
3065
3863
  constructor(agentId2) {
3066
3864
  this.agentId = agentId2;
3067
3865
  }
3068
- traceFromResponse(response, options) {
3866
+ traceFromResponse(response, options2) {
3069
3867
  const builder = new TraceBuilder(this.agentId);
3070
- if (options?.inputMessages) {
3071
- builder.setInput({ messages: options.inputMessages });
3868
+ if (options2?.inputMessages) {
3869
+ builder.setInput({ messages: options2.inputMessages });
3072
3870
  }
3073
3871
  const completionText = response.message?.content ?? "";
3074
3872
  builder.addLlmCall("completion", {
@@ -3082,7 +3880,7 @@ var OllamaAdapter = class {
3082
3880
  }
3083
3881
  builder.setMetadata({
3084
3882
  total_tokens: totalTokens,
3085
- latency_ms: options?.latencyMs,
3883
+ latency_ms: options2?.latencyMs,
3086
3884
  model: response.model
3087
3885
  });
3088
3886
  return builder.build();
@@ -3090,7 +3888,7 @@ var OllamaAdapter = class {
3090
3888
  };
3091
3889
 
3092
3890
  // src/adapters/otel.ts
3093
- import { spawnSync as spawnSync2 } from "child_process";
3891
+ import { spawn as spawn5 } from "child_process";
3094
3892
  var CONVERT_TIMEOUT_MS = 6e4;
3095
3893
  var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
3096
3894
  var STATUS_CODE_ERROR = 2;
@@ -3113,7 +3911,7 @@ var OTelAdapter = class _OTelAdapter {
3113
3911
  * @throws RangeError when a parsed export holds a non-finite number.
3114
3912
  * @throws OTLPConversionError when the engine times out, overflows the output buffer or
3115
3913
  * prints no trace.
3116
- * @throws Error from `spawnSync` when the engine cannot start.
3914
+ * @throws Error when the engine cannot start.
3117
3915
  */
3118
3916
  static async fromOtlp(otlp, agentId2) {
3119
3917
  const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
@@ -3125,33 +3923,9 @@ var OTelAdapter = class _OTelAdapter {
3125
3923
  const args = ["trace", "convert"];
3126
3924
  if (agentId2 !== void 0) args.push("--agent-id", agentId2);
3127
3925
  args.push("-");
3128
- const proc = spawnSync2(await resolveEngine(), args, {
3129
- input: payload,
3130
- timeout: CONVERT_TIMEOUT_MS,
3131
- maxBuffer: MAX_TRACE_OUTPUT_BYTES
3132
- });
3133
- if (proc.error) {
3134
- const code = proc.error.code;
3135
- if (code === "ETIMEDOUT") {
3136
- throw new OTLPConversionError(
3137
- `attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
3138
- );
3139
- }
3140
- if (code === "ENOBUFS") {
3141
- throw new OTLPConversionError(
3142
- `attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
3143
- );
3144
- }
3145
- throw proc.error;
3146
- }
3147
- if (proc.status !== 0) {
3148
- const message = proc.stderr.toString("utf8").trim();
3149
- throw new OTLPConversionError(
3150
- message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
3151
- );
3152
- }
3926
+ const stdout = await runConvert(await resolveEngine(), args, payload);
3153
3927
  try {
3154
- return JSON.parse(proc.stdout.toString("utf8"));
3928
+ return JSON.parse(stdout);
3155
3929
  } catch (err) {
3156
3930
  throw new OTLPConversionError(
3157
3931
  `attest-engine trace convert printed output that is not JSON: ${String(err)}`
@@ -3159,6 +3933,65 @@ var OTelAdapter = class _OTelAdapter {
3159
3933
  }
3160
3934
  }
3161
3935
  };
3936
+ function runConvert(engine, args, payload) {
3937
+ return new Promise((resolve4, reject) => {
3938
+ const child = spawn5(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
3939
+ let timer;
3940
+ child.on("error", (e) => {
3941
+ clearTimeout(timer);
3942
+ reject(e);
3943
+ });
3944
+ const { stdin, stdout, stderr } = child;
3945
+ if (stdin === null || stdout === null || stderr === null) return;
3946
+ const out = [];
3947
+ const err = [];
3948
+ let outBytes = 0;
3949
+ let failure = null;
3950
+ const fail = (e) => {
3951
+ failure ??= e;
3952
+ child.kill("SIGKILL");
3953
+ };
3954
+ timer = setTimeout(() => {
3955
+ fail(
3956
+ new OTLPConversionError(
3957
+ `attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
3958
+ )
3959
+ );
3960
+ }, CONVERT_TIMEOUT_MS);
3961
+ stdout.on("data", (chunk) => {
3962
+ outBytes += chunk.length;
3963
+ if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
3964
+ fail(
3965
+ new OTLPConversionError(
3966
+ `attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
3967
+ )
3968
+ );
3969
+ return;
3970
+ }
3971
+ out.push(chunk);
3972
+ });
3973
+ stderr.on("data", (chunk) => err.push(chunk));
3974
+ stdin.on("error", () => void 0);
3975
+ child.on("close", (code, signal) => {
3976
+ clearTimeout(timer);
3977
+ if (failure !== null) {
3978
+ reject(failure);
3979
+ return;
3980
+ }
3981
+ if (code !== 0) {
3982
+ const message = Buffer.concat(err).toString("utf8").trim();
3983
+ reject(
3984
+ new OTLPConversionError(
3985
+ message || `attest-engine trace convert exited ${String(code ?? signal)}`
3986
+ )
3987
+ );
3988
+ return;
3989
+ }
3990
+ resolve4(Buffer.concat(out).toString("utf8"));
3991
+ });
3992
+ stdin.end(payload);
3993
+ });
3994
+ }
3162
3995
  function hrTimeToNanos(time, spanName) {
3163
3996
  if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
3164
3997
  throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
@@ -3264,13 +4097,13 @@ var LangChainAdapter = class {
3264
4097
  * @param options.model - Model name override.
3265
4098
  * @returns A finalized Trace object.
3266
4099
  */
3267
- buildTrace(options) {
4100
+ buildTrace(options2) {
3268
4101
  const builder = new TraceBuilder(this.agentId);
3269
4102
  const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
3270
4103
  let totalTokens;
3271
4104
  for (const llm of this.llmCalls) {
3272
4105
  const args = {};
3273
- if (llm.model ?? options?.model) args.model = llm.model ?? options?.model;
4106
+ if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
3274
4107
  const result = { completion: llm.text };
3275
4108
  if (llm.tokens !== void 0) {
3276
4109
  result.tokens = llm.tokens;
@@ -3287,9 +4120,9 @@ var LangChainAdapter = class {
3287
4120
  builder.setOutput({ message: outputMessage });
3288
4121
  builder.setMetadata({
3289
4122
  total_tokens: totalTokens,
3290
- cost_usd: options?.costUsd,
4123
+ cost_usd: options2?.costUsd,
3291
4124
  latency_ms: latencyMs,
3292
- model: options?.model ?? this.llmCalls[0]?.model
4125
+ model: options2?.model ?? this.llmCalls[0]?.model
3293
4126
  });
3294
4127
  return builder.build();
3295
4128
  }
@@ -3303,6 +4136,7 @@ export {
3303
4136
  AttestClient,
3304
4137
  CONFUSED_USER,
3305
4138
  COOPERATIVE_USER,
4139
+ CaptureError,
3306
4140
  ContinuousEvalRunner,
3307
4141
  ENGINE_VERSION,
3308
4142
  ERR_ASSERTION_ERROR,
@@ -3311,6 +4145,7 @@ export {
3311
4145
  ERR_PROVIDER_ERROR,
3312
4146
  ERR_SESSION_ERROR,
3313
4147
  ERR_TIMEOUT,
4148
+ EgressBlockedError,
3314
4149
  EngineManager,
3315
4150
  EngineTimeoutError,
3316
4151
  ExpectChain,
@@ -3335,8 +4170,12 @@ export {
3335
4170
  REASON_MISSING_EVIDENCE,
3336
4171
  REASON_PROVIDER_NOT_CONFIGURED,
3337
4172
  REASON_PROVIDER_UNAVAILABLE,
4173
+ REASON_REPLAY_MISS,
3338
4174
  REASON_SHORT_CIRCUITED,
3339
4175
  REASON_SIMULATION,
4176
+ ReplayError,
4177
+ ReplayMissError,
4178
+ ReplayNotEvaluatedError,
3340
4179
  STATUS_HARD_FAIL,
3341
4180
  STATUS_NOT_EVALUATED,
3342
4181
  STATUS_PASS,
@@ -3351,6 +4190,7 @@ export {
3351
4190
  TIER_3,
3352
4191
  TYPE_CONSTRAINT,
3353
4192
  TYPE_CONTENT,
4193
+ TYPE_EFFECT,
3354
4194
  TYPE_EMBEDDING,
3355
4195
  TYPE_LLM_JUDGE,
3356
4196
  TYPE_PLUGIN,
@@ -3360,6 +4200,8 @@ export {
3360
4200
  TraceBuilder,
3361
4201
  TraceTree,
3362
4202
  VERSION,
4203
+ WorldChecks,
4204
+ WorldError,
3363
4205
  activeBuilder,
3364
4206
  addUsage,
3365
4207
  agent,
@@ -3370,28 +4212,43 @@ export {
3370
4212
  config,
3371
4213
  decodeResponse,
3372
4214
  delegate,
4215
+ dslCall,
4216
+ egressBlockRequested,
4217
+ emitVitest,
3373
4218
  encodeRequest,
3374
4219
  executePluginAssertion,
3375
4220
  extractId,
3376
4221
  extractResult,
4222
+ firstMiss,
3377
4223
  formatUsage,
3378
4224
  getSampleRate,
4225
+ installEgressGuard,
4226
+ isProviderHost,
3379
4227
  isSimulationMode,
3380
4228
  layerForType,
4229
+ liveLine,
4230
+ loadFixture,
3381
4231
  loadLabelsCSV,
3382
4232
  loadLabelsJSONL,
3383
4233
  parseUsage,
3384
4234
  previewLine,
3385
4235
  promptHash,
4236
+ record,
3386
4237
  renderAssertionFailure,
3387
4238
  renderDiagnostics,
3388
4239
  renderSummary,
4240
+ renderVitest,
3389
4241
  requireSimulationCapability,
3390
4242
  resetConfig,
4243
+ resolveMode,
3391
4244
  simulationEnvEnabled,
4245
+ startReplay,
4246
+ startWorld,
3392
4247
  sumUsage,
4248
+ testPathFor,
3393
4249
  tier,
3394
4250
  traceId,
4251
+ uninstallEgressGuard,
3395
4252
  usageSummaryLine,
3396
4253
  zeroUsage
3397
4254
  };