@attest-ai/core 1.0.0-rc.1 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -38,6 +38,7 @@ __export(index_exports, {
38
38
  AttestClient: () => AttestClient,
39
39
  CONFUSED_USER: () => CONFUSED_USER,
40
40
  COOPERATIVE_USER: () => COOPERATIVE_USER,
41
+ CaptureError: () => CaptureError,
41
42
  ContinuousEvalRunner: () => ContinuousEvalRunner,
42
43
  ENGINE_VERSION: () => ENGINE_VERSION,
43
44
  ERR_ASSERTION_ERROR: () => ERR_ASSERTION_ERROR,
@@ -46,6 +47,7 @@ __export(index_exports, {
46
47
  ERR_PROVIDER_ERROR: () => ERR_PROVIDER_ERROR,
47
48
  ERR_SESSION_ERROR: () => ERR_SESSION_ERROR,
48
49
  ERR_TIMEOUT: () => ERR_TIMEOUT,
50
+ EgressBlockedError: () => EgressBlockedError,
49
51
  EngineManager: () => EngineManager,
50
52
  EngineTimeoutError: () => EngineTimeoutError,
51
53
  ExpectChain: () => ExpectChain,
@@ -54,6 +56,7 @@ __export(index_exports, {
54
56
  GeminiAdapter: () => GeminiAdapter,
55
57
  LangChainAdapter: () => LangChainAdapter,
56
58
  ManualAdapter: () => ManualAdapter,
59
+ OTLPConversionError: () => OTLPConversionError,
57
60
  OTelAdapter: () => OTelAdapter,
58
61
  OUTCOME_FAIL: () => OUTCOME_FAIL,
59
62
  OUTCOME_PASS: () => OUTCOME_PASS,
@@ -64,11 +67,17 @@ __export(index_exports, {
64
67
  ProtocolDesyncError: () => ProtocolDesyncError,
65
68
  ProtocolDiagnosticBuffer: () => ProtocolDiagnosticBuffer,
66
69
  ProtocolError: () => ProtocolError,
70
+ REASON_BUDGET_EXHAUSTED: () => REASON_BUDGET_EXHAUSTED,
71
+ REASON_JUDGE_ABSTAINED: () => REASON_JUDGE_ABSTAINED,
67
72
  REASON_MISSING_EVIDENCE: () => REASON_MISSING_EVIDENCE,
68
73
  REASON_PROVIDER_NOT_CONFIGURED: () => REASON_PROVIDER_NOT_CONFIGURED,
69
74
  REASON_PROVIDER_UNAVAILABLE: () => REASON_PROVIDER_UNAVAILABLE,
75
+ REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
70
76
  REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
71
77
  REASON_SIMULATION: () => REASON_SIMULATION,
78
+ ReplayError: () => ReplayError,
79
+ ReplayMissError: () => ReplayMissError,
80
+ ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
72
81
  STATUS_HARD_FAIL: () => STATUS_HARD_FAIL,
73
82
  STATUS_NOT_EVALUATED: () => STATUS_NOT_EVALUATED,
74
83
  STATUS_PASS: () => STATUS_PASS,
@@ -83,6 +92,7 @@ __export(index_exports, {
83
92
  TIER_3: () => TIER_3,
84
93
  TYPE_CONSTRAINT: () => TYPE_CONSTRAINT,
85
94
  TYPE_CONTENT: () => TYPE_CONTENT,
95
+ TYPE_EFFECT: () => TYPE_EFFECT,
86
96
  TYPE_EMBEDDING: () => TYPE_EMBEDDING,
87
97
  TYPE_LLM_JUDGE: () => TYPE_LLM_JUDGE,
88
98
  TYPE_PLUGIN: () => TYPE_PLUGIN,
@@ -92,7 +102,10 @@ __export(index_exports, {
92
102
  TraceBuilder: () => TraceBuilder,
93
103
  TraceTree: () => TraceTree,
94
104
  VERSION: () => VERSION,
105
+ WorldChecks: () => WorldChecks,
106
+ WorldError: () => WorldError,
95
107
  activeBuilder: () => activeBuilder,
108
+ addUsage: () => addUsage,
96
109
  agent: () => agent,
97
110
  agentId: () => agentId,
98
111
  assertionId: () => assertionId,
@@ -101,25 +114,45 @@ __export(index_exports, {
101
114
  config: () => config,
102
115
  decodeResponse: () => decodeResponse,
103
116
  delegate: () => delegate,
117
+ dslCall: () => dslCall,
118
+ egressBlockRequested: () => egressBlockRequested,
119
+ emitVitest: () => emitVitest,
104
120
  encodeRequest: () => encodeRequest,
105
121
  executePluginAssertion: () => executePluginAssertion,
106
122
  extractId: () => extractId,
107
123
  extractResult: () => extractResult,
124
+ firstMiss: () => firstMiss,
125
+ formatUsage: () => formatUsage,
108
126
  getSampleRate: () => getSampleRate,
127
+ installEgressGuard: () => installEgressGuard,
128
+ isProviderHost: () => isProviderHost,
109
129
  isSimulationMode: () => isSimulationMode,
110
130
  layerForType: () => layerForType,
131
+ liveLine: () => liveLine,
132
+ loadFixture: () => loadFixture,
111
133
  loadLabelsCSV: () => loadLabelsCSV,
112
134
  loadLabelsJSONL: () => loadLabelsJSONL,
135
+ parseUsage: () => parseUsage,
113
136
  previewLine: () => previewLine,
114
137
  promptHash: () => promptHash,
138
+ record: () => record,
115
139
  renderAssertionFailure: () => renderAssertionFailure,
116
140
  renderDiagnostics: () => renderDiagnostics,
117
141
  renderSummary: () => renderSummary,
142
+ renderVitest: () => renderVitest,
118
143
  requireSimulationCapability: () => requireSimulationCapability,
119
144
  resetConfig: () => resetConfig,
145
+ resolveMode: () => resolveMode,
120
146
  simulationEnvEnabled: () => simulationEnvEnabled,
147
+ startReplay: () => startReplay,
148
+ startWorld: () => startWorld,
149
+ sumUsage: () => sumUsage,
150
+ testPathFor: () => testPathFor,
121
151
  tier: () => tier,
122
- traceId: () => traceId
152
+ traceId: () => traceId,
153
+ uninstallEgressGuard: () => uninstallEgressGuard,
154
+ usageSummaryLine: () => usageSummaryLine,
155
+ zeroUsage: () => zeroUsage
123
156
  });
124
157
  module.exports = __toCommonJS(index_exports);
125
158
 
@@ -137,12 +170,16 @@ var REASON_PROVIDER_NOT_CONFIGURED = "provider_not_configured";
137
170
  var REASON_SIMULATION = "simulation";
138
171
  var REASON_MISSING_EVIDENCE = "missing_evidence";
139
172
  var REASON_SHORT_CIRCUITED = "short_circuited";
173
+ var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
174
+ var REASON_JUDGE_ABSTAINED = "judge_abstained";
175
+ var REASON_REPLAY_MISS = "replay_miss";
140
176
  var OUTCOME_PASS = "pass";
141
177
  var OUTCOME_FAIL = "fail";
142
178
  var OUTCOME_SKIP = "skip";
143
179
  var TYPE_SCHEMA = "schema";
144
180
  var TYPE_CONSTRAINT = "constraint";
145
181
  var TYPE_TRACE = "trace";
182
+ var TYPE_EFFECT = "effect";
146
183
  var TYPE_CONTENT = "content";
147
184
  var TYPE_EMBEDDING = "embedding";
148
185
  var TYPE_LLM_JUDGE = "llm_judge";
@@ -180,6 +217,7 @@ function layerForType(assertionType) {
180
217
  return 2;
181
218
  case "trace":
182
219
  case "trace_tree":
220
+ case "effect":
183
221
  return 3;
184
222
  case "content":
185
223
  return 4;
@@ -227,6 +265,12 @@ var ProtocolDesyncError = class extends Error {
227
265
  this.diagnostics = diagnostics;
228
266
  }
229
267
  };
268
+ var OTLPConversionError = class extends Error {
269
+ constructor(message) {
270
+ super(message);
271
+ this.name = "OTLPConversionError";
272
+ }
273
+ };
230
274
 
231
275
  // src/proto/diagnostics.ts
232
276
  var DEFAULT_BUFFER_SIZE = 32;
@@ -333,8 +377,8 @@ function extractId(response) {
333
377
  }
334
378
 
335
379
  // src/version.ts
336
- var VERSION = "1.0.0-rc.1";
337
- var ENGINE_VERSION = "1.0.0-rc.1";
380
+ var VERSION = "1.0.0-rc.3";
381
+ var ENGINE_VERSION = "1.0.0-rc.3";
338
382
 
339
383
  // src/engine-manager.ts
340
384
  var import_node_child_process = require("child_process");
@@ -402,7 +446,7 @@ function httpsGet(url, redirectCount = 0) {
402
446
  new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
403
447
  );
404
448
  }
405
- return new Promise((resolve, reject) => {
449
+ return new Promise((resolve4, reject) => {
406
450
  const client = url.startsWith("https:") ? https : http;
407
451
  client.get(url, (res) => {
408
452
  const status = res.statusCode ?? 0;
@@ -413,7 +457,7 @@ function httpsGet(url, redirectCount = 0) {
413
457
  return;
414
458
  }
415
459
  res.resume();
416
- resolve(httpsGet(location, redirectCount + 1));
460
+ resolve4(httpsGet(location, redirectCount + 1));
417
461
  return;
418
462
  }
419
463
  if (status < 200 || status >= 300) {
@@ -423,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
423
467
  }
424
468
  const chunks = [];
425
469
  res.on("data", (chunk) => chunks.push(chunk));
426
- res.on("end", () => resolve(Buffer.concat(chunks)));
470
+ res.on("end", () => resolve4(Buffer.concat(chunks)));
427
471
  res.on("error", reject);
428
472
  }).on("error", reject);
429
473
  });
@@ -554,8 +598,8 @@ Install options:
554
598
  }
555
599
  return [await downloadEngine(), "download"];
556
600
  }
557
- async function resolveEngine(explicitPath, options = {}) {
558
- const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);
601
+ async function resolveEngine(explicitPath, options2 = {}) {
602
+ const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
559
603
  const found = engineVersion(enginePath);
560
604
  if (found !== ENGINE_VERSION) {
561
605
  throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
@@ -570,9 +614,9 @@ var EngineManager = class {
570
614
  initialized = false;
571
615
  requestId = 0;
572
616
  initResult = null;
573
- constructor(options) {
574
- this.enginePath = options?.enginePath ?? "";
575
- this.logLevel = options?.logLevel ?? "warn";
617
+ constructor(options2) {
618
+ this.enginePath = options2?.enginePath ?? "";
619
+ this.logLevel = options2?.logLevel ?? "warn";
576
620
  }
577
621
  async start() {
578
622
  this.enginePath = await resolveEngine(this.enginePath || void 0);
@@ -650,11 +694,11 @@ var EngineManager = class {
650
694
  }
651
695
  this.requestId += 1;
652
696
  const requestStr = encodeRequest(this.requestId, method, params);
653
- return new Promise((resolve, reject) => {
697
+ return new Promise((resolve4, reject) => {
654
698
  const onLine = (line) => {
655
699
  try {
656
700
  const response = decodeResponse(line);
657
- resolve(extractResult(response));
701
+ resolve4(extractResult(response));
658
702
  } catch (err) {
659
703
  reject(err);
660
704
  }
@@ -688,17 +732,17 @@ var EngineManager = class {
688
732
  // src/config.ts
689
733
  var _simulationMode = false;
690
734
  var _sampleRate = 0;
691
- function config(options) {
692
- if (options.simulation !== void 0) {
693
- _simulationMode = options.simulation;
735
+ function config(options2) {
736
+ if (options2.simulation !== void 0) {
737
+ _simulationMode = options2.simulation;
694
738
  }
695
- if (options.sampleRate !== void 0) {
696
- if (options.sampleRate < 0 || options.sampleRate > 1) {
739
+ if (options2.sampleRate !== void 0) {
740
+ if (options2.sampleRate < 0 || options2.sampleRate > 1) {
697
741
  throw new RangeError(
698
- `sampleRate must be between 0.0 and 1.0, got ${options.sampleRate}`
742
+ `sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
699
743
  );
700
744
  }
701
- _sampleRate = options.sampleRate;
745
+ _sampleRate = options2.sampleRate;
702
746
  }
703
747
  }
704
748
  function isSimulationMode() {
@@ -727,6 +771,76 @@ function resetConfig() {
727
771
  _sampleRate = 0;
728
772
  }
729
773
 
774
+ // src/usage.ts
775
+ var INT_FIELDS = [
776
+ "model_calls",
777
+ "judge_calls",
778
+ "input_tokens",
779
+ "output_tokens",
780
+ "cached_tokens",
781
+ "unpriced_calls",
782
+ "budget_refused"
783
+ ];
784
+ var USD_FIELDS = ["cost_usd", "retry_cost_usd"];
785
+ function zeroUsage() {
786
+ return {
787
+ model_calls: 0,
788
+ judge_calls: 0,
789
+ input_tokens: 0,
790
+ output_tokens: 0,
791
+ cached_tokens: 0,
792
+ cost_usd: 0,
793
+ retry_cost_usd: 0,
794
+ unpriced_calls: 0,
795
+ budget_refused: 0
796
+ };
797
+ }
798
+ function addUsage(a, b) {
799
+ return {
800
+ model_calls: a.model_calls + b.model_calls,
801
+ judge_calls: a.judge_calls + b.judge_calls,
802
+ input_tokens: a.input_tokens + b.input_tokens,
803
+ output_tokens: a.output_tokens + b.output_tokens,
804
+ cached_tokens: a.cached_tokens + b.cached_tokens,
805
+ cost_usd: a.cost_usd + b.cost_usd,
806
+ retry_cost_usd: a.retry_cost_usd + b.retry_cost_usd,
807
+ unpriced_calls: a.unpriced_calls + b.unpriced_calls,
808
+ budget_refused: a.budget_refused + b.budget_refused
809
+ };
810
+ }
811
+ function sumUsage(usages) {
812
+ let total = zeroUsage();
813
+ for (const u of usages) total = addUsage(total, u);
814
+ return total;
815
+ }
816
+ function parseUsage(raw) {
817
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
818
+ throw new TypeError(`evaluate_batch result: usage must be an object, got ${JSON.stringify(raw)}`);
819
+ }
820
+ const obj = raw;
821
+ const field = (name, integer) => {
822
+ const v = obj[name];
823
+ if (typeof v !== "number" || !Number.isFinite(v) || v < 0 || integer && !Number.isInteger(v)) {
824
+ const kind = integer ? "a non-negative integer" : "a non-negative number";
825
+ throw new TypeError(`evaluate_batch result: usage.${name} must be ${kind}, got ${JSON.stringify(v)}`);
826
+ }
827
+ return v;
828
+ };
829
+ const ints = Object.fromEntries(INT_FIELDS.map((f) => [f, field(f, true)]));
830
+ const usd = Object.fromEntries(USD_FIELDS.map((f) => [f, field(f, false)]));
831
+ return { ...ints, ...usd };
832
+ }
833
+ function formatUsage(u) {
834
+ let line = `${u.model_calls} model calls \xB7 ${u.input_tokens + u.output_tokens} tokens (${u.cached_tokens} cached) \xB7 $${u.cost_usd.toFixed(4)} \xB7 retry cost $${u.retry_cost_usd.toFixed(4)}`;
835
+ if (u.budget_refused > 0) line += ` \xB7 ${u.budget_refused} calls refused by budget`;
836
+ if (u.unpriced_calls > 0) line += ` \xB7 ${u.unpriced_calls} unpriced calls not in $`;
837
+ return line;
838
+ }
839
+ function usageSummaryLine(u) {
840
+ if (u.model_calls === 0 && u.budget_refused === 0) return void 0;
841
+ return `attest usage: ${formatUsage(u)}`;
842
+ }
843
+
730
844
  // src/client.ts
731
845
  var DEFAULT_TIMEOUT_MS = 3e4;
732
846
  var DEFAULT_DIAGNOSTIC_BUFFER = 32;
@@ -784,16 +898,16 @@ var AttestClient = class {
784
898
  desyncThreshold;
785
899
  desyncWindowMs;
786
900
  desynced = false;
787
- constructor(engine, options) {
901
+ constructor(engine, options2) {
788
902
  this.engine = engine;
789
- this.logger = options?.logger ?? envLoggerDefault();
903
+ this.logger = options2?.logger ?? envLoggerDefault();
790
904
  this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
791
- options?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
905
+ options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
792
906
  );
793
- this.desyncThreshold = options?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
794
- this.desyncWindowMs = options?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
795
- if (options?.onDiagnostic !== void 0) {
796
- this.diagnosticListeners.add(options.onDiagnostic);
907
+ this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
908
+ this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
909
+ if (options2?.onDiagnostic !== void 0) {
910
+ this.diagnosticListeners.add(options2.onDiagnostic);
797
911
  }
798
912
  }
799
913
  /** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
@@ -951,12 +1065,12 @@ var AttestClient = class {
951
1065
  throw new Error("Engine process not started.");
952
1066
  }
953
1067
  let capturedReqId;
954
- const requestPromise = new Promise((resolve, reject) => {
1068
+ const requestPromise = new Promise((resolve4, reject) => {
955
1069
  this.writeChain = this.writeChain.then(() => {
956
1070
  this.requestId += 1;
957
1071
  const reqId = this.requestId;
958
1072
  capturedReqId = reqId;
959
- this.pending.set(reqId, { resolve, reject });
1073
+ this.pending.set(reqId, { resolve: resolve4, reject });
960
1074
  const requestStr = encodeRequest(reqId, method, params);
961
1075
  cp.stdin.write(requestStr, (err) => {
962
1076
  if (err) {
@@ -974,22 +1088,23 @@ var AttestClient = class {
974
1088
  throw err;
975
1089
  });
976
1090
  }
977
- async evaluateBatch(trace, assertions, options) {
1091
+ async evaluateBatch(trace, assertions, options2) {
978
1092
  if (isSimulationMode()) {
979
1093
  requireSimulationCapability(this.engine["initResult"]);
980
1094
  }
981
1095
  const params = {
982
1096
  trace,
983
- assertions: [...assertions]
1097
+ assertions: [...assertions],
1098
+ ...options2?.budget !== void 0 ? { budget: options2.budget } : {}
984
1099
  };
985
1100
  const raw = await this.sendRequest(
986
1101
  "evaluate_batch",
987
1102
  params,
988
- options?.timeout
1103
+ options2?.timeout
989
1104
  );
990
- return raw;
1105
+ return { ...raw, usage: parseUsage(raw["usage"]) };
991
1106
  }
992
- async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options) {
1107
+ async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
993
1108
  const result = {
994
1109
  assertion_id: assertionId2,
995
1110
  status,
@@ -1007,11 +1122,11 @@ var AttestClient = class {
1007
1122
  const raw = await this.sendRequest(
1008
1123
  "submit_plugin_result",
1009
1124
  params,
1010
- options?.timeout
1125
+ options2?.timeout
1011
1126
  );
1012
1127
  return Boolean(raw?.accepted ?? false);
1013
1128
  }
1014
- async queryDrift(assertionId2, windowSize = 50, options) {
1129
+ async queryDrift(assertionId2, windowSize = 50, options2) {
1015
1130
  if (isSimulationMode()) {
1016
1131
  return {
1017
1132
  assertion_id: assertionId2,
@@ -1026,11 +1141,11 @@ var AttestClient = class {
1026
1141
  const raw = await this.sendRequest(
1027
1142
  "query_drift",
1028
1143
  { assertion_id: assertionId2, window_size: windowSize },
1029
- options?.timeout
1144
+ options2?.timeout
1030
1145
  );
1031
1146
  return raw.report;
1032
1147
  }
1033
- async generateUserMessage(persona, conversationHistory, faultConfig, options) {
1148
+ async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
1034
1149
  if (isSimulationMode()) {
1035
1150
  return `[simulation] Hello from ${persona.name}`;
1036
1151
  }
@@ -1044,7 +1159,7 @@ var AttestClient = class {
1044
1159
  const raw = await this.sendRequest(
1045
1160
  "generate_user_message",
1046
1161
  params,
1047
- options?.timeout
1162
+ options2?.timeout
1048
1163
  );
1049
1164
  return String(raw.message);
1050
1165
  }
@@ -1089,33 +1204,33 @@ var TraceBuilder = class {
1089
1204
  this.input = input;
1090
1205
  return this;
1091
1206
  }
1092
- addLlmCall(name, options) {
1207
+ addLlmCall(name, options2) {
1093
1208
  this.steps.push({
1094
1209
  type: STEP_LLM_CALL,
1095
1210
  name,
1096
- args: options?.args,
1097
- result: options?.result,
1098
- metadata: options?.metadata
1211
+ args: options2?.args,
1212
+ result: options2?.result,
1213
+ metadata: options2?.metadata
1099
1214
  });
1100
1215
  return this;
1101
1216
  }
1102
- addToolCall(name, options) {
1217
+ addToolCall(name, options2) {
1103
1218
  this.steps.push({
1104
1219
  type: STEP_TOOL_CALL,
1105
1220
  name,
1106
- args: options?.args,
1107
- result: options?.result,
1108
- metadata: options?.metadata
1221
+ args: options2?.args,
1222
+ result: options2?.result,
1223
+ metadata: options2?.metadata
1109
1224
  });
1110
1225
  return this;
1111
1226
  }
1112
- addRetrieval(name, options) {
1227
+ addRetrieval(name, options2) {
1113
1228
  this.steps.push({
1114
1229
  type: STEP_RETRIEVAL,
1115
1230
  name,
1116
- args: options?.args,
1117
- result: options?.result,
1118
- metadata: options?.metadata
1231
+ args: options2?.args,
1232
+ result: options2?.result,
1233
+ metadata: options2?.metadata
1119
1234
  });
1120
1235
  return this;
1121
1236
  }
@@ -1175,11 +1290,14 @@ var AgentResult = class {
1175
1290
  assertionResults;
1176
1291
  totalCost;
1177
1292
  totalDurationMs;
1178
- constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0) {
1293
+ /** Attest model usage of the evaluation that produced this result. */
1294
+ usage;
1295
+ constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0, usage = zeroUsage()) {
1179
1296
  this.trace = trace;
1180
1297
  this.assertionResults = assertionResults.map(normalizeAssertionResult);
1181
1298
  this.totalCost = totalCost;
1182
1299
  this.totalDurationMs = totalDurationMs;
1300
+ this.usage = usage;
1183
1301
  }
1184
1302
  /** True if every assertion has status "pass". */
1185
1303
  get passed() {
@@ -1190,10 +1308,10 @@ var AgentResult = class {
1190
1308
  * not_evaluated skips (fails when `strict`); otherwise pass, including
1191
1309
  * when only soft_fail rows are present (see `softFailures`).
1192
1310
  */
1193
- outcome(options) {
1311
+ outcome(options2) {
1194
1312
  if (this.hardFailures.length > 0) return OUTCOME_FAIL;
1195
1313
  if (this.notEvaluated.length > 0) {
1196
- return options?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
1314
+ return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
1197
1315
  }
1198
1316
  return OUTCOME_PASS;
1199
1317
  }
@@ -1303,9 +1421,9 @@ function truncate(value, limit = 280) {
1303
1421
  }
1304
1422
  return value.slice(0, limit - 3) + "...";
1305
1423
  }
1306
- function renderAssertionFailure(result, options = {}) {
1307
- const color = options.color ?? false;
1308
- const testFile = options.testFile;
1424
+ function renderAssertionFailure(result, options2 = {}) {
1425
+ const color = options2.color ?? false;
1426
+ const testFile = options2.testFile;
1309
1427
  const layer = result.layer ?? 0;
1310
1428
  const label = layerLabel(layer, result.type);
1311
1429
  const lines = [];
@@ -1358,7 +1476,8 @@ function appendJudgeMetadata(lines, meta) {
1358
1476
  if (meta.rubric_version) {
1359
1477
  rubric = `${rubric} @ ${meta.rubric_version}`;
1360
1478
  }
1361
- lines.push(` judge: ${meta.model} / ${rubric}`);
1479
+ const judge = meta.provider ? `${meta.provider}/${meta.model}` : meta.model;
1480
+ lines.push(` judge: ${judge} / ${rubric}`);
1362
1481
  }
1363
1482
  if (meta.prompt_hash) {
1364
1483
  lines.push(` prompt: #${meta.prompt_hash}`);
@@ -1385,8 +1504,8 @@ function appendJudgeMetadata(lines, meta) {
1385
1504
  );
1386
1505
  }
1387
1506
  }
1388
- function renderSummary(result, options = {}) {
1389
- const color = options.color ?? false;
1507
+ function renderSummary(result, options2 = {}) {
1508
+ const color = options2.color ?? false;
1390
1509
  const passN = result.passCount;
1391
1510
  const soft = result.assertionResults.filter(
1392
1511
  (r) => r.status === STATUS_SOFT_FAIL
@@ -1404,9 +1523,9 @@ function renderSummary(result, options = {}) {
1404
1523
  ];
1405
1524
  return parts.join(" | ");
1406
1525
  }
1407
- function renderDiagnostics(result, options = {}) {
1408
- const color = options.color ?? false;
1409
- const includePassing = options.includePassing ?? false;
1526
+ function renderDiagnostics(result, options2 = {}) {
1527
+ const color = options2.color ?? false;
1528
+ const includePassing = options2.includePassing ?? false;
1410
1529
  const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
1411
1530
  if (rows.length === 0) {
1412
1531
  return dim(" (no failures)\n", color);
@@ -1414,7 +1533,7 @@ function renderDiagnostics(result, options = {}) {
1414
1533
  const out = [""];
1415
1534
  out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
1416
1535
  for (const r of rows) {
1417
- out.push(renderAssertionFailure(r, options));
1536
+ out.push(renderAssertionFailure(r, options2));
1418
1537
  }
1419
1538
  out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
1420
1539
  return out.join("\n");
@@ -1531,9 +1650,64 @@ var TraceTree = class {
1531
1650
 
1532
1651
  // src/expect.ts
1533
1652
  var import_node_crypto2 = require("crypto");
1653
+ var MAX_WORLD_BYTES = 9 * 1024 * 1024;
1654
+ var WorldChecks = class {
1655
+ _snapshot;
1656
+ _add;
1657
+ constructor(snapshot, add) {
1658
+ this._snapshot = snapshot;
1659
+ this._add = add;
1660
+ }
1661
+ addCheck(spec, where) {
1662
+ const full = { ...spec, world: this._snapshot };
1663
+ if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
1664
+ return this._add(full);
1665
+ }
1666
+ /**
1667
+ * Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
1668
+ * `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
1669
+ *
1670
+ * @throws Error when no bound is given, or `equals` is combined with another bound.
1671
+ *
1672
+ * @example
1673
+ * ```ts
1674
+ * expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
1675
+ * ```
1676
+ */
1677
+ count(resource, opts) {
1678
+ if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
1679
+ throw new Error("count needs equals, atMost or atLeast");
1680
+ }
1681
+ if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
1682
+ throw new Error("count: equals excludes atMost and atLeast");
1683
+ }
1684
+ const spec = { check: "count", resource };
1685
+ if (opts.equals !== void 0) spec.equals = opts.equals;
1686
+ if (opts.atMost !== void 0) spec.at_most = opts.atMost;
1687
+ if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
1688
+ return this.addCheck(spec, opts.where);
1689
+ }
1690
+ /** Assert no two matching records share the same values for field(s) `by`. */
1691
+ unique(resource, opts) {
1692
+ const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
1693
+ if (fields.length === 0) throw new Error("unique needs at least one field in by");
1694
+ return this.addCheck({ check: "unique", resource, fields }, opts.where);
1695
+ }
1696
+ /**
1697
+ * Assert every matching record carries a distinct value in field `key` (default
1698
+ * `idempotency_key`), the record field that stores the idempotency key.
1699
+ */
1700
+ idempotent(resource, opts = {}) {
1701
+ return this.addCheck(
1702
+ { check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
1703
+ opts.where
1704
+ );
1705
+ }
1706
+ };
1534
1707
  var ExpectChain = class {
1535
1708
  _result;
1536
1709
  _assertions = [];
1710
+ _worldBytes = 0;
1537
1711
  constructor(result) {
1538
1712
  this._result = result;
1539
1713
  }
@@ -1882,6 +2056,104 @@ var ExpectChain = class {
1882
2056
  soft: opts?.soft ?? false
1883
2057
  });
1884
2058
  }
2059
+ // -- Layer 3: Behavior contracts (tool calls only, retries counted) --
2060
+ /**
2061
+ * Assert how many times a tool is called, per value of argument `per` when set.
2062
+ *
2063
+ * @example
2064
+ * ```ts
2065
+ * // fails with "issue_refund called 2× for order_id=O-1042 (max 1)"
2066
+ * expect(result).toolCardinality("issue_refund", { per: "order_id", atMost: 1 });
2067
+ * ```
2068
+ */
2069
+ toolCardinality(tool, opts) {
2070
+ if (opts.atMost === void 0 && opts.atLeast === void 0) {
2071
+ throw new Error("toolCardinality needs atMost, atLeast or both");
2072
+ }
2073
+ const spec = { check: "tool_cardinality", tool };
2074
+ if (opts.per !== void 0) spec.per = opts.per;
2075
+ if (opts.atLeast !== void 0) spec.min = opts.atLeast;
2076
+ if (opts.atMost !== void 0) spec.max = opts.atMost;
2077
+ spec.soft = opts.soft ?? false;
2078
+ return this.add(TYPE_TRACE, spec);
2079
+ }
2080
+ /**
2081
+ * Assert the tool-call trajectory matches `tools`: `strict` (same sequence,
2082
+ * the default), `unordered` (same calls, any order), `subset` (no call
2083
+ * outside `tools`) or `superset` (every call in `tools` made). Use
2084
+ * {@link trajectoryPartialOrder} for ordering constraints.
2085
+ */
2086
+ trajectory(tools, opts) {
2087
+ return this.add(TYPE_TRACE, {
2088
+ check: "trajectory",
2089
+ mode: opts?.mode ?? "strict",
2090
+ tools,
2091
+ soft: opts?.soft ?? false
2092
+ });
2093
+ }
2094
+ /**
2095
+ * Assert every tool in `tools` and `order` is called and each
2096
+ * `[before, after]` pair holds as in {@link toolCalledBefore}.
2097
+ */
2098
+ trajectoryPartialOrder(order, opts) {
2099
+ const spec = {
2100
+ check: "trajectory",
2101
+ mode: "partial_order",
2102
+ order: order.map(([before, after]) => [before, after])
2103
+ };
2104
+ if (opts?.tools !== void 0) spec.tools = opts.tools;
2105
+ spec.soft = opts?.soft ?? false;
2106
+ return this.add(TYPE_TRACE, spec);
2107
+ }
2108
+ /** Assert every call to `after` has an earlier call to `before`; holds when `after` is never called. */
2109
+ toolCalledBefore(before, after, opts) {
2110
+ return this.add(TYPE_TRACE, { check: "tool_order", before, after, soft: opts?.soft ?? false });
2111
+ }
2112
+ /** Assert no block of `period` (1–4) tool calls repeats `minRepeats` (default 2) times in a row; A→B→A→B fails period 2. */
2113
+ noToolCycles(period, opts) {
2114
+ return this.add(TYPE_TRACE, {
2115
+ check: "cycle",
2116
+ period,
2117
+ min_repeats: opts?.minRepeats ?? 2,
2118
+ soft: opts?.soft ?? false
2119
+ });
2120
+ }
2121
+ /** Assert at most `maxStreak` consecutive tool calls end in error (trace v2). */
2122
+ maxToolFailureStreak(maxStreak, opts) {
2123
+ return this.add(TYPE_TRACE, { check: "failure_streak", max: maxStreak, soft: opts?.soft ?? false });
2124
+ }
2125
+ /**
2126
+ * Assert every call to `tool` has argument `arg` (a dot path) satisfying
2127
+ * `op value`; holds when `tool` is never called.
2128
+ */
2129
+ toolArgSatisfies(tool, arg, op, value, opts) {
2130
+ return this.add(TYPE_TRACE, {
2131
+ check: "arg_predicate",
2132
+ tool,
2133
+ arg,
2134
+ op,
2135
+ value,
2136
+ soft: opts?.soft ?? false
2137
+ });
2138
+ }
2139
+ // -- Layer 3: Effect checks (the external system the agent acted on) --
2140
+ /**
2141
+ * Check effects in a world snapshot:
2142
+ * `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
2143
+ * a world helper: the checks are `not_evaluated` (`missing_evidence`).
2144
+ */
2145
+ world(snapshot) {
2146
+ const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
2147
+ return new WorldChecks(snapshot, (spec) => {
2148
+ this._worldBytes += size;
2149
+ if (this._worldBytes > MAX_WORLD_BYTES) {
2150
+ throw new Error(
2151
+ `effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
2152
+ );
2153
+ }
2154
+ return this.add(TYPE_EFFECT, spec);
2155
+ });
2156
+ }
1885
2157
  // -- Layer 4: Content --
1886
2158
  /**
1887
2159
  * Assert the output message contains the given string.
@@ -2375,6 +2647,728 @@ function attestExpect(result) {
2375
2647
  return new ExpectChain(new AgentResult(result));
2376
2648
  }
2377
2649
 
2650
+ // src/capture.ts
2651
+ var import_node_child_process2 = require("child_process");
2652
+ var fs2 = __toESM(require("fs"), 1);
2653
+ var path2 = __toESM(require("path"), 1);
2654
+ var import_node_url = require("url");
2655
+ var FIXTURE_VERSION = 1;
2656
+ var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
2657
+ var CaptureError = class extends Error {
2658
+ /** The fixture written, when the engine wrote one. */
2659
+ fixture;
2660
+ constructor(message, fixture = null) {
2661
+ super(message);
2662
+ this.name = "CaptureError";
2663
+ this.fixture = fixture;
2664
+ }
2665
+ };
2666
+ function toPath(p) {
2667
+ return p instanceof URL ? (0, import_node_url.fileURLToPath)(p) : p;
2668
+ }
2669
+ function loadFixture(file) {
2670
+ const p = toPath(file);
2671
+ const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
2672
+ if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
2673
+ throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
2674
+ }
2675
+ validate(p, doc);
2676
+ return doc;
2677
+ }
2678
+ var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
2679
+ function isObject(v) {
2680
+ return typeof v === "object" && v !== null && !Array.isArray(v);
2681
+ }
2682
+ function kindOf(v) {
2683
+ if (Array.isArray(v)) return "array";
2684
+ if (isObject(v)) return "object";
2685
+ if (typeof v === "string" || typeof v === "boolean") return typeof v;
2686
+ return "other";
2687
+ }
2688
+ function validate(p, doc) {
2689
+ const need = (where, obj, fields) => {
2690
+ if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
2691
+ for (const [key, kind] of Object.entries(fields)) {
2692
+ if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
2693
+ }
2694
+ return obj;
2695
+ };
2696
+ need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
2697
+ if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
2698
+ if (!NAME_PATTERN.test(doc["name"])) {
2699
+ throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
2700
+ }
2701
+ const v = need("fixture.verification", doc["verification"], {
2702
+ verified: "boolean",
2703
+ expect_fail: "boolean",
2704
+ results: "array",
2705
+ expected_failures: "array"
2706
+ });
2707
+ doc["contracts"].forEach(
2708
+ (c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
2709
+ );
2710
+ v["results"].forEach(
2711
+ (r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
2712
+ );
2713
+ }
2714
+ function objectKey(k) {
2715
+ if (k === "__proto__") return `[${JSON.stringify(k)}]`;
2716
+ return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
2717
+ }
2718
+ function comment(text) {
2719
+ return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
2720
+ }
2721
+ function ts(value) {
2722
+ if (value === null || typeof value === "boolean" || typeof value === "number") {
2723
+ return JSON.stringify(value);
2724
+ }
2725
+ if (typeof value === "string") return JSON.stringify(value);
2726
+ if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
2727
+ if (typeof value === "object") {
2728
+ const entries = Object.entries(value).map(
2729
+ ([k, v]) => `${objectKey(k)}: ${ts(v)}`
2730
+ );
2731
+ return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
2732
+ }
2733
+ throw new Error(`value ${String(value)} has no literal`);
2734
+ }
2735
+ function str(spec, key) {
2736
+ const v = spec[key];
2737
+ if (typeof v !== "string") throw new Error(`missing ${key}`);
2738
+ return v;
2739
+ }
2740
+ function options(entries) {
2741
+ const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
2742
+ return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
2743
+ }
2744
+ function call(method, args, opts) {
2745
+ return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
2746
+ }
2747
+ function dslCall(contract) {
2748
+ const s = contract.spec;
2749
+ const soft = s["soft"] === true ? true : void 0;
2750
+ try {
2751
+ if (contract.type === "trace") {
2752
+ const check = s["check"];
2753
+ if (check === "trajectory" && s["mode"] !== "partial_order") {
2754
+ const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
2755
+ if (!Array.isArray(s["tools"])) throw new Error("missing tools");
2756
+ return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
2757
+ }
2758
+ if (check === "tool_order") {
2759
+ return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
2760
+ }
2761
+ if (check === "tool_cardinality") {
2762
+ return call(
2763
+ "toolCardinality",
2764
+ [ts(str(s, "tool"))],
2765
+ options([
2766
+ ["per", s["per"]],
2767
+ ["atLeast", s["min"]],
2768
+ ["atMost", s["max"]],
2769
+ ["soft", soft]
2770
+ ])
2771
+ );
2772
+ }
2773
+ }
2774
+ if (contract.type === "schema" && s["target"] === "output.structured") {
2775
+ if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
2776
+ return call("outputMatchesSchema", [ts(s["schema"])], "");
2777
+ }
2778
+ if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
2779
+ if (typeof s["value"] !== "number") throw new Error("missing value");
2780
+ return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
2781
+ }
2782
+ } catch (err) {
2783
+ throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
2784
+ }
2785
+ throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
2786
+ }
2787
+ function testPathFor(fixturePath) {
2788
+ const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
2789
+ return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
2790
+ }
2791
+ function renderVitest(fixture, fixturePath, testPath) {
2792
+ const calls = [];
2793
+ const skipped = [];
2794
+ for (const c of fixture.contracts) {
2795
+ try {
2796
+ calls.push([c, dslCall(c)]);
2797
+ } catch (err) {
2798
+ skipped.push(err.message);
2799
+ }
2800
+ }
2801
+ const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
2802
+ const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
2803
+ const expectFail = fixture.verification.expect_fail;
2804
+ const lines = [
2805
+ `// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
2806
+ "//",
2807
+ "// Review and edit before committing: they check the captured trace without running",
2808
+ "// the agent. Re-capturing does not overwrite this file.",
2809
+ `import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
2810
+ `import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
2811
+ `import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
2812
+ "",
2813
+ `const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
2814
+ ];
2815
+ if (skipped.length > 0) {
2816
+ lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
2817
+ for (const s of skipped) lines.push(`// - ${comment(s)}`);
2818
+ }
2819
+ lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
2820
+ for (const [c, dsl] of calls) {
2821
+ lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
2822
+ }
2823
+ lines[lines.length - 1] += ";";
2824
+ lines.push("}");
2825
+ if (expectFail) {
2826
+ lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
2827
+ lines.push("const EXPECTED: Array<[string, string]> = [");
2828
+ for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
2829
+ lines.push(
2830
+ "];",
2831
+ "",
2832
+ `describe(${ts(fixture.name)}, () => {`,
2833
+ ` it("reproduces the incident", async () => {`,
2834
+ " const result = await inspect(contracts());",
2835
+ "",
2836
+ " expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
2837
+ " });",
2838
+ "});"
2839
+ );
2840
+ } else {
2841
+ lines.push(
2842
+ "",
2843
+ `describe(${ts(fixture.name)}, () => {`,
2844
+ ` it("keeps its drafted contracts", async () => {`,
2845
+ " await evaluate(contracts());",
2846
+ " });",
2847
+ "});"
2848
+ );
2849
+ }
2850
+ return `${lines.join("\n")}
2851
+ `;
2852
+ }
2853
+ function emitVitest(fixturePath) {
2854
+ const testPath = testPathFor(fixturePath);
2855
+ const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
2856
+ if (existing?.isSymbolicLink()) {
2857
+ throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
2858
+ }
2859
+ if (existing !== void 0) return null;
2860
+ const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
2861
+ fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
2862
+ try {
2863
+ fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
2864
+ } catch (err) {
2865
+ if (err.code === "EEXIST") {
2866
+ throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
2867
+ }
2868
+ throw err;
2869
+ }
2870
+ return testPath;
2871
+ }
2872
+ function serve(engine, args) {
2873
+ const child = (0, import_node_child_process2.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
2874
+ let stdout = "";
2875
+ let stderr = "";
2876
+ let resolveReady = () => void 0;
2877
+ const ready = new Promise((resolve4) => {
2878
+ resolveReady = resolve4;
2879
+ });
2880
+ const exited = new Promise((resolve4) => {
2881
+ child.on("close", (code, signal) => {
2882
+ resolveReady(null);
2883
+ resolve4({ code, signal });
2884
+ });
2885
+ child.on("error", (err) => {
2886
+ stderr += err.message;
2887
+ resolveReady(null);
2888
+ resolve4({ code: null, signal: null });
2889
+ });
2890
+ });
2891
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
2892
+ stdout += chunk;
2893
+ const nl = stdout.indexOf("\n");
2894
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
2895
+ });
2896
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
2897
+ stderr += chunk;
2898
+ });
2899
+ child.stdin.on("error", () => void 0);
2900
+ return {
2901
+ ready,
2902
+ async finish(finalize) {
2903
+ if (finalize) {
2904
+ const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
2905
+ child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
2906
+ `);
2907
+ }
2908
+ child.stdin.end();
2909
+ const { code, signal } = await exited;
2910
+ return { code, signal, stdout, stderr };
2911
+ }
2912
+ };
2913
+ }
2914
+ function readyEnv(line) {
2915
+ if (line === null) return null;
2916
+ let parsed;
2917
+ try {
2918
+ parsed = JSON.parse(line);
2919
+ } catch {
2920
+ return null;
2921
+ }
2922
+ const env = parsed?.env;
2923
+ if (typeof env !== "object" || env === null) return null;
2924
+ const entries = Object.entries(env);
2925
+ if (!entries.every(([, v]) => typeof v === "string")) return null;
2926
+ return Object.fromEntries(entries);
2927
+ }
2928
+ function exitDetail(exit) {
2929
+ const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
2930
+ const err = exit.stderr.trim();
2931
+ return err ? `${how}: ${err}` : how;
2932
+ }
2933
+ function restore(saved) {
2934
+ for (const [k, v] of saved) {
2935
+ if (v === void 0) delete process.env[k];
2936
+ else process.env[k] = v;
2937
+ }
2938
+ }
2939
+ var recording = false;
2940
+ async function record(name, fn, opts = {}) {
2941
+ if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
2942
+ recording = true;
2943
+ try {
2944
+ return await recordExclusive(name, fn, opts);
2945
+ } finally {
2946
+ recording = false;
2947
+ }
2948
+ }
2949
+ async function recordExclusive(name, fn, opts) {
2950
+ const engine = await resolveEngine(opts.enginePath);
2951
+ const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
2952
+ if (opts.expectFail) args.push("--expect-fail");
2953
+ if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
2954
+ if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
2955
+ const proc = serve(engine, args);
2956
+ const env = readyEnv(await proc.ready);
2957
+ if (env === null) {
2958
+ const exit2 = await proc.finish(false);
2959
+ throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
2960
+ }
2961
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
2962
+ Object.assign(process.env, env);
2963
+ let value;
2964
+ try {
2965
+ value = await fn({ name, env });
2966
+ if (opts.flush) await opts.flush();
2967
+ } catch (err) {
2968
+ restore(saved);
2969
+ await proc.finish(false);
2970
+ throw err;
2971
+ }
2972
+ restore(saved);
2973
+ const exit = await proc.finish(true);
2974
+ process.stderr.write(exit.stderr);
2975
+ const lines = exit.stdout.trim().split("\n");
2976
+ if (lines.length < 2) {
2977
+ throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
2978
+ }
2979
+ const outcome = JSON.parse(lines[lines.length - 1] ?? "");
2980
+ if (exit.code !== 0 || !outcome.verified) {
2981
+ throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
2982
+ }
2983
+ const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
2984
+ return { value, fixture: outcome.fixture, verified: true, testFile };
2985
+ }
2986
+
2987
+ // src/replay.ts
2988
+ var import_node_child_process3 = require("child_process");
2989
+ var path3 = __toESM(require("path"), 1);
2990
+
2991
+ // src/egress.ts
2992
+ var net = __toESM(require("net"), 1);
2993
+ var PROVIDER_HOSTS = [
2994
+ "api.openai.com",
2995
+ "api.anthropic.com",
2996
+ "generativelanguage.googleapis.com",
2997
+ /\.openai\.azure\.com$/
2998
+ ];
2999
+ var EgressBlockedError = class extends Error {
3000
+ constructor(host) {
3001
+ super(
3002
+ `attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
3003
+ );
3004
+ this.host = host;
3005
+ this.name = "EgressBlockedError";
3006
+ }
3007
+ host;
3008
+ };
3009
+ function isProviderHost(host) {
3010
+ const h = host.toLowerCase().replace(/\.$/, "");
3011
+ return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
3012
+ }
3013
+ function egressBlockRequested(env = process.env) {
3014
+ const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
3015
+ if (v !== "" && v !== "block" && v !== "allow") {
3016
+ throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
3017
+ }
3018
+ return v === "block";
3019
+ }
3020
+ var original = null;
3021
+ var installs = 0;
3022
+ function hostOf(args) {
3023
+ const first = Array.isArray(args[0]) ? args[0][0] : args[0];
3024
+ if (typeof first === "object" && first !== null) {
3025
+ const o = first;
3026
+ return typeof o.host === "string" ? o.host : null;
3027
+ }
3028
+ if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
3029
+ return typeof args[1] === "string" ? args[1] : null;
3030
+ }
3031
+ return null;
3032
+ }
3033
+ function installEgressGuard() {
3034
+ installs++;
3035
+ if (original === null) {
3036
+ const proto = net.Socket.prototype;
3037
+ const orig = proto.connect;
3038
+ original = orig;
3039
+ proto.connect = function(...args) {
3040
+ const host = hostOf(args);
3041
+ if (host !== null && isProviderHost(host)) {
3042
+ const err = new EgressBlockedError(host);
3043
+ process.nextTick(() => this.destroy(err));
3044
+ return this;
3045
+ }
3046
+ return orig.apply(this, args);
3047
+ };
3048
+ }
3049
+ let released = false;
3050
+ return () => {
3051
+ if (released) return;
3052
+ released = true;
3053
+ installs--;
3054
+ if (installs === 0) uninstallEgressGuard();
3055
+ };
3056
+ }
3057
+ function egressGuardActive() {
3058
+ return installs > 0;
3059
+ }
3060
+ function uninstallEgressGuard() {
3061
+ installs = 0;
3062
+ if (original === null) return;
3063
+ net.Socket.prototype.connect = original;
3064
+ original = null;
3065
+ }
3066
+
3067
+ // src/replay.ts
3068
+ var ReplayMissError = class extends Error {
3069
+ constructor(message, summary) {
3070
+ super(message);
3071
+ this.summary = summary;
3072
+ this.name = "ReplayMissError";
3073
+ }
3074
+ summary;
3075
+ };
3076
+ var ReplayNotEvaluatedError = class extends Error {
3077
+ constructor(message, summary) {
3078
+ super(message);
3079
+ this.summary = summary;
3080
+ this.name = "ReplayNotEvaluatedError";
3081
+ }
3082
+ summary;
3083
+ };
3084
+ function escapeControl(text) {
3085
+ return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
3086
+ }
3087
+ var ReplayError = class extends Error {
3088
+ constructor(message) {
3089
+ super(escapeControl(message));
3090
+ this.name = "ReplayError";
3091
+ }
3092
+ };
3093
+ function resolveMode(explicit, env = process.env) {
3094
+ if (explicit !== void 0) return checkMode(explicit, "mode");
3095
+ const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
3096
+ if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
3097
+ const ci = (env.CI ?? "").trim().toLowerCase();
3098
+ return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
3099
+ }
3100
+ function checkMode(value, where) {
3101
+ if (value === "strict" || value === "auto" || value === "off") return value;
3102
+ throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
3103
+ }
3104
+ var active = null;
3105
+ async function startReplay(name, opts = {}) {
3106
+ if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
3107
+ const resolved = resolveMode(opts.mode);
3108
+ if (resolved === "off") return offSession(name);
3109
+ const mode = resolved;
3110
+ if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
3111
+ active = name;
3112
+ try {
3113
+ return await start(name, mode, opts);
3114
+ } catch (err) {
3115
+ active = null;
3116
+ throw err;
3117
+ }
3118
+ }
3119
+ function offSession(name) {
3120
+ return {
3121
+ name,
3122
+ mode: "off",
3123
+ env: {},
3124
+ summary: null,
3125
+ finish: () => Promise.resolve(null),
3126
+ error: () => null,
3127
+ check: () => Promise.resolve()
3128
+ };
3129
+ }
3130
+ function missError(mode, summary) {
3131
+ if (summary.misses === 0) return null;
3132
+ const message = escapeControl(firstMiss(summary) ?? "replay miss");
3133
+ return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
3134
+ }
3135
+ async function start(name, mode, opts) {
3136
+ const engine = await resolveEngine(opts.enginePath);
3137
+ const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
3138
+ const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
3139
+ const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
3140
+ if (blocking) args.push("--no-live");
3141
+ const child = (0, import_node_child_process3.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
3142
+ let stdout = "";
3143
+ let stderr = "";
3144
+ let resolveReady = () => void 0;
3145
+ const ready = new Promise((resolve4) => {
3146
+ resolveReady = resolve4;
3147
+ });
3148
+ const exited = new Promise((resolve4) => {
3149
+ child.on("close", (code) => {
3150
+ resolveReady(null);
3151
+ resolve4(code);
3152
+ });
3153
+ child.on("error", (err) => {
3154
+ stderr += err.message;
3155
+ resolveReady(null);
3156
+ resolve4(null);
3157
+ });
3158
+ });
3159
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
3160
+ stdout += chunk;
3161
+ const nl = stdout.indexOf("\n");
3162
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
3163
+ });
3164
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
3165
+ stderr += chunk;
3166
+ });
3167
+ child.stdin.on("error", () => void 0);
3168
+ const stop = async () => {
3169
+ child.stdin.write("stop\n");
3170
+ child.stdin.end();
3171
+ return { code: await exited, stdout, stderr };
3172
+ };
3173
+ const env = parseEnv(await ready);
3174
+ if (env === null) {
3175
+ const out = await stop();
3176
+ active = null;
3177
+ throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
3178
+ }
3179
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
3180
+ Object.assign(process.env, env);
3181
+ let summary = null;
3182
+ let finishing = null;
3183
+ const finish = () => {
3184
+ finishing ??= (async () => {
3185
+ for (const [k, v] of saved) {
3186
+ if (v === void 0) delete process.env[k];
3187
+ else process.env[k] = v;
3188
+ }
3189
+ const out = await stop();
3190
+ active = null;
3191
+ const parsed = parseSummary(out.stdout);
3192
+ if (parsed === null) {
3193
+ throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
3194
+ }
3195
+ if (blocking && parsed.live > 0) {
3196
+ throw new ReplayError(
3197
+ `replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
3198
+ );
3199
+ }
3200
+ summary = parsed;
3201
+ return parsed;
3202
+ })();
3203
+ return finishing;
3204
+ };
3205
+ return {
3206
+ name,
3207
+ mode,
3208
+ env,
3209
+ get summary() {
3210
+ return summary;
3211
+ },
3212
+ finish,
3213
+ error: () => summary === null ? null : missError(mode, summary),
3214
+ check: async () => {
3215
+ await finish();
3216
+ const err = summary === null ? null : missError(mode, summary);
3217
+ if (err !== null) throw err;
3218
+ }
3219
+ };
3220
+ }
3221
+ function parseEnv(line) {
3222
+ if (line === null) return null;
3223
+ try {
3224
+ const env = JSON.parse(line).env;
3225
+ if (typeof env !== "object" || env === null) return null;
3226
+ const entries = Object.entries(env);
3227
+ if (!entries.every(([, v]) => typeof v === "string")) return null;
3228
+ return Object.fromEntries(entries);
3229
+ } catch {
3230
+ return null;
3231
+ }
3232
+ }
3233
+ function parseSummary(out) {
3234
+ const lines = out.trim().split("\n");
3235
+ try {
3236
+ const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
3237
+ if (s === void 0 || !Array.isArray(s.records)) return null;
3238
+ for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
3239
+ return s;
3240
+ } catch {
3241
+ return null;
3242
+ }
3243
+ }
3244
+ function firstMiss(summary) {
3245
+ return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
3246
+ }
3247
+ function liveLine(live, costUsd, unpriced = 0) {
3248
+ const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
3249
+ return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
3250
+ }
3251
+
3252
+ // src/world.ts
3253
+ var import_node_child_process4 = require("child_process");
3254
+ var path4 = __toESM(require("path"), 1);
3255
+ var STATE_PATH = "/_attest/state";
3256
+ var STATE_TIMEOUT_MS = 1e4;
3257
+ var WorldError = class extends Error {
3258
+ constructor(message) {
3259
+ super(escapeControl(message));
3260
+ this.name = "WorldError";
3261
+ }
3262
+ };
3263
+ var active2 = null;
3264
+ async function startWorld(worldFile, opts = {}) {
3265
+ if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
3266
+ const file = path4.resolve(worldFile);
3267
+ active2 = file;
3268
+ try {
3269
+ return await start2(file, opts);
3270
+ } catch (err) {
3271
+ active2 = null;
3272
+ throw err;
3273
+ }
3274
+ }
3275
+ async function start2(file, opts) {
3276
+ const engine = await resolveEngine(opts.enginePath);
3277
+ const child = (0, import_node_child_process4.spawn)(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
3278
+ let stdout = "";
3279
+ let stderr = "";
3280
+ let resolveReady = () => void 0;
3281
+ const ready = new Promise((resolve4) => {
3282
+ resolveReady = resolve4;
3283
+ });
3284
+ const exited = new Promise((resolve4) => {
3285
+ child.on("close", (code) => {
3286
+ resolveReady(null);
3287
+ resolve4(code);
3288
+ });
3289
+ child.on("error", (err) => {
3290
+ stderr += err.message;
3291
+ resolveReady(null);
3292
+ resolve4(null);
3293
+ });
3294
+ });
3295
+ child.stdout.setEncoding("utf8").on("data", (chunk) => {
3296
+ stdout += chunk;
3297
+ const nl = stdout.indexOf("\n");
3298
+ if (nl >= 0) resolveReady(stdout.slice(0, nl));
3299
+ });
3300
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
3301
+ stderr += chunk;
3302
+ });
3303
+ child.stdin.on("error", () => void 0);
3304
+ const halt = async () => {
3305
+ child.stdin.write("stop\n");
3306
+ child.stdin.end();
3307
+ return { code: await exited, stdout, stderr };
3308
+ };
3309
+ const env = parseEnv(await ready);
3310
+ if (env === null) {
3311
+ const out = await halt();
3312
+ throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
3313
+ }
3314
+ const urls = Object.values(env);
3315
+ if (urls.length !== 1) {
3316
+ await halt();
3317
+ throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
3318
+ }
3319
+ const baseUrl = urls[0];
3320
+ const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
3321
+ Object.assign(process.env, env);
3322
+ let stopped = false;
3323
+ let stopping = null;
3324
+ const stop = () => {
3325
+ stopped = true;
3326
+ stopping ??= (async () => {
3327
+ for (const [k, v] of saved) {
3328
+ if (v === void 0) delete process.env[k];
3329
+ else process.env[k] = v;
3330
+ }
3331
+ const out = await halt();
3332
+ active2 = null;
3333
+ const final = parseFinal(out.stdout);
3334
+ if (out.code !== 0 || final === null) {
3335
+ throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
3336
+ }
3337
+ return final;
3338
+ })();
3339
+ return stopping;
3340
+ };
3341
+ const state = async () => {
3342
+ if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
3343
+ const url = baseUrl + STATE_PATH;
3344
+ let res;
3345
+ try {
3346
+ res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
3347
+ } catch (err) {
3348
+ throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
3349
+ }
3350
+ if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
3351
+ const body = await res.json().catch(() => {
3352
+ throw new WorldError(`GET ${url}: the body is not JSON`);
3353
+ });
3354
+ if (typeof body !== "object" || body === null || Array.isArray(body)) {
3355
+ throw new WorldError(`GET ${url}: the body is not a snapshot object`);
3356
+ }
3357
+ return body;
3358
+ };
3359
+ return { path: file, env, baseUrl, state, stop };
3360
+ }
3361
+ function parseFinal(out) {
3362
+ const lines = out.trim().split("\n");
3363
+ try {
3364
+ const world = JSON.parse(lines[lines.length - 1] ?? "").world;
3365
+ if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
3366
+ return world;
3367
+ } catch {
3368
+ return null;
3369
+ }
3370
+ }
3371
+
2378
3372
  // src/context.ts
2379
3373
  var import_node_async_hooks = require("async_hooks");
2380
3374
  var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
@@ -2578,12 +3572,12 @@ var ContinuousEvalRunner = class {
2578
3572
  maxSize;
2579
3573
  running = false;
2580
3574
  loopTimer = null;
2581
- constructor(client, assertions, options) {
3575
+ constructor(client, assertions, options2) {
2582
3576
  this.client = client;
2583
3577
  this.assertions = assertions;
2584
- this.sampler = new Sampler(options?.sampleRate ?? 1);
2585
- this.dispatcher = new AlertDispatcher(options?.alertWebhook, options?.alertSlackUrl);
2586
- this.maxSize = options?.maxSize ?? resolveQueueSize();
3578
+ this.sampler = new Sampler(options2?.sampleRate ?? 1);
3579
+ this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
3580
+ this.maxSize = options2?.maxSize ?? resolveQueueSize();
2587
3581
  }
2588
3582
  get alertEmitter() {
2589
3583
  return this.dispatcher;
@@ -2694,13 +3688,13 @@ function loadLabelsCSV(text) {
2694
3688
  if (rows.length === 0) {
2695
3689
  throw new Error("empty CSV");
2696
3690
  }
2697
- let start = 0;
3691
+ let start3 = 0;
2698
3692
  if (rows[0].length >= 2) {
2699
3693
  const firstHuman = parseFloatStrict(rows[0][1]);
2700
- if (firstHuman === null) start = 1;
3694
+ if (firstHuman === null) start3 = 1;
2701
3695
  }
2702
3696
  const out = [];
2703
- for (let idx = start; idx < rows.length; idx++) {
3697
+ for (let idx = start3; idx < rows.length; idx++) {
2704
3698
  const row = rows[idx];
2705
3699
  const lineNum = idx + 1;
2706
3700
  if (row.length === 0) continue;
@@ -2909,10 +3903,10 @@ var OpenAIAdapter = class {
2909
3903
  constructor(agentId2) {
2910
3904
  this.agentId = agentId2;
2911
3905
  }
2912
- traceFromResponse(response, options) {
3906
+ traceFromResponse(response, options2) {
2913
3907
  const builder = new TraceBuilder(this.agentId);
2914
- if (options?.inputMessages) {
2915
- builder.setInput({ messages: options.inputMessages });
3908
+ if (options2?.inputMessages) {
3909
+ builder.setInput({ messages: options2.inputMessages });
2916
3910
  }
2917
3911
  const message = response.choices[0].message;
2918
3912
  const completionText = message.content ?? "";
@@ -2932,12 +3926,12 @@ var OpenAIAdapter = class {
2932
3926
  }
2933
3927
  builder.setOutput({
2934
3928
  message: completionText,
2935
- structured: options?.structuredOutput ?? {}
3929
+ structured: options2?.structuredOutput ?? {}
2936
3930
  });
2937
3931
  builder.setMetadata({
2938
3932
  total_tokens: response.usage?.total_tokens,
2939
- cost_usd: options?.costUsd,
2940
- latency_ms: options?.latencyMs,
3933
+ cost_usd: options2?.costUsd,
3934
+ latency_ms: options2?.latencyMs,
2941
3935
  model: response.model
2942
3936
  });
2943
3937
  return builder.build();
@@ -2950,10 +3944,10 @@ var AnthropicAdapter = class {
2950
3944
  constructor(agentId2) {
2951
3945
  this.agentId = agentId2;
2952
3946
  }
2953
- traceFromResponse(response, options) {
3947
+ traceFromResponse(response, options2) {
2954
3948
  const builder = new TraceBuilder(this.agentId);
2955
- if (options?.inputMessages) {
2956
- builder.setInput({ messages: options.inputMessages });
3949
+ if (options2?.inputMessages) {
3950
+ builder.setInput({ messages: options2.inputMessages });
2957
3951
  }
2958
3952
  const completionParts = [];
2959
3953
  for (const block of response.content) {
@@ -2976,8 +3970,8 @@ var AnthropicAdapter = class {
2976
3970
  }
2977
3971
  builder.setMetadata({
2978
3972
  total_tokens: totalTokens,
2979
- cost_usd: options?.costUsd,
2980
- latency_ms: options?.latencyMs,
3973
+ cost_usd: options2?.costUsd,
3974
+ latency_ms: options2?.latencyMs,
2981
3975
  model: response.model
2982
3976
  });
2983
3977
  return builder.build();
@@ -2990,10 +3984,10 @@ var GeminiAdapter = class {
2990
3984
  constructor(agentId2) {
2991
3985
  this.agentId = agentId2;
2992
3986
  }
2993
- traceFromResponse(response, options) {
3987
+ traceFromResponse(response, options2) {
2994
3988
  const builder = new TraceBuilder(this.agentId);
2995
- if (options?.inputText) {
2996
- builder.setInput({ text: options.inputText });
3989
+ if (options2?.inputText) {
3990
+ builder.setInput({ text: options2.inputText });
2997
3991
  }
2998
3992
  let completionText = "";
2999
3993
  if (response.text != null) {
@@ -3013,9 +4007,9 @@ var GeminiAdapter = class {
3013
4007
  builder.addLlmCall("completion", { result: { completion: completionText } });
3014
4008
  builder.setOutput({ message: completionText });
3015
4009
  builder.setMetadata({
3016
- cost_usd: options?.costUsd,
3017
- latency_ms: options?.latencyMs,
3018
- model: options?.model
4010
+ cost_usd: options2?.costUsd,
4011
+ latency_ms: options2?.latencyMs,
4012
+ model: options2?.model
3019
4013
  });
3020
4014
  return builder.build();
3021
4015
  }
@@ -3027,10 +4021,10 @@ var OllamaAdapter = class {
3027
4021
  constructor(agentId2) {
3028
4022
  this.agentId = agentId2;
3029
4023
  }
3030
- traceFromResponse(response, options) {
4024
+ traceFromResponse(response, options2) {
3031
4025
  const builder = new TraceBuilder(this.agentId);
3032
- if (options?.inputMessages) {
3033
- builder.setInput({ messages: options.inputMessages });
4026
+ if (options2?.inputMessages) {
4027
+ builder.setInput({ messages: options2.inputMessages });
3034
4028
  }
3035
4029
  const completionText = response.message?.content ?? "";
3036
4030
  builder.addLlmCall("completion", {
@@ -3044,7 +4038,7 @@ var OllamaAdapter = class {
3044
4038
  }
3045
4039
  builder.setMetadata({
3046
4040
  total_tokens: totalTokens,
3047
- latency_ms: options?.latencyMs,
4041
+ latency_ms: options2?.latencyMs,
3048
4042
  model: response.model
3049
4043
  });
3050
4044
  return builder.build();
@@ -3052,119 +4046,148 @@ var OllamaAdapter = class {
3052
4046
  };
3053
4047
 
3054
4048
  // src/adapters/otel.ts
4049
+ var import_node_child_process5 = require("child_process");
4050
+ var CONVERT_TIMEOUT_MS = 6e4;
4051
+ var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
4052
+ var STATUS_CODE_ERROR = 2;
3055
4053
  var OTelAdapter = class _OTelAdapter {
3056
- agentId;
3057
- constructor(agentId2) {
3058
- this.agentId = agentId2;
4054
+ /**
4055
+ * Build a trace from finished OpenTelemetry spans of one trace.
4056
+ *
4057
+ * @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
4058
+ * @throws OTLPConversionError when the engine rejects the spans.
4059
+ */
4060
+ static async fromSpans(spans, agentId2) {
4061
+ return _OTelAdapter.fromOtlp(spansToOtlp(spans), agentId2);
3059
4062
  }
3060
- static fromSpans(spans, agentId2) {
3061
- const adapter = new _OTelAdapter(agentId2);
3062
- return adapter.buildTrace(spans);
4063
+ /**
4064
+ * Convert an OTLP/JSON export (`resourceSpans` → `scopeSpans` → `spans`).
4065
+ *
4066
+ * @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
4067
+ * @throws OTLPConversionError when the engine rejects the export (malformed JSON, invalid
4068
+ * ids or timestamps, more than one trace, non-numeric token counts, ...).
4069
+ * @throws RangeError when a parsed export holds a non-finite number.
4070
+ * @throws OTLPConversionError when the engine times out, overflows the output buffer or
4071
+ * prints no trace.
4072
+ * @throws Error when the engine cannot start.
4073
+ */
4074
+ static async fromOtlp(otlp, agentId2) {
4075
+ const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
4076
+ if (typeof v === "number" && !Number.isFinite(v)) {
4077
+ throw new RangeError(`OTLP export holds a non-finite number: ${String(v)}`);
4078
+ }
4079
+ return v;
4080
+ });
4081
+ const args = ["trace", "convert"];
4082
+ if (agentId2 !== void 0) args.push("--agent-id", agentId2);
4083
+ args.push("-");
4084
+ const stdout = await runConvert(await resolveEngine(), args, payload);
4085
+ try {
4086
+ return JSON.parse(stdout);
4087
+ } catch (err) {
4088
+ throw new OTLPConversionError(
4089
+ `attest-engine trace convert printed output that is not JSON: ${String(err)}`
4090
+ );
4091
+ }
3063
4092
  }
3064
- buildTrace(spans) {
3065
- const sorted = [...spans].sort((a, b) => {
3066
- const aTime = a.startTime[0] * 1e9 + a.startTime[1];
3067
- const bTime = b.startTime[0] * 1e9 + b.startTime[1];
3068
- return aTime - bTime;
4093
+ };
4094
+ function runConvert(engine, args, payload) {
4095
+ return new Promise((resolve4, reject) => {
4096
+ const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
4097
+ let timer;
4098
+ child.on("error", (e) => {
4099
+ clearTimeout(timer);
4100
+ reject(e);
3069
4101
  });
3070
- const builder = new TraceBuilder(this.agentId);
3071
- const rootSpan = this.findRootSpan(sorted);
3072
- if (rootSpan) {
3073
- const traceId2 = rootSpan.spanContext().traceId;
3074
- if (traceId2) {
3075
- builder.setTraceId(`otel_${traceId2.slice(0, 16)}`);
4102
+ const { stdin, stdout, stderr } = child;
4103
+ if (stdin === null || stdout === null || stderr === null) return;
4104
+ const out = [];
4105
+ const err = [];
4106
+ let outBytes = 0;
4107
+ let failure = null;
4108
+ const fail = (e) => {
4109
+ failure ??= e;
4110
+ child.kill("SIGKILL");
4111
+ };
4112
+ timer = setTimeout(() => {
4113
+ fail(
4114
+ new OTLPConversionError(
4115
+ `attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
4116
+ )
4117
+ );
4118
+ }, CONVERT_TIMEOUT_MS);
4119
+ stdout.on("data", (chunk) => {
4120
+ outBytes += chunk.length;
4121
+ if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
4122
+ fail(
4123
+ new OTLPConversionError(
4124
+ `attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
4125
+ )
4126
+ );
4127
+ return;
3076
4128
  }
3077
- }
3078
- let outputMessage = "";
3079
- let totalTokens;
3080
- let latencyMs;
3081
- let model;
3082
- for (const span of sorted) {
3083
- const attrs = span.attributes ?? {};
3084
- const stepType = this.classifySpan(attrs, span.name);
3085
- if (stepType === "llm_call") {
3086
- const [stepArgs, stepResult] = this.extractLlmStep(attrs);
3087
- builder.addLlmCall(span.name, {
3088
- args: stepArgs,
3089
- result: stepResult,
3090
- metadata: this.spanMetadata(span)
3091
- });
3092
- const completion = attrs["gen_ai.completion"];
3093
- if (completion) outputMessage = String(completion);
3094
- const inputTokens = Number(attrs["gen_ai.usage.input_tokens"] ?? 0);
3095
- const outputTokens = Number(attrs["gen_ai.usage.output_tokens"] ?? 0);
3096
- const spanTokens = inputTokens + outputTokens;
3097
- if (spanTokens > 0) totalTokens = (totalTokens ?? 0) + spanTokens;
3098
- if (model === void 0) {
3099
- const responseModel = attrs["gen_ai.response.model"];
3100
- const requestModel = attrs["gen_ai.request.model"];
3101
- if (responseModel) model = String(responseModel);
3102
- else if (requestModel) model = String(requestModel);
3103
- }
3104
- } else if (stepType === "tool_call") {
3105
- const [stepArgs, stepResult] = this.extractToolStep(attrs);
3106
- const toolName = String(attrs["gen_ai.tool.name"] ?? span.name);
3107
- builder.addToolCall(toolName, {
3108
- args: stepArgs,
3109
- result: stepResult,
3110
- metadata: this.spanMetadata(span)
3111
- });
4129
+ out.push(chunk);
4130
+ });
4131
+ stderr.on("data", (chunk) => err.push(chunk));
4132
+ stdin.on("error", () => void 0);
4133
+ child.on("close", (code, signal) => {
4134
+ clearTimeout(timer);
4135
+ if (failure !== null) {
4136
+ reject(failure);
4137
+ return;
3112
4138
  }
3113
- }
3114
- if (rootSpan) {
3115
- const startNs = rootSpan.startTime[0] * 1e9 + rootSpan.startTime[1];
3116
- const endNs = rootSpan.endTime[0] * 1e9 + rootSpan.endTime[1];
3117
- latencyMs = Math.floor((endNs - startNs) / 1e6);
3118
- }
3119
- builder.setOutput({ message: outputMessage });
3120
- builder.setMetadata({ total_tokens: totalTokens, latency_ms: latencyMs, model });
3121
- return builder.build();
4139
+ if (code !== 0) {
4140
+ const message = Buffer.concat(err).toString("utf8").trim();
4141
+ reject(
4142
+ new OTLPConversionError(
4143
+ message || `attest-engine trace convert exited ${String(code ?? signal)}`
4144
+ )
4145
+ );
4146
+ return;
4147
+ }
4148
+ resolve4(Buffer.concat(out).toString("utf8"));
4149
+ });
4150
+ stdin.end(payload);
4151
+ });
4152
+ }
4153
+ function hrTimeToNanos(time, spanName) {
4154
+ if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
4155
+ throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
3122
4156
  }
3123
- findRootSpan(spans) {
3124
- for (const span of spans) {
3125
- if (!span.parentSpanId) return span;
4157
+ return (BigInt(time[0]) * 1000000000n + BigInt(time[1])).toString();
4158
+ }
4159
+ function anyValue(value) {
4160
+ if (value === null || value === void 0) return {};
4161
+ if (typeof value === "boolean") return { boolValue: value };
4162
+ if (typeof value === "string") return { stringValue: value };
4163
+ if (typeof value === "number") {
4164
+ if (!Number.isFinite(value)) {
4165
+ throw new RangeError(`OTel attribute value must be finite, got ${String(value)}`);
3126
4166
  }
3127
- return spans[0];
4167
+ return Number.isSafeInteger(value) ? { intValue: String(value) } : { doubleValue: value };
3128
4168
  }
3129
- classifySpan(attrs, name) {
3130
- const op = String(attrs["gen_ai.operation.name"] ?? "");
3131
- if (op === "chat" || op === "completion" || op === "generate_content" || "gen_ai.completion" in attrs) {
3132
- return "llm_call";
3133
- }
3134
- if (op === "tool" || "gen_ai.tool.name" in attrs) {
3135
- return "tool_call";
4169
+ return { arrayValue: { values: value.map((v) => anyValue(v)) } };
4170
+ }
4171
+ function spansToOtlp(spans) {
4172
+ const encoded = spans.map((span) => {
4173
+ const ctx = span.spanContext();
4174
+ const out = {
4175
+ traceId: ctx.traceId,
4176
+ spanId: ctx.spanId,
4177
+ name: span.name,
4178
+ startTimeUnixNano: hrTimeToNanos(span.startTime, span.name),
4179
+ endTimeUnixNano: hrTimeToNanos(span.endTime, span.name),
4180
+ attributes: Object.entries(span.attributes ?? {}).filter(([, v]) => v !== void 0).map(([key, v]) => ({ key, value: anyValue(v) }))
4181
+ };
4182
+ const parent = span.parentSpanContext?.spanId ?? span.parentSpanId;
4183
+ if (parent) out["parentSpanId"] = parent;
4184
+ if (span.status?.code === STATUS_CODE_ERROR) {
4185
+ out["status"] = { code: STATUS_CODE_ERROR, message: span.status.message ?? "" };
3136
4186
  }
3137
- const lower = name.toLowerCase();
3138
- if (lower.includes("completion") || lower.includes("chat")) return "llm_call";
3139
- if (lower.includes("tool")) return "tool_call";
3140
- return void 0;
3141
- }
3142
- extractLlmStep(attrs) {
3143
- const args = {};
3144
- const result = {};
3145
- if ("gen_ai.request.model" in attrs) args.model = String(attrs["gen_ai.request.model"]);
3146
- if ("gen_ai.system" in attrs) args.system = String(attrs["gen_ai.system"]);
3147
- if ("gen_ai.prompt" in attrs) args.prompt = String(attrs["gen_ai.prompt"]);
3148
- if ("gen_ai.completion" in attrs) result.completion = String(attrs["gen_ai.completion"]);
3149
- if ("gen_ai.usage.input_tokens" in attrs) result.input_tokens = Number(attrs["gen_ai.usage.input_tokens"]);
3150
- if ("gen_ai.usage.output_tokens" in attrs) result.output_tokens = Number(attrs["gen_ai.usage.output_tokens"]);
3151
- if ("gen_ai.response.model" in attrs) result.model = String(attrs["gen_ai.response.model"]);
3152
- return [args, result];
3153
- }
3154
- extractToolStep(attrs) {
3155
- const args = {};
3156
- const result = {};
3157
- if ("gen_ai.tool.call.id" in attrs) args.call_id = String(attrs["gen_ai.tool.call.id"]);
3158
- if ("gen_ai.tool.parameters" in attrs) args.parameters = attrs["gen_ai.tool.parameters"];
3159
- if ("gen_ai.tool.output" in attrs) result.output = attrs["gen_ai.tool.output"];
3160
- return [args, result];
3161
- }
3162
- spanMetadata(span) {
3163
- const startNs = span.startTime[0] * 1e9 + span.startTime[1];
3164
- const endNs = span.endTime[0] * 1e9 + span.endTime[1];
3165
- return { duration_ms: Math.floor((endNs - startNs) / 1e6) };
3166
- }
3167
- };
4187
+ return out;
4188
+ });
4189
+ return { resourceSpans: [{ scopeSpans: [{ spans: encoded }] }] };
4190
+ }
3168
4191
 
3169
4192
  // src/adapters/langchain.ts
3170
4193
  var LangChainAdapter = class {
@@ -3232,13 +4255,13 @@ var LangChainAdapter = class {
3232
4255
  * @param options.model - Model name override.
3233
4256
  * @returns A finalized Trace object.
3234
4257
  */
3235
- buildTrace(options) {
4258
+ buildTrace(options2) {
3236
4259
  const builder = new TraceBuilder(this.agentId);
3237
4260
  const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
3238
4261
  let totalTokens;
3239
4262
  for (const llm of this.llmCalls) {
3240
4263
  const args = {};
3241
- if (llm.model ?? options?.model) args.model = llm.model ?? options?.model;
4264
+ if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
3242
4265
  const result = { completion: llm.text };
3243
4266
  if (llm.tokens !== void 0) {
3244
4267
  result.tokens = llm.tokens;
@@ -3255,9 +4278,9 @@ var LangChainAdapter = class {
3255
4278
  builder.setOutput({ message: outputMessage });
3256
4279
  builder.setMetadata({
3257
4280
  total_tokens: totalTokens,
3258
- cost_usd: options?.costUsd,
4281
+ cost_usd: options2?.costUsd,
3259
4282
  latency_ms: latencyMs,
3260
- model: options?.model ?? this.llmCalls[0]?.model
4283
+ model: options2?.model ?? this.llmCalls[0]?.model
3261
4284
  });
3262
4285
  return builder.build();
3263
4286
  }
@@ -3272,6 +4295,7 @@ var LangChainAdapter = class {
3272
4295
  AttestClient,
3273
4296
  CONFUSED_USER,
3274
4297
  COOPERATIVE_USER,
4298
+ CaptureError,
3275
4299
  ContinuousEvalRunner,
3276
4300
  ENGINE_VERSION,
3277
4301
  ERR_ASSERTION_ERROR,
@@ -3280,6 +4304,7 @@ var LangChainAdapter = class {
3280
4304
  ERR_PROVIDER_ERROR,
3281
4305
  ERR_SESSION_ERROR,
3282
4306
  ERR_TIMEOUT,
4307
+ EgressBlockedError,
3283
4308
  EngineManager,
3284
4309
  EngineTimeoutError,
3285
4310
  ExpectChain,
@@ -3288,6 +4313,7 @@ var LangChainAdapter = class {
3288
4313
  GeminiAdapter,
3289
4314
  LangChainAdapter,
3290
4315
  ManualAdapter,
4316
+ OTLPConversionError,
3291
4317
  OTelAdapter,
3292
4318
  OUTCOME_FAIL,
3293
4319
  OUTCOME_PASS,
@@ -3298,11 +4324,17 @@ var LangChainAdapter = class {
3298
4324
  ProtocolDesyncError,
3299
4325
  ProtocolDiagnosticBuffer,
3300
4326
  ProtocolError,
4327
+ REASON_BUDGET_EXHAUSTED,
4328
+ REASON_JUDGE_ABSTAINED,
3301
4329
  REASON_MISSING_EVIDENCE,
3302
4330
  REASON_PROVIDER_NOT_CONFIGURED,
3303
4331
  REASON_PROVIDER_UNAVAILABLE,
4332
+ REASON_REPLAY_MISS,
3304
4333
  REASON_SHORT_CIRCUITED,
3305
4334
  REASON_SIMULATION,
4335
+ ReplayError,
4336
+ ReplayMissError,
4337
+ ReplayNotEvaluatedError,
3306
4338
  STATUS_HARD_FAIL,
3307
4339
  STATUS_NOT_EVALUATED,
3308
4340
  STATUS_PASS,
@@ -3317,6 +4349,7 @@ var LangChainAdapter = class {
3317
4349
  TIER_3,
3318
4350
  TYPE_CONSTRAINT,
3319
4351
  TYPE_CONTENT,
4352
+ TYPE_EFFECT,
3320
4353
  TYPE_EMBEDDING,
3321
4354
  TYPE_LLM_JUDGE,
3322
4355
  TYPE_PLUGIN,
@@ -3326,7 +4359,10 @@ var LangChainAdapter = class {
3326
4359
  TraceBuilder,
3327
4360
  TraceTree,
3328
4361
  VERSION,
4362
+ WorldChecks,
4363
+ WorldError,
3329
4364
  activeBuilder,
4365
+ addUsage,
3330
4366
  agent,
3331
4367
  agentId,
3332
4368
  assertionId,
@@ -3335,24 +4371,44 @@ var LangChainAdapter = class {
3335
4371
  config,
3336
4372
  decodeResponse,
3337
4373
  delegate,
4374
+ dslCall,
4375
+ egressBlockRequested,
4376
+ emitVitest,
3338
4377
  encodeRequest,
3339
4378
  executePluginAssertion,
3340
4379
  extractId,
3341
4380
  extractResult,
4381
+ firstMiss,
4382
+ formatUsage,
3342
4383
  getSampleRate,
4384
+ installEgressGuard,
4385
+ isProviderHost,
3343
4386
  isSimulationMode,
3344
4387
  layerForType,
4388
+ liveLine,
4389
+ loadFixture,
3345
4390
  loadLabelsCSV,
3346
4391
  loadLabelsJSONL,
4392
+ parseUsage,
3347
4393
  previewLine,
3348
4394
  promptHash,
4395
+ record,
3349
4396
  renderAssertionFailure,
3350
4397
  renderDiagnostics,
3351
4398
  renderSummary,
4399
+ renderVitest,
3352
4400
  requireSimulationCapability,
3353
4401
  resetConfig,
4402
+ resolveMode,
3354
4403
  simulationEnvEnabled,
4404
+ startReplay,
4405
+ startWorld,
4406
+ sumUsage,
4407
+ testPathFor,
3355
4408
  tier,
3356
- traceId
4409
+ traceId,
4410
+ uninstallEgressGuard,
4411
+ usageSummaryLine,
4412
+ zeroUsage
3357
4413
  });
3358
4414
  //# sourceMappingURL=index.cjs.map