@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +1001 -120
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +313 -5
- package/dist/cjs/proto/index.cjs +7 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +18 -2
- package/dist/esm/cli.d.ts +6 -1
- package/dist/esm/cli.js +310 -20
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +313 -5
- package/dist/esm/index.js +977 -120
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +18 -2
- package/dist/esm/proto/index.js +5 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +3 -3
package/dist/cjs/index.cjs
CHANGED
|
@@ -38,6 +38,7 @@ __export(index_exports, {
|
|
|
38
38
|
AttestClient: () => AttestClient,
|
|
39
39
|
CONFUSED_USER: () => CONFUSED_USER,
|
|
40
40
|
COOPERATIVE_USER: () => COOPERATIVE_USER,
|
|
41
|
+
CaptureError: () => CaptureError,
|
|
41
42
|
ContinuousEvalRunner: () => ContinuousEvalRunner,
|
|
42
43
|
ENGINE_VERSION: () => ENGINE_VERSION,
|
|
43
44
|
ERR_ASSERTION_ERROR: () => ERR_ASSERTION_ERROR,
|
|
@@ -46,6 +47,7 @@ __export(index_exports, {
|
|
|
46
47
|
ERR_PROVIDER_ERROR: () => ERR_PROVIDER_ERROR,
|
|
47
48
|
ERR_SESSION_ERROR: () => ERR_SESSION_ERROR,
|
|
48
49
|
ERR_TIMEOUT: () => ERR_TIMEOUT,
|
|
50
|
+
EgressBlockedError: () => EgressBlockedError,
|
|
49
51
|
EngineManager: () => EngineManager,
|
|
50
52
|
EngineTimeoutError: () => EngineTimeoutError,
|
|
51
53
|
ExpectChain: () => ExpectChain,
|
|
@@ -70,8 +72,12 @@ __export(index_exports, {
|
|
|
70
72
|
REASON_MISSING_EVIDENCE: () => REASON_MISSING_EVIDENCE,
|
|
71
73
|
REASON_PROVIDER_NOT_CONFIGURED: () => REASON_PROVIDER_NOT_CONFIGURED,
|
|
72
74
|
REASON_PROVIDER_UNAVAILABLE: () => REASON_PROVIDER_UNAVAILABLE,
|
|
75
|
+
REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
|
|
73
76
|
REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
|
|
74
77
|
REASON_SIMULATION: () => REASON_SIMULATION,
|
|
78
|
+
ReplayError: () => ReplayError,
|
|
79
|
+
ReplayMissError: () => ReplayMissError,
|
|
80
|
+
ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
|
|
75
81
|
STATUS_HARD_FAIL: () => STATUS_HARD_FAIL,
|
|
76
82
|
STATUS_NOT_EVALUATED: () => STATUS_NOT_EVALUATED,
|
|
77
83
|
STATUS_PASS: () => STATUS_PASS,
|
|
@@ -86,6 +92,7 @@ __export(index_exports, {
|
|
|
86
92
|
TIER_3: () => TIER_3,
|
|
87
93
|
TYPE_CONSTRAINT: () => TYPE_CONSTRAINT,
|
|
88
94
|
TYPE_CONTENT: () => TYPE_CONTENT,
|
|
95
|
+
TYPE_EFFECT: () => TYPE_EFFECT,
|
|
89
96
|
TYPE_EMBEDDING: () => TYPE_EMBEDDING,
|
|
90
97
|
TYPE_LLM_JUDGE: () => TYPE_LLM_JUDGE,
|
|
91
98
|
TYPE_PLUGIN: () => TYPE_PLUGIN,
|
|
@@ -95,6 +102,8 @@ __export(index_exports, {
|
|
|
95
102
|
TraceBuilder: () => TraceBuilder,
|
|
96
103
|
TraceTree: () => TraceTree,
|
|
97
104
|
VERSION: () => VERSION,
|
|
105
|
+
WorldChecks: () => WorldChecks,
|
|
106
|
+
WorldError: () => WorldError,
|
|
98
107
|
activeBuilder: () => activeBuilder,
|
|
99
108
|
addUsage: () => addUsage,
|
|
100
109
|
agent: () => agent,
|
|
@@ -105,28 +114,43 @@ __export(index_exports, {
|
|
|
105
114
|
config: () => config,
|
|
106
115
|
decodeResponse: () => decodeResponse,
|
|
107
116
|
delegate: () => delegate,
|
|
117
|
+
dslCall: () => dslCall,
|
|
118
|
+
egressBlockRequested: () => egressBlockRequested,
|
|
119
|
+
emitVitest: () => emitVitest,
|
|
108
120
|
encodeRequest: () => encodeRequest,
|
|
109
121
|
executePluginAssertion: () => executePluginAssertion,
|
|
110
122
|
extractId: () => extractId,
|
|
111
123
|
extractResult: () => extractResult,
|
|
124
|
+
firstMiss: () => firstMiss,
|
|
112
125
|
formatUsage: () => formatUsage,
|
|
113
126
|
getSampleRate: () => getSampleRate,
|
|
127
|
+
installEgressGuard: () => installEgressGuard,
|
|
128
|
+
isProviderHost: () => isProviderHost,
|
|
114
129
|
isSimulationMode: () => isSimulationMode,
|
|
115
130
|
layerForType: () => layerForType,
|
|
131
|
+
liveLine: () => liveLine,
|
|
132
|
+
loadFixture: () => loadFixture,
|
|
116
133
|
loadLabelsCSV: () => loadLabelsCSV,
|
|
117
134
|
loadLabelsJSONL: () => loadLabelsJSONL,
|
|
118
135
|
parseUsage: () => parseUsage,
|
|
119
136
|
previewLine: () => previewLine,
|
|
120
137
|
promptHash: () => promptHash,
|
|
138
|
+
record: () => record,
|
|
121
139
|
renderAssertionFailure: () => renderAssertionFailure,
|
|
122
140
|
renderDiagnostics: () => renderDiagnostics,
|
|
123
141
|
renderSummary: () => renderSummary,
|
|
142
|
+
renderVitest: () => renderVitest,
|
|
124
143
|
requireSimulationCapability: () => requireSimulationCapability,
|
|
125
144
|
resetConfig: () => resetConfig,
|
|
145
|
+
resolveMode: () => resolveMode,
|
|
126
146
|
simulationEnvEnabled: () => simulationEnvEnabled,
|
|
147
|
+
startReplay: () => startReplay,
|
|
148
|
+
startWorld: () => startWorld,
|
|
127
149
|
sumUsage: () => sumUsage,
|
|
150
|
+
testPathFor: () => testPathFor,
|
|
128
151
|
tier: () => tier,
|
|
129
152
|
traceId: () => traceId,
|
|
153
|
+
uninstallEgressGuard: () => uninstallEgressGuard,
|
|
130
154
|
usageSummaryLine: () => usageSummaryLine,
|
|
131
155
|
zeroUsage: () => zeroUsage
|
|
132
156
|
});
|
|
@@ -148,12 +172,14 @@ var REASON_MISSING_EVIDENCE = "missing_evidence";
|
|
|
148
172
|
var REASON_SHORT_CIRCUITED = "short_circuited";
|
|
149
173
|
var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
|
|
150
174
|
var REASON_JUDGE_ABSTAINED = "judge_abstained";
|
|
175
|
+
var REASON_REPLAY_MISS = "replay_miss";
|
|
151
176
|
var OUTCOME_PASS = "pass";
|
|
152
177
|
var OUTCOME_FAIL = "fail";
|
|
153
178
|
var OUTCOME_SKIP = "skip";
|
|
154
179
|
var TYPE_SCHEMA = "schema";
|
|
155
180
|
var TYPE_CONSTRAINT = "constraint";
|
|
156
181
|
var TYPE_TRACE = "trace";
|
|
182
|
+
var TYPE_EFFECT = "effect";
|
|
157
183
|
var TYPE_CONTENT = "content";
|
|
158
184
|
var TYPE_EMBEDDING = "embedding";
|
|
159
185
|
var TYPE_LLM_JUDGE = "llm_judge";
|
|
@@ -191,6 +217,7 @@ function layerForType(assertionType) {
|
|
|
191
217
|
return 2;
|
|
192
218
|
case "trace":
|
|
193
219
|
case "trace_tree":
|
|
220
|
+
case "effect":
|
|
194
221
|
return 3;
|
|
195
222
|
case "content":
|
|
196
223
|
return 4;
|
|
@@ -350,8 +377,8 @@ function extractId(response) {
|
|
|
350
377
|
}
|
|
351
378
|
|
|
352
379
|
// src/version.ts
|
|
353
|
-
var VERSION = "1.0.0-rc.
|
|
354
|
-
var ENGINE_VERSION = "1.0.0-rc.
|
|
380
|
+
var VERSION = "1.0.0-rc.3";
|
|
381
|
+
var ENGINE_VERSION = "1.0.0-rc.3";
|
|
355
382
|
|
|
356
383
|
// src/engine-manager.ts
|
|
357
384
|
var import_node_child_process = require("child_process");
|
|
@@ -419,7 +446,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
419
446
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
420
447
|
);
|
|
421
448
|
}
|
|
422
|
-
return new Promise((
|
|
449
|
+
return new Promise((resolve4, reject) => {
|
|
423
450
|
const client = url.startsWith("https:") ? https : http;
|
|
424
451
|
client.get(url, (res) => {
|
|
425
452
|
const status = res.statusCode ?? 0;
|
|
@@ -430,7 +457,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
430
457
|
return;
|
|
431
458
|
}
|
|
432
459
|
res.resume();
|
|
433
|
-
|
|
460
|
+
resolve4(httpsGet(location, redirectCount + 1));
|
|
434
461
|
return;
|
|
435
462
|
}
|
|
436
463
|
if (status < 200 || status >= 300) {
|
|
@@ -440,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
440
467
|
}
|
|
441
468
|
const chunks = [];
|
|
442
469
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
443
|
-
res.on("end", () =>
|
|
470
|
+
res.on("end", () => resolve4(Buffer.concat(chunks)));
|
|
444
471
|
res.on("error", reject);
|
|
445
472
|
}).on("error", reject);
|
|
446
473
|
});
|
|
@@ -571,8 +598,8 @@ Install options:
|
|
|
571
598
|
}
|
|
572
599
|
return [await downloadEngine(), "download"];
|
|
573
600
|
}
|
|
574
|
-
async function resolveEngine(explicitPath,
|
|
575
|
-
const [enginePath, source] = await locateEngine(explicitPath,
|
|
601
|
+
async function resolveEngine(explicitPath, options2 = {}) {
|
|
602
|
+
const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
|
|
576
603
|
const found = engineVersion(enginePath);
|
|
577
604
|
if (found !== ENGINE_VERSION) {
|
|
578
605
|
throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
|
|
@@ -587,9 +614,9 @@ var EngineManager = class {
|
|
|
587
614
|
initialized = false;
|
|
588
615
|
requestId = 0;
|
|
589
616
|
initResult = null;
|
|
590
|
-
constructor(
|
|
591
|
-
this.enginePath =
|
|
592
|
-
this.logLevel =
|
|
617
|
+
constructor(options2) {
|
|
618
|
+
this.enginePath = options2?.enginePath ?? "";
|
|
619
|
+
this.logLevel = options2?.logLevel ?? "warn";
|
|
593
620
|
}
|
|
594
621
|
async start() {
|
|
595
622
|
this.enginePath = await resolveEngine(this.enginePath || void 0);
|
|
@@ -667,11 +694,11 @@ var EngineManager = class {
|
|
|
667
694
|
}
|
|
668
695
|
this.requestId += 1;
|
|
669
696
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
670
|
-
return new Promise((
|
|
697
|
+
return new Promise((resolve4, reject) => {
|
|
671
698
|
const onLine = (line) => {
|
|
672
699
|
try {
|
|
673
700
|
const response = decodeResponse(line);
|
|
674
|
-
|
|
701
|
+
resolve4(extractResult(response));
|
|
675
702
|
} catch (err) {
|
|
676
703
|
reject(err);
|
|
677
704
|
}
|
|
@@ -705,17 +732,17 @@ var EngineManager = class {
|
|
|
705
732
|
// src/config.ts
|
|
706
733
|
var _simulationMode = false;
|
|
707
734
|
var _sampleRate = 0;
|
|
708
|
-
function config(
|
|
709
|
-
if (
|
|
710
|
-
_simulationMode =
|
|
735
|
+
function config(options2) {
|
|
736
|
+
if (options2.simulation !== void 0) {
|
|
737
|
+
_simulationMode = options2.simulation;
|
|
711
738
|
}
|
|
712
|
-
if (
|
|
713
|
-
if (
|
|
739
|
+
if (options2.sampleRate !== void 0) {
|
|
740
|
+
if (options2.sampleRate < 0 || options2.sampleRate > 1) {
|
|
714
741
|
throw new RangeError(
|
|
715
|
-
`sampleRate must be between 0.0 and 1.0, got ${
|
|
742
|
+
`sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
|
|
716
743
|
);
|
|
717
744
|
}
|
|
718
|
-
_sampleRate =
|
|
745
|
+
_sampleRate = options2.sampleRate;
|
|
719
746
|
}
|
|
720
747
|
}
|
|
721
748
|
function isSimulationMode() {
|
|
@@ -871,16 +898,16 @@ var AttestClient = class {
|
|
|
871
898
|
desyncThreshold;
|
|
872
899
|
desyncWindowMs;
|
|
873
900
|
desynced = false;
|
|
874
|
-
constructor(engine,
|
|
901
|
+
constructor(engine, options2) {
|
|
875
902
|
this.engine = engine;
|
|
876
|
-
this.logger =
|
|
903
|
+
this.logger = options2?.logger ?? envLoggerDefault();
|
|
877
904
|
this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
|
|
878
|
-
|
|
905
|
+
options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
|
|
879
906
|
);
|
|
880
|
-
this.desyncThreshold =
|
|
881
|
-
this.desyncWindowMs =
|
|
882
|
-
if (
|
|
883
|
-
this.diagnosticListeners.add(
|
|
907
|
+
this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
|
|
908
|
+
this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
|
|
909
|
+
if (options2?.onDiagnostic !== void 0) {
|
|
910
|
+
this.diagnosticListeners.add(options2.onDiagnostic);
|
|
884
911
|
}
|
|
885
912
|
}
|
|
886
913
|
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
@@ -1038,12 +1065,12 @@ var AttestClient = class {
|
|
|
1038
1065
|
throw new Error("Engine process not started.");
|
|
1039
1066
|
}
|
|
1040
1067
|
let capturedReqId;
|
|
1041
|
-
const requestPromise = new Promise((
|
|
1068
|
+
const requestPromise = new Promise((resolve4, reject) => {
|
|
1042
1069
|
this.writeChain = this.writeChain.then(() => {
|
|
1043
1070
|
this.requestId += 1;
|
|
1044
1071
|
const reqId = this.requestId;
|
|
1045
1072
|
capturedReqId = reqId;
|
|
1046
|
-
this.pending.set(reqId, { resolve, reject });
|
|
1073
|
+
this.pending.set(reqId, { resolve: resolve4, reject });
|
|
1047
1074
|
const requestStr = encodeRequest(reqId, method, params);
|
|
1048
1075
|
cp.stdin.write(requestStr, (err) => {
|
|
1049
1076
|
if (err) {
|
|
@@ -1061,23 +1088,23 @@ var AttestClient = class {
|
|
|
1061
1088
|
throw err;
|
|
1062
1089
|
});
|
|
1063
1090
|
}
|
|
1064
|
-
async evaluateBatch(trace, assertions,
|
|
1091
|
+
async evaluateBatch(trace, assertions, options2) {
|
|
1065
1092
|
if (isSimulationMode()) {
|
|
1066
1093
|
requireSimulationCapability(this.engine["initResult"]);
|
|
1067
1094
|
}
|
|
1068
1095
|
const params = {
|
|
1069
1096
|
trace,
|
|
1070
1097
|
assertions: [...assertions],
|
|
1071
|
-
...
|
|
1098
|
+
...options2?.budget !== void 0 ? { budget: options2.budget } : {}
|
|
1072
1099
|
};
|
|
1073
1100
|
const raw = await this.sendRequest(
|
|
1074
1101
|
"evaluate_batch",
|
|
1075
1102
|
params,
|
|
1076
|
-
|
|
1103
|
+
options2?.timeout
|
|
1077
1104
|
);
|
|
1078
1105
|
return { ...raw, usage: parseUsage(raw["usage"]) };
|
|
1079
1106
|
}
|
|
1080
|
-
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation,
|
|
1107
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
|
|
1081
1108
|
const result = {
|
|
1082
1109
|
assertion_id: assertionId2,
|
|
1083
1110
|
status,
|
|
@@ -1095,11 +1122,11 @@ var AttestClient = class {
|
|
|
1095
1122
|
const raw = await this.sendRequest(
|
|
1096
1123
|
"submit_plugin_result",
|
|
1097
1124
|
params,
|
|
1098
|
-
|
|
1125
|
+
options2?.timeout
|
|
1099
1126
|
);
|
|
1100
1127
|
return Boolean(raw?.accepted ?? false);
|
|
1101
1128
|
}
|
|
1102
|
-
async queryDrift(assertionId2, windowSize = 50,
|
|
1129
|
+
async queryDrift(assertionId2, windowSize = 50, options2) {
|
|
1103
1130
|
if (isSimulationMode()) {
|
|
1104
1131
|
return {
|
|
1105
1132
|
assertion_id: assertionId2,
|
|
@@ -1114,11 +1141,11 @@ var AttestClient = class {
|
|
|
1114
1141
|
const raw = await this.sendRequest(
|
|
1115
1142
|
"query_drift",
|
|
1116
1143
|
{ assertion_id: assertionId2, window_size: windowSize },
|
|
1117
|
-
|
|
1144
|
+
options2?.timeout
|
|
1118
1145
|
);
|
|
1119
1146
|
return raw.report;
|
|
1120
1147
|
}
|
|
1121
|
-
async generateUserMessage(persona, conversationHistory, faultConfig,
|
|
1148
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
|
|
1122
1149
|
if (isSimulationMode()) {
|
|
1123
1150
|
return `[simulation] Hello from ${persona.name}`;
|
|
1124
1151
|
}
|
|
@@ -1132,7 +1159,7 @@ var AttestClient = class {
|
|
|
1132
1159
|
const raw = await this.sendRequest(
|
|
1133
1160
|
"generate_user_message",
|
|
1134
1161
|
params,
|
|
1135
|
-
|
|
1162
|
+
options2?.timeout
|
|
1136
1163
|
);
|
|
1137
1164
|
return String(raw.message);
|
|
1138
1165
|
}
|
|
@@ -1177,33 +1204,33 @@ var TraceBuilder = class {
|
|
|
1177
1204
|
this.input = input;
|
|
1178
1205
|
return this;
|
|
1179
1206
|
}
|
|
1180
|
-
addLlmCall(name,
|
|
1207
|
+
addLlmCall(name, options2) {
|
|
1181
1208
|
this.steps.push({
|
|
1182
1209
|
type: STEP_LLM_CALL,
|
|
1183
1210
|
name,
|
|
1184
|
-
args:
|
|
1185
|
-
result:
|
|
1186
|
-
metadata:
|
|
1211
|
+
args: options2?.args,
|
|
1212
|
+
result: options2?.result,
|
|
1213
|
+
metadata: options2?.metadata
|
|
1187
1214
|
});
|
|
1188
1215
|
return this;
|
|
1189
1216
|
}
|
|
1190
|
-
addToolCall(name,
|
|
1217
|
+
addToolCall(name, options2) {
|
|
1191
1218
|
this.steps.push({
|
|
1192
1219
|
type: STEP_TOOL_CALL,
|
|
1193
1220
|
name,
|
|
1194
|
-
args:
|
|
1195
|
-
result:
|
|
1196
|
-
metadata:
|
|
1221
|
+
args: options2?.args,
|
|
1222
|
+
result: options2?.result,
|
|
1223
|
+
metadata: options2?.metadata
|
|
1197
1224
|
});
|
|
1198
1225
|
return this;
|
|
1199
1226
|
}
|
|
1200
|
-
addRetrieval(name,
|
|
1227
|
+
addRetrieval(name, options2) {
|
|
1201
1228
|
this.steps.push({
|
|
1202
1229
|
type: STEP_RETRIEVAL,
|
|
1203
1230
|
name,
|
|
1204
|
-
args:
|
|
1205
|
-
result:
|
|
1206
|
-
metadata:
|
|
1231
|
+
args: options2?.args,
|
|
1232
|
+
result: options2?.result,
|
|
1233
|
+
metadata: options2?.metadata
|
|
1207
1234
|
});
|
|
1208
1235
|
return this;
|
|
1209
1236
|
}
|
|
@@ -1281,10 +1308,10 @@ var AgentResult = class {
|
|
|
1281
1308
|
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
1282
1309
|
* when only soft_fail rows are present (see `softFailures`).
|
|
1283
1310
|
*/
|
|
1284
|
-
outcome(
|
|
1311
|
+
outcome(options2) {
|
|
1285
1312
|
if (this.hardFailures.length > 0) return OUTCOME_FAIL;
|
|
1286
1313
|
if (this.notEvaluated.length > 0) {
|
|
1287
|
-
return
|
|
1314
|
+
return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
|
|
1288
1315
|
}
|
|
1289
1316
|
return OUTCOME_PASS;
|
|
1290
1317
|
}
|
|
@@ -1394,9 +1421,9 @@ function truncate(value, limit = 280) {
|
|
|
1394
1421
|
}
|
|
1395
1422
|
return value.slice(0, limit - 3) + "...";
|
|
1396
1423
|
}
|
|
1397
|
-
function renderAssertionFailure(result,
|
|
1398
|
-
const color =
|
|
1399
|
-
const testFile =
|
|
1424
|
+
function renderAssertionFailure(result, options2 = {}) {
|
|
1425
|
+
const color = options2.color ?? false;
|
|
1426
|
+
const testFile = options2.testFile;
|
|
1400
1427
|
const layer = result.layer ?? 0;
|
|
1401
1428
|
const label = layerLabel(layer, result.type);
|
|
1402
1429
|
const lines = [];
|
|
@@ -1477,8 +1504,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1477
1504
|
);
|
|
1478
1505
|
}
|
|
1479
1506
|
}
|
|
1480
|
-
function renderSummary(result,
|
|
1481
|
-
const color =
|
|
1507
|
+
function renderSummary(result, options2 = {}) {
|
|
1508
|
+
const color = options2.color ?? false;
|
|
1482
1509
|
const passN = result.passCount;
|
|
1483
1510
|
const soft = result.assertionResults.filter(
|
|
1484
1511
|
(r) => r.status === STATUS_SOFT_FAIL
|
|
@@ -1496,9 +1523,9 @@ function renderSummary(result, options = {}) {
|
|
|
1496
1523
|
];
|
|
1497
1524
|
return parts.join(" | ");
|
|
1498
1525
|
}
|
|
1499
|
-
function renderDiagnostics(result,
|
|
1500
|
-
const color =
|
|
1501
|
-
const includePassing =
|
|
1526
|
+
function renderDiagnostics(result, options2 = {}) {
|
|
1527
|
+
const color = options2.color ?? false;
|
|
1528
|
+
const includePassing = options2.includePassing ?? false;
|
|
1502
1529
|
const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
1503
1530
|
if (rows.length === 0) {
|
|
1504
1531
|
return dim(" (no failures)\n", color);
|
|
@@ -1506,7 +1533,7 @@ function renderDiagnostics(result, options = {}) {
|
|
|
1506
1533
|
const out = [""];
|
|
1507
1534
|
out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
|
|
1508
1535
|
for (const r of rows) {
|
|
1509
|
-
out.push(renderAssertionFailure(r,
|
|
1536
|
+
out.push(renderAssertionFailure(r, options2));
|
|
1510
1537
|
}
|
|
1511
1538
|
out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
|
|
1512
1539
|
return out.join("\n");
|
|
@@ -1623,9 +1650,64 @@ var TraceTree = class {
|
|
|
1623
1650
|
|
|
1624
1651
|
// src/expect.ts
|
|
1625
1652
|
var import_node_crypto2 = require("crypto");
|
|
1653
|
+
var MAX_WORLD_BYTES = 9 * 1024 * 1024;
|
|
1654
|
+
var WorldChecks = class {
|
|
1655
|
+
_snapshot;
|
|
1656
|
+
_add;
|
|
1657
|
+
constructor(snapshot, add) {
|
|
1658
|
+
this._snapshot = snapshot;
|
|
1659
|
+
this._add = add;
|
|
1660
|
+
}
|
|
1661
|
+
addCheck(spec, where) {
|
|
1662
|
+
const full = { ...spec, world: this._snapshot };
|
|
1663
|
+
if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
|
|
1664
|
+
return this._add(full);
|
|
1665
|
+
}
|
|
1666
|
+
/**
|
|
1667
|
+
* Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
|
|
1668
|
+
* `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
|
|
1669
|
+
*
|
|
1670
|
+
* @throws Error when no bound is given, or `equals` is combined with another bound.
|
|
1671
|
+
*
|
|
1672
|
+
* @example
|
|
1673
|
+
* ```ts
|
|
1674
|
+
* expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
|
|
1675
|
+
* ```
|
|
1676
|
+
*/
|
|
1677
|
+
count(resource, opts) {
|
|
1678
|
+
if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
1679
|
+
throw new Error("count needs equals, atMost or atLeast");
|
|
1680
|
+
}
|
|
1681
|
+
if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
|
|
1682
|
+
throw new Error("count: equals excludes atMost and atLeast");
|
|
1683
|
+
}
|
|
1684
|
+
const spec = { check: "count", resource };
|
|
1685
|
+
if (opts.equals !== void 0) spec.equals = opts.equals;
|
|
1686
|
+
if (opts.atMost !== void 0) spec.at_most = opts.atMost;
|
|
1687
|
+
if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
|
|
1688
|
+
return this.addCheck(spec, opts.where);
|
|
1689
|
+
}
|
|
1690
|
+
/** Assert no two matching records share the same values for field(s) `by`. */
|
|
1691
|
+
unique(resource, opts) {
|
|
1692
|
+
const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
|
|
1693
|
+
if (fields.length === 0) throw new Error("unique needs at least one field in by");
|
|
1694
|
+
return this.addCheck({ check: "unique", resource, fields }, opts.where);
|
|
1695
|
+
}
|
|
1696
|
+
/**
|
|
1697
|
+
* Assert every matching record carries a distinct value in field `key` (default
|
|
1698
|
+
* `idempotency_key`), the record field that stores the idempotency key.
|
|
1699
|
+
*/
|
|
1700
|
+
idempotent(resource, opts = {}) {
|
|
1701
|
+
return this.addCheck(
|
|
1702
|
+
{ check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
|
|
1703
|
+
opts.where
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
};
|
|
1626
1707
|
var ExpectChain = class {
|
|
1627
1708
|
_result;
|
|
1628
1709
|
_assertions = [];
|
|
1710
|
+
_worldBytes = 0;
|
|
1629
1711
|
constructor(result) {
|
|
1630
1712
|
this._result = result;
|
|
1631
1713
|
}
|
|
@@ -2054,6 +2136,24 @@ var ExpectChain = class {
|
|
|
2054
2136
|
soft: opts?.soft ?? false
|
|
2055
2137
|
});
|
|
2056
2138
|
}
|
|
2139
|
+
// -- Layer 3: Effect checks (the external system the agent acted on) --
|
|
2140
|
+
/**
|
|
2141
|
+
* Check effects in a world snapshot:
|
|
2142
|
+
* `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
|
|
2143
|
+
* a world helper: the checks are `not_evaluated` (`missing_evidence`).
|
|
2144
|
+
*/
|
|
2145
|
+
world(snapshot) {
|
|
2146
|
+
const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
|
|
2147
|
+
return new WorldChecks(snapshot, (spec) => {
|
|
2148
|
+
this._worldBytes += size;
|
|
2149
|
+
if (this._worldBytes > MAX_WORLD_BYTES) {
|
|
2150
|
+
throw new Error(
|
|
2151
|
+
`effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
|
|
2152
|
+
);
|
|
2153
|
+
}
|
|
2154
|
+
return this.add(TYPE_EFFECT, spec);
|
|
2155
|
+
});
|
|
2156
|
+
}
|
|
2057
2157
|
// -- Layer 4: Content --
|
|
2058
2158
|
/**
|
|
2059
2159
|
* Assert the output message contains the given string.
|
|
@@ -2547,6 +2647,728 @@ function attestExpect(result) {
|
|
|
2547
2647
|
return new ExpectChain(new AgentResult(result));
|
|
2548
2648
|
}
|
|
2549
2649
|
|
|
2650
|
+
// src/capture.ts
|
|
2651
|
+
var import_node_child_process2 = require("child_process");
|
|
2652
|
+
var fs2 = __toESM(require("fs"), 1);
|
|
2653
|
+
var path2 = __toESM(require("path"), 1);
|
|
2654
|
+
var import_node_url = require("url");
|
|
2655
|
+
var FIXTURE_VERSION = 1;
|
|
2656
|
+
var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
|
|
2657
|
+
var CaptureError = class extends Error {
|
|
2658
|
+
/** The fixture written, when the engine wrote one. */
|
|
2659
|
+
fixture;
|
|
2660
|
+
constructor(message, fixture = null) {
|
|
2661
|
+
super(message);
|
|
2662
|
+
this.name = "CaptureError";
|
|
2663
|
+
this.fixture = fixture;
|
|
2664
|
+
}
|
|
2665
|
+
};
|
|
2666
|
+
function toPath(p) {
|
|
2667
|
+
return p instanceof URL ? (0, import_node_url.fileURLToPath)(p) : p;
|
|
2668
|
+
}
|
|
2669
|
+
function loadFixture(file) {
|
|
2670
|
+
const p = toPath(file);
|
|
2671
|
+
const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
|
|
2672
|
+
if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
|
|
2673
|
+
throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
|
|
2674
|
+
}
|
|
2675
|
+
validate(p, doc);
|
|
2676
|
+
return doc;
|
|
2677
|
+
}
|
|
2678
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
2679
|
+
function isObject(v) {
|
|
2680
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2681
|
+
}
|
|
2682
|
+
function kindOf(v) {
|
|
2683
|
+
if (Array.isArray(v)) return "array";
|
|
2684
|
+
if (isObject(v)) return "object";
|
|
2685
|
+
if (typeof v === "string" || typeof v === "boolean") return typeof v;
|
|
2686
|
+
return "other";
|
|
2687
|
+
}
|
|
2688
|
+
function validate(p, doc) {
|
|
2689
|
+
const need = (where, obj, fields) => {
|
|
2690
|
+
if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
|
|
2691
|
+
for (const [key, kind] of Object.entries(fields)) {
|
|
2692
|
+
if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
|
|
2693
|
+
}
|
|
2694
|
+
return obj;
|
|
2695
|
+
};
|
|
2696
|
+
need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
|
|
2697
|
+
if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
|
|
2698
|
+
if (!NAME_PATTERN.test(doc["name"])) {
|
|
2699
|
+
throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
|
|
2700
|
+
}
|
|
2701
|
+
const v = need("fixture.verification", doc["verification"], {
|
|
2702
|
+
verified: "boolean",
|
|
2703
|
+
expect_fail: "boolean",
|
|
2704
|
+
results: "array",
|
|
2705
|
+
expected_failures: "array"
|
|
2706
|
+
});
|
|
2707
|
+
doc["contracts"].forEach(
|
|
2708
|
+
(c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
|
|
2709
|
+
);
|
|
2710
|
+
v["results"].forEach(
|
|
2711
|
+
(r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
|
|
2712
|
+
);
|
|
2713
|
+
}
|
|
2714
|
+
function objectKey(k) {
|
|
2715
|
+
if (k === "__proto__") return `[${JSON.stringify(k)}]`;
|
|
2716
|
+
return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
|
|
2717
|
+
}
|
|
2718
|
+
function comment(text) {
|
|
2719
|
+
return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
|
|
2720
|
+
}
|
|
2721
|
+
function ts(value) {
|
|
2722
|
+
if (value === null || typeof value === "boolean" || typeof value === "number") {
|
|
2723
|
+
return JSON.stringify(value);
|
|
2724
|
+
}
|
|
2725
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
2726
|
+
if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
|
|
2727
|
+
if (typeof value === "object") {
|
|
2728
|
+
const entries = Object.entries(value).map(
|
|
2729
|
+
([k, v]) => `${objectKey(k)}: ${ts(v)}`
|
|
2730
|
+
);
|
|
2731
|
+
return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
|
|
2732
|
+
}
|
|
2733
|
+
throw new Error(`value ${String(value)} has no literal`);
|
|
2734
|
+
}
|
|
2735
|
+
function str(spec, key) {
|
|
2736
|
+
const v = spec[key];
|
|
2737
|
+
if (typeof v !== "string") throw new Error(`missing ${key}`);
|
|
2738
|
+
return v;
|
|
2739
|
+
}
|
|
2740
|
+
function options(entries) {
|
|
2741
|
+
const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
|
|
2742
|
+
return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
|
|
2743
|
+
}
|
|
2744
|
+
function call(method, args, opts) {
|
|
2745
|
+
return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
|
|
2746
|
+
}
|
|
2747
|
+
function dslCall(contract) {
|
|
2748
|
+
const s = contract.spec;
|
|
2749
|
+
const soft = s["soft"] === true ? true : void 0;
|
|
2750
|
+
try {
|
|
2751
|
+
if (contract.type === "trace") {
|
|
2752
|
+
const check = s["check"];
|
|
2753
|
+
if (check === "trajectory" && s["mode"] !== "partial_order") {
|
|
2754
|
+
const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
|
|
2755
|
+
if (!Array.isArray(s["tools"])) throw new Error("missing tools");
|
|
2756
|
+
return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
|
|
2757
|
+
}
|
|
2758
|
+
if (check === "tool_order") {
|
|
2759
|
+
return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
|
|
2760
|
+
}
|
|
2761
|
+
if (check === "tool_cardinality") {
|
|
2762
|
+
return call(
|
|
2763
|
+
"toolCardinality",
|
|
2764
|
+
[ts(str(s, "tool"))],
|
|
2765
|
+
options([
|
|
2766
|
+
["per", s["per"]],
|
|
2767
|
+
["atLeast", s["min"]],
|
|
2768
|
+
["atMost", s["max"]],
|
|
2769
|
+
["soft", soft]
|
|
2770
|
+
])
|
|
2771
|
+
);
|
|
2772
|
+
}
|
|
2773
|
+
}
|
|
2774
|
+
if (contract.type === "schema" && s["target"] === "output.structured") {
|
|
2775
|
+
if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
|
|
2776
|
+
return call("outputMatchesSchema", [ts(s["schema"])], "");
|
|
2777
|
+
}
|
|
2778
|
+
if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
|
|
2779
|
+
if (typeof s["value"] !== "number") throw new Error("missing value");
|
|
2780
|
+
return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
|
|
2781
|
+
}
|
|
2782
|
+
} catch (err) {
|
|
2783
|
+
throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
|
|
2784
|
+
}
|
|
2785
|
+
throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
|
|
2786
|
+
}
|
|
2787
|
+
function testPathFor(fixturePath) {
|
|
2788
|
+
const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
|
|
2789
|
+
return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
|
|
2790
|
+
}
|
|
2791
|
+
function renderVitest(fixture, fixturePath, testPath) {
|
|
2792
|
+
const calls = [];
|
|
2793
|
+
const skipped = [];
|
|
2794
|
+
for (const c of fixture.contracts) {
|
|
2795
|
+
try {
|
|
2796
|
+
calls.push([c, dslCall(c)]);
|
|
2797
|
+
} catch (err) {
|
|
2798
|
+
skipped.push(err.message);
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
|
|
2802
|
+
const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
|
|
2803
|
+
const expectFail = fixture.verification.expect_fail;
|
|
2804
|
+
const lines = [
|
|
2805
|
+
`// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
|
|
2806
|
+
"//",
|
|
2807
|
+
"// Review and edit before committing: they check the captured trace without running",
|
|
2808
|
+
"// the agent. Re-capturing does not overwrite this file.",
|
|
2809
|
+
`import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
|
|
2810
|
+
`import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
|
|
2811
|
+
`import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
|
|
2812
|
+
"",
|
|
2813
|
+
`const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
|
|
2814
|
+
];
|
|
2815
|
+
if (skipped.length > 0) {
|
|
2816
|
+
lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
|
|
2817
|
+
for (const s of skipped) lines.push(`// - ${comment(s)}`);
|
|
2818
|
+
}
|
|
2819
|
+
lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
|
|
2820
|
+
for (const [c, dsl] of calls) {
|
|
2821
|
+
lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
|
|
2822
|
+
}
|
|
2823
|
+
lines[lines.length - 1] += ";";
|
|
2824
|
+
lines.push("}");
|
|
2825
|
+
if (expectFail) {
|
|
2826
|
+
lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
|
|
2827
|
+
lines.push("const EXPECTED: Array<[string, string]> = [");
|
|
2828
|
+
for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
|
|
2829
|
+
lines.push(
|
|
2830
|
+
"];",
|
|
2831
|
+
"",
|
|
2832
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2833
|
+
` it("reproduces the incident", async () => {`,
|
|
2834
|
+
" const result = await inspect(contracts());",
|
|
2835
|
+
"",
|
|
2836
|
+
" expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
|
|
2837
|
+
" });",
|
|
2838
|
+
"});"
|
|
2839
|
+
);
|
|
2840
|
+
} else {
|
|
2841
|
+
lines.push(
|
|
2842
|
+
"",
|
|
2843
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2844
|
+
` it("keeps its drafted contracts", async () => {`,
|
|
2845
|
+
" await evaluate(contracts());",
|
|
2846
|
+
" });",
|
|
2847
|
+
"});"
|
|
2848
|
+
);
|
|
2849
|
+
}
|
|
2850
|
+
return `${lines.join("\n")}
|
|
2851
|
+
`;
|
|
2852
|
+
}
|
|
2853
|
+
function emitVitest(fixturePath) {
|
|
2854
|
+
const testPath = testPathFor(fixturePath);
|
|
2855
|
+
const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
|
|
2856
|
+
if (existing?.isSymbolicLink()) {
|
|
2857
|
+
throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
|
|
2858
|
+
}
|
|
2859
|
+
if (existing !== void 0) return null;
|
|
2860
|
+
const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
|
|
2861
|
+
fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
|
|
2862
|
+
try {
|
|
2863
|
+
fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
|
|
2864
|
+
} catch (err) {
|
|
2865
|
+
if (err.code === "EEXIST") {
|
|
2866
|
+
throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
|
|
2867
|
+
}
|
|
2868
|
+
throw err;
|
|
2869
|
+
}
|
|
2870
|
+
return testPath;
|
|
2871
|
+
}
|
|
2872
|
+
function serve(engine, args) {
|
|
2873
|
+
const child = (0, import_node_child_process2.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2874
|
+
let stdout = "";
|
|
2875
|
+
let stderr = "";
|
|
2876
|
+
let resolveReady = () => void 0;
|
|
2877
|
+
const ready = new Promise((resolve4) => {
|
|
2878
|
+
resolveReady = resolve4;
|
|
2879
|
+
});
|
|
2880
|
+
const exited = new Promise((resolve4) => {
|
|
2881
|
+
child.on("close", (code, signal) => {
|
|
2882
|
+
resolveReady(null);
|
|
2883
|
+
resolve4({ code, signal });
|
|
2884
|
+
});
|
|
2885
|
+
child.on("error", (err) => {
|
|
2886
|
+
stderr += err.message;
|
|
2887
|
+
resolveReady(null);
|
|
2888
|
+
resolve4({ code: null, signal: null });
|
|
2889
|
+
});
|
|
2890
|
+
});
|
|
2891
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
2892
|
+
stdout += chunk;
|
|
2893
|
+
const nl = stdout.indexOf("\n");
|
|
2894
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
2895
|
+
});
|
|
2896
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
2897
|
+
stderr += chunk;
|
|
2898
|
+
});
|
|
2899
|
+
child.stdin.on("error", () => void 0);
|
|
2900
|
+
return {
|
|
2901
|
+
ready,
|
|
2902
|
+
async finish(finalize) {
|
|
2903
|
+
if (finalize) {
|
|
2904
|
+
const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
|
|
2905
|
+
child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
|
|
2906
|
+
`);
|
|
2907
|
+
}
|
|
2908
|
+
child.stdin.end();
|
|
2909
|
+
const { code, signal } = await exited;
|
|
2910
|
+
return { code, signal, stdout, stderr };
|
|
2911
|
+
}
|
|
2912
|
+
};
|
|
2913
|
+
}
|
|
2914
|
+
function readyEnv(line) {
|
|
2915
|
+
if (line === null) return null;
|
|
2916
|
+
let parsed;
|
|
2917
|
+
try {
|
|
2918
|
+
parsed = JSON.parse(line);
|
|
2919
|
+
} catch {
|
|
2920
|
+
return null;
|
|
2921
|
+
}
|
|
2922
|
+
const env = parsed?.env;
|
|
2923
|
+
if (typeof env !== "object" || env === null) return null;
|
|
2924
|
+
const entries = Object.entries(env);
|
|
2925
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
2926
|
+
return Object.fromEntries(entries);
|
|
2927
|
+
}
|
|
2928
|
+
function exitDetail(exit) {
|
|
2929
|
+
const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
|
|
2930
|
+
const err = exit.stderr.trim();
|
|
2931
|
+
return err ? `${how}: ${err}` : how;
|
|
2932
|
+
}
|
|
2933
|
+
function restore(saved) {
|
|
2934
|
+
for (const [k, v] of saved) {
|
|
2935
|
+
if (v === void 0) delete process.env[k];
|
|
2936
|
+
else process.env[k] = v;
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
var recording = false;
|
|
2940
|
+
async function record(name, fn, opts = {}) {
|
|
2941
|
+
if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
|
|
2942
|
+
recording = true;
|
|
2943
|
+
try {
|
|
2944
|
+
return await recordExclusive(name, fn, opts);
|
|
2945
|
+
} finally {
|
|
2946
|
+
recording = false;
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
async function recordExclusive(name, fn, opts) {
|
|
2950
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2951
|
+
const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
|
|
2952
|
+
if (opts.expectFail) args.push("--expect-fail");
|
|
2953
|
+
if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
|
|
2954
|
+
if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
|
|
2955
|
+
const proc = serve(engine, args);
|
|
2956
|
+
const env = readyEnv(await proc.ready);
|
|
2957
|
+
if (env === null) {
|
|
2958
|
+
const exit2 = await proc.finish(false);
|
|
2959
|
+
throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
|
|
2960
|
+
}
|
|
2961
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
2962
|
+
Object.assign(process.env, env);
|
|
2963
|
+
let value;
|
|
2964
|
+
try {
|
|
2965
|
+
value = await fn({ name, env });
|
|
2966
|
+
if (opts.flush) await opts.flush();
|
|
2967
|
+
} catch (err) {
|
|
2968
|
+
restore(saved);
|
|
2969
|
+
await proc.finish(false);
|
|
2970
|
+
throw err;
|
|
2971
|
+
}
|
|
2972
|
+
restore(saved);
|
|
2973
|
+
const exit = await proc.finish(true);
|
|
2974
|
+
process.stderr.write(exit.stderr);
|
|
2975
|
+
const lines = exit.stdout.trim().split("\n");
|
|
2976
|
+
if (lines.length < 2) {
|
|
2977
|
+
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2978
|
+
}
|
|
2979
|
+
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2980
|
+
if (exit.code !== 0 || !outcome.verified) {
|
|
2981
|
+
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2982
|
+
}
|
|
2983
|
+
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2984
|
+
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
2985
|
+
}
|
|
2986
|
+
|
|
2987
|
+
// src/replay.ts
|
|
2988
|
+
var import_node_child_process3 = require("child_process");
|
|
2989
|
+
var path3 = __toESM(require("path"), 1);
|
|
2990
|
+
|
|
2991
|
+
// src/egress.ts
|
|
2992
|
+
var net = __toESM(require("net"), 1);
|
|
2993
|
+
var PROVIDER_HOSTS = [
|
|
2994
|
+
"api.openai.com",
|
|
2995
|
+
"api.anthropic.com",
|
|
2996
|
+
"generativelanguage.googleapis.com",
|
|
2997
|
+
/\.openai\.azure\.com$/
|
|
2998
|
+
];
|
|
2999
|
+
var EgressBlockedError = class extends Error {
|
|
3000
|
+
constructor(host) {
|
|
3001
|
+
super(
|
|
3002
|
+
`attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
|
|
3003
|
+
);
|
|
3004
|
+
this.host = host;
|
|
3005
|
+
this.name = "EgressBlockedError";
|
|
3006
|
+
}
|
|
3007
|
+
host;
|
|
3008
|
+
};
|
|
3009
|
+
function isProviderHost(host) {
|
|
3010
|
+
const h = host.toLowerCase().replace(/\.$/, "");
|
|
3011
|
+
return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
|
|
3012
|
+
}
|
|
3013
|
+
function egressBlockRequested(env = process.env) {
|
|
3014
|
+
const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
|
|
3015
|
+
if (v !== "" && v !== "block" && v !== "allow") {
|
|
3016
|
+
throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
|
|
3017
|
+
}
|
|
3018
|
+
return v === "block";
|
|
3019
|
+
}
|
|
3020
|
+
var original = null;
|
|
3021
|
+
var installs = 0;
|
|
3022
|
+
function hostOf(args) {
|
|
3023
|
+
const first = Array.isArray(args[0]) ? args[0][0] : args[0];
|
|
3024
|
+
if (typeof first === "object" && first !== null) {
|
|
3025
|
+
const o = first;
|
|
3026
|
+
return typeof o.host === "string" ? o.host : null;
|
|
3027
|
+
}
|
|
3028
|
+
if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
|
|
3029
|
+
return typeof args[1] === "string" ? args[1] : null;
|
|
3030
|
+
}
|
|
3031
|
+
return null;
|
|
3032
|
+
}
|
|
3033
|
+
function installEgressGuard() {
|
|
3034
|
+
installs++;
|
|
3035
|
+
if (original === null) {
|
|
3036
|
+
const proto = net.Socket.prototype;
|
|
3037
|
+
const orig = proto.connect;
|
|
3038
|
+
original = orig;
|
|
3039
|
+
proto.connect = function(...args) {
|
|
3040
|
+
const host = hostOf(args);
|
|
3041
|
+
if (host !== null && isProviderHost(host)) {
|
|
3042
|
+
const err = new EgressBlockedError(host);
|
|
3043
|
+
process.nextTick(() => this.destroy(err));
|
|
3044
|
+
return this;
|
|
3045
|
+
}
|
|
3046
|
+
return orig.apply(this, args);
|
|
3047
|
+
};
|
|
3048
|
+
}
|
|
3049
|
+
let released = false;
|
|
3050
|
+
return () => {
|
|
3051
|
+
if (released) return;
|
|
3052
|
+
released = true;
|
|
3053
|
+
installs--;
|
|
3054
|
+
if (installs === 0) uninstallEgressGuard();
|
|
3055
|
+
};
|
|
3056
|
+
}
|
|
3057
|
+
function egressGuardActive() {
|
|
3058
|
+
return installs > 0;
|
|
3059
|
+
}
|
|
3060
|
+
function uninstallEgressGuard() {
|
|
3061
|
+
installs = 0;
|
|
3062
|
+
if (original === null) return;
|
|
3063
|
+
net.Socket.prototype.connect = original;
|
|
3064
|
+
original = null;
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
// src/replay.ts
|
|
3068
|
+
var ReplayMissError = class extends Error {
|
|
3069
|
+
constructor(message, summary) {
|
|
3070
|
+
super(message);
|
|
3071
|
+
this.summary = summary;
|
|
3072
|
+
this.name = "ReplayMissError";
|
|
3073
|
+
}
|
|
3074
|
+
summary;
|
|
3075
|
+
};
|
|
3076
|
+
var ReplayNotEvaluatedError = class extends Error {
|
|
3077
|
+
constructor(message, summary) {
|
|
3078
|
+
super(message);
|
|
3079
|
+
this.summary = summary;
|
|
3080
|
+
this.name = "ReplayNotEvaluatedError";
|
|
3081
|
+
}
|
|
3082
|
+
summary;
|
|
3083
|
+
};
|
|
3084
|
+
function escapeControl(text) {
|
|
3085
|
+
return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
|
|
3086
|
+
}
|
|
3087
|
+
var ReplayError = class extends Error {
|
|
3088
|
+
constructor(message) {
|
|
3089
|
+
super(escapeControl(message));
|
|
3090
|
+
this.name = "ReplayError";
|
|
3091
|
+
}
|
|
3092
|
+
};
|
|
3093
|
+
function resolveMode(explicit, env = process.env) {
|
|
3094
|
+
if (explicit !== void 0) return checkMode(explicit, "mode");
|
|
3095
|
+
const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
|
|
3096
|
+
if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
|
|
3097
|
+
const ci = (env.CI ?? "").trim().toLowerCase();
|
|
3098
|
+
return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
|
|
3099
|
+
}
|
|
3100
|
+
function checkMode(value, where) {
|
|
3101
|
+
if (value === "strict" || value === "auto" || value === "off") return value;
|
|
3102
|
+
throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
|
|
3103
|
+
}
|
|
3104
|
+
var active = null;
|
|
3105
|
+
async function startReplay(name, opts = {}) {
|
|
3106
|
+
if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
|
|
3107
|
+
const resolved = resolveMode(opts.mode);
|
|
3108
|
+
if (resolved === "off") return offSession(name);
|
|
3109
|
+
const mode = resolved;
|
|
3110
|
+
if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
|
|
3111
|
+
active = name;
|
|
3112
|
+
try {
|
|
3113
|
+
return await start(name, mode, opts);
|
|
3114
|
+
} catch (err) {
|
|
3115
|
+
active = null;
|
|
3116
|
+
throw err;
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
function offSession(name) {
|
|
3120
|
+
return {
|
|
3121
|
+
name,
|
|
3122
|
+
mode: "off",
|
|
3123
|
+
env: {},
|
|
3124
|
+
summary: null,
|
|
3125
|
+
finish: () => Promise.resolve(null),
|
|
3126
|
+
error: () => null,
|
|
3127
|
+
check: () => Promise.resolve()
|
|
3128
|
+
};
|
|
3129
|
+
}
|
|
3130
|
+
function missError(mode, summary) {
|
|
3131
|
+
if (summary.misses === 0) return null;
|
|
3132
|
+
const message = escapeControl(firstMiss(summary) ?? "replay miss");
|
|
3133
|
+
return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
|
|
3134
|
+
}
|
|
3135
|
+
async function start(name, mode, opts) {
|
|
3136
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3137
|
+
const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
|
|
3138
|
+
const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
|
|
3139
|
+
const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
|
|
3140
|
+
if (blocking) args.push("--no-live");
|
|
3141
|
+
const child = (0, import_node_child_process3.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3142
|
+
let stdout = "";
|
|
3143
|
+
let stderr = "";
|
|
3144
|
+
let resolveReady = () => void 0;
|
|
3145
|
+
const ready = new Promise((resolve4) => {
|
|
3146
|
+
resolveReady = resolve4;
|
|
3147
|
+
});
|
|
3148
|
+
const exited = new Promise((resolve4) => {
|
|
3149
|
+
child.on("close", (code) => {
|
|
3150
|
+
resolveReady(null);
|
|
3151
|
+
resolve4(code);
|
|
3152
|
+
});
|
|
3153
|
+
child.on("error", (err) => {
|
|
3154
|
+
stderr += err.message;
|
|
3155
|
+
resolveReady(null);
|
|
3156
|
+
resolve4(null);
|
|
3157
|
+
});
|
|
3158
|
+
});
|
|
3159
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3160
|
+
stdout += chunk;
|
|
3161
|
+
const nl = stdout.indexOf("\n");
|
|
3162
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3163
|
+
});
|
|
3164
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3165
|
+
stderr += chunk;
|
|
3166
|
+
});
|
|
3167
|
+
child.stdin.on("error", () => void 0);
|
|
3168
|
+
const stop = async () => {
|
|
3169
|
+
child.stdin.write("stop\n");
|
|
3170
|
+
child.stdin.end();
|
|
3171
|
+
return { code: await exited, stdout, stderr };
|
|
3172
|
+
};
|
|
3173
|
+
const env = parseEnv(await ready);
|
|
3174
|
+
if (env === null) {
|
|
3175
|
+
const out = await stop();
|
|
3176
|
+
active = null;
|
|
3177
|
+
throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3178
|
+
}
|
|
3179
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3180
|
+
Object.assign(process.env, env);
|
|
3181
|
+
let summary = null;
|
|
3182
|
+
let finishing = null;
|
|
3183
|
+
const finish = () => {
|
|
3184
|
+
finishing ??= (async () => {
|
|
3185
|
+
for (const [k, v] of saved) {
|
|
3186
|
+
if (v === void 0) delete process.env[k];
|
|
3187
|
+
else process.env[k] = v;
|
|
3188
|
+
}
|
|
3189
|
+
const out = await stop();
|
|
3190
|
+
active = null;
|
|
3191
|
+
const parsed = parseSummary(out.stdout);
|
|
3192
|
+
if (parsed === null) {
|
|
3193
|
+
throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3194
|
+
}
|
|
3195
|
+
if (blocking && parsed.live > 0) {
|
|
3196
|
+
throw new ReplayError(
|
|
3197
|
+
`replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
summary = parsed;
|
|
3201
|
+
return parsed;
|
|
3202
|
+
})();
|
|
3203
|
+
return finishing;
|
|
3204
|
+
};
|
|
3205
|
+
return {
|
|
3206
|
+
name,
|
|
3207
|
+
mode,
|
|
3208
|
+
env,
|
|
3209
|
+
get summary() {
|
|
3210
|
+
return summary;
|
|
3211
|
+
},
|
|
3212
|
+
finish,
|
|
3213
|
+
error: () => summary === null ? null : missError(mode, summary),
|
|
3214
|
+
check: async () => {
|
|
3215
|
+
await finish();
|
|
3216
|
+
const err = summary === null ? null : missError(mode, summary);
|
|
3217
|
+
if (err !== null) throw err;
|
|
3218
|
+
}
|
|
3219
|
+
};
|
|
3220
|
+
}
|
|
3221
|
+
function parseEnv(line) {
|
|
3222
|
+
if (line === null) return null;
|
|
3223
|
+
try {
|
|
3224
|
+
const env = JSON.parse(line).env;
|
|
3225
|
+
if (typeof env !== "object" || env === null) return null;
|
|
3226
|
+
const entries = Object.entries(env);
|
|
3227
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
3228
|
+
return Object.fromEntries(entries);
|
|
3229
|
+
} catch {
|
|
3230
|
+
return null;
|
|
3231
|
+
}
|
|
3232
|
+
}
|
|
3233
|
+
function parseSummary(out) {
|
|
3234
|
+
const lines = out.trim().split("\n");
|
|
3235
|
+
try {
|
|
3236
|
+
const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
|
|
3237
|
+
if (s === void 0 || !Array.isArray(s.records)) return null;
|
|
3238
|
+
for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
|
|
3239
|
+
return s;
|
|
3240
|
+
} catch {
|
|
3241
|
+
return null;
|
|
3242
|
+
}
|
|
3243
|
+
}
|
|
3244
|
+
function firstMiss(summary) {
|
|
3245
|
+
return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
|
|
3246
|
+
}
|
|
3247
|
+
function liveLine(live, costUsd, unpriced = 0) {
|
|
3248
|
+
const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
|
|
3249
|
+
return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
|
|
3250
|
+
}
|
|
3251
|
+
|
|
3252
|
+
// src/world.ts
|
|
3253
|
+
var import_node_child_process4 = require("child_process");
|
|
3254
|
+
var path4 = __toESM(require("path"), 1);
|
|
3255
|
+
var STATE_PATH = "/_attest/state";
|
|
3256
|
+
var STATE_TIMEOUT_MS = 1e4;
|
|
3257
|
+
var WorldError = class extends Error {
|
|
3258
|
+
constructor(message) {
|
|
3259
|
+
super(escapeControl(message));
|
|
3260
|
+
this.name = "WorldError";
|
|
3261
|
+
}
|
|
3262
|
+
};
|
|
3263
|
+
var active2 = null;
|
|
3264
|
+
async function startWorld(worldFile, opts = {}) {
|
|
3265
|
+
if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
|
|
3266
|
+
const file = path4.resolve(worldFile);
|
|
3267
|
+
active2 = file;
|
|
3268
|
+
try {
|
|
3269
|
+
return await start2(file, opts);
|
|
3270
|
+
} catch (err) {
|
|
3271
|
+
active2 = null;
|
|
3272
|
+
throw err;
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
async function start2(file, opts) {
|
|
3276
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3277
|
+
const child = (0, import_node_child_process4.spawn)(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
3278
|
+
let stdout = "";
|
|
3279
|
+
let stderr = "";
|
|
3280
|
+
let resolveReady = () => void 0;
|
|
3281
|
+
const ready = new Promise((resolve4) => {
|
|
3282
|
+
resolveReady = resolve4;
|
|
3283
|
+
});
|
|
3284
|
+
const exited = new Promise((resolve4) => {
|
|
3285
|
+
child.on("close", (code) => {
|
|
3286
|
+
resolveReady(null);
|
|
3287
|
+
resolve4(code);
|
|
3288
|
+
});
|
|
3289
|
+
child.on("error", (err) => {
|
|
3290
|
+
stderr += err.message;
|
|
3291
|
+
resolveReady(null);
|
|
3292
|
+
resolve4(null);
|
|
3293
|
+
});
|
|
3294
|
+
});
|
|
3295
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3296
|
+
stdout += chunk;
|
|
3297
|
+
const nl = stdout.indexOf("\n");
|
|
3298
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3299
|
+
});
|
|
3300
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3301
|
+
stderr += chunk;
|
|
3302
|
+
});
|
|
3303
|
+
child.stdin.on("error", () => void 0);
|
|
3304
|
+
const halt = async () => {
|
|
3305
|
+
child.stdin.write("stop\n");
|
|
3306
|
+
child.stdin.end();
|
|
3307
|
+
return { code: await exited, stdout, stderr };
|
|
3308
|
+
};
|
|
3309
|
+
const env = parseEnv(await ready);
|
|
3310
|
+
if (env === null) {
|
|
3311
|
+
const out = await halt();
|
|
3312
|
+
throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3313
|
+
}
|
|
3314
|
+
const urls = Object.values(env);
|
|
3315
|
+
if (urls.length !== 1) {
|
|
3316
|
+
await halt();
|
|
3317
|
+
throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
|
|
3318
|
+
}
|
|
3319
|
+
const baseUrl = urls[0];
|
|
3320
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3321
|
+
Object.assign(process.env, env);
|
|
3322
|
+
let stopped = false;
|
|
3323
|
+
let stopping = null;
|
|
3324
|
+
const stop = () => {
|
|
3325
|
+
stopped = true;
|
|
3326
|
+
stopping ??= (async () => {
|
|
3327
|
+
for (const [k, v] of saved) {
|
|
3328
|
+
if (v === void 0) delete process.env[k];
|
|
3329
|
+
else process.env[k] = v;
|
|
3330
|
+
}
|
|
3331
|
+
const out = await halt();
|
|
3332
|
+
active2 = null;
|
|
3333
|
+
const final = parseFinal(out.stdout);
|
|
3334
|
+
if (out.code !== 0 || final === null) {
|
|
3335
|
+
throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3336
|
+
}
|
|
3337
|
+
return final;
|
|
3338
|
+
})();
|
|
3339
|
+
return stopping;
|
|
3340
|
+
};
|
|
3341
|
+
const state = async () => {
|
|
3342
|
+
if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
|
|
3343
|
+
const url = baseUrl + STATE_PATH;
|
|
3344
|
+
let res;
|
|
3345
|
+
try {
|
|
3346
|
+
res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
|
|
3347
|
+
} catch (err) {
|
|
3348
|
+
throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3349
|
+
}
|
|
3350
|
+
if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
|
|
3351
|
+
const body = await res.json().catch(() => {
|
|
3352
|
+
throw new WorldError(`GET ${url}: the body is not JSON`);
|
|
3353
|
+
});
|
|
3354
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
3355
|
+
throw new WorldError(`GET ${url}: the body is not a snapshot object`);
|
|
3356
|
+
}
|
|
3357
|
+
return body;
|
|
3358
|
+
};
|
|
3359
|
+
return { path: file, env, baseUrl, state, stop };
|
|
3360
|
+
}
|
|
3361
|
+
function parseFinal(out) {
|
|
3362
|
+
const lines = out.trim().split("\n");
|
|
3363
|
+
try {
|
|
3364
|
+
const world = JSON.parse(lines[lines.length - 1] ?? "").world;
|
|
3365
|
+
if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
|
|
3366
|
+
return world;
|
|
3367
|
+
} catch {
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
|
|
2550
3372
|
// src/context.ts
|
|
2551
3373
|
var import_node_async_hooks = require("async_hooks");
|
|
2552
3374
|
var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
|
|
@@ -2750,12 +3572,12 @@ var ContinuousEvalRunner = class {
|
|
|
2750
3572
|
maxSize;
|
|
2751
3573
|
running = false;
|
|
2752
3574
|
loopTimer = null;
|
|
2753
|
-
constructor(client, assertions,
|
|
3575
|
+
constructor(client, assertions, options2) {
|
|
2754
3576
|
this.client = client;
|
|
2755
3577
|
this.assertions = assertions;
|
|
2756
|
-
this.sampler = new Sampler(
|
|
2757
|
-
this.dispatcher = new AlertDispatcher(
|
|
2758
|
-
this.maxSize =
|
|
3578
|
+
this.sampler = new Sampler(options2?.sampleRate ?? 1);
|
|
3579
|
+
this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
|
|
3580
|
+
this.maxSize = options2?.maxSize ?? resolveQueueSize();
|
|
2759
3581
|
}
|
|
2760
3582
|
get alertEmitter() {
|
|
2761
3583
|
return this.dispatcher;
|
|
@@ -2866,13 +3688,13 @@ function loadLabelsCSV(text) {
|
|
|
2866
3688
|
if (rows.length === 0) {
|
|
2867
3689
|
throw new Error("empty CSV");
|
|
2868
3690
|
}
|
|
2869
|
-
let
|
|
3691
|
+
let start3 = 0;
|
|
2870
3692
|
if (rows[0].length >= 2) {
|
|
2871
3693
|
const firstHuman = parseFloatStrict(rows[0][1]);
|
|
2872
|
-
if (firstHuman === null)
|
|
3694
|
+
if (firstHuman === null) start3 = 1;
|
|
2873
3695
|
}
|
|
2874
3696
|
const out = [];
|
|
2875
|
-
for (let idx =
|
|
3697
|
+
for (let idx = start3; idx < rows.length; idx++) {
|
|
2876
3698
|
const row = rows[idx];
|
|
2877
3699
|
const lineNum = idx + 1;
|
|
2878
3700
|
if (row.length === 0) continue;
|
|
@@ -3081,10 +3903,10 @@ var OpenAIAdapter = class {
|
|
|
3081
3903
|
constructor(agentId2) {
|
|
3082
3904
|
this.agentId = agentId2;
|
|
3083
3905
|
}
|
|
3084
|
-
traceFromResponse(response,
|
|
3906
|
+
traceFromResponse(response, options2) {
|
|
3085
3907
|
const builder = new TraceBuilder(this.agentId);
|
|
3086
|
-
if (
|
|
3087
|
-
builder.setInput({ messages:
|
|
3908
|
+
if (options2?.inputMessages) {
|
|
3909
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3088
3910
|
}
|
|
3089
3911
|
const message = response.choices[0].message;
|
|
3090
3912
|
const completionText = message.content ?? "";
|
|
@@ -3104,12 +3926,12 @@ var OpenAIAdapter = class {
|
|
|
3104
3926
|
}
|
|
3105
3927
|
builder.setOutput({
|
|
3106
3928
|
message: completionText,
|
|
3107
|
-
structured:
|
|
3929
|
+
structured: options2?.structuredOutput ?? {}
|
|
3108
3930
|
});
|
|
3109
3931
|
builder.setMetadata({
|
|
3110
3932
|
total_tokens: response.usage?.total_tokens,
|
|
3111
|
-
cost_usd:
|
|
3112
|
-
latency_ms:
|
|
3933
|
+
cost_usd: options2?.costUsd,
|
|
3934
|
+
latency_ms: options2?.latencyMs,
|
|
3113
3935
|
model: response.model
|
|
3114
3936
|
});
|
|
3115
3937
|
return builder.build();
|
|
@@ -3122,10 +3944,10 @@ var AnthropicAdapter = class {
|
|
|
3122
3944
|
constructor(agentId2) {
|
|
3123
3945
|
this.agentId = agentId2;
|
|
3124
3946
|
}
|
|
3125
|
-
traceFromResponse(response,
|
|
3947
|
+
traceFromResponse(response, options2) {
|
|
3126
3948
|
const builder = new TraceBuilder(this.agentId);
|
|
3127
|
-
if (
|
|
3128
|
-
builder.setInput({ messages:
|
|
3949
|
+
if (options2?.inputMessages) {
|
|
3950
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3129
3951
|
}
|
|
3130
3952
|
const completionParts = [];
|
|
3131
3953
|
for (const block of response.content) {
|
|
@@ -3148,8 +3970,8 @@ var AnthropicAdapter = class {
|
|
|
3148
3970
|
}
|
|
3149
3971
|
builder.setMetadata({
|
|
3150
3972
|
total_tokens: totalTokens,
|
|
3151
|
-
cost_usd:
|
|
3152
|
-
latency_ms:
|
|
3973
|
+
cost_usd: options2?.costUsd,
|
|
3974
|
+
latency_ms: options2?.latencyMs,
|
|
3153
3975
|
model: response.model
|
|
3154
3976
|
});
|
|
3155
3977
|
return builder.build();
|
|
@@ -3162,10 +3984,10 @@ var GeminiAdapter = class {
|
|
|
3162
3984
|
constructor(agentId2) {
|
|
3163
3985
|
this.agentId = agentId2;
|
|
3164
3986
|
}
|
|
3165
|
-
traceFromResponse(response,
|
|
3987
|
+
traceFromResponse(response, options2) {
|
|
3166
3988
|
const builder = new TraceBuilder(this.agentId);
|
|
3167
|
-
if (
|
|
3168
|
-
builder.setInput({ text:
|
|
3989
|
+
if (options2?.inputText) {
|
|
3990
|
+
builder.setInput({ text: options2.inputText });
|
|
3169
3991
|
}
|
|
3170
3992
|
let completionText = "";
|
|
3171
3993
|
if (response.text != null) {
|
|
@@ -3185,9 +4007,9 @@ var GeminiAdapter = class {
|
|
|
3185
4007
|
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
3186
4008
|
builder.setOutput({ message: completionText });
|
|
3187
4009
|
builder.setMetadata({
|
|
3188
|
-
cost_usd:
|
|
3189
|
-
latency_ms:
|
|
3190
|
-
model:
|
|
4010
|
+
cost_usd: options2?.costUsd,
|
|
4011
|
+
latency_ms: options2?.latencyMs,
|
|
4012
|
+
model: options2?.model
|
|
3191
4013
|
});
|
|
3192
4014
|
return builder.build();
|
|
3193
4015
|
}
|
|
@@ -3199,10 +4021,10 @@ var OllamaAdapter = class {
|
|
|
3199
4021
|
constructor(agentId2) {
|
|
3200
4022
|
this.agentId = agentId2;
|
|
3201
4023
|
}
|
|
3202
|
-
traceFromResponse(response,
|
|
4024
|
+
traceFromResponse(response, options2) {
|
|
3203
4025
|
const builder = new TraceBuilder(this.agentId);
|
|
3204
|
-
if (
|
|
3205
|
-
builder.setInput({ messages:
|
|
4026
|
+
if (options2?.inputMessages) {
|
|
4027
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3206
4028
|
}
|
|
3207
4029
|
const completionText = response.message?.content ?? "";
|
|
3208
4030
|
builder.addLlmCall("completion", {
|
|
@@ -3216,7 +4038,7 @@ var OllamaAdapter = class {
|
|
|
3216
4038
|
}
|
|
3217
4039
|
builder.setMetadata({
|
|
3218
4040
|
total_tokens: totalTokens,
|
|
3219
|
-
latency_ms:
|
|
4041
|
+
latency_ms: options2?.latencyMs,
|
|
3220
4042
|
model: response.model
|
|
3221
4043
|
});
|
|
3222
4044
|
return builder.build();
|
|
@@ -3224,7 +4046,7 @@ var OllamaAdapter = class {
|
|
|
3224
4046
|
};
|
|
3225
4047
|
|
|
3226
4048
|
// src/adapters/otel.ts
|
|
3227
|
-
var
|
|
4049
|
+
var import_node_child_process5 = require("child_process");
|
|
3228
4050
|
var CONVERT_TIMEOUT_MS = 6e4;
|
|
3229
4051
|
var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
3230
4052
|
var STATUS_CODE_ERROR = 2;
|
|
@@ -3247,7 +4069,7 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3247
4069
|
* @throws RangeError when a parsed export holds a non-finite number.
|
|
3248
4070
|
* @throws OTLPConversionError when the engine times out, overflows the output buffer or
|
|
3249
4071
|
* prints no trace.
|
|
3250
|
-
* @throws Error
|
|
4072
|
+
* @throws Error when the engine cannot start.
|
|
3251
4073
|
*/
|
|
3252
4074
|
static async fromOtlp(otlp, agentId2) {
|
|
3253
4075
|
const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
|
|
@@ -3259,33 +4081,9 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3259
4081
|
const args = ["trace", "convert"];
|
|
3260
4082
|
if (agentId2 !== void 0) args.push("--agent-id", agentId2);
|
|
3261
4083
|
args.push("-");
|
|
3262
|
-
const
|
|
3263
|
-
input: payload,
|
|
3264
|
-
timeout: CONVERT_TIMEOUT_MS,
|
|
3265
|
-
maxBuffer: MAX_TRACE_OUTPUT_BYTES
|
|
3266
|
-
});
|
|
3267
|
-
if (proc.error) {
|
|
3268
|
-
const code = proc.error.code;
|
|
3269
|
-
if (code === "ETIMEDOUT") {
|
|
3270
|
-
throw new OTLPConversionError(
|
|
3271
|
-
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3272
|
-
);
|
|
3273
|
-
}
|
|
3274
|
-
if (code === "ENOBUFS") {
|
|
3275
|
-
throw new OTLPConversionError(
|
|
3276
|
-
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3277
|
-
);
|
|
3278
|
-
}
|
|
3279
|
-
throw proc.error;
|
|
3280
|
-
}
|
|
3281
|
-
if (proc.status !== 0) {
|
|
3282
|
-
const message = proc.stderr.toString("utf8").trim();
|
|
3283
|
-
throw new OTLPConversionError(
|
|
3284
|
-
message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
|
|
3285
|
-
);
|
|
3286
|
-
}
|
|
4084
|
+
const stdout = await runConvert(await resolveEngine(), args, payload);
|
|
3287
4085
|
try {
|
|
3288
|
-
return JSON.parse(
|
|
4086
|
+
return JSON.parse(stdout);
|
|
3289
4087
|
} catch (err) {
|
|
3290
4088
|
throw new OTLPConversionError(
|
|
3291
4089
|
`attest-engine trace convert printed output that is not JSON: ${String(err)}`
|
|
@@ -3293,6 +4091,65 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3293
4091
|
}
|
|
3294
4092
|
}
|
|
3295
4093
|
};
|
|
4094
|
+
function runConvert(engine, args, payload) {
|
|
4095
|
+
return new Promise((resolve4, reject) => {
|
|
4096
|
+
const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
4097
|
+
let timer;
|
|
4098
|
+
child.on("error", (e) => {
|
|
4099
|
+
clearTimeout(timer);
|
|
4100
|
+
reject(e);
|
|
4101
|
+
});
|
|
4102
|
+
const { stdin, stdout, stderr } = child;
|
|
4103
|
+
if (stdin === null || stdout === null || stderr === null) return;
|
|
4104
|
+
const out = [];
|
|
4105
|
+
const err = [];
|
|
4106
|
+
let outBytes = 0;
|
|
4107
|
+
let failure = null;
|
|
4108
|
+
const fail = (e) => {
|
|
4109
|
+
failure ??= e;
|
|
4110
|
+
child.kill("SIGKILL");
|
|
4111
|
+
};
|
|
4112
|
+
timer = setTimeout(() => {
|
|
4113
|
+
fail(
|
|
4114
|
+
new OTLPConversionError(
|
|
4115
|
+
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
4116
|
+
)
|
|
4117
|
+
);
|
|
4118
|
+
}, CONVERT_TIMEOUT_MS);
|
|
4119
|
+
stdout.on("data", (chunk) => {
|
|
4120
|
+
outBytes += chunk.length;
|
|
4121
|
+
if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
|
|
4122
|
+
fail(
|
|
4123
|
+
new OTLPConversionError(
|
|
4124
|
+
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
4125
|
+
)
|
|
4126
|
+
);
|
|
4127
|
+
return;
|
|
4128
|
+
}
|
|
4129
|
+
out.push(chunk);
|
|
4130
|
+
});
|
|
4131
|
+
stderr.on("data", (chunk) => err.push(chunk));
|
|
4132
|
+
stdin.on("error", () => void 0);
|
|
4133
|
+
child.on("close", (code, signal) => {
|
|
4134
|
+
clearTimeout(timer);
|
|
4135
|
+
if (failure !== null) {
|
|
4136
|
+
reject(failure);
|
|
4137
|
+
return;
|
|
4138
|
+
}
|
|
4139
|
+
if (code !== 0) {
|
|
4140
|
+
const message = Buffer.concat(err).toString("utf8").trim();
|
|
4141
|
+
reject(
|
|
4142
|
+
new OTLPConversionError(
|
|
4143
|
+
message || `attest-engine trace convert exited ${String(code ?? signal)}`
|
|
4144
|
+
)
|
|
4145
|
+
);
|
|
4146
|
+
return;
|
|
4147
|
+
}
|
|
4148
|
+
resolve4(Buffer.concat(out).toString("utf8"));
|
|
4149
|
+
});
|
|
4150
|
+
stdin.end(payload);
|
|
4151
|
+
});
|
|
4152
|
+
}
|
|
3296
4153
|
function hrTimeToNanos(time, spanName) {
|
|
3297
4154
|
if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
|
|
3298
4155
|
throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
|
|
@@ -3398,13 +4255,13 @@ var LangChainAdapter = class {
|
|
|
3398
4255
|
* @param options.model - Model name override.
|
|
3399
4256
|
* @returns A finalized Trace object.
|
|
3400
4257
|
*/
|
|
3401
|
-
buildTrace(
|
|
4258
|
+
buildTrace(options2) {
|
|
3402
4259
|
const builder = new TraceBuilder(this.agentId);
|
|
3403
4260
|
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
3404
4261
|
let totalTokens;
|
|
3405
4262
|
for (const llm of this.llmCalls) {
|
|
3406
4263
|
const args = {};
|
|
3407
|
-
if (llm.model ??
|
|
4264
|
+
if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
|
|
3408
4265
|
const result = { completion: llm.text };
|
|
3409
4266
|
if (llm.tokens !== void 0) {
|
|
3410
4267
|
result.tokens = llm.tokens;
|
|
@@ -3421,9 +4278,9 @@ var LangChainAdapter = class {
|
|
|
3421
4278
|
builder.setOutput({ message: outputMessage });
|
|
3422
4279
|
builder.setMetadata({
|
|
3423
4280
|
total_tokens: totalTokens,
|
|
3424
|
-
cost_usd:
|
|
4281
|
+
cost_usd: options2?.costUsd,
|
|
3425
4282
|
latency_ms: latencyMs,
|
|
3426
|
-
model:
|
|
4283
|
+
model: options2?.model ?? this.llmCalls[0]?.model
|
|
3427
4284
|
});
|
|
3428
4285
|
return builder.build();
|
|
3429
4286
|
}
|
|
@@ -3438,6 +4295,7 @@ var LangChainAdapter = class {
|
|
|
3438
4295
|
AttestClient,
|
|
3439
4296
|
CONFUSED_USER,
|
|
3440
4297
|
COOPERATIVE_USER,
|
|
4298
|
+
CaptureError,
|
|
3441
4299
|
ContinuousEvalRunner,
|
|
3442
4300
|
ENGINE_VERSION,
|
|
3443
4301
|
ERR_ASSERTION_ERROR,
|
|
@@ -3446,6 +4304,7 @@ var LangChainAdapter = class {
|
|
|
3446
4304
|
ERR_PROVIDER_ERROR,
|
|
3447
4305
|
ERR_SESSION_ERROR,
|
|
3448
4306
|
ERR_TIMEOUT,
|
|
4307
|
+
EgressBlockedError,
|
|
3449
4308
|
EngineManager,
|
|
3450
4309
|
EngineTimeoutError,
|
|
3451
4310
|
ExpectChain,
|
|
@@ -3470,8 +4329,12 @@ var LangChainAdapter = class {
|
|
|
3470
4329
|
REASON_MISSING_EVIDENCE,
|
|
3471
4330
|
REASON_PROVIDER_NOT_CONFIGURED,
|
|
3472
4331
|
REASON_PROVIDER_UNAVAILABLE,
|
|
4332
|
+
REASON_REPLAY_MISS,
|
|
3473
4333
|
REASON_SHORT_CIRCUITED,
|
|
3474
4334
|
REASON_SIMULATION,
|
|
4335
|
+
ReplayError,
|
|
4336
|
+
ReplayMissError,
|
|
4337
|
+
ReplayNotEvaluatedError,
|
|
3475
4338
|
STATUS_HARD_FAIL,
|
|
3476
4339
|
STATUS_NOT_EVALUATED,
|
|
3477
4340
|
STATUS_PASS,
|
|
@@ -3486,6 +4349,7 @@ var LangChainAdapter = class {
|
|
|
3486
4349
|
TIER_3,
|
|
3487
4350
|
TYPE_CONSTRAINT,
|
|
3488
4351
|
TYPE_CONTENT,
|
|
4352
|
+
TYPE_EFFECT,
|
|
3489
4353
|
TYPE_EMBEDDING,
|
|
3490
4354
|
TYPE_LLM_JUDGE,
|
|
3491
4355
|
TYPE_PLUGIN,
|
|
@@ -3495,6 +4359,8 @@ var LangChainAdapter = class {
|
|
|
3495
4359
|
TraceBuilder,
|
|
3496
4360
|
TraceTree,
|
|
3497
4361
|
VERSION,
|
|
4362
|
+
WorldChecks,
|
|
4363
|
+
WorldError,
|
|
3498
4364
|
activeBuilder,
|
|
3499
4365
|
addUsage,
|
|
3500
4366
|
agent,
|
|
@@ -3505,28 +4371,43 @@ var LangChainAdapter = class {
|
|
|
3505
4371
|
config,
|
|
3506
4372
|
decodeResponse,
|
|
3507
4373
|
delegate,
|
|
4374
|
+
dslCall,
|
|
4375
|
+
egressBlockRequested,
|
|
4376
|
+
emitVitest,
|
|
3508
4377
|
encodeRequest,
|
|
3509
4378
|
executePluginAssertion,
|
|
3510
4379
|
extractId,
|
|
3511
4380
|
extractResult,
|
|
4381
|
+
firstMiss,
|
|
3512
4382
|
formatUsage,
|
|
3513
4383
|
getSampleRate,
|
|
4384
|
+
installEgressGuard,
|
|
4385
|
+
isProviderHost,
|
|
3514
4386
|
isSimulationMode,
|
|
3515
4387
|
layerForType,
|
|
4388
|
+
liveLine,
|
|
4389
|
+
loadFixture,
|
|
3516
4390
|
loadLabelsCSV,
|
|
3517
4391
|
loadLabelsJSONL,
|
|
3518
4392
|
parseUsage,
|
|
3519
4393
|
previewLine,
|
|
3520
4394
|
promptHash,
|
|
4395
|
+
record,
|
|
3521
4396
|
renderAssertionFailure,
|
|
3522
4397
|
renderDiagnostics,
|
|
3523
4398
|
renderSummary,
|
|
4399
|
+
renderVitest,
|
|
3524
4400
|
requireSimulationCapability,
|
|
3525
4401
|
resetConfig,
|
|
4402
|
+
resolveMode,
|
|
3526
4403
|
simulationEnvEnabled,
|
|
4404
|
+
startReplay,
|
|
4405
|
+
startWorld,
|
|
3527
4406
|
sumUsage,
|
|
4407
|
+
testPathFor,
|
|
3528
4408
|
tier,
|
|
3529
4409
|
traceId,
|
|
4410
|
+
uninstallEgressGuard,
|
|
3530
4411
|
usageSummaryLine,
|
|
3531
4412
|
zeroUsage
|
|
3532
4413
|
});
|