@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +1002 -120
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +315 -6
- package/dist/cjs/proto/index.cjs +7 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +31 -2
- package/dist/esm/cli.d.ts +8 -1
- package/dist/esm/cli.js +536 -47
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +315 -6
- package/dist/esm/index.js +978 -120
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +31 -2
- package/dist/esm/proto/index.js +5 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +3 -3
package/dist/cjs/index.cjs
CHANGED
|
@@ -38,6 +38,7 @@ __export(index_exports, {
|
|
|
38
38
|
AttestClient: () => AttestClient,
|
|
39
39
|
CONFUSED_USER: () => CONFUSED_USER,
|
|
40
40
|
COOPERATIVE_USER: () => COOPERATIVE_USER,
|
|
41
|
+
CaptureError: () => CaptureError,
|
|
41
42
|
ContinuousEvalRunner: () => ContinuousEvalRunner,
|
|
42
43
|
ENGINE_VERSION: () => ENGINE_VERSION,
|
|
43
44
|
ERR_ASSERTION_ERROR: () => ERR_ASSERTION_ERROR,
|
|
@@ -46,6 +47,7 @@ __export(index_exports, {
|
|
|
46
47
|
ERR_PROVIDER_ERROR: () => ERR_PROVIDER_ERROR,
|
|
47
48
|
ERR_SESSION_ERROR: () => ERR_SESSION_ERROR,
|
|
48
49
|
ERR_TIMEOUT: () => ERR_TIMEOUT,
|
|
50
|
+
EgressBlockedError: () => EgressBlockedError,
|
|
49
51
|
EngineManager: () => EngineManager,
|
|
50
52
|
EngineTimeoutError: () => EngineTimeoutError,
|
|
51
53
|
ExpectChain: () => ExpectChain,
|
|
@@ -70,8 +72,12 @@ __export(index_exports, {
|
|
|
70
72
|
REASON_MISSING_EVIDENCE: () => REASON_MISSING_EVIDENCE,
|
|
71
73
|
REASON_PROVIDER_NOT_CONFIGURED: () => REASON_PROVIDER_NOT_CONFIGURED,
|
|
72
74
|
REASON_PROVIDER_UNAVAILABLE: () => REASON_PROVIDER_UNAVAILABLE,
|
|
75
|
+
REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
|
|
73
76
|
REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
|
|
74
77
|
REASON_SIMULATION: () => REASON_SIMULATION,
|
|
78
|
+
ReplayError: () => ReplayError,
|
|
79
|
+
ReplayMissError: () => ReplayMissError,
|
|
80
|
+
ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
|
|
75
81
|
STATUS_HARD_FAIL: () => STATUS_HARD_FAIL,
|
|
76
82
|
STATUS_NOT_EVALUATED: () => STATUS_NOT_EVALUATED,
|
|
77
83
|
STATUS_PASS: () => STATUS_PASS,
|
|
@@ -86,6 +92,7 @@ __export(index_exports, {
|
|
|
86
92
|
TIER_3: () => TIER_3,
|
|
87
93
|
TYPE_CONSTRAINT: () => TYPE_CONSTRAINT,
|
|
88
94
|
TYPE_CONTENT: () => TYPE_CONTENT,
|
|
95
|
+
TYPE_EFFECT: () => TYPE_EFFECT,
|
|
89
96
|
TYPE_EMBEDDING: () => TYPE_EMBEDDING,
|
|
90
97
|
TYPE_LLM_JUDGE: () => TYPE_LLM_JUDGE,
|
|
91
98
|
TYPE_PLUGIN: () => TYPE_PLUGIN,
|
|
@@ -95,6 +102,8 @@ __export(index_exports, {
|
|
|
95
102
|
TraceBuilder: () => TraceBuilder,
|
|
96
103
|
TraceTree: () => TraceTree,
|
|
97
104
|
VERSION: () => VERSION,
|
|
105
|
+
WorldChecks: () => WorldChecks,
|
|
106
|
+
WorldError: () => WorldError,
|
|
98
107
|
activeBuilder: () => activeBuilder,
|
|
99
108
|
addUsage: () => addUsage,
|
|
100
109
|
agent: () => agent,
|
|
@@ -105,28 +114,43 @@ __export(index_exports, {
|
|
|
105
114
|
config: () => config,
|
|
106
115
|
decodeResponse: () => decodeResponse,
|
|
107
116
|
delegate: () => delegate,
|
|
117
|
+
dslCall: () => dslCall,
|
|
118
|
+
egressBlockRequested: () => egressBlockRequested,
|
|
119
|
+
emitVitest: () => emitVitest,
|
|
108
120
|
encodeRequest: () => encodeRequest,
|
|
109
121
|
executePluginAssertion: () => executePluginAssertion,
|
|
110
122
|
extractId: () => extractId,
|
|
111
123
|
extractResult: () => extractResult,
|
|
124
|
+
firstMiss: () => firstMiss,
|
|
112
125
|
formatUsage: () => formatUsage,
|
|
113
126
|
getSampleRate: () => getSampleRate,
|
|
127
|
+
installEgressGuard: () => installEgressGuard,
|
|
128
|
+
isProviderHost: () => isProviderHost,
|
|
114
129
|
isSimulationMode: () => isSimulationMode,
|
|
115
130
|
layerForType: () => layerForType,
|
|
131
|
+
liveLine: () => liveLine,
|
|
132
|
+
loadFixture: () => loadFixture,
|
|
116
133
|
loadLabelsCSV: () => loadLabelsCSV,
|
|
117
134
|
loadLabelsJSONL: () => loadLabelsJSONL,
|
|
118
135
|
parseUsage: () => parseUsage,
|
|
119
136
|
previewLine: () => previewLine,
|
|
120
137
|
promptHash: () => promptHash,
|
|
138
|
+
record: () => record,
|
|
121
139
|
renderAssertionFailure: () => renderAssertionFailure,
|
|
122
140
|
renderDiagnostics: () => renderDiagnostics,
|
|
123
141
|
renderSummary: () => renderSummary,
|
|
142
|
+
renderVitest: () => renderVitest,
|
|
124
143
|
requireSimulationCapability: () => requireSimulationCapability,
|
|
125
144
|
resetConfig: () => resetConfig,
|
|
145
|
+
resolveMode: () => resolveMode,
|
|
126
146
|
simulationEnvEnabled: () => simulationEnvEnabled,
|
|
147
|
+
startReplay: () => startReplay,
|
|
148
|
+
startWorld: () => startWorld,
|
|
127
149
|
sumUsage: () => sumUsage,
|
|
150
|
+
testPathFor: () => testPathFor,
|
|
128
151
|
tier: () => tier,
|
|
129
152
|
traceId: () => traceId,
|
|
153
|
+
uninstallEgressGuard: () => uninstallEgressGuard,
|
|
130
154
|
usageSummaryLine: () => usageSummaryLine,
|
|
131
155
|
zeroUsage: () => zeroUsage
|
|
132
156
|
});
|
|
@@ -148,12 +172,14 @@ var REASON_MISSING_EVIDENCE = "missing_evidence";
|
|
|
148
172
|
var REASON_SHORT_CIRCUITED = "short_circuited";
|
|
149
173
|
var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
|
|
150
174
|
var REASON_JUDGE_ABSTAINED = "judge_abstained";
|
|
175
|
+
var REASON_REPLAY_MISS = "replay_miss";
|
|
151
176
|
var OUTCOME_PASS = "pass";
|
|
152
177
|
var OUTCOME_FAIL = "fail";
|
|
153
178
|
var OUTCOME_SKIP = "skip";
|
|
154
179
|
var TYPE_SCHEMA = "schema";
|
|
155
180
|
var TYPE_CONSTRAINT = "constraint";
|
|
156
181
|
var TYPE_TRACE = "trace";
|
|
182
|
+
var TYPE_EFFECT = "effect";
|
|
157
183
|
var TYPE_CONTENT = "content";
|
|
158
184
|
var TYPE_EMBEDDING = "embedding";
|
|
159
185
|
var TYPE_LLM_JUDGE = "llm_judge";
|
|
@@ -191,6 +217,7 @@ function layerForType(assertionType) {
|
|
|
191
217
|
return 2;
|
|
192
218
|
case "trace":
|
|
193
219
|
case "trace_tree":
|
|
220
|
+
case "effect":
|
|
194
221
|
return 3;
|
|
195
222
|
case "content":
|
|
196
223
|
return 4;
|
|
@@ -350,8 +377,8 @@ function extractId(response) {
|
|
|
350
377
|
}
|
|
351
378
|
|
|
352
379
|
// src/version.ts
|
|
353
|
-
var VERSION = "1.0.0-rc.
|
|
354
|
-
var ENGINE_VERSION = "1.0.0-rc.
|
|
380
|
+
var VERSION = "1.0.0-rc.4";
|
|
381
|
+
var ENGINE_VERSION = "1.0.0-rc.4";
|
|
355
382
|
|
|
356
383
|
// src/engine-manager.ts
|
|
357
384
|
var import_node_child_process = require("child_process");
|
|
@@ -419,7 +446,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
419
446
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
420
447
|
);
|
|
421
448
|
}
|
|
422
|
-
return new Promise((
|
|
449
|
+
return new Promise((resolve4, reject) => {
|
|
423
450
|
const client = url.startsWith("https:") ? https : http;
|
|
424
451
|
client.get(url, (res) => {
|
|
425
452
|
const status = res.statusCode ?? 0;
|
|
@@ -430,7 +457,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
430
457
|
return;
|
|
431
458
|
}
|
|
432
459
|
res.resume();
|
|
433
|
-
|
|
460
|
+
resolve4(httpsGet(location, redirectCount + 1));
|
|
434
461
|
return;
|
|
435
462
|
}
|
|
436
463
|
if (status < 200 || status >= 300) {
|
|
@@ -440,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
440
467
|
}
|
|
441
468
|
const chunks = [];
|
|
442
469
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
443
|
-
res.on("end", () =>
|
|
470
|
+
res.on("end", () => resolve4(Buffer.concat(chunks)));
|
|
444
471
|
res.on("error", reject);
|
|
445
472
|
}).on("error", reject);
|
|
446
473
|
});
|
|
@@ -571,8 +598,8 @@ Install options:
|
|
|
571
598
|
}
|
|
572
599
|
return [await downloadEngine(), "download"];
|
|
573
600
|
}
|
|
574
|
-
async function resolveEngine(explicitPath,
|
|
575
|
-
const [enginePath, source] = await locateEngine(explicitPath,
|
|
601
|
+
async function resolveEngine(explicitPath, options2 = {}) {
|
|
602
|
+
const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
|
|
576
603
|
const found = engineVersion(enginePath);
|
|
577
604
|
if (found !== ENGINE_VERSION) {
|
|
578
605
|
throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
|
|
@@ -587,9 +614,9 @@ var EngineManager = class {
|
|
|
587
614
|
initialized = false;
|
|
588
615
|
requestId = 0;
|
|
589
616
|
initResult = null;
|
|
590
|
-
constructor(
|
|
591
|
-
this.enginePath =
|
|
592
|
-
this.logLevel =
|
|
617
|
+
constructor(options2) {
|
|
618
|
+
this.enginePath = options2?.enginePath ?? "";
|
|
619
|
+
this.logLevel = options2?.logLevel ?? "warn";
|
|
593
620
|
}
|
|
594
621
|
async start() {
|
|
595
622
|
this.enginePath = await resolveEngine(this.enginePath || void 0);
|
|
@@ -667,11 +694,11 @@ var EngineManager = class {
|
|
|
667
694
|
}
|
|
668
695
|
this.requestId += 1;
|
|
669
696
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
670
|
-
return new Promise((
|
|
697
|
+
return new Promise((resolve4, reject) => {
|
|
671
698
|
const onLine = (line) => {
|
|
672
699
|
try {
|
|
673
700
|
const response = decodeResponse(line);
|
|
674
|
-
|
|
701
|
+
resolve4(extractResult(response));
|
|
675
702
|
} catch (err) {
|
|
676
703
|
reject(err);
|
|
677
704
|
}
|
|
@@ -705,17 +732,17 @@ var EngineManager = class {
|
|
|
705
732
|
// src/config.ts
|
|
706
733
|
var _simulationMode = false;
|
|
707
734
|
var _sampleRate = 0;
|
|
708
|
-
function config(
|
|
709
|
-
if (
|
|
710
|
-
_simulationMode =
|
|
735
|
+
function config(options2) {
|
|
736
|
+
if (options2.simulation !== void 0) {
|
|
737
|
+
_simulationMode = options2.simulation;
|
|
711
738
|
}
|
|
712
|
-
if (
|
|
713
|
-
if (
|
|
739
|
+
if (options2.sampleRate !== void 0) {
|
|
740
|
+
if (options2.sampleRate < 0 || options2.sampleRate > 1) {
|
|
714
741
|
throw new RangeError(
|
|
715
|
-
`sampleRate must be between 0.0 and 1.0, got ${
|
|
742
|
+
`sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
|
|
716
743
|
);
|
|
717
744
|
}
|
|
718
|
-
_sampleRate =
|
|
745
|
+
_sampleRate = options2.sampleRate;
|
|
719
746
|
}
|
|
720
747
|
}
|
|
721
748
|
function isSimulationMode() {
|
|
@@ -871,16 +898,16 @@ var AttestClient = class {
|
|
|
871
898
|
desyncThreshold;
|
|
872
899
|
desyncWindowMs;
|
|
873
900
|
desynced = false;
|
|
874
|
-
constructor(engine,
|
|
901
|
+
constructor(engine, options2) {
|
|
875
902
|
this.engine = engine;
|
|
876
|
-
this.logger =
|
|
903
|
+
this.logger = options2?.logger ?? envLoggerDefault();
|
|
877
904
|
this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
|
|
878
|
-
|
|
905
|
+
options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
|
|
879
906
|
);
|
|
880
|
-
this.desyncThreshold =
|
|
881
|
-
this.desyncWindowMs =
|
|
882
|
-
if (
|
|
883
|
-
this.diagnosticListeners.add(
|
|
907
|
+
this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
|
|
908
|
+
this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
|
|
909
|
+
if (options2?.onDiagnostic !== void 0) {
|
|
910
|
+
this.diagnosticListeners.add(options2.onDiagnostic);
|
|
884
911
|
}
|
|
885
912
|
}
|
|
886
913
|
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
@@ -1038,12 +1065,12 @@ var AttestClient = class {
|
|
|
1038
1065
|
throw new Error("Engine process not started.");
|
|
1039
1066
|
}
|
|
1040
1067
|
let capturedReqId;
|
|
1041
|
-
const requestPromise = new Promise((
|
|
1068
|
+
const requestPromise = new Promise((resolve4, reject) => {
|
|
1042
1069
|
this.writeChain = this.writeChain.then(() => {
|
|
1043
1070
|
this.requestId += 1;
|
|
1044
1071
|
const reqId = this.requestId;
|
|
1045
1072
|
capturedReqId = reqId;
|
|
1046
|
-
this.pending.set(reqId, { resolve, reject });
|
|
1073
|
+
this.pending.set(reqId, { resolve: resolve4, reject });
|
|
1047
1074
|
const requestStr = encodeRequest(reqId, method, params);
|
|
1048
1075
|
cp.stdin.write(requestStr, (err) => {
|
|
1049
1076
|
if (err) {
|
|
@@ -1061,23 +1088,24 @@ var AttestClient = class {
|
|
|
1061
1088
|
throw err;
|
|
1062
1089
|
});
|
|
1063
1090
|
}
|
|
1064
|
-
async evaluateBatch(trace, assertions,
|
|
1091
|
+
async evaluateBatch(trace, assertions, options2) {
|
|
1065
1092
|
if (isSimulationMode()) {
|
|
1066
1093
|
requireSimulationCapability(this.engine["initResult"]);
|
|
1067
1094
|
}
|
|
1068
1095
|
const params = {
|
|
1069
1096
|
trace,
|
|
1070
1097
|
assertions: [...assertions],
|
|
1071
|
-
...
|
|
1098
|
+
...options2?.budget !== void 0 ? { budget: options2.budget } : {},
|
|
1099
|
+
...options2?.context !== void 0 ? { context: options2.context } : {}
|
|
1072
1100
|
};
|
|
1073
1101
|
const raw = await this.sendRequest(
|
|
1074
1102
|
"evaluate_batch",
|
|
1075
1103
|
params,
|
|
1076
|
-
|
|
1104
|
+
options2?.timeout
|
|
1077
1105
|
);
|
|
1078
1106
|
return { ...raw, usage: parseUsage(raw["usage"]) };
|
|
1079
1107
|
}
|
|
1080
|
-
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation,
|
|
1108
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
|
|
1081
1109
|
const result = {
|
|
1082
1110
|
assertion_id: assertionId2,
|
|
1083
1111
|
status,
|
|
@@ -1095,11 +1123,11 @@ var AttestClient = class {
|
|
|
1095
1123
|
const raw = await this.sendRequest(
|
|
1096
1124
|
"submit_plugin_result",
|
|
1097
1125
|
params,
|
|
1098
|
-
|
|
1126
|
+
options2?.timeout
|
|
1099
1127
|
);
|
|
1100
1128
|
return Boolean(raw?.accepted ?? false);
|
|
1101
1129
|
}
|
|
1102
|
-
async queryDrift(assertionId2, windowSize = 50,
|
|
1130
|
+
async queryDrift(assertionId2, windowSize = 50, options2) {
|
|
1103
1131
|
if (isSimulationMode()) {
|
|
1104
1132
|
return {
|
|
1105
1133
|
assertion_id: assertionId2,
|
|
@@ -1114,11 +1142,11 @@ var AttestClient = class {
|
|
|
1114
1142
|
const raw = await this.sendRequest(
|
|
1115
1143
|
"query_drift",
|
|
1116
1144
|
{ assertion_id: assertionId2, window_size: windowSize },
|
|
1117
|
-
|
|
1145
|
+
options2?.timeout
|
|
1118
1146
|
);
|
|
1119
1147
|
return raw.report;
|
|
1120
1148
|
}
|
|
1121
|
-
async generateUserMessage(persona, conversationHistory, faultConfig,
|
|
1149
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
|
|
1122
1150
|
if (isSimulationMode()) {
|
|
1123
1151
|
return `[simulation] Hello from ${persona.name}`;
|
|
1124
1152
|
}
|
|
@@ -1132,7 +1160,7 @@ var AttestClient = class {
|
|
|
1132
1160
|
const raw = await this.sendRequest(
|
|
1133
1161
|
"generate_user_message",
|
|
1134
1162
|
params,
|
|
1135
|
-
|
|
1163
|
+
options2?.timeout
|
|
1136
1164
|
);
|
|
1137
1165
|
return String(raw.message);
|
|
1138
1166
|
}
|
|
@@ -1177,33 +1205,33 @@ var TraceBuilder = class {
|
|
|
1177
1205
|
this.input = input;
|
|
1178
1206
|
return this;
|
|
1179
1207
|
}
|
|
1180
|
-
addLlmCall(name,
|
|
1208
|
+
addLlmCall(name, options2) {
|
|
1181
1209
|
this.steps.push({
|
|
1182
1210
|
type: STEP_LLM_CALL,
|
|
1183
1211
|
name,
|
|
1184
|
-
args:
|
|
1185
|
-
result:
|
|
1186
|
-
metadata:
|
|
1212
|
+
args: options2?.args,
|
|
1213
|
+
result: options2?.result,
|
|
1214
|
+
metadata: options2?.metadata
|
|
1187
1215
|
});
|
|
1188
1216
|
return this;
|
|
1189
1217
|
}
|
|
1190
|
-
addToolCall(name,
|
|
1218
|
+
addToolCall(name, options2) {
|
|
1191
1219
|
this.steps.push({
|
|
1192
1220
|
type: STEP_TOOL_CALL,
|
|
1193
1221
|
name,
|
|
1194
|
-
args:
|
|
1195
|
-
result:
|
|
1196
|
-
metadata:
|
|
1222
|
+
args: options2?.args,
|
|
1223
|
+
result: options2?.result,
|
|
1224
|
+
metadata: options2?.metadata
|
|
1197
1225
|
});
|
|
1198
1226
|
return this;
|
|
1199
1227
|
}
|
|
1200
|
-
addRetrieval(name,
|
|
1228
|
+
addRetrieval(name, options2) {
|
|
1201
1229
|
this.steps.push({
|
|
1202
1230
|
type: STEP_RETRIEVAL,
|
|
1203
1231
|
name,
|
|
1204
|
-
args:
|
|
1205
|
-
result:
|
|
1206
|
-
metadata:
|
|
1232
|
+
args: options2?.args,
|
|
1233
|
+
result: options2?.result,
|
|
1234
|
+
metadata: options2?.metadata
|
|
1207
1235
|
});
|
|
1208
1236
|
return this;
|
|
1209
1237
|
}
|
|
@@ -1281,10 +1309,10 @@ var AgentResult = class {
|
|
|
1281
1309
|
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
1282
1310
|
* when only soft_fail rows are present (see `softFailures`).
|
|
1283
1311
|
*/
|
|
1284
|
-
outcome(
|
|
1312
|
+
outcome(options2) {
|
|
1285
1313
|
if (this.hardFailures.length > 0) return OUTCOME_FAIL;
|
|
1286
1314
|
if (this.notEvaluated.length > 0) {
|
|
1287
|
-
return
|
|
1315
|
+
return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
|
|
1288
1316
|
}
|
|
1289
1317
|
return OUTCOME_PASS;
|
|
1290
1318
|
}
|
|
@@ -1394,9 +1422,9 @@ function truncate(value, limit = 280) {
|
|
|
1394
1422
|
}
|
|
1395
1423
|
return value.slice(0, limit - 3) + "...";
|
|
1396
1424
|
}
|
|
1397
|
-
function renderAssertionFailure(result,
|
|
1398
|
-
const color =
|
|
1399
|
-
const testFile =
|
|
1425
|
+
function renderAssertionFailure(result, options2 = {}) {
|
|
1426
|
+
const color = options2.color ?? false;
|
|
1427
|
+
const testFile = options2.testFile;
|
|
1400
1428
|
const layer = result.layer ?? 0;
|
|
1401
1429
|
const label = layerLabel(layer, result.type);
|
|
1402
1430
|
const lines = [];
|
|
@@ -1477,8 +1505,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1477
1505
|
);
|
|
1478
1506
|
}
|
|
1479
1507
|
}
|
|
1480
|
-
function renderSummary(result,
|
|
1481
|
-
const color =
|
|
1508
|
+
function renderSummary(result, options2 = {}) {
|
|
1509
|
+
const color = options2.color ?? false;
|
|
1482
1510
|
const passN = result.passCount;
|
|
1483
1511
|
const soft = result.assertionResults.filter(
|
|
1484
1512
|
(r) => r.status === STATUS_SOFT_FAIL
|
|
@@ -1496,9 +1524,9 @@ function renderSummary(result, options = {}) {
|
|
|
1496
1524
|
];
|
|
1497
1525
|
return parts.join(" | ");
|
|
1498
1526
|
}
|
|
1499
|
-
function renderDiagnostics(result,
|
|
1500
|
-
const color =
|
|
1501
|
-
const includePassing =
|
|
1527
|
+
function renderDiagnostics(result, options2 = {}) {
|
|
1528
|
+
const color = options2.color ?? false;
|
|
1529
|
+
const includePassing = options2.includePassing ?? false;
|
|
1502
1530
|
const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
1503
1531
|
if (rows.length === 0) {
|
|
1504
1532
|
return dim(" (no failures)\n", color);
|
|
@@ -1506,7 +1534,7 @@ function renderDiagnostics(result, options = {}) {
|
|
|
1506
1534
|
const out = [""];
|
|
1507
1535
|
out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
|
|
1508
1536
|
for (const r of rows) {
|
|
1509
|
-
out.push(renderAssertionFailure(r,
|
|
1537
|
+
out.push(renderAssertionFailure(r, options2));
|
|
1510
1538
|
}
|
|
1511
1539
|
out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
|
|
1512
1540
|
return out.join("\n");
|
|
@@ -1623,9 +1651,64 @@ var TraceTree = class {
|
|
|
1623
1651
|
|
|
1624
1652
|
// src/expect.ts
|
|
1625
1653
|
var import_node_crypto2 = require("crypto");
|
|
1654
|
+
var MAX_WORLD_BYTES = 9 * 1024 * 1024;
|
|
1655
|
+
var WorldChecks = class {
|
|
1656
|
+
_snapshot;
|
|
1657
|
+
_add;
|
|
1658
|
+
constructor(snapshot, add) {
|
|
1659
|
+
this._snapshot = snapshot;
|
|
1660
|
+
this._add = add;
|
|
1661
|
+
}
|
|
1662
|
+
addCheck(spec, where) {
|
|
1663
|
+
const full = { ...spec, world: this._snapshot };
|
|
1664
|
+
if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
|
|
1665
|
+
return this._add(full);
|
|
1666
|
+
}
|
|
1667
|
+
/**
|
|
1668
|
+
* Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
|
|
1669
|
+
* `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
|
|
1670
|
+
*
|
|
1671
|
+
* @throws Error when no bound is given, or `equals` is combined with another bound.
|
|
1672
|
+
*
|
|
1673
|
+
* @example
|
|
1674
|
+
* ```ts
|
|
1675
|
+
* expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
|
|
1676
|
+
* ```
|
|
1677
|
+
*/
|
|
1678
|
+
count(resource, opts) {
|
|
1679
|
+
if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
1680
|
+
throw new Error("count needs equals, atMost or atLeast");
|
|
1681
|
+
}
|
|
1682
|
+
if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
|
|
1683
|
+
throw new Error("count: equals excludes atMost and atLeast");
|
|
1684
|
+
}
|
|
1685
|
+
const spec = { check: "count", resource };
|
|
1686
|
+
if (opts.equals !== void 0) spec.equals = opts.equals;
|
|
1687
|
+
if (opts.atMost !== void 0) spec.at_most = opts.atMost;
|
|
1688
|
+
if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
|
|
1689
|
+
return this.addCheck(spec, opts.where);
|
|
1690
|
+
}
|
|
1691
|
+
/** Assert no two matching records share the same values for field(s) `by`. */
|
|
1692
|
+
unique(resource, opts) {
|
|
1693
|
+
const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
|
|
1694
|
+
if (fields.length === 0) throw new Error("unique needs at least one field in by");
|
|
1695
|
+
return this.addCheck({ check: "unique", resource, fields }, opts.where);
|
|
1696
|
+
}
|
|
1697
|
+
/**
|
|
1698
|
+
* Assert every matching record carries a distinct value in field `key` (default
|
|
1699
|
+
* `idempotency_key`), the record field that stores the idempotency key.
|
|
1700
|
+
*/
|
|
1701
|
+
idempotent(resource, opts = {}) {
|
|
1702
|
+
return this.addCheck(
|
|
1703
|
+
{ check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
|
|
1704
|
+
opts.where
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
};
|
|
1626
1708
|
var ExpectChain = class {
|
|
1627
1709
|
_result;
|
|
1628
1710
|
_assertions = [];
|
|
1711
|
+
_worldBytes = 0;
|
|
1629
1712
|
constructor(result) {
|
|
1630
1713
|
this._result = result;
|
|
1631
1714
|
}
|
|
@@ -2054,6 +2137,24 @@ var ExpectChain = class {
|
|
|
2054
2137
|
soft: opts?.soft ?? false
|
|
2055
2138
|
});
|
|
2056
2139
|
}
|
|
2140
|
+
// -- Layer 3: Effect checks (the external system the agent acted on) --
|
|
2141
|
+
/**
|
|
2142
|
+
* Check effects in a world snapshot:
|
|
2143
|
+
* `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
|
|
2144
|
+
* a world helper: the checks are `not_evaluated` (`missing_evidence`).
|
|
2145
|
+
*/
|
|
2146
|
+
world(snapshot) {
|
|
2147
|
+
const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
|
|
2148
|
+
return new WorldChecks(snapshot, (spec) => {
|
|
2149
|
+
this._worldBytes += size;
|
|
2150
|
+
if (this._worldBytes > MAX_WORLD_BYTES) {
|
|
2151
|
+
throw new Error(
|
|
2152
|
+
`effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
return this.add(TYPE_EFFECT, spec);
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2057
2158
|
// -- Layer 4: Content --
|
|
2058
2159
|
/**
|
|
2059
2160
|
* Assert the output message contains the given string.
|
|
@@ -2547,6 +2648,728 @@ function attestExpect(result) {
|
|
|
2547
2648
|
return new ExpectChain(new AgentResult(result));
|
|
2548
2649
|
}
|
|
2549
2650
|
|
|
2651
|
+
// src/capture.ts
|
|
2652
|
+
var import_node_child_process2 = require("child_process");
|
|
2653
|
+
var fs2 = __toESM(require("fs"), 1);
|
|
2654
|
+
var path2 = __toESM(require("path"), 1);
|
|
2655
|
+
var import_node_url = require("url");
|
|
2656
|
+
var FIXTURE_VERSION = 1;
|
|
2657
|
+
var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
|
|
2658
|
+
var CaptureError = class extends Error {
|
|
2659
|
+
/** The fixture written, when the engine wrote one. */
|
|
2660
|
+
fixture;
|
|
2661
|
+
constructor(message, fixture = null) {
|
|
2662
|
+
super(message);
|
|
2663
|
+
this.name = "CaptureError";
|
|
2664
|
+
this.fixture = fixture;
|
|
2665
|
+
}
|
|
2666
|
+
};
|
|
2667
|
+
function toPath(p) {
|
|
2668
|
+
return p instanceof URL ? (0, import_node_url.fileURLToPath)(p) : p;
|
|
2669
|
+
}
|
|
2670
|
+
function loadFixture(file) {
|
|
2671
|
+
const p = toPath(file);
|
|
2672
|
+
const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
|
|
2673
|
+
if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
|
|
2674
|
+
throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
|
|
2675
|
+
}
|
|
2676
|
+
validate(p, doc);
|
|
2677
|
+
return doc;
|
|
2678
|
+
}
|
|
2679
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
2680
|
+
function isObject(v) {
|
|
2681
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2682
|
+
}
|
|
2683
|
+
function kindOf(v) {
|
|
2684
|
+
if (Array.isArray(v)) return "array";
|
|
2685
|
+
if (isObject(v)) return "object";
|
|
2686
|
+
if (typeof v === "string" || typeof v === "boolean") return typeof v;
|
|
2687
|
+
return "other";
|
|
2688
|
+
}
|
|
2689
|
+
function validate(p, doc) {
|
|
2690
|
+
const need = (where, obj, fields) => {
|
|
2691
|
+
if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
|
|
2692
|
+
for (const [key, kind] of Object.entries(fields)) {
|
|
2693
|
+
if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
|
|
2694
|
+
}
|
|
2695
|
+
return obj;
|
|
2696
|
+
};
|
|
2697
|
+
need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
|
|
2698
|
+
if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
|
|
2699
|
+
if (!NAME_PATTERN.test(doc["name"])) {
|
|
2700
|
+
throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
|
|
2701
|
+
}
|
|
2702
|
+
const v = need("fixture.verification", doc["verification"], {
|
|
2703
|
+
verified: "boolean",
|
|
2704
|
+
expect_fail: "boolean",
|
|
2705
|
+
results: "array",
|
|
2706
|
+
expected_failures: "array"
|
|
2707
|
+
});
|
|
2708
|
+
doc["contracts"].forEach(
|
|
2709
|
+
(c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
|
|
2710
|
+
);
|
|
2711
|
+
v["results"].forEach(
|
|
2712
|
+
(r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
|
|
2713
|
+
);
|
|
2714
|
+
}
|
|
2715
|
+
function objectKey(k) {
|
|
2716
|
+
if (k === "__proto__") return `[${JSON.stringify(k)}]`;
|
|
2717
|
+
return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
|
|
2718
|
+
}
|
|
2719
|
+
function comment(text) {
|
|
2720
|
+
return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
|
|
2721
|
+
}
|
|
2722
|
+
function ts(value) {
|
|
2723
|
+
if (value === null || typeof value === "boolean" || typeof value === "number") {
|
|
2724
|
+
return JSON.stringify(value);
|
|
2725
|
+
}
|
|
2726
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
2727
|
+
if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
|
|
2728
|
+
if (typeof value === "object") {
|
|
2729
|
+
const entries = Object.entries(value).map(
|
|
2730
|
+
([k, v]) => `${objectKey(k)}: ${ts(v)}`
|
|
2731
|
+
);
|
|
2732
|
+
return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
|
|
2733
|
+
}
|
|
2734
|
+
throw new Error(`value ${String(value)} has no literal`);
|
|
2735
|
+
}
|
|
2736
|
+
function str(spec, key) {
|
|
2737
|
+
const v = spec[key];
|
|
2738
|
+
if (typeof v !== "string") throw new Error(`missing ${key}`);
|
|
2739
|
+
return v;
|
|
2740
|
+
}
|
|
2741
|
+
function options(entries) {
|
|
2742
|
+
const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
|
|
2743
|
+
return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
|
|
2744
|
+
}
|
|
2745
|
+
function call(method, args, opts) {
|
|
2746
|
+
return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
|
|
2747
|
+
}
|
|
2748
|
+
function dslCall(contract) {
|
|
2749
|
+
const s = contract.spec;
|
|
2750
|
+
const soft = s["soft"] === true ? true : void 0;
|
|
2751
|
+
try {
|
|
2752
|
+
if (contract.type === "trace") {
|
|
2753
|
+
const check = s["check"];
|
|
2754
|
+
if (check === "trajectory" && s["mode"] !== "partial_order") {
|
|
2755
|
+
const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
|
|
2756
|
+
if (!Array.isArray(s["tools"])) throw new Error("missing tools");
|
|
2757
|
+
return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
|
|
2758
|
+
}
|
|
2759
|
+
if (check === "tool_order") {
|
|
2760
|
+
return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
|
|
2761
|
+
}
|
|
2762
|
+
if (check === "tool_cardinality") {
|
|
2763
|
+
return call(
|
|
2764
|
+
"toolCardinality",
|
|
2765
|
+
[ts(str(s, "tool"))],
|
|
2766
|
+
options([
|
|
2767
|
+
["per", s["per"]],
|
|
2768
|
+
["atLeast", s["min"]],
|
|
2769
|
+
["atMost", s["max"]],
|
|
2770
|
+
["soft", soft]
|
|
2771
|
+
])
|
|
2772
|
+
);
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
if (contract.type === "schema" && s["target"] === "output.structured") {
|
|
2776
|
+
if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
|
|
2777
|
+
return call("outputMatchesSchema", [ts(s["schema"])], "");
|
|
2778
|
+
}
|
|
2779
|
+
if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
|
|
2780
|
+
if (typeof s["value"] !== "number") throw new Error("missing value");
|
|
2781
|
+
return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
|
|
2782
|
+
}
|
|
2783
|
+
} catch (err) {
|
|
2784
|
+
throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
|
|
2785
|
+
}
|
|
2786
|
+
throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
|
|
2787
|
+
}
|
|
2788
|
+
function testPathFor(fixturePath) {
|
|
2789
|
+
const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
|
|
2790
|
+
return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
|
|
2791
|
+
}
|
|
2792
|
+
function renderVitest(fixture, fixturePath, testPath) {
|
|
2793
|
+
const calls = [];
|
|
2794
|
+
const skipped = [];
|
|
2795
|
+
for (const c of fixture.contracts) {
|
|
2796
|
+
try {
|
|
2797
|
+
calls.push([c, dslCall(c)]);
|
|
2798
|
+
} catch (err) {
|
|
2799
|
+
skipped.push(err.message);
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
|
|
2803
|
+
const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
|
|
2804
|
+
const expectFail = fixture.verification.expect_fail;
|
|
2805
|
+
const lines = [
|
|
2806
|
+
`// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
|
|
2807
|
+
"//",
|
|
2808
|
+
"// Review and edit before committing: they check the captured trace without running",
|
|
2809
|
+
"// the agent. Re-capturing does not overwrite this file.",
|
|
2810
|
+
`import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
|
|
2811
|
+
`import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
|
|
2812
|
+
`import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
|
|
2813
|
+
"",
|
|
2814
|
+
`const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
|
|
2815
|
+
];
|
|
2816
|
+
if (skipped.length > 0) {
|
|
2817
|
+
lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
|
|
2818
|
+
for (const s of skipped) lines.push(`// - ${comment(s)}`);
|
|
2819
|
+
}
|
|
2820
|
+
lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
|
|
2821
|
+
for (const [c, dsl] of calls) {
|
|
2822
|
+
lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
|
|
2823
|
+
}
|
|
2824
|
+
lines[lines.length - 1] += ";";
|
|
2825
|
+
lines.push("}");
|
|
2826
|
+
if (expectFail) {
|
|
2827
|
+
lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
|
|
2828
|
+
lines.push("const EXPECTED: Array<[string, string]> = [");
|
|
2829
|
+
for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
|
|
2830
|
+
lines.push(
|
|
2831
|
+
"];",
|
|
2832
|
+
"",
|
|
2833
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2834
|
+
` it("reproduces the incident", async () => {`,
|
|
2835
|
+
" const result = await inspect(contracts());",
|
|
2836
|
+
"",
|
|
2837
|
+
" expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
|
|
2838
|
+
" });",
|
|
2839
|
+
"});"
|
|
2840
|
+
);
|
|
2841
|
+
} else {
|
|
2842
|
+
lines.push(
|
|
2843
|
+
"",
|
|
2844
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2845
|
+
` it("keeps its drafted contracts", async () => {`,
|
|
2846
|
+
" await evaluate(contracts());",
|
|
2847
|
+
" });",
|
|
2848
|
+
"});"
|
|
2849
|
+
);
|
|
2850
|
+
}
|
|
2851
|
+
return `${lines.join("\n")}
|
|
2852
|
+
`;
|
|
2853
|
+
}
|
|
2854
|
+
function emitVitest(fixturePath) {
|
|
2855
|
+
const testPath = testPathFor(fixturePath);
|
|
2856
|
+
const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
|
|
2857
|
+
if (existing?.isSymbolicLink()) {
|
|
2858
|
+
throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
|
|
2859
|
+
}
|
|
2860
|
+
if (existing !== void 0) return null;
|
|
2861
|
+
const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
|
|
2862
|
+
fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
|
|
2863
|
+
try {
|
|
2864
|
+
fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
|
|
2865
|
+
} catch (err) {
|
|
2866
|
+
if (err.code === "EEXIST") {
|
|
2867
|
+
throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
|
|
2868
|
+
}
|
|
2869
|
+
throw err;
|
|
2870
|
+
}
|
|
2871
|
+
return testPath;
|
|
2872
|
+
}
|
|
2873
|
+
function serve(engine, args) {
|
|
2874
|
+
const child = (0, import_node_child_process2.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2875
|
+
let stdout = "";
|
|
2876
|
+
let stderr = "";
|
|
2877
|
+
let resolveReady = () => void 0;
|
|
2878
|
+
const ready = new Promise((resolve4) => {
|
|
2879
|
+
resolveReady = resolve4;
|
|
2880
|
+
});
|
|
2881
|
+
const exited = new Promise((resolve4) => {
|
|
2882
|
+
child.on("close", (code, signal) => {
|
|
2883
|
+
resolveReady(null);
|
|
2884
|
+
resolve4({ code, signal });
|
|
2885
|
+
});
|
|
2886
|
+
child.on("error", (err) => {
|
|
2887
|
+
stderr += err.message;
|
|
2888
|
+
resolveReady(null);
|
|
2889
|
+
resolve4({ code: null, signal: null });
|
|
2890
|
+
});
|
|
2891
|
+
});
|
|
2892
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
2893
|
+
stdout += chunk;
|
|
2894
|
+
const nl = stdout.indexOf("\n");
|
|
2895
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
2896
|
+
});
|
|
2897
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
2898
|
+
stderr += chunk;
|
|
2899
|
+
});
|
|
2900
|
+
child.stdin.on("error", () => void 0);
|
|
2901
|
+
return {
|
|
2902
|
+
ready,
|
|
2903
|
+
async finish(finalize) {
|
|
2904
|
+
if (finalize) {
|
|
2905
|
+
const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
|
|
2906
|
+
child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
|
|
2907
|
+
`);
|
|
2908
|
+
}
|
|
2909
|
+
child.stdin.end();
|
|
2910
|
+
const { code, signal } = await exited;
|
|
2911
|
+
return { code, signal, stdout, stderr };
|
|
2912
|
+
}
|
|
2913
|
+
};
|
|
2914
|
+
}
|
|
2915
|
+
function readyEnv(line) {
|
|
2916
|
+
if (line === null) return null;
|
|
2917
|
+
let parsed;
|
|
2918
|
+
try {
|
|
2919
|
+
parsed = JSON.parse(line);
|
|
2920
|
+
} catch {
|
|
2921
|
+
return null;
|
|
2922
|
+
}
|
|
2923
|
+
const env = parsed?.env;
|
|
2924
|
+
if (typeof env !== "object" || env === null) return null;
|
|
2925
|
+
const entries = Object.entries(env);
|
|
2926
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
2927
|
+
return Object.fromEntries(entries);
|
|
2928
|
+
}
|
|
2929
|
+
function exitDetail(exit) {
|
|
2930
|
+
const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
|
|
2931
|
+
const err = exit.stderr.trim();
|
|
2932
|
+
return err ? `${how}: ${err}` : how;
|
|
2933
|
+
}
|
|
2934
|
+
function restore(saved) {
|
|
2935
|
+
for (const [k, v] of saved) {
|
|
2936
|
+
if (v === void 0) delete process.env[k];
|
|
2937
|
+
else process.env[k] = v;
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
var recording = false;
|
|
2941
|
+
async function record(name, fn, opts = {}) {
|
|
2942
|
+
if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
|
|
2943
|
+
recording = true;
|
|
2944
|
+
try {
|
|
2945
|
+
return await recordExclusive(name, fn, opts);
|
|
2946
|
+
} finally {
|
|
2947
|
+
recording = false;
|
|
2948
|
+
}
|
|
2949
|
+
}
|
|
2950
|
+
async function recordExclusive(name, fn, opts) {
|
|
2951
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2952
|
+
const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
|
|
2953
|
+
if (opts.expectFail) args.push("--expect-fail");
|
|
2954
|
+
if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
|
|
2955
|
+
if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
|
|
2956
|
+
const proc = serve(engine, args);
|
|
2957
|
+
const env = readyEnv(await proc.ready);
|
|
2958
|
+
if (env === null) {
|
|
2959
|
+
const exit2 = await proc.finish(false);
|
|
2960
|
+
throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
|
|
2961
|
+
}
|
|
2962
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
2963
|
+
Object.assign(process.env, env);
|
|
2964
|
+
let value;
|
|
2965
|
+
try {
|
|
2966
|
+
value = await fn({ name, env });
|
|
2967
|
+
if (opts.flush) await opts.flush();
|
|
2968
|
+
} catch (err) {
|
|
2969
|
+
restore(saved);
|
|
2970
|
+
await proc.finish(false);
|
|
2971
|
+
throw err;
|
|
2972
|
+
}
|
|
2973
|
+
restore(saved);
|
|
2974
|
+
const exit = await proc.finish(true);
|
|
2975
|
+
process.stderr.write(exit.stderr);
|
|
2976
|
+
const lines = exit.stdout.trim().split("\n");
|
|
2977
|
+
if (lines.length < 2) {
|
|
2978
|
+
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2979
|
+
}
|
|
2980
|
+
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2981
|
+
if (exit.code !== 0 || !outcome.verified) {
|
|
2982
|
+
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2983
|
+
}
|
|
2984
|
+
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2985
|
+
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
// src/replay.ts
|
|
2989
|
+
var import_node_child_process3 = require("child_process");
|
|
2990
|
+
var path3 = __toESM(require("path"), 1);
|
|
2991
|
+
|
|
2992
|
+
// src/egress.ts
|
|
2993
|
+
var net = __toESM(require("net"), 1);
|
|
2994
|
+
var PROVIDER_HOSTS = [
|
|
2995
|
+
"api.openai.com",
|
|
2996
|
+
"api.anthropic.com",
|
|
2997
|
+
"generativelanguage.googleapis.com",
|
|
2998
|
+
/\.openai\.azure\.com$/
|
|
2999
|
+
];
|
|
3000
|
+
var EgressBlockedError = class extends Error {
|
|
3001
|
+
constructor(host) {
|
|
3002
|
+
super(
|
|
3003
|
+
`attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
|
|
3004
|
+
);
|
|
3005
|
+
this.host = host;
|
|
3006
|
+
this.name = "EgressBlockedError";
|
|
3007
|
+
}
|
|
3008
|
+
host;
|
|
3009
|
+
};
|
|
3010
|
+
function isProviderHost(host) {
|
|
3011
|
+
const h = host.toLowerCase().replace(/\.$/, "");
|
|
3012
|
+
return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
|
|
3013
|
+
}
|
|
3014
|
+
function egressBlockRequested(env = process.env) {
|
|
3015
|
+
const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
|
|
3016
|
+
if (v !== "" && v !== "block" && v !== "allow") {
|
|
3017
|
+
throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
|
|
3018
|
+
}
|
|
3019
|
+
return v === "block";
|
|
3020
|
+
}
|
|
3021
|
+
var original = null;
|
|
3022
|
+
var installs = 0;
|
|
3023
|
+
function hostOf(args) {
|
|
3024
|
+
const first = Array.isArray(args[0]) ? args[0][0] : args[0];
|
|
3025
|
+
if (typeof first === "object" && first !== null) {
|
|
3026
|
+
const o = first;
|
|
3027
|
+
return typeof o.host === "string" ? o.host : null;
|
|
3028
|
+
}
|
|
3029
|
+
if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
|
|
3030
|
+
return typeof args[1] === "string" ? args[1] : null;
|
|
3031
|
+
}
|
|
3032
|
+
return null;
|
|
3033
|
+
}
|
|
3034
|
+
function installEgressGuard() {
|
|
3035
|
+
installs++;
|
|
3036
|
+
if (original === null) {
|
|
3037
|
+
const proto = net.Socket.prototype;
|
|
3038
|
+
const orig = proto.connect;
|
|
3039
|
+
original = orig;
|
|
3040
|
+
proto.connect = function(...args) {
|
|
3041
|
+
const host = hostOf(args);
|
|
3042
|
+
if (host !== null && isProviderHost(host)) {
|
|
3043
|
+
const err = new EgressBlockedError(host);
|
|
3044
|
+
process.nextTick(() => this.destroy(err));
|
|
3045
|
+
return this;
|
|
3046
|
+
}
|
|
3047
|
+
return orig.apply(this, args);
|
|
3048
|
+
};
|
|
3049
|
+
}
|
|
3050
|
+
let released = false;
|
|
3051
|
+
return () => {
|
|
3052
|
+
if (released) return;
|
|
3053
|
+
released = true;
|
|
3054
|
+
installs--;
|
|
3055
|
+
if (installs === 0) uninstallEgressGuard();
|
|
3056
|
+
};
|
|
3057
|
+
}
|
|
3058
|
+
function egressGuardActive() {
|
|
3059
|
+
return installs > 0;
|
|
3060
|
+
}
|
|
3061
|
+
function uninstallEgressGuard() {
|
|
3062
|
+
installs = 0;
|
|
3063
|
+
if (original === null) return;
|
|
3064
|
+
net.Socket.prototype.connect = original;
|
|
3065
|
+
original = null;
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
// src/replay.ts
|
|
3069
|
+
var ReplayMissError = class extends Error {
|
|
3070
|
+
constructor(message, summary) {
|
|
3071
|
+
super(message);
|
|
3072
|
+
this.summary = summary;
|
|
3073
|
+
this.name = "ReplayMissError";
|
|
3074
|
+
}
|
|
3075
|
+
summary;
|
|
3076
|
+
};
|
|
3077
|
+
var ReplayNotEvaluatedError = class extends Error {
|
|
3078
|
+
constructor(message, summary) {
|
|
3079
|
+
super(message);
|
|
3080
|
+
this.summary = summary;
|
|
3081
|
+
this.name = "ReplayNotEvaluatedError";
|
|
3082
|
+
}
|
|
3083
|
+
summary;
|
|
3084
|
+
};
|
|
3085
|
+
function escapeControl(text) {
|
|
3086
|
+
return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
|
|
3087
|
+
}
|
|
3088
|
+
var ReplayError = class extends Error {
|
|
3089
|
+
constructor(message) {
|
|
3090
|
+
super(escapeControl(message));
|
|
3091
|
+
this.name = "ReplayError";
|
|
3092
|
+
}
|
|
3093
|
+
};
|
|
3094
|
+
function resolveMode(explicit, env = process.env) {
|
|
3095
|
+
if (explicit !== void 0) return checkMode(explicit, "mode");
|
|
3096
|
+
const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
|
|
3097
|
+
if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
|
|
3098
|
+
const ci = (env.CI ?? "").trim().toLowerCase();
|
|
3099
|
+
return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
|
|
3100
|
+
}
|
|
3101
|
+
function checkMode(value, where) {
|
|
3102
|
+
if (value === "strict" || value === "auto" || value === "off") return value;
|
|
3103
|
+
throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
|
|
3104
|
+
}
|
|
3105
|
+
var active = null;
|
|
3106
|
+
async function startReplay(name, opts = {}) {
|
|
3107
|
+
if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
|
|
3108
|
+
const resolved = resolveMode(opts.mode);
|
|
3109
|
+
if (resolved === "off") return offSession(name);
|
|
3110
|
+
const mode = resolved;
|
|
3111
|
+
if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
|
|
3112
|
+
active = name;
|
|
3113
|
+
try {
|
|
3114
|
+
return await start(name, mode, opts);
|
|
3115
|
+
} catch (err) {
|
|
3116
|
+
active = null;
|
|
3117
|
+
throw err;
|
|
3118
|
+
}
|
|
3119
|
+
}
|
|
3120
|
+
function offSession(name) {
|
|
3121
|
+
return {
|
|
3122
|
+
name,
|
|
3123
|
+
mode: "off",
|
|
3124
|
+
env: {},
|
|
3125
|
+
summary: null,
|
|
3126
|
+
finish: () => Promise.resolve(null),
|
|
3127
|
+
error: () => null,
|
|
3128
|
+
check: () => Promise.resolve()
|
|
3129
|
+
};
|
|
3130
|
+
}
|
|
3131
|
+
function missError(mode, summary) {
|
|
3132
|
+
if (summary.misses === 0) return null;
|
|
3133
|
+
const message = escapeControl(firstMiss(summary) ?? "replay miss");
|
|
3134
|
+
return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
|
|
3135
|
+
}
|
|
3136
|
+
async function start(name, mode, opts) {
|
|
3137
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3138
|
+
const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
|
|
3139
|
+
const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
|
|
3140
|
+
const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
|
|
3141
|
+
if (blocking) args.push("--no-live");
|
|
3142
|
+
const child = (0, import_node_child_process3.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3143
|
+
let stdout = "";
|
|
3144
|
+
let stderr = "";
|
|
3145
|
+
let resolveReady = () => void 0;
|
|
3146
|
+
const ready = new Promise((resolve4) => {
|
|
3147
|
+
resolveReady = resolve4;
|
|
3148
|
+
});
|
|
3149
|
+
const exited = new Promise((resolve4) => {
|
|
3150
|
+
child.on("close", (code) => {
|
|
3151
|
+
resolveReady(null);
|
|
3152
|
+
resolve4(code);
|
|
3153
|
+
});
|
|
3154
|
+
child.on("error", (err) => {
|
|
3155
|
+
stderr += err.message;
|
|
3156
|
+
resolveReady(null);
|
|
3157
|
+
resolve4(null);
|
|
3158
|
+
});
|
|
3159
|
+
});
|
|
3160
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3161
|
+
stdout += chunk;
|
|
3162
|
+
const nl = stdout.indexOf("\n");
|
|
3163
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3164
|
+
});
|
|
3165
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3166
|
+
stderr += chunk;
|
|
3167
|
+
});
|
|
3168
|
+
child.stdin.on("error", () => void 0);
|
|
3169
|
+
const stop = async () => {
|
|
3170
|
+
child.stdin.write("stop\n");
|
|
3171
|
+
child.stdin.end();
|
|
3172
|
+
return { code: await exited, stdout, stderr };
|
|
3173
|
+
};
|
|
3174
|
+
const env = parseEnv(await ready);
|
|
3175
|
+
if (env === null) {
|
|
3176
|
+
const out = await stop();
|
|
3177
|
+
active = null;
|
|
3178
|
+
throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3179
|
+
}
|
|
3180
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3181
|
+
Object.assign(process.env, env);
|
|
3182
|
+
let summary = null;
|
|
3183
|
+
let finishing = null;
|
|
3184
|
+
const finish = () => {
|
|
3185
|
+
finishing ??= (async () => {
|
|
3186
|
+
for (const [k, v] of saved) {
|
|
3187
|
+
if (v === void 0) delete process.env[k];
|
|
3188
|
+
else process.env[k] = v;
|
|
3189
|
+
}
|
|
3190
|
+
const out = await stop();
|
|
3191
|
+
active = null;
|
|
3192
|
+
const parsed = parseSummary(out.stdout);
|
|
3193
|
+
if (parsed === null) {
|
|
3194
|
+
throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3195
|
+
}
|
|
3196
|
+
if (blocking && parsed.live > 0) {
|
|
3197
|
+
throw new ReplayError(
|
|
3198
|
+
`replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
summary = parsed;
|
|
3202
|
+
return parsed;
|
|
3203
|
+
})();
|
|
3204
|
+
return finishing;
|
|
3205
|
+
};
|
|
3206
|
+
return {
|
|
3207
|
+
name,
|
|
3208
|
+
mode,
|
|
3209
|
+
env,
|
|
3210
|
+
get summary() {
|
|
3211
|
+
return summary;
|
|
3212
|
+
},
|
|
3213
|
+
finish,
|
|
3214
|
+
error: () => summary === null ? null : missError(mode, summary),
|
|
3215
|
+
check: async () => {
|
|
3216
|
+
await finish();
|
|
3217
|
+
const err = summary === null ? null : missError(mode, summary);
|
|
3218
|
+
if (err !== null) throw err;
|
|
3219
|
+
}
|
|
3220
|
+
};
|
|
3221
|
+
}
|
|
3222
|
+
function parseEnv(line) {
|
|
3223
|
+
if (line === null) return null;
|
|
3224
|
+
try {
|
|
3225
|
+
const env = JSON.parse(line).env;
|
|
3226
|
+
if (typeof env !== "object" || env === null) return null;
|
|
3227
|
+
const entries = Object.entries(env);
|
|
3228
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
3229
|
+
return Object.fromEntries(entries);
|
|
3230
|
+
} catch {
|
|
3231
|
+
return null;
|
|
3232
|
+
}
|
|
3233
|
+
}
|
|
3234
|
+
function parseSummary(out) {
|
|
3235
|
+
const lines = out.trim().split("\n");
|
|
3236
|
+
try {
|
|
3237
|
+
const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
|
|
3238
|
+
if (s === void 0 || !Array.isArray(s.records)) return null;
|
|
3239
|
+
for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
|
|
3240
|
+
return s;
|
|
3241
|
+
} catch {
|
|
3242
|
+
return null;
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3245
|
+
function firstMiss(summary) {
|
|
3246
|
+
return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
|
|
3247
|
+
}
|
|
3248
|
+
function liveLine(live, costUsd, unpriced = 0) {
|
|
3249
|
+
const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
|
|
3250
|
+
return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
|
|
3251
|
+
}
|
|
3252
|
+
|
|
3253
|
+
// src/world.ts
|
|
3254
|
+
var import_node_child_process4 = require("child_process");
|
|
3255
|
+
var path4 = __toESM(require("path"), 1);
|
|
3256
|
+
var STATE_PATH = "/_attest/state";
|
|
3257
|
+
var STATE_TIMEOUT_MS = 1e4;
|
|
3258
|
+
var WorldError = class extends Error {
|
|
3259
|
+
constructor(message) {
|
|
3260
|
+
super(escapeControl(message));
|
|
3261
|
+
this.name = "WorldError";
|
|
3262
|
+
}
|
|
3263
|
+
};
|
|
3264
|
+
var active2 = null;
|
|
3265
|
+
async function startWorld(worldFile, opts = {}) {
|
|
3266
|
+
if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
|
|
3267
|
+
const file = path4.resolve(worldFile);
|
|
3268
|
+
active2 = file;
|
|
3269
|
+
try {
|
|
3270
|
+
return await start2(file, opts);
|
|
3271
|
+
} catch (err) {
|
|
3272
|
+
active2 = null;
|
|
3273
|
+
throw err;
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
async function start2(file, opts) {
|
|
3277
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3278
|
+
const child = (0, import_node_child_process4.spawn)(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
3279
|
+
let stdout = "";
|
|
3280
|
+
let stderr = "";
|
|
3281
|
+
let resolveReady = () => void 0;
|
|
3282
|
+
const ready = new Promise((resolve4) => {
|
|
3283
|
+
resolveReady = resolve4;
|
|
3284
|
+
});
|
|
3285
|
+
const exited = new Promise((resolve4) => {
|
|
3286
|
+
child.on("close", (code) => {
|
|
3287
|
+
resolveReady(null);
|
|
3288
|
+
resolve4(code);
|
|
3289
|
+
});
|
|
3290
|
+
child.on("error", (err) => {
|
|
3291
|
+
stderr += err.message;
|
|
3292
|
+
resolveReady(null);
|
|
3293
|
+
resolve4(null);
|
|
3294
|
+
});
|
|
3295
|
+
});
|
|
3296
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3297
|
+
stdout += chunk;
|
|
3298
|
+
const nl = stdout.indexOf("\n");
|
|
3299
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3300
|
+
});
|
|
3301
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3302
|
+
stderr += chunk;
|
|
3303
|
+
});
|
|
3304
|
+
child.stdin.on("error", () => void 0);
|
|
3305
|
+
const halt = async () => {
|
|
3306
|
+
child.stdin.write("stop\n");
|
|
3307
|
+
child.stdin.end();
|
|
3308
|
+
return { code: await exited, stdout, stderr };
|
|
3309
|
+
};
|
|
3310
|
+
const env = parseEnv(await ready);
|
|
3311
|
+
if (env === null) {
|
|
3312
|
+
const out = await halt();
|
|
3313
|
+
throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3314
|
+
}
|
|
3315
|
+
const urls = Object.values(env);
|
|
3316
|
+
if (urls.length !== 1) {
|
|
3317
|
+
await halt();
|
|
3318
|
+
throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
|
|
3319
|
+
}
|
|
3320
|
+
const baseUrl = urls[0];
|
|
3321
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3322
|
+
Object.assign(process.env, env);
|
|
3323
|
+
let stopped = false;
|
|
3324
|
+
let stopping = null;
|
|
3325
|
+
const stop = () => {
|
|
3326
|
+
stopped = true;
|
|
3327
|
+
stopping ??= (async () => {
|
|
3328
|
+
for (const [k, v] of saved) {
|
|
3329
|
+
if (v === void 0) delete process.env[k];
|
|
3330
|
+
else process.env[k] = v;
|
|
3331
|
+
}
|
|
3332
|
+
const out = await halt();
|
|
3333
|
+
active2 = null;
|
|
3334
|
+
const final = parseFinal(out.stdout);
|
|
3335
|
+
if (out.code !== 0 || final === null) {
|
|
3336
|
+
throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3337
|
+
}
|
|
3338
|
+
return final;
|
|
3339
|
+
})();
|
|
3340
|
+
return stopping;
|
|
3341
|
+
};
|
|
3342
|
+
const state = async () => {
|
|
3343
|
+
if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
|
|
3344
|
+
const url = baseUrl + STATE_PATH;
|
|
3345
|
+
let res;
|
|
3346
|
+
try {
|
|
3347
|
+
res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
|
|
3348
|
+
} catch (err) {
|
|
3349
|
+
throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3350
|
+
}
|
|
3351
|
+
if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
|
|
3352
|
+
const body = await res.json().catch(() => {
|
|
3353
|
+
throw new WorldError(`GET ${url}: the body is not JSON`);
|
|
3354
|
+
});
|
|
3355
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
3356
|
+
throw new WorldError(`GET ${url}: the body is not a snapshot object`);
|
|
3357
|
+
}
|
|
3358
|
+
return body;
|
|
3359
|
+
};
|
|
3360
|
+
return { path: file, env, baseUrl, state, stop };
|
|
3361
|
+
}
|
|
3362
|
+
function parseFinal(out) {
|
|
3363
|
+
const lines = out.trim().split("\n");
|
|
3364
|
+
try {
|
|
3365
|
+
const world = JSON.parse(lines[lines.length - 1] ?? "").world;
|
|
3366
|
+
if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
|
|
3367
|
+
return world;
|
|
3368
|
+
} catch {
|
|
3369
|
+
return null;
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
|
|
2550
3373
|
// src/context.ts
|
|
2551
3374
|
var import_node_async_hooks = require("async_hooks");
|
|
2552
3375
|
var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
|
|
@@ -2750,12 +3573,12 @@ var ContinuousEvalRunner = class {
|
|
|
2750
3573
|
maxSize;
|
|
2751
3574
|
running = false;
|
|
2752
3575
|
loopTimer = null;
|
|
2753
|
-
constructor(client, assertions,
|
|
3576
|
+
constructor(client, assertions, options2) {
|
|
2754
3577
|
this.client = client;
|
|
2755
3578
|
this.assertions = assertions;
|
|
2756
|
-
this.sampler = new Sampler(
|
|
2757
|
-
this.dispatcher = new AlertDispatcher(
|
|
2758
|
-
this.maxSize =
|
|
3579
|
+
this.sampler = new Sampler(options2?.sampleRate ?? 1);
|
|
3580
|
+
this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
|
|
3581
|
+
this.maxSize = options2?.maxSize ?? resolveQueueSize();
|
|
2759
3582
|
}
|
|
2760
3583
|
get alertEmitter() {
|
|
2761
3584
|
return this.dispatcher;
|
|
@@ -2866,13 +3689,13 @@ function loadLabelsCSV(text) {
|
|
|
2866
3689
|
if (rows.length === 0) {
|
|
2867
3690
|
throw new Error("empty CSV");
|
|
2868
3691
|
}
|
|
2869
|
-
let
|
|
3692
|
+
let start3 = 0;
|
|
2870
3693
|
if (rows[0].length >= 2) {
|
|
2871
3694
|
const firstHuman = parseFloatStrict(rows[0][1]);
|
|
2872
|
-
if (firstHuman === null)
|
|
3695
|
+
if (firstHuman === null) start3 = 1;
|
|
2873
3696
|
}
|
|
2874
3697
|
const out = [];
|
|
2875
|
-
for (let idx =
|
|
3698
|
+
for (let idx = start3; idx < rows.length; idx++) {
|
|
2876
3699
|
const row = rows[idx];
|
|
2877
3700
|
const lineNum = idx + 1;
|
|
2878
3701
|
if (row.length === 0) continue;
|
|
@@ -3081,10 +3904,10 @@ var OpenAIAdapter = class {
|
|
|
3081
3904
|
constructor(agentId2) {
|
|
3082
3905
|
this.agentId = agentId2;
|
|
3083
3906
|
}
|
|
3084
|
-
traceFromResponse(response,
|
|
3907
|
+
traceFromResponse(response, options2) {
|
|
3085
3908
|
const builder = new TraceBuilder(this.agentId);
|
|
3086
|
-
if (
|
|
3087
|
-
builder.setInput({ messages:
|
|
3909
|
+
if (options2?.inputMessages) {
|
|
3910
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3088
3911
|
}
|
|
3089
3912
|
const message = response.choices[0].message;
|
|
3090
3913
|
const completionText = message.content ?? "";
|
|
@@ -3104,12 +3927,12 @@ var OpenAIAdapter = class {
|
|
|
3104
3927
|
}
|
|
3105
3928
|
builder.setOutput({
|
|
3106
3929
|
message: completionText,
|
|
3107
|
-
structured:
|
|
3930
|
+
structured: options2?.structuredOutput ?? {}
|
|
3108
3931
|
});
|
|
3109
3932
|
builder.setMetadata({
|
|
3110
3933
|
total_tokens: response.usage?.total_tokens,
|
|
3111
|
-
cost_usd:
|
|
3112
|
-
latency_ms:
|
|
3934
|
+
cost_usd: options2?.costUsd,
|
|
3935
|
+
latency_ms: options2?.latencyMs,
|
|
3113
3936
|
model: response.model
|
|
3114
3937
|
});
|
|
3115
3938
|
return builder.build();
|
|
@@ -3122,10 +3945,10 @@ var AnthropicAdapter = class {
|
|
|
3122
3945
|
constructor(agentId2) {
|
|
3123
3946
|
this.agentId = agentId2;
|
|
3124
3947
|
}
|
|
3125
|
-
traceFromResponse(response,
|
|
3948
|
+
traceFromResponse(response, options2) {
|
|
3126
3949
|
const builder = new TraceBuilder(this.agentId);
|
|
3127
|
-
if (
|
|
3128
|
-
builder.setInput({ messages:
|
|
3950
|
+
if (options2?.inputMessages) {
|
|
3951
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3129
3952
|
}
|
|
3130
3953
|
const completionParts = [];
|
|
3131
3954
|
for (const block of response.content) {
|
|
@@ -3148,8 +3971,8 @@ var AnthropicAdapter = class {
|
|
|
3148
3971
|
}
|
|
3149
3972
|
builder.setMetadata({
|
|
3150
3973
|
total_tokens: totalTokens,
|
|
3151
|
-
cost_usd:
|
|
3152
|
-
latency_ms:
|
|
3974
|
+
cost_usd: options2?.costUsd,
|
|
3975
|
+
latency_ms: options2?.latencyMs,
|
|
3153
3976
|
model: response.model
|
|
3154
3977
|
});
|
|
3155
3978
|
return builder.build();
|
|
@@ -3162,10 +3985,10 @@ var GeminiAdapter = class {
|
|
|
3162
3985
|
constructor(agentId2) {
|
|
3163
3986
|
this.agentId = agentId2;
|
|
3164
3987
|
}
|
|
3165
|
-
traceFromResponse(response,
|
|
3988
|
+
traceFromResponse(response, options2) {
|
|
3166
3989
|
const builder = new TraceBuilder(this.agentId);
|
|
3167
|
-
if (
|
|
3168
|
-
builder.setInput({ text:
|
|
3990
|
+
if (options2?.inputText) {
|
|
3991
|
+
builder.setInput({ text: options2.inputText });
|
|
3169
3992
|
}
|
|
3170
3993
|
let completionText = "";
|
|
3171
3994
|
if (response.text != null) {
|
|
@@ -3185,9 +4008,9 @@ var GeminiAdapter = class {
|
|
|
3185
4008
|
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
3186
4009
|
builder.setOutput({ message: completionText });
|
|
3187
4010
|
builder.setMetadata({
|
|
3188
|
-
cost_usd:
|
|
3189
|
-
latency_ms:
|
|
3190
|
-
model:
|
|
4011
|
+
cost_usd: options2?.costUsd,
|
|
4012
|
+
latency_ms: options2?.latencyMs,
|
|
4013
|
+
model: options2?.model
|
|
3191
4014
|
});
|
|
3192
4015
|
return builder.build();
|
|
3193
4016
|
}
|
|
@@ -3199,10 +4022,10 @@ var OllamaAdapter = class {
|
|
|
3199
4022
|
constructor(agentId2) {
|
|
3200
4023
|
this.agentId = agentId2;
|
|
3201
4024
|
}
|
|
3202
|
-
traceFromResponse(response,
|
|
4025
|
+
traceFromResponse(response, options2) {
|
|
3203
4026
|
const builder = new TraceBuilder(this.agentId);
|
|
3204
|
-
if (
|
|
3205
|
-
builder.setInput({ messages:
|
|
4027
|
+
if (options2?.inputMessages) {
|
|
4028
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3206
4029
|
}
|
|
3207
4030
|
const completionText = response.message?.content ?? "";
|
|
3208
4031
|
builder.addLlmCall("completion", {
|
|
@@ -3216,7 +4039,7 @@ var OllamaAdapter = class {
|
|
|
3216
4039
|
}
|
|
3217
4040
|
builder.setMetadata({
|
|
3218
4041
|
total_tokens: totalTokens,
|
|
3219
|
-
latency_ms:
|
|
4042
|
+
latency_ms: options2?.latencyMs,
|
|
3220
4043
|
model: response.model
|
|
3221
4044
|
});
|
|
3222
4045
|
return builder.build();
|
|
@@ -3224,7 +4047,7 @@ var OllamaAdapter = class {
|
|
|
3224
4047
|
};
|
|
3225
4048
|
|
|
3226
4049
|
// src/adapters/otel.ts
|
|
3227
|
-
var
|
|
4050
|
+
var import_node_child_process5 = require("child_process");
|
|
3228
4051
|
var CONVERT_TIMEOUT_MS = 6e4;
|
|
3229
4052
|
var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
3230
4053
|
var STATUS_CODE_ERROR = 2;
|
|
@@ -3247,7 +4070,7 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3247
4070
|
* @throws RangeError when a parsed export holds a non-finite number.
|
|
3248
4071
|
* @throws OTLPConversionError when the engine times out, overflows the output buffer or
|
|
3249
4072
|
* prints no trace.
|
|
3250
|
-
* @throws Error
|
|
4073
|
+
* @throws Error when the engine cannot start.
|
|
3251
4074
|
*/
|
|
3252
4075
|
static async fromOtlp(otlp, agentId2) {
|
|
3253
4076
|
const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
|
|
@@ -3259,33 +4082,9 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3259
4082
|
const args = ["trace", "convert"];
|
|
3260
4083
|
if (agentId2 !== void 0) args.push("--agent-id", agentId2);
|
|
3261
4084
|
args.push("-");
|
|
3262
|
-
const
|
|
3263
|
-
input: payload,
|
|
3264
|
-
timeout: CONVERT_TIMEOUT_MS,
|
|
3265
|
-
maxBuffer: MAX_TRACE_OUTPUT_BYTES
|
|
3266
|
-
});
|
|
3267
|
-
if (proc.error) {
|
|
3268
|
-
const code = proc.error.code;
|
|
3269
|
-
if (code === "ETIMEDOUT") {
|
|
3270
|
-
throw new OTLPConversionError(
|
|
3271
|
-
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
3272
|
-
);
|
|
3273
|
-
}
|
|
3274
|
-
if (code === "ENOBUFS") {
|
|
3275
|
-
throw new OTLPConversionError(
|
|
3276
|
-
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
3277
|
-
);
|
|
3278
|
-
}
|
|
3279
|
-
throw proc.error;
|
|
3280
|
-
}
|
|
3281
|
-
if (proc.status !== 0) {
|
|
3282
|
-
const message = proc.stderr.toString("utf8").trim();
|
|
3283
|
-
throw new OTLPConversionError(
|
|
3284
|
-
message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
|
|
3285
|
-
);
|
|
3286
|
-
}
|
|
4085
|
+
const stdout = await runConvert(await resolveEngine(), args, payload);
|
|
3287
4086
|
try {
|
|
3288
|
-
return JSON.parse(
|
|
4087
|
+
return JSON.parse(stdout);
|
|
3289
4088
|
} catch (err) {
|
|
3290
4089
|
throw new OTLPConversionError(
|
|
3291
4090
|
`attest-engine trace convert printed output that is not JSON: ${String(err)}`
|
|
@@ -3293,6 +4092,65 @@ var OTelAdapter = class _OTelAdapter {
|
|
|
3293
4092
|
}
|
|
3294
4093
|
}
|
|
3295
4094
|
};
|
|
4095
|
+
function runConvert(engine, args, payload) {
|
|
4096
|
+
return new Promise((resolve4, reject) => {
|
|
4097
|
+
const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
4098
|
+
let timer;
|
|
4099
|
+
child.on("error", (e) => {
|
|
4100
|
+
clearTimeout(timer);
|
|
4101
|
+
reject(e);
|
|
4102
|
+
});
|
|
4103
|
+
const { stdin, stdout, stderr } = child;
|
|
4104
|
+
if (stdin === null || stdout === null || stderr === null) return;
|
|
4105
|
+
const out = [];
|
|
4106
|
+
const err = [];
|
|
4107
|
+
let outBytes = 0;
|
|
4108
|
+
let failure = null;
|
|
4109
|
+
const fail = (e) => {
|
|
4110
|
+
failure ??= e;
|
|
4111
|
+
child.kill("SIGKILL");
|
|
4112
|
+
};
|
|
4113
|
+
timer = setTimeout(() => {
|
|
4114
|
+
fail(
|
|
4115
|
+
new OTLPConversionError(
|
|
4116
|
+
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
4117
|
+
)
|
|
4118
|
+
);
|
|
4119
|
+
}, CONVERT_TIMEOUT_MS);
|
|
4120
|
+
stdout.on("data", (chunk) => {
|
|
4121
|
+
outBytes += chunk.length;
|
|
4122
|
+
if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
|
|
4123
|
+
fail(
|
|
4124
|
+
new OTLPConversionError(
|
|
4125
|
+
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
4126
|
+
)
|
|
4127
|
+
);
|
|
4128
|
+
return;
|
|
4129
|
+
}
|
|
4130
|
+
out.push(chunk);
|
|
4131
|
+
});
|
|
4132
|
+
stderr.on("data", (chunk) => err.push(chunk));
|
|
4133
|
+
stdin.on("error", () => void 0);
|
|
4134
|
+
child.on("close", (code, signal) => {
|
|
4135
|
+
clearTimeout(timer);
|
|
4136
|
+
if (failure !== null) {
|
|
4137
|
+
reject(failure);
|
|
4138
|
+
return;
|
|
4139
|
+
}
|
|
4140
|
+
if (code !== 0) {
|
|
4141
|
+
const message = Buffer.concat(err).toString("utf8").trim();
|
|
4142
|
+
reject(
|
|
4143
|
+
new OTLPConversionError(
|
|
4144
|
+
message || `attest-engine trace convert exited ${String(code ?? signal)}`
|
|
4145
|
+
)
|
|
4146
|
+
);
|
|
4147
|
+
return;
|
|
4148
|
+
}
|
|
4149
|
+
resolve4(Buffer.concat(out).toString("utf8"));
|
|
4150
|
+
});
|
|
4151
|
+
stdin.end(payload);
|
|
4152
|
+
});
|
|
4153
|
+
}
|
|
3296
4154
|
function hrTimeToNanos(time, spanName) {
|
|
3297
4155
|
if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
|
|
3298
4156
|
throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
|
|
@@ -3398,13 +4256,13 @@ var LangChainAdapter = class {
|
|
|
3398
4256
|
* @param options.model - Model name override.
|
|
3399
4257
|
* @returns A finalized Trace object.
|
|
3400
4258
|
*/
|
|
3401
|
-
buildTrace(
|
|
4259
|
+
buildTrace(options2) {
|
|
3402
4260
|
const builder = new TraceBuilder(this.agentId);
|
|
3403
4261
|
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
3404
4262
|
let totalTokens;
|
|
3405
4263
|
for (const llm of this.llmCalls) {
|
|
3406
4264
|
const args = {};
|
|
3407
|
-
if (llm.model ??
|
|
4265
|
+
if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
|
|
3408
4266
|
const result = { completion: llm.text };
|
|
3409
4267
|
if (llm.tokens !== void 0) {
|
|
3410
4268
|
result.tokens = llm.tokens;
|
|
@@ -3421,9 +4279,9 @@ var LangChainAdapter = class {
|
|
|
3421
4279
|
builder.setOutput({ message: outputMessage });
|
|
3422
4280
|
builder.setMetadata({
|
|
3423
4281
|
total_tokens: totalTokens,
|
|
3424
|
-
cost_usd:
|
|
4282
|
+
cost_usd: options2?.costUsd,
|
|
3425
4283
|
latency_ms: latencyMs,
|
|
3426
|
-
model:
|
|
4284
|
+
model: options2?.model ?? this.llmCalls[0]?.model
|
|
3427
4285
|
});
|
|
3428
4286
|
return builder.build();
|
|
3429
4287
|
}
|
|
@@ -3438,6 +4296,7 @@ var LangChainAdapter = class {
|
|
|
3438
4296
|
AttestClient,
|
|
3439
4297
|
CONFUSED_USER,
|
|
3440
4298
|
COOPERATIVE_USER,
|
|
4299
|
+
CaptureError,
|
|
3441
4300
|
ContinuousEvalRunner,
|
|
3442
4301
|
ENGINE_VERSION,
|
|
3443
4302
|
ERR_ASSERTION_ERROR,
|
|
@@ -3446,6 +4305,7 @@ var LangChainAdapter = class {
|
|
|
3446
4305
|
ERR_PROVIDER_ERROR,
|
|
3447
4306
|
ERR_SESSION_ERROR,
|
|
3448
4307
|
ERR_TIMEOUT,
|
|
4308
|
+
EgressBlockedError,
|
|
3449
4309
|
EngineManager,
|
|
3450
4310
|
EngineTimeoutError,
|
|
3451
4311
|
ExpectChain,
|
|
@@ -3470,8 +4330,12 @@ var LangChainAdapter = class {
|
|
|
3470
4330
|
REASON_MISSING_EVIDENCE,
|
|
3471
4331
|
REASON_PROVIDER_NOT_CONFIGURED,
|
|
3472
4332
|
REASON_PROVIDER_UNAVAILABLE,
|
|
4333
|
+
REASON_REPLAY_MISS,
|
|
3473
4334
|
REASON_SHORT_CIRCUITED,
|
|
3474
4335
|
REASON_SIMULATION,
|
|
4336
|
+
ReplayError,
|
|
4337
|
+
ReplayMissError,
|
|
4338
|
+
ReplayNotEvaluatedError,
|
|
3475
4339
|
STATUS_HARD_FAIL,
|
|
3476
4340
|
STATUS_NOT_EVALUATED,
|
|
3477
4341
|
STATUS_PASS,
|
|
@@ -3486,6 +4350,7 @@ var LangChainAdapter = class {
|
|
|
3486
4350
|
TIER_3,
|
|
3487
4351
|
TYPE_CONSTRAINT,
|
|
3488
4352
|
TYPE_CONTENT,
|
|
4353
|
+
TYPE_EFFECT,
|
|
3489
4354
|
TYPE_EMBEDDING,
|
|
3490
4355
|
TYPE_LLM_JUDGE,
|
|
3491
4356
|
TYPE_PLUGIN,
|
|
@@ -3495,6 +4360,8 @@ var LangChainAdapter = class {
|
|
|
3495
4360
|
TraceBuilder,
|
|
3496
4361
|
TraceTree,
|
|
3497
4362
|
VERSION,
|
|
4363
|
+
WorldChecks,
|
|
4364
|
+
WorldError,
|
|
3498
4365
|
activeBuilder,
|
|
3499
4366
|
addUsage,
|
|
3500
4367
|
agent,
|
|
@@ -3505,28 +4372,43 @@ var LangChainAdapter = class {
|
|
|
3505
4372
|
config,
|
|
3506
4373
|
decodeResponse,
|
|
3507
4374
|
delegate,
|
|
4375
|
+
dslCall,
|
|
4376
|
+
egressBlockRequested,
|
|
4377
|
+
emitVitest,
|
|
3508
4378
|
encodeRequest,
|
|
3509
4379
|
executePluginAssertion,
|
|
3510
4380
|
extractId,
|
|
3511
4381
|
extractResult,
|
|
4382
|
+
firstMiss,
|
|
3512
4383
|
formatUsage,
|
|
3513
4384
|
getSampleRate,
|
|
4385
|
+
installEgressGuard,
|
|
4386
|
+
isProviderHost,
|
|
3514
4387
|
isSimulationMode,
|
|
3515
4388
|
layerForType,
|
|
4389
|
+
liveLine,
|
|
4390
|
+
loadFixture,
|
|
3516
4391
|
loadLabelsCSV,
|
|
3517
4392
|
loadLabelsJSONL,
|
|
3518
4393
|
parseUsage,
|
|
3519
4394
|
previewLine,
|
|
3520
4395
|
promptHash,
|
|
4396
|
+
record,
|
|
3521
4397
|
renderAssertionFailure,
|
|
3522
4398
|
renderDiagnostics,
|
|
3523
4399
|
renderSummary,
|
|
4400
|
+
renderVitest,
|
|
3524
4401
|
requireSimulationCapability,
|
|
3525
4402
|
resetConfig,
|
|
4403
|
+
resolveMode,
|
|
3526
4404
|
simulationEnvEnabled,
|
|
4405
|
+
startReplay,
|
|
4406
|
+
startWorld,
|
|
3527
4407
|
sumUsage,
|
|
4408
|
+
testPathFor,
|
|
3528
4409
|
tier,
|
|
3529
4410
|
traceId,
|
|
4411
|
+
uninstallEgressGuard,
|
|
3530
4412
|
usageSummaryLine,
|
|
3531
4413
|
zeroUsage
|
|
3532
4414
|
});
|