@attest-ai/core 1.0.0-rc.1 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +1256 -200
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +437 -24
- package/dist/cjs/proto/index.cjs +21 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +91 -2
- package/dist/esm/cli.d.ts +6 -1
- package/dist/esm/cli.js +310 -20
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +437 -24
- package/dist/esm/index.js +1222 -199
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +91 -2
- package/dist/esm/proto/index.js +16 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +3 -3
package/dist/cjs/index.cjs
CHANGED
|
@@ -38,6 +38,7 @@ __export(index_exports, {
|
|
|
38
38
|
AttestClient: () => AttestClient,
|
|
39
39
|
CONFUSED_USER: () => CONFUSED_USER,
|
|
40
40
|
COOPERATIVE_USER: () => COOPERATIVE_USER,
|
|
41
|
+
CaptureError: () => CaptureError,
|
|
41
42
|
ContinuousEvalRunner: () => ContinuousEvalRunner,
|
|
42
43
|
ENGINE_VERSION: () => ENGINE_VERSION,
|
|
43
44
|
ERR_ASSERTION_ERROR: () => ERR_ASSERTION_ERROR,
|
|
@@ -46,6 +47,7 @@ __export(index_exports, {
|
|
|
46
47
|
ERR_PROVIDER_ERROR: () => ERR_PROVIDER_ERROR,
|
|
47
48
|
ERR_SESSION_ERROR: () => ERR_SESSION_ERROR,
|
|
48
49
|
ERR_TIMEOUT: () => ERR_TIMEOUT,
|
|
50
|
+
EgressBlockedError: () => EgressBlockedError,
|
|
49
51
|
EngineManager: () => EngineManager,
|
|
50
52
|
EngineTimeoutError: () => EngineTimeoutError,
|
|
51
53
|
ExpectChain: () => ExpectChain,
|
|
@@ -54,6 +56,7 @@ __export(index_exports, {
|
|
|
54
56
|
GeminiAdapter: () => GeminiAdapter,
|
|
55
57
|
LangChainAdapter: () => LangChainAdapter,
|
|
56
58
|
ManualAdapter: () => ManualAdapter,
|
|
59
|
+
OTLPConversionError: () => OTLPConversionError,
|
|
57
60
|
OTelAdapter: () => OTelAdapter,
|
|
58
61
|
OUTCOME_FAIL: () => OUTCOME_FAIL,
|
|
59
62
|
OUTCOME_PASS: () => OUTCOME_PASS,
|
|
@@ -64,11 +67,17 @@ __export(index_exports, {
|
|
|
64
67
|
ProtocolDesyncError: () => ProtocolDesyncError,
|
|
65
68
|
ProtocolDiagnosticBuffer: () => ProtocolDiagnosticBuffer,
|
|
66
69
|
ProtocolError: () => ProtocolError,
|
|
70
|
+
REASON_BUDGET_EXHAUSTED: () => REASON_BUDGET_EXHAUSTED,
|
|
71
|
+
REASON_JUDGE_ABSTAINED: () => REASON_JUDGE_ABSTAINED,
|
|
67
72
|
REASON_MISSING_EVIDENCE: () => REASON_MISSING_EVIDENCE,
|
|
68
73
|
REASON_PROVIDER_NOT_CONFIGURED: () => REASON_PROVIDER_NOT_CONFIGURED,
|
|
69
74
|
REASON_PROVIDER_UNAVAILABLE: () => REASON_PROVIDER_UNAVAILABLE,
|
|
75
|
+
REASON_REPLAY_MISS: () => REASON_REPLAY_MISS,
|
|
70
76
|
REASON_SHORT_CIRCUITED: () => REASON_SHORT_CIRCUITED,
|
|
71
77
|
REASON_SIMULATION: () => REASON_SIMULATION,
|
|
78
|
+
ReplayError: () => ReplayError,
|
|
79
|
+
ReplayMissError: () => ReplayMissError,
|
|
80
|
+
ReplayNotEvaluatedError: () => ReplayNotEvaluatedError,
|
|
72
81
|
STATUS_HARD_FAIL: () => STATUS_HARD_FAIL,
|
|
73
82
|
STATUS_NOT_EVALUATED: () => STATUS_NOT_EVALUATED,
|
|
74
83
|
STATUS_PASS: () => STATUS_PASS,
|
|
@@ -83,6 +92,7 @@ __export(index_exports, {
|
|
|
83
92
|
TIER_3: () => TIER_3,
|
|
84
93
|
TYPE_CONSTRAINT: () => TYPE_CONSTRAINT,
|
|
85
94
|
TYPE_CONTENT: () => TYPE_CONTENT,
|
|
95
|
+
TYPE_EFFECT: () => TYPE_EFFECT,
|
|
86
96
|
TYPE_EMBEDDING: () => TYPE_EMBEDDING,
|
|
87
97
|
TYPE_LLM_JUDGE: () => TYPE_LLM_JUDGE,
|
|
88
98
|
TYPE_PLUGIN: () => TYPE_PLUGIN,
|
|
@@ -92,7 +102,10 @@ __export(index_exports, {
|
|
|
92
102
|
TraceBuilder: () => TraceBuilder,
|
|
93
103
|
TraceTree: () => TraceTree,
|
|
94
104
|
VERSION: () => VERSION,
|
|
105
|
+
WorldChecks: () => WorldChecks,
|
|
106
|
+
WorldError: () => WorldError,
|
|
95
107
|
activeBuilder: () => activeBuilder,
|
|
108
|
+
addUsage: () => addUsage,
|
|
96
109
|
agent: () => agent,
|
|
97
110
|
agentId: () => agentId,
|
|
98
111
|
assertionId: () => assertionId,
|
|
@@ -101,25 +114,45 @@ __export(index_exports, {
|
|
|
101
114
|
config: () => config,
|
|
102
115
|
decodeResponse: () => decodeResponse,
|
|
103
116
|
delegate: () => delegate,
|
|
117
|
+
dslCall: () => dslCall,
|
|
118
|
+
egressBlockRequested: () => egressBlockRequested,
|
|
119
|
+
emitVitest: () => emitVitest,
|
|
104
120
|
encodeRequest: () => encodeRequest,
|
|
105
121
|
executePluginAssertion: () => executePluginAssertion,
|
|
106
122
|
extractId: () => extractId,
|
|
107
123
|
extractResult: () => extractResult,
|
|
124
|
+
firstMiss: () => firstMiss,
|
|
125
|
+
formatUsage: () => formatUsage,
|
|
108
126
|
getSampleRate: () => getSampleRate,
|
|
127
|
+
installEgressGuard: () => installEgressGuard,
|
|
128
|
+
isProviderHost: () => isProviderHost,
|
|
109
129
|
isSimulationMode: () => isSimulationMode,
|
|
110
130
|
layerForType: () => layerForType,
|
|
131
|
+
liveLine: () => liveLine,
|
|
132
|
+
loadFixture: () => loadFixture,
|
|
111
133
|
loadLabelsCSV: () => loadLabelsCSV,
|
|
112
134
|
loadLabelsJSONL: () => loadLabelsJSONL,
|
|
135
|
+
parseUsage: () => parseUsage,
|
|
113
136
|
previewLine: () => previewLine,
|
|
114
137
|
promptHash: () => promptHash,
|
|
138
|
+
record: () => record,
|
|
115
139
|
renderAssertionFailure: () => renderAssertionFailure,
|
|
116
140
|
renderDiagnostics: () => renderDiagnostics,
|
|
117
141
|
renderSummary: () => renderSummary,
|
|
142
|
+
renderVitest: () => renderVitest,
|
|
118
143
|
requireSimulationCapability: () => requireSimulationCapability,
|
|
119
144
|
resetConfig: () => resetConfig,
|
|
145
|
+
resolveMode: () => resolveMode,
|
|
120
146
|
simulationEnvEnabled: () => simulationEnvEnabled,
|
|
147
|
+
startReplay: () => startReplay,
|
|
148
|
+
startWorld: () => startWorld,
|
|
149
|
+
sumUsage: () => sumUsage,
|
|
150
|
+
testPathFor: () => testPathFor,
|
|
121
151
|
tier: () => tier,
|
|
122
|
-
traceId: () => traceId
|
|
152
|
+
traceId: () => traceId,
|
|
153
|
+
uninstallEgressGuard: () => uninstallEgressGuard,
|
|
154
|
+
usageSummaryLine: () => usageSummaryLine,
|
|
155
|
+
zeroUsage: () => zeroUsage
|
|
123
156
|
});
|
|
124
157
|
module.exports = __toCommonJS(index_exports);
|
|
125
158
|
|
|
@@ -137,12 +170,16 @@ var REASON_PROVIDER_NOT_CONFIGURED = "provider_not_configured";
|
|
|
137
170
|
var REASON_SIMULATION = "simulation";
|
|
138
171
|
var REASON_MISSING_EVIDENCE = "missing_evidence";
|
|
139
172
|
var REASON_SHORT_CIRCUITED = "short_circuited";
|
|
173
|
+
var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
|
|
174
|
+
var REASON_JUDGE_ABSTAINED = "judge_abstained";
|
|
175
|
+
var REASON_REPLAY_MISS = "replay_miss";
|
|
140
176
|
var OUTCOME_PASS = "pass";
|
|
141
177
|
var OUTCOME_FAIL = "fail";
|
|
142
178
|
var OUTCOME_SKIP = "skip";
|
|
143
179
|
var TYPE_SCHEMA = "schema";
|
|
144
180
|
var TYPE_CONSTRAINT = "constraint";
|
|
145
181
|
var TYPE_TRACE = "trace";
|
|
182
|
+
var TYPE_EFFECT = "effect";
|
|
146
183
|
var TYPE_CONTENT = "content";
|
|
147
184
|
var TYPE_EMBEDDING = "embedding";
|
|
148
185
|
var TYPE_LLM_JUDGE = "llm_judge";
|
|
@@ -180,6 +217,7 @@ function layerForType(assertionType) {
|
|
|
180
217
|
return 2;
|
|
181
218
|
case "trace":
|
|
182
219
|
case "trace_tree":
|
|
220
|
+
case "effect":
|
|
183
221
|
return 3;
|
|
184
222
|
case "content":
|
|
185
223
|
return 4;
|
|
@@ -227,6 +265,12 @@ var ProtocolDesyncError = class extends Error {
|
|
|
227
265
|
this.diagnostics = diagnostics;
|
|
228
266
|
}
|
|
229
267
|
};
|
|
268
|
+
var OTLPConversionError = class extends Error {
|
|
269
|
+
constructor(message) {
|
|
270
|
+
super(message);
|
|
271
|
+
this.name = "OTLPConversionError";
|
|
272
|
+
}
|
|
273
|
+
};
|
|
230
274
|
|
|
231
275
|
// src/proto/diagnostics.ts
|
|
232
276
|
var DEFAULT_BUFFER_SIZE = 32;
|
|
@@ -333,8 +377,8 @@ function extractId(response) {
|
|
|
333
377
|
}
|
|
334
378
|
|
|
335
379
|
// src/version.ts
|
|
336
|
-
var VERSION = "1.0.0-rc.
|
|
337
|
-
var ENGINE_VERSION = "1.0.0-rc.
|
|
380
|
+
var VERSION = "1.0.0-rc.3";
|
|
381
|
+
var ENGINE_VERSION = "1.0.0-rc.3";
|
|
338
382
|
|
|
339
383
|
// src/engine-manager.ts
|
|
340
384
|
var import_node_child_process = require("child_process");
|
|
@@ -402,7 +446,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
402
446
|
new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
|
|
403
447
|
);
|
|
404
448
|
}
|
|
405
|
-
return new Promise((
|
|
449
|
+
return new Promise((resolve4, reject) => {
|
|
406
450
|
const client = url.startsWith("https:") ? https : http;
|
|
407
451
|
client.get(url, (res) => {
|
|
408
452
|
const status = res.statusCode ?? 0;
|
|
@@ -413,7 +457,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
413
457
|
return;
|
|
414
458
|
}
|
|
415
459
|
res.resume();
|
|
416
|
-
|
|
460
|
+
resolve4(httpsGet(location, redirectCount + 1));
|
|
417
461
|
return;
|
|
418
462
|
}
|
|
419
463
|
if (status < 200 || status >= 300) {
|
|
@@ -423,7 +467,7 @@ function httpsGet(url, redirectCount = 0) {
|
|
|
423
467
|
}
|
|
424
468
|
const chunks = [];
|
|
425
469
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
426
|
-
res.on("end", () =>
|
|
470
|
+
res.on("end", () => resolve4(Buffer.concat(chunks)));
|
|
427
471
|
res.on("error", reject);
|
|
428
472
|
}).on("error", reject);
|
|
429
473
|
});
|
|
@@ -554,8 +598,8 @@ Install options:
|
|
|
554
598
|
}
|
|
555
599
|
return [await downloadEngine(), "download"];
|
|
556
600
|
}
|
|
557
|
-
async function resolveEngine(explicitPath,
|
|
558
|
-
const [enginePath, source] = await locateEngine(explicitPath,
|
|
601
|
+
async function resolveEngine(explicitPath, options2 = {}) {
|
|
602
|
+
const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
|
|
559
603
|
const found = engineVersion(enginePath);
|
|
560
604
|
if (found !== ENGINE_VERSION) {
|
|
561
605
|
throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
|
|
@@ -570,9 +614,9 @@ var EngineManager = class {
|
|
|
570
614
|
initialized = false;
|
|
571
615
|
requestId = 0;
|
|
572
616
|
initResult = null;
|
|
573
|
-
constructor(
|
|
574
|
-
this.enginePath =
|
|
575
|
-
this.logLevel =
|
|
617
|
+
constructor(options2) {
|
|
618
|
+
this.enginePath = options2?.enginePath ?? "";
|
|
619
|
+
this.logLevel = options2?.logLevel ?? "warn";
|
|
576
620
|
}
|
|
577
621
|
async start() {
|
|
578
622
|
this.enginePath = await resolveEngine(this.enginePath || void 0);
|
|
@@ -650,11 +694,11 @@ var EngineManager = class {
|
|
|
650
694
|
}
|
|
651
695
|
this.requestId += 1;
|
|
652
696
|
const requestStr = encodeRequest(this.requestId, method, params);
|
|
653
|
-
return new Promise((
|
|
697
|
+
return new Promise((resolve4, reject) => {
|
|
654
698
|
const onLine = (line) => {
|
|
655
699
|
try {
|
|
656
700
|
const response = decodeResponse(line);
|
|
657
|
-
|
|
701
|
+
resolve4(extractResult(response));
|
|
658
702
|
} catch (err) {
|
|
659
703
|
reject(err);
|
|
660
704
|
}
|
|
@@ -688,17 +732,17 @@ var EngineManager = class {
|
|
|
688
732
|
// src/config.ts
|
|
689
733
|
var _simulationMode = false;
|
|
690
734
|
var _sampleRate = 0;
|
|
691
|
-
function config(
|
|
692
|
-
if (
|
|
693
|
-
_simulationMode =
|
|
735
|
+
function config(options2) {
|
|
736
|
+
if (options2.simulation !== void 0) {
|
|
737
|
+
_simulationMode = options2.simulation;
|
|
694
738
|
}
|
|
695
|
-
if (
|
|
696
|
-
if (
|
|
739
|
+
if (options2.sampleRate !== void 0) {
|
|
740
|
+
if (options2.sampleRate < 0 || options2.sampleRate > 1) {
|
|
697
741
|
throw new RangeError(
|
|
698
|
-
`sampleRate must be between 0.0 and 1.0, got ${
|
|
742
|
+
`sampleRate must be between 0.0 and 1.0, got ${options2.sampleRate}`
|
|
699
743
|
);
|
|
700
744
|
}
|
|
701
|
-
_sampleRate =
|
|
745
|
+
_sampleRate = options2.sampleRate;
|
|
702
746
|
}
|
|
703
747
|
}
|
|
704
748
|
function isSimulationMode() {
|
|
@@ -727,6 +771,76 @@ function resetConfig() {
|
|
|
727
771
|
_sampleRate = 0;
|
|
728
772
|
}
|
|
729
773
|
|
|
774
|
+
// src/usage.ts
|
|
775
|
+
var INT_FIELDS = [
|
|
776
|
+
"model_calls",
|
|
777
|
+
"judge_calls",
|
|
778
|
+
"input_tokens",
|
|
779
|
+
"output_tokens",
|
|
780
|
+
"cached_tokens",
|
|
781
|
+
"unpriced_calls",
|
|
782
|
+
"budget_refused"
|
|
783
|
+
];
|
|
784
|
+
var USD_FIELDS = ["cost_usd", "retry_cost_usd"];
|
|
785
|
+
function zeroUsage() {
|
|
786
|
+
return {
|
|
787
|
+
model_calls: 0,
|
|
788
|
+
judge_calls: 0,
|
|
789
|
+
input_tokens: 0,
|
|
790
|
+
output_tokens: 0,
|
|
791
|
+
cached_tokens: 0,
|
|
792
|
+
cost_usd: 0,
|
|
793
|
+
retry_cost_usd: 0,
|
|
794
|
+
unpriced_calls: 0,
|
|
795
|
+
budget_refused: 0
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
function addUsage(a, b) {
|
|
799
|
+
return {
|
|
800
|
+
model_calls: a.model_calls + b.model_calls,
|
|
801
|
+
judge_calls: a.judge_calls + b.judge_calls,
|
|
802
|
+
input_tokens: a.input_tokens + b.input_tokens,
|
|
803
|
+
output_tokens: a.output_tokens + b.output_tokens,
|
|
804
|
+
cached_tokens: a.cached_tokens + b.cached_tokens,
|
|
805
|
+
cost_usd: a.cost_usd + b.cost_usd,
|
|
806
|
+
retry_cost_usd: a.retry_cost_usd + b.retry_cost_usd,
|
|
807
|
+
unpriced_calls: a.unpriced_calls + b.unpriced_calls,
|
|
808
|
+
budget_refused: a.budget_refused + b.budget_refused
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
function sumUsage(usages) {
|
|
812
|
+
let total = zeroUsage();
|
|
813
|
+
for (const u of usages) total = addUsage(total, u);
|
|
814
|
+
return total;
|
|
815
|
+
}
|
|
816
|
+
function parseUsage(raw) {
|
|
817
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
818
|
+
throw new TypeError(`evaluate_batch result: usage must be an object, got ${JSON.stringify(raw)}`);
|
|
819
|
+
}
|
|
820
|
+
const obj = raw;
|
|
821
|
+
const field = (name, integer) => {
|
|
822
|
+
const v = obj[name];
|
|
823
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0 || integer && !Number.isInteger(v)) {
|
|
824
|
+
const kind = integer ? "a non-negative integer" : "a non-negative number";
|
|
825
|
+
throw new TypeError(`evaluate_batch result: usage.${name} must be ${kind}, got ${JSON.stringify(v)}`);
|
|
826
|
+
}
|
|
827
|
+
return v;
|
|
828
|
+
};
|
|
829
|
+
const ints = Object.fromEntries(INT_FIELDS.map((f) => [f, field(f, true)]));
|
|
830
|
+
const usd = Object.fromEntries(USD_FIELDS.map((f) => [f, field(f, false)]));
|
|
831
|
+
return { ...ints, ...usd };
|
|
832
|
+
}
|
|
833
|
+
function formatUsage(u) {
|
|
834
|
+
let line = `${u.model_calls} model calls \xB7 ${u.input_tokens + u.output_tokens} tokens (${u.cached_tokens} cached) \xB7 $${u.cost_usd.toFixed(4)} \xB7 retry cost $${u.retry_cost_usd.toFixed(4)}`;
|
|
835
|
+
if (u.budget_refused > 0) line += ` \xB7 ${u.budget_refused} calls refused by budget`;
|
|
836
|
+
if (u.unpriced_calls > 0) line += ` \xB7 ${u.unpriced_calls} unpriced calls not in $`;
|
|
837
|
+
return line;
|
|
838
|
+
}
|
|
839
|
+
function usageSummaryLine(u) {
|
|
840
|
+
if (u.model_calls === 0 && u.budget_refused === 0) return void 0;
|
|
841
|
+
return `attest usage: ${formatUsage(u)}`;
|
|
842
|
+
}
|
|
843
|
+
|
|
730
844
|
// src/client.ts
|
|
731
845
|
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
732
846
|
var DEFAULT_DIAGNOSTIC_BUFFER = 32;
|
|
@@ -784,16 +898,16 @@ var AttestClient = class {
|
|
|
784
898
|
desyncThreshold;
|
|
785
899
|
desyncWindowMs;
|
|
786
900
|
desynced = false;
|
|
787
|
-
constructor(engine,
|
|
901
|
+
constructor(engine, options2) {
|
|
788
902
|
this.engine = engine;
|
|
789
|
-
this.logger =
|
|
903
|
+
this.logger = options2?.logger ?? envLoggerDefault();
|
|
790
904
|
this.diagnosticBuffer = new ProtocolDiagnosticBuffer(
|
|
791
|
-
|
|
905
|
+
options2?.diagnosticBufferSize ?? DEFAULT_DIAGNOSTIC_BUFFER
|
|
792
906
|
);
|
|
793
|
-
this.desyncThreshold =
|
|
794
|
-
this.desyncWindowMs =
|
|
795
|
-
if (
|
|
796
|
-
this.diagnosticListeners.add(
|
|
907
|
+
this.desyncThreshold = options2?.desyncThreshold ?? DEFAULT_DESYNC_THRESHOLD;
|
|
908
|
+
this.desyncWindowMs = options2?.desyncWindowMs ?? DEFAULT_DESYNC_WINDOW_MS;
|
|
909
|
+
if (options2?.onDiagnostic !== void 0) {
|
|
910
|
+
this.diagnosticListeners.add(options2.onDiagnostic);
|
|
797
911
|
}
|
|
798
912
|
}
|
|
799
913
|
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
@@ -951,12 +1065,12 @@ var AttestClient = class {
|
|
|
951
1065
|
throw new Error("Engine process not started.");
|
|
952
1066
|
}
|
|
953
1067
|
let capturedReqId;
|
|
954
|
-
const requestPromise = new Promise((
|
|
1068
|
+
const requestPromise = new Promise((resolve4, reject) => {
|
|
955
1069
|
this.writeChain = this.writeChain.then(() => {
|
|
956
1070
|
this.requestId += 1;
|
|
957
1071
|
const reqId = this.requestId;
|
|
958
1072
|
capturedReqId = reqId;
|
|
959
|
-
this.pending.set(reqId, { resolve, reject });
|
|
1073
|
+
this.pending.set(reqId, { resolve: resolve4, reject });
|
|
960
1074
|
const requestStr = encodeRequest(reqId, method, params);
|
|
961
1075
|
cp.stdin.write(requestStr, (err) => {
|
|
962
1076
|
if (err) {
|
|
@@ -974,22 +1088,23 @@ var AttestClient = class {
|
|
|
974
1088
|
throw err;
|
|
975
1089
|
});
|
|
976
1090
|
}
|
|
977
|
-
async evaluateBatch(trace, assertions,
|
|
1091
|
+
async evaluateBatch(trace, assertions, options2) {
|
|
978
1092
|
if (isSimulationMode()) {
|
|
979
1093
|
requireSimulationCapability(this.engine["initResult"]);
|
|
980
1094
|
}
|
|
981
1095
|
const params = {
|
|
982
1096
|
trace,
|
|
983
|
-
assertions: [...assertions]
|
|
1097
|
+
assertions: [...assertions],
|
|
1098
|
+
...options2?.budget !== void 0 ? { budget: options2.budget } : {}
|
|
984
1099
|
};
|
|
985
1100
|
const raw = await this.sendRequest(
|
|
986
1101
|
"evaluate_batch",
|
|
987
1102
|
params,
|
|
988
|
-
|
|
1103
|
+
options2?.timeout
|
|
989
1104
|
);
|
|
990
|
-
return raw;
|
|
1105
|
+
return { ...raw, usage: parseUsage(raw["usage"]) };
|
|
991
1106
|
}
|
|
992
|
-
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation,
|
|
1107
|
+
async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options2) {
|
|
993
1108
|
const result = {
|
|
994
1109
|
assertion_id: assertionId2,
|
|
995
1110
|
status,
|
|
@@ -1007,11 +1122,11 @@ var AttestClient = class {
|
|
|
1007
1122
|
const raw = await this.sendRequest(
|
|
1008
1123
|
"submit_plugin_result",
|
|
1009
1124
|
params,
|
|
1010
|
-
|
|
1125
|
+
options2?.timeout
|
|
1011
1126
|
);
|
|
1012
1127
|
return Boolean(raw?.accepted ?? false);
|
|
1013
1128
|
}
|
|
1014
|
-
async queryDrift(assertionId2, windowSize = 50,
|
|
1129
|
+
async queryDrift(assertionId2, windowSize = 50, options2) {
|
|
1015
1130
|
if (isSimulationMode()) {
|
|
1016
1131
|
return {
|
|
1017
1132
|
assertion_id: assertionId2,
|
|
@@ -1026,11 +1141,11 @@ var AttestClient = class {
|
|
|
1026
1141
|
const raw = await this.sendRequest(
|
|
1027
1142
|
"query_drift",
|
|
1028
1143
|
{ assertion_id: assertionId2, window_size: windowSize },
|
|
1029
|
-
|
|
1144
|
+
options2?.timeout
|
|
1030
1145
|
);
|
|
1031
1146
|
return raw.report;
|
|
1032
1147
|
}
|
|
1033
|
-
async generateUserMessage(persona, conversationHistory, faultConfig,
|
|
1148
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options2) {
|
|
1034
1149
|
if (isSimulationMode()) {
|
|
1035
1150
|
return `[simulation] Hello from ${persona.name}`;
|
|
1036
1151
|
}
|
|
@@ -1044,7 +1159,7 @@ var AttestClient = class {
|
|
|
1044
1159
|
const raw = await this.sendRequest(
|
|
1045
1160
|
"generate_user_message",
|
|
1046
1161
|
params,
|
|
1047
|
-
|
|
1162
|
+
options2?.timeout
|
|
1048
1163
|
);
|
|
1049
1164
|
return String(raw.message);
|
|
1050
1165
|
}
|
|
@@ -1089,33 +1204,33 @@ var TraceBuilder = class {
|
|
|
1089
1204
|
this.input = input;
|
|
1090
1205
|
return this;
|
|
1091
1206
|
}
|
|
1092
|
-
addLlmCall(name,
|
|
1207
|
+
addLlmCall(name, options2) {
|
|
1093
1208
|
this.steps.push({
|
|
1094
1209
|
type: STEP_LLM_CALL,
|
|
1095
1210
|
name,
|
|
1096
|
-
args:
|
|
1097
|
-
result:
|
|
1098
|
-
metadata:
|
|
1211
|
+
args: options2?.args,
|
|
1212
|
+
result: options2?.result,
|
|
1213
|
+
metadata: options2?.metadata
|
|
1099
1214
|
});
|
|
1100
1215
|
return this;
|
|
1101
1216
|
}
|
|
1102
|
-
addToolCall(name,
|
|
1217
|
+
addToolCall(name, options2) {
|
|
1103
1218
|
this.steps.push({
|
|
1104
1219
|
type: STEP_TOOL_CALL,
|
|
1105
1220
|
name,
|
|
1106
|
-
args:
|
|
1107
|
-
result:
|
|
1108
|
-
metadata:
|
|
1221
|
+
args: options2?.args,
|
|
1222
|
+
result: options2?.result,
|
|
1223
|
+
metadata: options2?.metadata
|
|
1109
1224
|
});
|
|
1110
1225
|
return this;
|
|
1111
1226
|
}
|
|
1112
|
-
addRetrieval(name,
|
|
1227
|
+
addRetrieval(name, options2) {
|
|
1113
1228
|
this.steps.push({
|
|
1114
1229
|
type: STEP_RETRIEVAL,
|
|
1115
1230
|
name,
|
|
1116
|
-
args:
|
|
1117
|
-
result:
|
|
1118
|
-
metadata:
|
|
1231
|
+
args: options2?.args,
|
|
1232
|
+
result: options2?.result,
|
|
1233
|
+
metadata: options2?.metadata
|
|
1119
1234
|
});
|
|
1120
1235
|
return this;
|
|
1121
1236
|
}
|
|
@@ -1175,11 +1290,14 @@ var AgentResult = class {
|
|
|
1175
1290
|
assertionResults;
|
|
1176
1291
|
totalCost;
|
|
1177
1292
|
totalDurationMs;
|
|
1178
|
-
|
|
1293
|
+
/** Attest model usage of the evaluation that produced this result. */
|
|
1294
|
+
usage;
|
|
1295
|
+
constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0, usage = zeroUsage()) {
|
|
1179
1296
|
this.trace = trace;
|
|
1180
1297
|
this.assertionResults = assertionResults.map(normalizeAssertionResult);
|
|
1181
1298
|
this.totalCost = totalCost;
|
|
1182
1299
|
this.totalDurationMs = totalDurationMs;
|
|
1300
|
+
this.usage = usage;
|
|
1183
1301
|
}
|
|
1184
1302
|
/** True if every assertion has status "pass". */
|
|
1185
1303
|
get passed() {
|
|
@@ -1190,10 +1308,10 @@ var AgentResult = class {
|
|
|
1190
1308
|
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
1191
1309
|
* when only soft_fail rows are present (see `softFailures`).
|
|
1192
1310
|
*/
|
|
1193
|
-
outcome(
|
|
1311
|
+
outcome(options2) {
|
|
1194
1312
|
if (this.hardFailures.length > 0) return OUTCOME_FAIL;
|
|
1195
1313
|
if (this.notEvaluated.length > 0) {
|
|
1196
|
-
return
|
|
1314
|
+
return options2?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
|
|
1197
1315
|
}
|
|
1198
1316
|
return OUTCOME_PASS;
|
|
1199
1317
|
}
|
|
@@ -1303,9 +1421,9 @@ function truncate(value, limit = 280) {
|
|
|
1303
1421
|
}
|
|
1304
1422
|
return value.slice(0, limit - 3) + "...";
|
|
1305
1423
|
}
|
|
1306
|
-
function renderAssertionFailure(result,
|
|
1307
|
-
const color =
|
|
1308
|
-
const testFile =
|
|
1424
|
+
function renderAssertionFailure(result, options2 = {}) {
|
|
1425
|
+
const color = options2.color ?? false;
|
|
1426
|
+
const testFile = options2.testFile;
|
|
1309
1427
|
const layer = result.layer ?? 0;
|
|
1310
1428
|
const label = layerLabel(layer, result.type);
|
|
1311
1429
|
const lines = [];
|
|
@@ -1358,7 +1476,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1358
1476
|
if (meta.rubric_version) {
|
|
1359
1477
|
rubric = `${rubric} @ ${meta.rubric_version}`;
|
|
1360
1478
|
}
|
|
1361
|
-
|
|
1479
|
+
const judge = meta.provider ? `${meta.provider}/${meta.model}` : meta.model;
|
|
1480
|
+
lines.push(` judge: ${judge} / ${rubric}`);
|
|
1362
1481
|
}
|
|
1363
1482
|
if (meta.prompt_hash) {
|
|
1364
1483
|
lines.push(` prompt: #${meta.prompt_hash}`);
|
|
@@ -1385,8 +1504,8 @@ function appendJudgeMetadata(lines, meta) {
|
|
|
1385
1504
|
);
|
|
1386
1505
|
}
|
|
1387
1506
|
}
|
|
1388
|
-
function renderSummary(result,
|
|
1389
|
-
const color =
|
|
1507
|
+
function renderSummary(result, options2 = {}) {
|
|
1508
|
+
const color = options2.color ?? false;
|
|
1390
1509
|
const passN = result.passCount;
|
|
1391
1510
|
const soft = result.assertionResults.filter(
|
|
1392
1511
|
(r) => r.status === STATUS_SOFT_FAIL
|
|
@@ -1404,9 +1523,9 @@ function renderSummary(result, options = {}) {
|
|
|
1404
1523
|
];
|
|
1405
1524
|
return parts.join(" | ");
|
|
1406
1525
|
}
|
|
1407
|
-
function renderDiagnostics(result,
|
|
1408
|
-
const color =
|
|
1409
|
-
const includePassing =
|
|
1526
|
+
function renderDiagnostics(result, options2 = {}) {
|
|
1527
|
+
const color = options2.color ?? false;
|
|
1528
|
+
const includePassing = options2.includePassing ?? false;
|
|
1410
1529
|
const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
|
|
1411
1530
|
if (rows.length === 0) {
|
|
1412
1531
|
return dim(" (no failures)\n", color);
|
|
@@ -1414,7 +1533,7 @@ function renderDiagnostics(result, options = {}) {
|
|
|
1414
1533
|
const out = [""];
|
|
1415
1534
|
out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
|
|
1416
1535
|
for (const r of rows) {
|
|
1417
|
-
out.push(renderAssertionFailure(r,
|
|
1536
|
+
out.push(renderAssertionFailure(r, options2));
|
|
1418
1537
|
}
|
|
1419
1538
|
out.push(dim(` Summary: ${renderSummary(result, { color })}`, color));
|
|
1420
1539
|
return out.join("\n");
|
|
@@ -1531,9 +1650,64 @@ var TraceTree = class {
|
|
|
1531
1650
|
|
|
1532
1651
|
// src/expect.ts
|
|
1533
1652
|
var import_node_crypto2 = require("crypto");
|
|
1653
|
+
var MAX_WORLD_BYTES = 9 * 1024 * 1024;
|
|
1654
|
+
var WorldChecks = class {
|
|
1655
|
+
_snapshot;
|
|
1656
|
+
_add;
|
|
1657
|
+
constructor(snapshot, add) {
|
|
1658
|
+
this._snapshot = snapshot;
|
|
1659
|
+
this._add = add;
|
|
1660
|
+
}
|
|
1661
|
+
addCheck(spec, where) {
|
|
1662
|
+
const full = { ...spec, world: this._snapshot };
|
|
1663
|
+
if (where !== void 0 && Object.keys(where).length > 0) full.where = where;
|
|
1664
|
+
return this._add(full);
|
|
1665
|
+
}
|
|
1666
|
+
/**
|
|
1667
|
+
* Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
|
|
1668
|
+
* `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
|
|
1669
|
+
*
|
|
1670
|
+
* @throws Error when no bound is given, or `equals` is combined with another bound.
|
|
1671
|
+
*
|
|
1672
|
+
* @example
|
|
1673
|
+
* ```ts
|
|
1674
|
+
* expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
|
|
1675
|
+
* ```
|
|
1676
|
+
*/
|
|
1677
|
+
count(resource, opts) {
|
|
1678
|
+
if (opts.equals === void 0 && opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
1679
|
+
throw new Error("count needs equals, atMost or atLeast");
|
|
1680
|
+
}
|
|
1681
|
+
if (opts.equals !== void 0 && (opts.atMost !== void 0 || opts.atLeast !== void 0)) {
|
|
1682
|
+
throw new Error("count: equals excludes atMost and atLeast");
|
|
1683
|
+
}
|
|
1684
|
+
const spec = { check: "count", resource };
|
|
1685
|
+
if (opts.equals !== void 0) spec.equals = opts.equals;
|
|
1686
|
+
if (opts.atMost !== void 0) spec.at_most = opts.atMost;
|
|
1687
|
+
if (opts.atLeast !== void 0) spec.at_least = opts.atLeast;
|
|
1688
|
+
return this.addCheck(spec, opts.where);
|
|
1689
|
+
}
|
|
1690
|
+
/** Assert no two matching records share the same values for field(s) `by`. */
|
|
1691
|
+
unique(resource, opts) {
|
|
1692
|
+
const fields = typeof opts.by === "string" ? [opts.by] : [...opts.by];
|
|
1693
|
+
if (fields.length === 0) throw new Error("unique needs at least one field in by");
|
|
1694
|
+
return this.addCheck({ check: "unique", resource, fields }, opts.where);
|
|
1695
|
+
}
|
|
1696
|
+
/**
|
|
1697
|
+
* Assert every matching record carries a distinct value in field `key` (default
|
|
1698
|
+
* `idempotency_key`), the record field that stores the idempotency key.
|
|
1699
|
+
*/
|
|
1700
|
+
idempotent(resource, opts = {}) {
|
|
1701
|
+
return this.addCheck(
|
|
1702
|
+
{ check: "idempotent", resource, field: opts.key ?? "idempotency_key" },
|
|
1703
|
+
opts.where
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
};
|
|
1534
1707
|
var ExpectChain = class {
|
|
1535
1708
|
_result;
|
|
1536
1709
|
_assertions = [];
|
|
1710
|
+
_worldBytes = 0;
|
|
1537
1711
|
constructor(result) {
|
|
1538
1712
|
this._result = result;
|
|
1539
1713
|
}
|
|
@@ -1882,6 +2056,104 @@ var ExpectChain = class {
|
|
|
1882
2056
|
soft: opts?.soft ?? false
|
|
1883
2057
|
});
|
|
1884
2058
|
}
|
|
2059
|
+
// -- Layer 3: Behavior contracts (tool calls only, retries counted) --
|
|
2060
|
+
/**
|
|
2061
|
+
* Assert how many times a tool is called, per value of argument `per` when set.
|
|
2062
|
+
*
|
|
2063
|
+
* @example
|
|
2064
|
+
* ```ts
|
|
2065
|
+
* // fails with "issue_refund called 2× for order_id=O-1042 (max 1)"
|
|
2066
|
+
* expect(result).toolCardinality("issue_refund", { per: "order_id", atMost: 1 });
|
|
2067
|
+
* ```
|
|
2068
|
+
*/
|
|
2069
|
+
toolCardinality(tool, opts) {
|
|
2070
|
+
if (opts.atMost === void 0 && opts.atLeast === void 0) {
|
|
2071
|
+
throw new Error("toolCardinality needs atMost, atLeast or both");
|
|
2072
|
+
}
|
|
2073
|
+
const spec = { check: "tool_cardinality", tool };
|
|
2074
|
+
if (opts.per !== void 0) spec.per = opts.per;
|
|
2075
|
+
if (opts.atLeast !== void 0) spec.min = opts.atLeast;
|
|
2076
|
+
if (opts.atMost !== void 0) spec.max = opts.atMost;
|
|
2077
|
+
spec.soft = opts.soft ?? false;
|
|
2078
|
+
return this.add(TYPE_TRACE, spec);
|
|
2079
|
+
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Assert the tool-call trajectory matches `tools`: `strict` (same sequence,
|
|
2082
|
+
* the default), `unordered` (same calls, any order), `subset` (no call
|
|
2083
|
+
* outside `tools`) or `superset` (every call in `tools` made). Use
|
|
2084
|
+
* {@link trajectoryPartialOrder} for ordering constraints.
|
|
2085
|
+
*/
|
|
2086
|
+
trajectory(tools, opts) {
|
|
2087
|
+
return this.add(TYPE_TRACE, {
|
|
2088
|
+
check: "trajectory",
|
|
2089
|
+
mode: opts?.mode ?? "strict",
|
|
2090
|
+
tools,
|
|
2091
|
+
soft: opts?.soft ?? false
|
|
2092
|
+
});
|
|
2093
|
+
}
|
|
2094
|
+
/**
|
|
2095
|
+
* Assert every tool in `tools` and `order` is called and each
|
|
2096
|
+
* `[before, after]` pair holds as in {@link toolCalledBefore}.
|
|
2097
|
+
*/
|
|
2098
|
+
trajectoryPartialOrder(order, opts) {
|
|
2099
|
+
const spec = {
|
|
2100
|
+
check: "trajectory",
|
|
2101
|
+
mode: "partial_order",
|
|
2102
|
+
order: order.map(([before, after]) => [before, after])
|
|
2103
|
+
};
|
|
2104
|
+
if (opts?.tools !== void 0) spec.tools = opts.tools;
|
|
2105
|
+
spec.soft = opts?.soft ?? false;
|
|
2106
|
+
return this.add(TYPE_TRACE, spec);
|
|
2107
|
+
}
|
|
2108
|
+
/** Assert every call to `after` has an earlier call to `before`; holds when `after` is never called. */
|
|
2109
|
+
toolCalledBefore(before, after, opts) {
|
|
2110
|
+
return this.add(TYPE_TRACE, { check: "tool_order", before, after, soft: opts?.soft ?? false });
|
|
2111
|
+
}
|
|
2112
|
+
/** Assert no block of `period` (1–4) tool calls repeats `minRepeats` (default 2) times in a row; A→B→A→B fails period 2. */
|
|
2113
|
+
noToolCycles(period, opts) {
|
|
2114
|
+
return this.add(TYPE_TRACE, {
|
|
2115
|
+
check: "cycle",
|
|
2116
|
+
period,
|
|
2117
|
+
min_repeats: opts?.minRepeats ?? 2,
|
|
2118
|
+
soft: opts?.soft ?? false
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
/** Assert at most `maxStreak` consecutive tool calls end in error (trace v2). */
|
|
2122
|
+
maxToolFailureStreak(maxStreak, opts) {
|
|
2123
|
+
return this.add(TYPE_TRACE, { check: "failure_streak", max: maxStreak, soft: opts?.soft ?? false });
|
|
2124
|
+
}
|
|
2125
|
+
/**
|
|
2126
|
+
* Assert every call to `tool` has argument `arg` (a dot path) satisfying
|
|
2127
|
+
* `op value`; holds when `tool` is never called.
|
|
2128
|
+
*/
|
|
2129
|
+
toolArgSatisfies(tool, arg, op, value, opts) {
|
|
2130
|
+
return this.add(TYPE_TRACE, {
|
|
2131
|
+
check: "arg_predicate",
|
|
2132
|
+
tool,
|
|
2133
|
+
arg,
|
|
2134
|
+
op,
|
|
2135
|
+
value,
|
|
2136
|
+
soft: opts?.soft ?? false
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
// -- Layer 3: Effect checks (the external system the agent acted on) --
|
|
2140
|
+
/**
|
|
2141
|
+
* Check effects in a world snapshot:
|
|
2142
|
+
* `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
|
|
2143
|
+
* a world helper: the checks are `not_evaluated` (`missing_evidence`).
|
|
2144
|
+
*/
|
|
2145
|
+
world(snapshot) {
|
|
2146
|
+
const size = new TextEncoder().encode(JSON.stringify(snapshot)).length;
|
|
2147
|
+
return new WorldChecks(snapshot, (spec) => {
|
|
2148
|
+
this._worldBytes += size;
|
|
2149
|
+
if (this._worldBytes > MAX_WORLD_BYTES) {
|
|
2150
|
+
throw new Error(
|
|
2151
|
+
`effect checks would send the ${size}-byte world snapshot ${Math.floor(this._worldBytes / Math.max(size, 1))} times (${this._worldBytes} bytes), over the ${MAX_WORLD_BYTES}-byte limit of one request; use fewer checks per test or scope the world to less state`
|
|
2152
|
+
);
|
|
2153
|
+
}
|
|
2154
|
+
return this.add(TYPE_EFFECT, spec);
|
|
2155
|
+
});
|
|
2156
|
+
}
|
|
1885
2157
|
// -- Layer 4: Content --
|
|
1886
2158
|
/**
|
|
1887
2159
|
* Assert the output message contains the given string.
|
|
@@ -2375,6 +2647,728 @@ function attestExpect(result) {
|
|
|
2375
2647
|
return new ExpectChain(new AgentResult(result));
|
|
2376
2648
|
}
|
|
2377
2649
|
|
|
2650
|
+
// src/capture.ts
|
|
2651
|
+
var import_node_child_process2 = require("child_process");
|
|
2652
|
+
var fs2 = __toESM(require("fs"), 1);
|
|
2653
|
+
var path2 = __toESM(require("path"), 1);
|
|
2654
|
+
var import_node_url = require("url");
|
|
2655
|
+
var FIXTURE_VERSION = 1;
|
|
2656
|
+
var DEFAULT_FIXTURE_DIR = path2.join("tests", "attest", "fixtures");
|
|
2657
|
+
var CaptureError = class extends Error {
|
|
2658
|
+
/** The fixture written, when the engine wrote one. */
|
|
2659
|
+
fixture;
|
|
2660
|
+
constructor(message, fixture = null) {
|
|
2661
|
+
super(message);
|
|
2662
|
+
this.name = "CaptureError";
|
|
2663
|
+
this.fixture = fixture;
|
|
2664
|
+
}
|
|
2665
|
+
};
|
|
2666
|
+
function toPath(p) {
|
|
2667
|
+
return p instanceof URL ? (0, import_node_url.fileURLToPath)(p) : p;
|
|
2668
|
+
}
|
|
2669
|
+
function loadFixture(file) {
|
|
2670
|
+
const p = toPath(file);
|
|
2671
|
+
const doc = JSON.parse(fs2.readFileSync(p, "utf8"));
|
|
2672
|
+
if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
|
|
2673
|
+
throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
|
|
2674
|
+
}
|
|
2675
|
+
validate(p, doc);
|
|
2676
|
+
return doc;
|
|
2677
|
+
}
|
|
2678
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
2679
|
+
function isObject(v) {
|
|
2680
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2681
|
+
}
|
|
2682
|
+
function kindOf(v) {
|
|
2683
|
+
if (Array.isArray(v)) return "array";
|
|
2684
|
+
if (isObject(v)) return "object";
|
|
2685
|
+
if (typeof v === "string" || typeof v === "boolean") return typeof v;
|
|
2686
|
+
return "other";
|
|
2687
|
+
}
|
|
2688
|
+
function validate(p, doc) {
|
|
2689
|
+
const need = (where, obj, fields) => {
|
|
2690
|
+
if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
|
|
2691
|
+
for (const [key, kind] of Object.entries(fields)) {
|
|
2692
|
+
if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
|
|
2693
|
+
}
|
|
2694
|
+
return obj;
|
|
2695
|
+
};
|
|
2696
|
+
need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
|
|
2697
|
+
if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
|
|
2698
|
+
if (!NAME_PATTERN.test(doc["name"])) {
|
|
2699
|
+
throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
|
|
2700
|
+
}
|
|
2701
|
+
const v = need("fixture.verification", doc["verification"], {
|
|
2702
|
+
verified: "boolean",
|
|
2703
|
+
expect_fail: "boolean",
|
|
2704
|
+
results: "array",
|
|
2705
|
+
expected_failures: "array"
|
|
2706
|
+
});
|
|
2707
|
+
doc["contracts"].forEach(
|
|
2708
|
+
(c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
|
|
2709
|
+
);
|
|
2710
|
+
v["results"].forEach(
|
|
2711
|
+
(r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
|
|
2712
|
+
);
|
|
2713
|
+
}
|
|
2714
|
+
function objectKey(k) {
|
|
2715
|
+
if (k === "__proto__") return `[${JSON.stringify(k)}]`;
|
|
2716
|
+
return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
|
|
2717
|
+
}
|
|
2718
|
+
function comment(text) {
|
|
2719
|
+
return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
|
|
2720
|
+
}
|
|
2721
|
+
function ts(value) {
|
|
2722
|
+
if (value === null || typeof value === "boolean" || typeof value === "number") {
|
|
2723
|
+
return JSON.stringify(value);
|
|
2724
|
+
}
|
|
2725
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
2726
|
+
if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
|
|
2727
|
+
if (typeof value === "object") {
|
|
2728
|
+
const entries = Object.entries(value).map(
|
|
2729
|
+
([k, v]) => `${objectKey(k)}: ${ts(v)}`
|
|
2730
|
+
);
|
|
2731
|
+
return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
|
|
2732
|
+
}
|
|
2733
|
+
throw new Error(`value ${String(value)} has no literal`);
|
|
2734
|
+
}
|
|
2735
|
+
function str(spec, key) {
|
|
2736
|
+
const v = spec[key];
|
|
2737
|
+
if (typeof v !== "string") throw new Error(`missing ${key}`);
|
|
2738
|
+
return v;
|
|
2739
|
+
}
|
|
2740
|
+
function options(entries) {
|
|
2741
|
+
const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
|
|
2742
|
+
return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
|
|
2743
|
+
}
|
|
2744
|
+
function call(method, args, opts) {
|
|
2745
|
+
return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
|
|
2746
|
+
}
|
|
2747
|
+
function dslCall(contract) {
|
|
2748
|
+
const s = contract.spec;
|
|
2749
|
+
const soft = s["soft"] === true ? true : void 0;
|
|
2750
|
+
try {
|
|
2751
|
+
if (contract.type === "trace") {
|
|
2752
|
+
const check = s["check"];
|
|
2753
|
+
if (check === "trajectory" && s["mode"] !== "partial_order") {
|
|
2754
|
+
const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
|
|
2755
|
+
if (!Array.isArray(s["tools"])) throw new Error("missing tools");
|
|
2756
|
+
return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
|
|
2757
|
+
}
|
|
2758
|
+
if (check === "tool_order") {
|
|
2759
|
+
return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
|
|
2760
|
+
}
|
|
2761
|
+
if (check === "tool_cardinality") {
|
|
2762
|
+
return call(
|
|
2763
|
+
"toolCardinality",
|
|
2764
|
+
[ts(str(s, "tool"))],
|
|
2765
|
+
options([
|
|
2766
|
+
["per", s["per"]],
|
|
2767
|
+
["atLeast", s["min"]],
|
|
2768
|
+
["atMost", s["max"]],
|
|
2769
|
+
["soft", soft]
|
|
2770
|
+
])
|
|
2771
|
+
);
|
|
2772
|
+
}
|
|
2773
|
+
}
|
|
2774
|
+
if (contract.type === "schema" && s["target"] === "output.structured") {
|
|
2775
|
+
if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
|
|
2776
|
+
return call("outputMatchesSchema", [ts(s["schema"])], "");
|
|
2777
|
+
}
|
|
2778
|
+
if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
|
|
2779
|
+
if (typeof s["value"] !== "number") throw new Error("missing value");
|
|
2780
|
+
return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
|
|
2781
|
+
}
|
|
2782
|
+
} catch (err) {
|
|
2783
|
+
throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
|
|
2784
|
+
}
|
|
2785
|
+
throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
|
|
2786
|
+
}
|
|
2787
|
+
function testPathFor(fixturePath) {
|
|
2788
|
+
const name = path2.basename(fixturePath).replace(/\.attest\.json$/, "");
|
|
2789
|
+
return path2.join(path2.dirname(path2.dirname(fixturePath)), `${name}.test.ts`);
|
|
2790
|
+
}
|
|
2791
|
+
function renderVitest(fixture, fixturePath, testPath) {
|
|
2792
|
+
const calls = [];
|
|
2793
|
+
const skipped = [];
|
|
2794
|
+
for (const c of fixture.contracts) {
|
|
2795
|
+
try {
|
|
2796
|
+
calls.push([c, dslCall(c)]);
|
|
2797
|
+
} catch (err) {
|
|
2798
|
+
skipped.push(err.message);
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
const rel = path2.relative(path2.dirname(testPath), fixturePath).split(path2.sep).join("/");
|
|
2802
|
+
const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
|
|
2803
|
+
const expectFail = fixture.verification.expect_fail;
|
|
2804
|
+
const lines = [
|
|
2805
|
+
`// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
|
|
2806
|
+
"//",
|
|
2807
|
+
"// Review and edit before committing: they check the captured trace without running",
|
|
2808
|
+
"// the agent. Re-capturing does not overwrite this file.",
|
|
2809
|
+
`import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
|
|
2810
|
+
`import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
|
|
2811
|
+
`import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
|
|
2812
|
+
"",
|
|
2813
|
+
`const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
|
|
2814
|
+
];
|
|
2815
|
+
if (skipped.length > 0) {
|
|
2816
|
+
lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
|
|
2817
|
+
for (const s of skipped) lines.push(`// - ${comment(s)}`);
|
|
2818
|
+
}
|
|
2819
|
+
lines.push("", "function contracts(): ExpectChain {", " return attestExpect(FIXTURE.trace)");
|
|
2820
|
+
for (const [c, dsl] of calls) {
|
|
2821
|
+
lines.push(` // ${comment(`${c.id}: ${c.description}`)}`, ` ${dsl}`);
|
|
2822
|
+
}
|
|
2823
|
+
lines[lines.length - 1] += ";";
|
|
2824
|
+
lines.push("}");
|
|
2825
|
+
if (expectFail) {
|
|
2826
|
+
lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
|
|
2827
|
+
lines.push("const EXPECTED: Array<[string, string]> = [");
|
|
2828
|
+
for (const [c] of calls) lines.push(` [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
|
|
2829
|
+
lines.push(
|
|
2830
|
+
"];",
|
|
2831
|
+
"",
|
|
2832
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2833
|
+
` it("reproduces the incident", async () => {`,
|
|
2834
|
+
" const result = await inspect(contracts());",
|
|
2835
|
+
"",
|
|
2836
|
+
" expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
|
|
2837
|
+
" });",
|
|
2838
|
+
"});"
|
|
2839
|
+
);
|
|
2840
|
+
} else {
|
|
2841
|
+
lines.push(
|
|
2842
|
+
"",
|
|
2843
|
+
`describe(${ts(fixture.name)}, () => {`,
|
|
2844
|
+
` it("keeps its drafted contracts", async () => {`,
|
|
2845
|
+
" await evaluate(contracts());",
|
|
2846
|
+
" });",
|
|
2847
|
+
"});"
|
|
2848
|
+
);
|
|
2849
|
+
}
|
|
2850
|
+
return `${lines.join("\n")}
|
|
2851
|
+
`;
|
|
2852
|
+
}
|
|
2853
|
+
function emitVitest(fixturePath) {
|
|
2854
|
+
const testPath = testPathFor(fixturePath);
|
|
2855
|
+
const existing = fs2.lstatSync(testPath, { throwIfNoEntry: false });
|
|
2856
|
+
if (existing?.isSymbolicLink()) {
|
|
2857
|
+
throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
|
|
2858
|
+
}
|
|
2859
|
+
if (existing !== void 0) return null;
|
|
2860
|
+
const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
|
|
2861
|
+
fs2.mkdirSync(path2.dirname(testPath), { recursive: true });
|
|
2862
|
+
try {
|
|
2863
|
+
fs2.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
|
|
2864
|
+
} catch (err) {
|
|
2865
|
+
if (err.code === "EEXIST") {
|
|
2866
|
+
throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
|
|
2867
|
+
}
|
|
2868
|
+
throw err;
|
|
2869
|
+
}
|
|
2870
|
+
return testPath;
|
|
2871
|
+
}
|
|
2872
|
+
function serve(engine, args) {
|
|
2873
|
+
const child = (0, import_node_child_process2.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
2874
|
+
let stdout = "";
|
|
2875
|
+
let stderr = "";
|
|
2876
|
+
let resolveReady = () => void 0;
|
|
2877
|
+
const ready = new Promise((resolve4) => {
|
|
2878
|
+
resolveReady = resolve4;
|
|
2879
|
+
});
|
|
2880
|
+
const exited = new Promise((resolve4) => {
|
|
2881
|
+
child.on("close", (code, signal) => {
|
|
2882
|
+
resolveReady(null);
|
|
2883
|
+
resolve4({ code, signal });
|
|
2884
|
+
});
|
|
2885
|
+
child.on("error", (err) => {
|
|
2886
|
+
stderr += err.message;
|
|
2887
|
+
resolveReady(null);
|
|
2888
|
+
resolve4({ code: null, signal: null });
|
|
2889
|
+
});
|
|
2890
|
+
});
|
|
2891
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
2892
|
+
stdout += chunk;
|
|
2893
|
+
const nl = stdout.indexOf("\n");
|
|
2894
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
2895
|
+
});
|
|
2896
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
2897
|
+
stderr += chunk;
|
|
2898
|
+
});
|
|
2899
|
+
child.stdin.on("error", () => void 0);
|
|
2900
|
+
return {
|
|
2901
|
+
ready,
|
|
2902
|
+
async finish(finalize) {
|
|
2903
|
+
if (finalize) {
|
|
2904
|
+
const environ = Object.entries(process.env).flatMap(([k, v]) => v === void 0 ? [] : [`${k}=${v}`]);
|
|
2905
|
+
child.stdin.write(`${JSON.stringify({ finalize: true, environ })}
|
|
2906
|
+
`);
|
|
2907
|
+
}
|
|
2908
|
+
child.stdin.end();
|
|
2909
|
+
const { code, signal } = await exited;
|
|
2910
|
+
return { code, signal, stdout, stderr };
|
|
2911
|
+
}
|
|
2912
|
+
};
|
|
2913
|
+
}
|
|
2914
|
+
function readyEnv(line) {
|
|
2915
|
+
if (line === null) return null;
|
|
2916
|
+
let parsed;
|
|
2917
|
+
try {
|
|
2918
|
+
parsed = JSON.parse(line);
|
|
2919
|
+
} catch {
|
|
2920
|
+
return null;
|
|
2921
|
+
}
|
|
2922
|
+
const env = parsed?.env;
|
|
2923
|
+
if (typeof env !== "object" || env === null) return null;
|
|
2924
|
+
const entries = Object.entries(env);
|
|
2925
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
2926
|
+
return Object.fromEntries(entries);
|
|
2927
|
+
}
|
|
2928
|
+
function exitDetail(exit) {
|
|
2929
|
+
const how = exit.signal !== null ? `killed by ${exit.signal}` : `exit ${String(exit.code)}`;
|
|
2930
|
+
const err = exit.stderr.trim();
|
|
2931
|
+
return err ? `${how}: ${err}` : how;
|
|
2932
|
+
}
|
|
2933
|
+
function restore(saved) {
|
|
2934
|
+
for (const [k, v] of saved) {
|
|
2935
|
+
if (v === void 0) delete process.env[k];
|
|
2936
|
+
else process.env[k] = v;
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
var recording = false;
|
|
2940
|
+
async function record(name, fn, opts = {}) {
|
|
2941
|
+
if (recording) throw new CaptureError("record() is already running; captures cannot overlap");
|
|
2942
|
+
recording = true;
|
|
2943
|
+
try {
|
|
2944
|
+
return await recordExclusive(name, fn, opts);
|
|
2945
|
+
} finally {
|
|
2946
|
+
recording = false;
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
async function recordExclusive(name, fn, opts) {
|
|
2950
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
2951
|
+
const args = ["capture", "--serve", "--name", name, "--out-dir", opts.outDir ?? DEFAULT_FIXTURE_DIR];
|
|
2952
|
+
if (opts.expectFail) args.push("--expect-fail");
|
|
2953
|
+
if (opts.agentId !== void 0) args.push("--agent-id", opts.agentId);
|
|
2954
|
+
if (opts.world !== void 0) args.push("--world", path2.resolve(opts.world));
|
|
2955
|
+
const proc = serve(engine, args);
|
|
2956
|
+
const env = readyEnv(await proc.ready);
|
|
2957
|
+
if (env === null) {
|
|
2958
|
+
const exit2 = await proc.finish(false);
|
|
2959
|
+
throw new CaptureError(`attest-engine capture --serve did not start (${exitDetail(exit2)})`);
|
|
2960
|
+
}
|
|
2961
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
2962
|
+
Object.assign(process.env, env);
|
|
2963
|
+
let value;
|
|
2964
|
+
try {
|
|
2965
|
+
value = await fn({ name, env });
|
|
2966
|
+
if (opts.flush) await opts.flush();
|
|
2967
|
+
} catch (err) {
|
|
2968
|
+
restore(saved);
|
|
2969
|
+
await proc.finish(false);
|
|
2970
|
+
throw err;
|
|
2971
|
+
}
|
|
2972
|
+
restore(saved);
|
|
2973
|
+
const exit = await proc.finish(true);
|
|
2974
|
+
process.stderr.write(exit.stderr);
|
|
2975
|
+
const lines = exit.stdout.trim().split("\n");
|
|
2976
|
+
if (lines.length < 2) {
|
|
2977
|
+
throw new CaptureError(`capture ${name} wrote no fixture (${exitDetail(exit)})`);
|
|
2978
|
+
}
|
|
2979
|
+
const outcome = JSON.parse(lines[lines.length - 1] ?? "");
|
|
2980
|
+
if (exit.code !== 0 || !outcome.verified) {
|
|
2981
|
+
throw new CaptureError(`capture ${name}: fixture ${outcome.fixture} is not verified; no test drafted`, outcome.fixture);
|
|
2982
|
+
}
|
|
2983
|
+
const testFile = opts.emitTest === false ? null : emitVitest(outcome.fixture);
|
|
2984
|
+
return { value, fixture: outcome.fixture, verified: true, testFile };
|
|
2985
|
+
}
|
|
2986
|
+
|
|
2987
|
+
// src/replay.ts
|
|
2988
|
+
var import_node_child_process3 = require("child_process");
|
|
2989
|
+
var path3 = __toESM(require("path"), 1);
|
|
2990
|
+
|
|
2991
|
+
// src/egress.ts
|
|
2992
|
+
var net = __toESM(require("net"), 1);
|
|
2993
|
+
var PROVIDER_HOSTS = [
|
|
2994
|
+
"api.openai.com",
|
|
2995
|
+
"api.anthropic.com",
|
|
2996
|
+
"generativelanguage.googleapis.com",
|
|
2997
|
+
/\.openai\.azure\.com$/
|
|
2998
|
+
];
|
|
2999
|
+
var EgressBlockedError = class extends Error {
|
|
3000
|
+
constructor(host) {
|
|
3001
|
+
super(
|
|
3002
|
+
`attest egress guard: connection to model provider ${host} blocked; the agent must reach models through the replay proxy (OPENAI_BASE_URL / ANTHROPIC_BASE_URL)`
|
|
3003
|
+
);
|
|
3004
|
+
this.host = host;
|
|
3005
|
+
this.name = "EgressBlockedError";
|
|
3006
|
+
}
|
|
3007
|
+
host;
|
|
3008
|
+
};
|
|
3009
|
+
function isProviderHost(host) {
|
|
3010
|
+
const h = host.toLowerCase().replace(/\.$/, "");
|
|
3011
|
+
return PROVIDER_HOSTS.some((p) => typeof p === "string" ? p === h : p.test(h));
|
|
3012
|
+
}
|
|
3013
|
+
function egressBlockRequested(env = process.env) {
|
|
3014
|
+
const v = (env.ATTEST_EGRESS ?? "").trim().toLowerCase();
|
|
3015
|
+
if (v !== "" && v !== "block" && v !== "allow") {
|
|
3016
|
+
throw new Error(`ATTEST_EGRESS must be block or allow, got ${JSON.stringify(env.ATTEST_EGRESS)}`);
|
|
3017
|
+
}
|
|
3018
|
+
return v === "block";
|
|
3019
|
+
}
|
|
3020
|
+
var original = null;
|
|
3021
|
+
var installs = 0;
|
|
3022
|
+
function hostOf(args) {
|
|
3023
|
+
const first = Array.isArray(args[0]) ? args[0][0] : args[0];
|
|
3024
|
+
if (typeof first === "object" && first !== null) {
|
|
3025
|
+
const o = first;
|
|
3026
|
+
return typeof o.host === "string" ? o.host : null;
|
|
3027
|
+
}
|
|
3028
|
+
if (typeof first === "number" || typeof first === "string" && /^\d+$/.test(first)) {
|
|
3029
|
+
return typeof args[1] === "string" ? args[1] : null;
|
|
3030
|
+
}
|
|
3031
|
+
return null;
|
|
3032
|
+
}
|
|
3033
|
+
function installEgressGuard() {
|
|
3034
|
+
installs++;
|
|
3035
|
+
if (original === null) {
|
|
3036
|
+
const proto = net.Socket.prototype;
|
|
3037
|
+
const orig = proto.connect;
|
|
3038
|
+
original = orig;
|
|
3039
|
+
proto.connect = function(...args) {
|
|
3040
|
+
const host = hostOf(args);
|
|
3041
|
+
if (host !== null && isProviderHost(host)) {
|
|
3042
|
+
const err = new EgressBlockedError(host);
|
|
3043
|
+
process.nextTick(() => this.destroy(err));
|
|
3044
|
+
return this;
|
|
3045
|
+
}
|
|
3046
|
+
return orig.apply(this, args);
|
|
3047
|
+
};
|
|
3048
|
+
}
|
|
3049
|
+
let released = false;
|
|
3050
|
+
return () => {
|
|
3051
|
+
if (released) return;
|
|
3052
|
+
released = true;
|
|
3053
|
+
installs--;
|
|
3054
|
+
if (installs === 0) uninstallEgressGuard();
|
|
3055
|
+
};
|
|
3056
|
+
}
|
|
3057
|
+
function egressGuardActive() {
|
|
3058
|
+
return installs > 0;
|
|
3059
|
+
}
|
|
3060
|
+
function uninstallEgressGuard() {
|
|
3061
|
+
installs = 0;
|
|
3062
|
+
if (original === null) return;
|
|
3063
|
+
net.Socket.prototype.connect = original;
|
|
3064
|
+
original = null;
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
// src/replay.ts
|
|
3068
|
+
var ReplayMissError = class extends Error {
|
|
3069
|
+
constructor(message, summary) {
|
|
3070
|
+
super(message);
|
|
3071
|
+
this.summary = summary;
|
|
3072
|
+
this.name = "ReplayMissError";
|
|
3073
|
+
}
|
|
3074
|
+
summary;
|
|
3075
|
+
};
|
|
3076
|
+
var ReplayNotEvaluatedError = class extends Error {
|
|
3077
|
+
constructor(message, summary) {
|
|
3078
|
+
super(message);
|
|
3079
|
+
this.summary = summary;
|
|
3080
|
+
this.name = "ReplayNotEvaluatedError";
|
|
3081
|
+
}
|
|
3082
|
+
summary;
|
|
3083
|
+
};
|
|
3084
|
+
function escapeControl(text) {
|
|
3085
|
+
return text.replace(/[\p{Cc}\p{Cf}]/gu, (c) => `\\u${(c.codePointAt(0) ?? 0).toString(16).padStart(4, "0")}`);
|
|
3086
|
+
}
|
|
3087
|
+
var ReplayError = class extends Error {
|
|
3088
|
+
constructor(message) {
|
|
3089
|
+
super(escapeControl(message));
|
|
3090
|
+
this.name = "ReplayError";
|
|
3091
|
+
}
|
|
3092
|
+
};
|
|
3093
|
+
function resolveMode(explicit, env = process.env) {
|
|
3094
|
+
if (explicit !== void 0) return checkMode(explicit, "mode");
|
|
3095
|
+
const fromEnv = (env.ATTEST_REPLAY ?? "").trim().toLowerCase();
|
|
3096
|
+
if (fromEnv !== "") return checkMode(fromEnv, "ATTEST_REPLAY");
|
|
3097
|
+
const ci = (env.CI ?? "").trim().toLowerCase();
|
|
3098
|
+
return ci !== "" && ci !== "false" && ci !== "0" ? "strict" : "auto";
|
|
3099
|
+
}
|
|
3100
|
+
function checkMode(value, where) {
|
|
3101
|
+
if (value === "strict" || value === "auto" || value === "off") return value;
|
|
3102
|
+
throw new ReplayError(`${where} must be strict, auto or off, got ${JSON.stringify(value)}`);
|
|
3103
|
+
}
|
|
3104
|
+
var active = null;
|
|
3105
|
+
async function startReplay(name, opts = {}) {
|
|
3106
|
+
if (!NAME_PATTERN.test(name)) throw new ReplayError(`invalid fixture name ${JSON.stringify(name)}`);
|
|
3107
|
+
const resolved = resolveMode(opts.mode);
|
|
3108
|
+
if (resolved === "off") return offSession(name);
|
|
3109
|
+
const mode = resolved;
|
|
3110
|
+
if (active !== null) throw new ReplayError(`replay of ${active} is already running; sessions cannot overlap`);
|
|
3111
|
+
active = name;
|
|
3112
|
+
try {
|
|
3113
|
+
return await start(name, mode, opts);
|
|
3114
|
+
} catch (err) {
|
|
3115
|
+
active = null;
|
|
3116
|
+
throw err;
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
function offSession(name) {
|
|
3120
|
+
return {
|
|
3121
|
+
name,
|
|
3122
|
+
mode: "off",
|
|
3123
|
+
env: {},
|
|
3124
|
+
summary: null,
|
|
3125
|
+
finish: () => Promise.resolve(null),
|
|
3126
|
+
error: () => null,
|
|
3127
|
+
check: () => Promise.resolve()
|
|
3128
|
+
};
|
|
3129
|
+
}
|
|
3130
|
+
function missError(mode, summary) {
|
|
3131
|
+
if (summary.misses === 0) return null;
|
|
3132
|
+
const message = escapeControl(firstMiss(summary) ?? "replay miss");
|
|
3133
|
+
return mode === "auto" ? new ReplayNotEvaluatedError(message, summary) : new ReplayMissError(message, summary);
|
|
3134
|
+
}
|
|
3135
|
+
async function start(name, mode, opts) {
|
|
3136
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3137
|
+
const fixturesDir = path3.resolve(opts.fixturesDir ?? DEFAULT_FIXTURE_DIR);
|
|
3138
|
+
const blocking = egressGuardActive() || (opts.noLive ?? egressBlockRequested());
|
|
3139
|
+
const args = ["replay", "--serve", "--name", name, "--fixtures-dir", fixturesDir, "--mode", mode];
|
|
3140
|
+
if (blocking) args.push("--no-live");
|
|
3141
|
+
const child = (0, import_node_child_process3.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3142
|
+
let stdout = "";
|
|
3143
|
+
let stderr = "";
|
|
3144
|
+
let resolveReady = () => void 0;
|
|
3145
|
+
const ready = new Promise((resolve4) => {
|
|
3146
|
+
resolveReady = resolve4;
|
|
3147
|
+
});
|
|
3148
|
+
const exited = new Promise((resolve4) => {
|
|
3149
|
+
child.on("close", (code) => {
|
|
3150
|
+
resolveReady(null);
|
|
3151
|
+
resolve4(code);
|
|
3152
|
+
});
|
|
3153
|
+
child.on("error", (err) => {
|
|
3154
|
+
stderr += err.message;
|
|
3155
|
+
resolveReady(null);
|
|
3156
|
+
resolve4(null);
|
|
3157
|
+
});
|
|
3158
|
+
});
|
|
3159
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3160
|
+
stdout += chunk;
|
|
3161
|
+
const nl = stdout.indexOf("\n");
|
|
3162
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3163
|
+
});
|
|
3164
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3165
|
+
stderr += chunk;
|
|
3166
|
+
});
|
|
3167
|
+
child.stdin.on("error", () => void 0);
|
|
3168
|
+
const stop = async () => {
|
|
3169
|
+
child.stdin.write("stop\n");
|
|
3170
|
+
child.stdin.end();
|
|
3171
|
+
return { code: await exited, stdout, stderr };
|
|
3172
|
+
};
|
|
3173
|
+
const env = parseEnv(await ready);
|
|
3174
|
+
if (env === null) {
|
|
3175
|
+
const out = await stop();
|
|
3176
|
+
active = null;
|
|
3177
|
+
throw new ReplayError(`attest-engine replay --serve did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3178
|
+
}
|
|
3179
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3180
|
+
Object.assign(process.env, env);
|
|
3181
|
+
let summary = null;
|
|
3182
|
+
let finishing = null;
|
|
3183
|
+
const finish = () => {
|
|
3184
|
+
finishing ??= (async () => {
|
|
3185
|
+
for (const [k, v] of saved) {
|
|
3186
|
+
if (v === void 0) delete process.env[k];
|
|
3187
|
+
else process.env[k] = v;
|
|
3188
|
+
}
|
|
3189
|
+
const out = await stop();
|
|
3190
|
+
active = null;
|
|
3191
|
+
const parsed = parseSummary(out.stdout);
|
|
3192
|
+
if (parsed === null) {
|
|
3193
|
+
throw new ReplayError(`replay ${name} wrote no summary (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3194
|
+
}
|
|
3195
|
+
if (blocking && parsed.live > 0) {
|
|
3196
|
+
throw new ReplayError(
|
|
3197
|
+
`replay ${name}: ${String(parsed.live)} live model calls while the egress guard is blocking`
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
summary = parsed;
|
|
3201
|
+
return parsed;
|
|
3202
|
+
})();
|
|
3203
|
+
return finishing;
|
|
3204
|
+
};
|
|
3205
|
+
return {
|
|
3206
|
+
name,
|
|
3207
|
+
mode,
|
|
3208
|
+
env,
|
|
3209
|
+
get summary() {
|
|
3210
|
+
return summary;
|
|
3211
|
+
},
|
|
3212
|
+
finish,
|
|
3213
|
+
error: () => summary === null ? null : missError(mode, summary),
|
|
3214
|
+
check: async () => {
|
|
3215
|
+
await finish();
|
|
3216
|
+
const err = summary === null ? null : missError(mode, summary);
|
|
3217
|
+
if (err !== null) throw err;
|
|
3218
|
+
}
|
|
3219
|
+
};
|
|
3220
|
+
}
|
|
3221
|
+
function parseEnv(line) {
|
|
3222
|
+
if (line === null) return null;
|
|
3223
|
+
try {
|
|
3224
|
+
const env = JSON.parse(line).env;
|
|
3225
|
+
if (typeof env !== "object" || env === null) return null;
|
|
3226
|
+
const entries = Object.entries(env);
|
|
3227
|
+
if (!entries.every(([, v]) => typeof v === "string")) return null;
|
|
3228
|
+
return Object.fromEntries(entries);
|
|
3229
|
+
} catch {
|
|
3230
|
+
return null;
|
|
3231
|
+
}
|
|
3232
|
+
}
|
|
3233
|
+
function parseSummary(out) {
|
|
3234
|
+
const lines = out.trim().split("\n");
|
|
3235
|
+
try {
|
|
3236
|
+
const s = JSON.parse(lines[lines.length - 1] ?? "").summary;
|
|
3237
|
+
if (s === void 0 || !Array.isArray(s.records)) return null;
|
|
3238
|
+
for (const r of s.records) if (r.message !== void 0) r.message = escapeControl(r.message);
|
|
3239
|
+
return s;
|
|
3240
|
+
} catch {
|
|
3241
|
+
return null;
|
|
3242
|
+
}
|
|
3243
|
+
}
|
|
3244
|
+
function firstMiss(summary) {
|
|
3245
|
+
return summary.records.find((r) => r.outcome === "miss")?.message ?? null;
|
|
3246
|
+
}
|
|
3247
|
+
function liveLine(live, costUsd, unpriced = 0) {
|
|
3248
|
+
const line = `${String(live)} live model ${live === 1 ? "call" : "calls"} \xB7 $${costUsd.toFixed(2)}`;
|
|
3249
|
+
return unpriced > 0 ? `${line} (+${String(unpriced)} unpriced)` : line;
|
|
3250
|
+
}
|
|
3251
|
+
|
|
3252
|
+
// src/world.ts
|
|
3253
|
+
var import_node_child_process4 = require("child_process");
|
|
3254
|
+
var path4 = __toESM(require("path"), 1);
|
|
3255
|
+
var STATE_PATH = "/_attest/state";
|
|
3256
|
+
var STATE_TIMEOUT_MS = 1e4;
|
|
3257
|
+
var WorldError = class extends Error {
|
|
3258
|
+
constructor(message) {
|
|
3259
|
+
super(escapeControl(message));
|
|
3260
|
+
this.name = "WorldError";
|
|
3261
|
+
}
|
|
3262
|
+
};
|
|
3263
|
+
var active2 = null;
|
|
3264
|
+
async function startWorld(worldFile, opts = {}) {
|
|
3265
|
+
if (active2 !== null) throw new WorldError(`world ${active2} is already running; worlds cannot overlap`);
|
|
3266
|
+
const file = path4.resolve(worldFile);
|
|
3267
|
+
active2 = file;
|
|
3268
|
+
try {
|
|
3269
|
+
return await start2(file, opts);
|
|
3270
|
+
} catch (err) {
|
|
3271
|
+
active2 = null;
|
|
3272
|
+
throw err;
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
async function start2(file, opts) {
|
|
3276
|
+
const engine = await resolveEngine(opts.enginePath);
|
|
3277
|
+
const child = (0, import_node_child_process4.spawn)(engine, ["world", "--file", file, "--serve"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
3278
|
+
let stdout = "";
|
|
3279
|
+
let stderr = "";
|
|
3280
|
+
let resolveReady = () => void 0;
|
|
3281
|
+
const ready = new Promise((resolve4) => {
|
|
3282
|
+
resolveReady = resolve4;
|
|
3283
|
+
});
|
|
3284
|
+
const exited = new Promise((resolve4) => {
|
|
3285
|
+
child.on("close", (code) => {
|
|
3286
|
+
resolveReady(null);
|
|
3287
|
+
resolve4(code);
|
|
3288
|
+
});
|
|
3289
|
+
child.on("error", (err) => {
|
|
3290
|
+
stderr += err.message;
|
|
3291
|
+
resolveReady(null);
|
|
3292
|
+
resolve4(null);
|
|
3293
|
+
});
|
|
3294
|
+
});
|
|
3295
|
+
child.stdout.setEncoding("utf8").on("data", (chunk) => {
|
|
3296
|
+
stdout += chunk;
|
|
3297
|
+
const nl = stdout.indexOf("\n");
|
|
3298
|
+
if (nl >= 0) resolveReady(stdout.slice(0, nl));
|
|
3299
|
+
});
|
|
3300
|
+
child.stderr.setEncoding("utf8").on("data", (chunk) => {
|
|
3301
|
+
stderr += chunk;
|
|
3302
|
+
});
|
|
3303
|
+
child.stdin.on("error", () => void 0);
|
|
3304
|
+
const halt = async () => {
|
|
3305
|
+
child.stdin.write("stop\n");
|
|
3306
|
+
child.stdin.end();
|
|
3307
|
+
return { code: await exited, stdout, stderr };
|
|
3308
|
+
};
|
|
3309
|
+
const env = parseEnv(await ready);
|
|
3310
|
+
if (env === null) {
|
|
3311
|
+
const out = await halt();
|
|
3312
|
+
throw new WorldError(`attest-engine world ${file} did not start: ${out.stderr.trim() || `exit ${String(out.code)}`}`);
|
|
3313
|
+
}
|
|
3314
|
+
const urls = Object.values(env);
|
|
3315
|
+
if (urls.length !== 1) {
|
|
3316
|
+
await halt();
|
|
3317
|
+
throw new WorldError(`attest-engine world ${file}: want one env variable, got ${JSON.stringify(env)}`);
|
|
3318
|
+
}
|
|
3319
|
+
const baseUrl = urls[0];
|
|
3320
|
+
const saved = new Map(Object.keys(env).map((k) => [k, process.env[k]]));
|
|
3321
|
+
Object.assign(process.env, env);
|
|
3322
|
+
let stopped = false;
|
|
3323
|
+
let stopping = null;
|
|
3324
|
+
const stop = () => {
|
|
3325
|
+
stopped = true;
|
|
3326
|
+
stopping ??= (async () => {
|
|
3327
|
+
for (const [k, v] of saved) {
|
|
3328
|
+
if (v === void 0) delete process.env[k];
|
|
3329
|
+
else process.env[k] = v;
|
|
3330
|
+
}
|
|
3331
|
+
const out = await halt();
|
|
3332
|
+
active2 = null;
|
|
3333
|
+
const final = parseFinal(out.stdout);
|
|
3334
|
+
if (out.code !== 0 || final === null) {
|
|
3335
|
+
throw new WorldError(`attest-engine world ${file} failed (exit ${String(out.code)}): ${out.stderr.trim()}`);
|
|
3336
|
+
}
|
|
3337
|
+
return final;
|
|
3338
|
+
})();
|
|
3339
|
+
return stopping;
|
|
3340
|
+
};
|
|
3341
|
+
const state = async () => {
|
|
3342
|
+
if (stopped) throw new WorldError(`world ${file} is stopped; use the snapshot stop() returned`);
|
|
3343
|
+
const url = baseUrl + STATE_PATH;
|
|
3344
|
+
let res;
|
|
3345
|
+
try {
|
|
3346
|
+
res = await fetch(url, { signal: AbortSignal.timeout(STATE_TIMEOUT_MS) });
|
|
3347
|
+
} catch (err) {
|
|
3348
|
+
throw new WorldError(`GET ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3349
|
+
}
|
|
3350
|
+
if (res.status !== 200) throw new WorldError(`GET ${url}: HTTP ${String(res.status)}`);
|
|
3351
|
+
const body = await res.json().catch(() => {
|
|
3352
|
+
throw new WorldError(`GET ${url}: the body is not JSON`);
|
|
3353
|
+
});
|
|
3354
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
3355
|
+
throw new WorldError(`GET ${url}: the body is not a snapshot object`);
|
|
3356
|
+
}
|
|
3357
|
+
return body;
|
|
3358
|
+
};
|
|
3359
|
+
return { path: file, env, baseUrl, state, stop };
|
|
3360
|
+
}
|
|
3361
|
+
function parseFinal(out) {
|
|
3362
|
+
const lines = out.trim().split("\n");
|
|
3363
|
+
try {
|
|
3364
|
+
const world = JSON.parse(lines[lines.length - 1] ?? "").world;
|
|
3365
|
+
if (typeof world !== "object" || world === null || Array.isArray(world)) return null;
|
|
3366
|
+
return world;
|
|
3367
|
+
} catch {
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
|
|
2378
3372
|
// src/context.ts
|
|
2379
3373
|
var import_node_async_hooks = require("async_hooks");
|
|
2380
3374
|
var activeBuilder = new import_node_async_hooks.AsyncLocalStorage();
|
|
@@ -2578,12 +3572,12 @@ var ContinuousEvalRunner = class {
|
|
|
2578
3572
|
maxSize;
|
|
2579
3573
|
running = false;
|
|
2580
3574
|
loopTimer = null;
|
|
2581
|
-
constructor(client, assertions,
|
|
3575
|
+
constructor(client, assertions, options2) {
|
|
2582
3576
|
this.client = client;
|
|
2583
3577
|
this.assertions = assertions;
|
|
2584
|
-
this.sampler = new Sampler(
|
|
2585
|
-
this.dispatcher = new AlertDispatcher(
|
|
2586
|
-
this.maxSize =
|
|
3578
|
+
this.sampler = new Sampler(options2?.sampleRate ?? 1);
|
|
3579
|
+
this.dispatcher = new AlertDispatcher(options2?.alertWebhook, options2?.alertSlackUrl);
|
|
3580
|
+
this.maxSize = options2?.maxSize ?? resolveQueueSize();
|
|
2587
3581
|
}
|
|
2588
3582
|
get alertEmitter() {
|
|
2589
3583
|
return this.dispatcher;
|
|
@@ -2694,13 +3688,13 @@ function loadLabelsCSV(text) {
|
|
|
2694
3688
|
if (rows.length === 0) {
|
|
2695
3689
|
throw new Error("empty CSV");
|
|
2696
3690
|
}
|
|
2697
|
-
let
|
|
3691
|
+
let start3 = 0;
|
|
2698
3692
|
if (rows[0].length >= 2) {
|
|
2699
3693
|
const firstHuman = parseFloatStrict(rows[0][1]);
|
|
2700
|
-
if (firstHuman === null)
|
|
3694
|
+
if (firstHuman === null) start3 = 1;
|
|
2701
3695
|
}
|
|
2702
3696
|
const out = [];
|
|
2703
|
-
for (let idx =
|
|
3697
|
+
for (let idx = start3; idx < rows.length; idx++) {
|
|
2704
3698
|
const row = rows[idx];
|
|
2705
3699
|
const lineNum = idx + 1;
|
|
2706
3700
|
if (row.length === 0) continue;
|
|
@@ -2909,10 +3903,10 @@ var OpenAIAdapter = class {
|
|
|
2909
3903
|
constructor(agentId2) {
|
|
2910
3904
|
this.agentId = agentId2;
|
|
2911
3905
|
}
|
|
2912
|
-
traceFromResponse(response,
|
|
3906
|
+
traceFromResponse(response, options2) {
|
|
2913
3907
|
const builder = new TraceBuilder(this.agentId);
|
|
2914
|
-
if (
|
|
2915
|
-
builder.setInput({ messages:
|
|
3908
|
+
if (options2?.inputMessages) {
|
|
3909
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2916
3910
|
}
|
|
2917
3911
|
const message = response.choices[0].message;
|
|
2918
3912
|
const completionText = message.content ?? "";
|
|
@@ -2932,12 +3926,12 @@ var OpenAIAdapter = class {
|
|
|
2932
3926
|
}
|
|
2933
3927
|
builder.setOutput({
|
|
2934
3928
|
message: completionText,
|
|
2935
|
-
structured:
|
|
3929
|
+
structured: options2?.structuredOutput ?? {}
|
|
2936
3930
|
});
|
|
2937
3931
|
builder.setMetadata({
|
|
2938
3932
|
total_tokens: response.usage?.total_tokens,
|
|
2939
|
-
cost_usd:
|
|
2940
|
-
latency_ms:
|
|
3933
|
+
cost_usd: options2?.costUsd,
|
|
3934
|
+
latency_ms: options2?.latencyMs,
|
|
2941
3935
|
model: response.model
|
|
2942
3936
|
});
|
|
2943
3937
|
return builder.build();
|
|
@@ -2950,10 +3944,10 @@ var AnthropicAdapter = class {
|
|
|
2950
3944
|
constructor(agentId2) {
|
|
2951
3945
|
this.agentId = agentId2;
|
|
2952
3946
|
}
|
|
2953
|
-
traceFromResponse(response,
|
|
3947
|
+
traceFromResponse(response, options2) {
|
|
2954
3948
|
const builder = new TraceBuilder(this.agentId);
|
|
2955
|
-
if (
|
|
2956
|
-
builder.setInput({ messages:
|
|
3949
|
+
if (options2?.inputMessages) {
|
|
3950
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
2957
3951
|
}
|
|
2958
3952
|
const completionParts = [];
|
|
2959
3953
|
for (const block of response.content) {
|
|
@@ -2976,8 +3970,8 @@ var AnthropicAdapter = class {
|
|
|
2976
3970
|
}
|
|
2977
3971
|
builder.setMetadata({
|
|
2978
3972
|
total_tokens: totalTokens,
|
|
2979
|
-
cost_usd:
|
|
2980
|
-
latency_ms:
|
|
3973
|
+
cost_usd: options2?.costUsd,
|
|
3974
|
+
latency_ms: options2?.latencyMs,
|
|
2981
3975
|
model: response.model
|
|
2982
3976
|
});
|
|
2983
3977
|
return builder.build();
|
|
@@ -2990,10 +3984,10 @@ var GeminiAdapter = class {
|
|
|
2990
3984
|
constructor(agentId2) {
|
|
2991
3985
|
this.agentId = agentId2;
|
|
2992
3986
|
}
|
|
2993
|
-
traceFromResponse(response,
|
|
3987
|
+
traceFromResponse(response, options2) {
|
|
2994
3988
|
const builder = new TraceBuilder(this.agentId);
|
|
2995
|
-
if (
|
|
2996
|
-
builder.setInput({ text:
|
|
3989
|
+
if (options2?.inputText) {
|
|
3990
|
+
builder.setInput({ text: options2.inputText });
|
|
2997
3991
|
}
|
|
2998
3992
|
let completionText = "";
|
|
2999
3993
|
if (response.text != null) {
|
|
@@ -3013,9 +4007,9 @@ var GeminiAdapter = class {
|
|
|
3013
4007
|
builder.addLlmCall("completion", { result: { completion: completionText } });
|
|
3014
4008
|
builder.setOutput({ message: completionText });
|
|
3015
4009
|
builder.setMetadata({
|
|
3016
|
-
cost_usd:
|
|
3017
|
-
latency_ms:
|
|
3018
|
-
model:
|
|
4010
|
+
cost_usd: options2?.costUsd,
|
|
4011
|
+
latency_ms: options2?.latencyMs,
|
|
4012
|
+
model: options2?.model
|
|
3019
4013
|
});
|
|
3020
4014
|
return builder.build();
|
|
3021
4015
|
}
|
|
@@ -3027,10 +4021,10 @@ var OllamaAdapter = class {
|
|
|
3027
4021
|
constructor(agentId2) {
|
|
3028
4022
|
this.agentId = agentId2;
|
|
3029
4023
|
}
|
|
3030
|
-
traceFromResponse(response,
|
|
4024
|
+
traceFromResponse(response, options2) {
|
|
3031
4025
|
const builder = new TraceBuilder(this.agentId);
|
|
3032
|
-
if (
|
|
3033
|
-
builder.setInput({ messages:
|
|
4026
|
+
if (options2?.inputMessages) {
|
|
4027
|
+
builder.setInput({ messages: options2.inputMessages });
|
|
3034
4028
|
}
|
|
3035
4029
|
const completionText = response.message?.content ?? "";
|
|
3036
4030
|
builder.addLlmCall("completion", {
|
|
@@ -3044,7 +4038,7 @@ var OllamaAdapter = class {
|
|
|
3044
4038
|
}
|
|
3045
4039
|
builder.setMetadata({
|
|
3046
4040
|
total_tokens: totalTokens,
|
|
3047
|
-
latency_ms:
|
|
4041
|
+
latency_ms: options2?.latencyMs,
|
|
3048
4042
|
model: response.model
|
|
3049
4043
|
});
|
|
3050
4044
|
return builder.build();
|
|
@@ -3052,119 +4046,148 @@ var OllamaAdapter = class {
|
|
|
3052
4046
|
};
|
|
3053
4047
|
|
|
3054
4048
|
// src/adapters/otel.ts
|
|
4049
|
+
var import_node_child_process5 = require("child_process");
|
|
4050
|
+
var CONVERT_TIMEOUT_MS = 6e4;
|
|
4051
|
+
var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
4052
|
+
var STATUS_CODE_ERROR = 2;
|
|
3055
4053
|
var OTelAdapter = class _OTelAdapter {
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
4054
|
+
/**
|
|
4055
|
+
* Build a trace from finished OpenTelemetry spans of one trace.
|
|
4056
|
+
*
|
|
4057
|
+
* @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
|
|
4058
|
+
* @throws OTLPConversionError when the engine rejects the spans.
|
|
4059
|
+
*/
|
|
4060
|
+
static async fromSpans(spans, agentId2) {
|
|
4061
|
+
return _OTelAdapter.fromOtlp(spansToOtlp(spans), agentId2);
|
|
3059
4062
|
}
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
4063
|
+
/**
|
|
4064
|
+
* Convert an OTLP/JSON export (`resourceSpans` → `scopeSpans` → `spans`).
|
|
4065
|
+
*
|
|
4066
|
+
* @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
|
|
4067
|
+
* @throws OTLPConversionError when the engine rejects the export (malformed JSON, invalid
|
|
4068
|
+
* ids or timestamps, more than one trace, non-numeric token counts, ...).
|
|
4069
|
+
* @throws RangeError when a parsed export holds a non-finite number.
|
|
4070
|
+
* @throws OTLPConversionError when the engine times out, overflows the output buffer or
|
|
4071
|
+
* prints no trace.
|
|
4072
|
+
* @throws Error when the engine cannot start.
|
|
4073
|
+
*/
|
|
4074
|
+
static async fromOtlp(otlp, agentId2) {
|
|
4075
|
+
const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
|
|
4076
|
+
if (typeof v === "number" && !Number.isFinite(v)) {
|
|
4077
|
+
throw new RangeError(`OTLP export holds a non-finite number: ${String(v)}`);
|
|
4078
|
+
}
|
|
4079
|
+
return v;
|
|
4080
|
+
});
|
|
4081
|
+
const args = ["trace", "convert"];
|
|
4082
|
+
if (agentId2 !== void 0) args.push("--agent-id", agentId2);
|
|
4083
|
+
args.push("-");
|
|
4084
|
+
const stdout = await runConvert(await resolveEngine(), args, payload);
|
|
4085
|
+
try {
|
|
4086
|
+
return JSON.parse(stdout);
|
|
4087
|
+
} catch (err) {
|
|
4088
|
+
throw new OTLPConversionError(
|
|
4089
|
+
`attest-engine trace convert printed output that is not JSON: ${String(err)}`
|
|
4090
|
+
);
|
|
4091
|
+
}
|
|
3063
4092
|
}
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
4093
|
+
};
|
|
4094
|
+
function runConvert(engine, args, payload) {
|
|
4095
|
+
return new Promise((resolve4, reject) => {
|
|
4096
|
+
const child = (0, import_node_child_process5.spawn)(engine, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
4097
|
+
let timer;
|
|
4098
|
+
child.on("error", (e) => {
|
|
4099
|
+
clearTimeout(timer);
|
|
4100
|
+
reject(e);
|
|
3069
4101
|
});
|
|
3070
|
-
const
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
4102
|
+
const { stdin, stdout, stderr } = child;
|
|
4103
|
+
if (stdin === null || stdout === null || stderr === null) return;
|
|
4104
|
+
const out = [];
|
|
4105
|
+
const err = [];
|
|
4106
|
+
let outBytes = 0;
|
|
4107
|
+
let failure = null;
|
|
4108
|
+
const fail = (e) => {
|
|
4109
|
+
failure ??= e;
|
|
4110
|
+
child.kill("SIGKILL");
|
|
4111
|
+
};
|
|
4112
|
+
timer = setTimeout(() => {
|
|
4113
|
+
fail(
|
|
4114
|
+
new OTLPConversionError(
|
|
4115
|
+
`attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
|
|
4116
|
+
)
|
|
4117
|
+
);
|
|
4118
|
+
}, CONVERT_TIMEOUT_MS);
|
|
4119
|
+
stdout.on("data", (chunk) => {
|
|
4120
|
+
outBytes += chunk.length;
|
|
4121
|
+
if (outBytes > MAX_TRACE_OUTPUT_BYTES) {
|
|
4122
|
+
fail(
|
|
4123
|
+
new OTLPConversionError(
|
|
4124
|
+
`attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
|
|
4125
|
+
)
|
|
4126
|
+
);
|
|
4127
|
+
return;
|
|
3076
4128
|
}
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
const [stepArgs, stepResult] = this.extractLlmStep(attrs);
|
|
3087
|
-
builder.addLlmCall(span.name, {
|
|
3088
|
-
args: stepArgs,
|
|
3089
|
-
result: stepResult,
|
|
3090
|
-
metadata: this.spanMetadata(span)
|
|
3091
|
-
});
|
|
3092
|
-
const completion = attrs["gen_ai.completion"];
|
|
3093
|
-
if (completion) outputMessage = String(completion);
|
|
3094
|
-
const inputTokens = Number(attrs["gen_ai.usage.input_tokens"] ?? 0);
|
|
3095
|
-
const outputTokens = Number(attrs["gen_ai.usage.output_tokens"] ?? 0);
|
|
3096
|
-
const spanTokens = inputTokens + outputTokens;
|
|
3097
|
-
if (spanTokens > 0) totalTokens = (totalTokens ?? 0) + spanTokens;
|
|
3098
|
-
if (model === void 0) {
|
|
3099
|
-
const responseModel = attrs["gen_ai.response.model"];
|
|
3100
|
-
const requestModel = attrs["gen_ai.request.model"];
|
|
3101
|
-
if (responseModel) model = String(responseModel);
|
|
3102
|
-
else if (requestModel) model = String(requestModel);
|
|
3103
|
-
}
|
|
3104
|
-
} else if (stepType === "tool_call") {
|
|
3105
|
-
const [stepArgs, stepResult] = this.extractToolStep(attrs);
|
|
3106
|
-
const toolName = String(attrs["gen_ai.tool.name"] ?? span.name);
|
|
3107
|
-
builder.addToolCall(toolName, {
|
|
3108
|
-
args: stepArgs,
|
|
3109
|
-
result: stepResult,
|
|
3110
|
-
metadata: this.spanMetadata(span)
|
|
3111
|
-
});
|
|
4129
|
+
out.push(chunk);
|
|
4130
|
+
});
|
|
4131
|
+
stderr.on("data", (chunk) => err.push(chunk));
|
|
4132
|
+
stdin.on("error", () => void 0);
|
|
4133
|
+
child.on("close", (code, signal) => {
|
|
4134
|
+
clearTimeout(timer);
|
|
4135
|
+
if (failure !== null) {
|
|
4136
|
+
reject(failure);
|
|
4137
|
+
return;
|
|
3112
4138
|
}
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
4139
|
+
if (code !== 0) {
|
|
4140
|
+
const message = Buffer.concat(err).toString("utf8").trim();
|
|
4141
|
+
reject(
|
|
4142
|
+
new OTLPConversionError(
|
|
4143
|
+
message || `attest-engine trace convert exited ${String(code ?? signal)}`
|
|
4144
|
+
)
|
|
4145
|
+
);
|
|
4146
|
+
return;
|
|
4147
|
+
}
|
|
4148
|
+
resolve4(Buffer.concat(out).toString("utf8"));
|
|
4149
|
+
});
|
|
4150
|
+
stdin.end(payload);
|
|
4151
|
+
});
|
|
4152
|
+
}
|
|
4153
|
+
function hrTimeToNanos(time, spanName) {
|
|
4154
|
+
if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
|
|
4155
|
+
throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
|
|
3122
4156
|
}
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
4157
|
+
return (BigInt(time[0]) * 1000000000n + BigInt(time[1])).toString();
|
|
4158
|
+
}
|
|
4159
|
+
function anyValue(value) {
|
|
4160
|
+
if (value === null || value === void 0) return {};
|
|
4161
|
+
if (typeof value === "boolean") return { boolValue: value };
|
|
4162
|
+
if (typeof value === "string") return { stringValue: value };
|
|
4163
|
+
if (typeof value === "number") {
|
|
4164
|
+
if (!Number.isFinite(value)) {
|
|
4165
|
+
throw new RangeError(`OTel attribute value must be finite, got ${String(value)}`);
|
|
3126
4166
|
}
|
|
3127
|
-
return
|
|
4167
|
+
return Number.isSafeInteger(value) ? { intValue: String(value) } : { doubleValue: value };
|
|
3128
4168
|
}
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
4169
|
+
return { arrayValue: { values: value.map((v) => anyValue(v)) } };
|
|
4170
|
+
}
|
|
4171
|
+
function spansToOtlp(spans) {
|
|
4172
|
+
const encoded = spans.map((span) => {
|
|
4173
|
+
const ctx = span.spanContext();
|
|
4174
|
+
const out = {
|
|
4175
|
+
traceId: ctx.traceId,
|
|
4176
|
+
spanId: ctx.spanId,
|
|
4177
|
+
name: span.name,
|
|
4178
|
+
startTimeUnixNano: hrTimeToNanos(span.startTime, span.name),
|
|
4179
|
+
endTimeUnixNano: hrTimeToNanos(span.endTime, span.name),
|
|
4180
|
+
attributes: Object.entries(span.attributes ?? {}).filter(([, v]) => v !== void 0).map(([key, v]) => ({ key, value: anyValue(v) }))
|
|
4181
|
+
};
|
|
4182
|
+
const parent = span.parentSpanContext?.spanId ?? span.parentSpanId;
|
|
4183
|
+
if (parent) out["parentSpanId"] = parent;
|
|
4184
|
+
if (span.status?.code === STATUS_CODE_ERROR) {
|
|
4185
|
+
out["status"] = { code: STATUS_CODE_ERROR, message: span.status.message ?? "" };
|
|
3136
4186
|
}
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
}
|
|
3142
|
-
extractLlmStep(attrs) {
|
|
3143
|
-
const args = {};
|
|
3144
|
-
const result = {};
|
|
3145
|
-
if ("gen_ai.request.model" in attrs) args.model = String(attrs["gen_ai.request.model"]);
|
|
3146
|
-
if ("gen_ai.system" in attrs) args.system = String(attrs["gen_ai.system"]);
|
|
3147
|
-
if ("gen_ai.prompt" in attrs) args.prompt = String(attrs["gen_ai.prompt"]);
|
|
3148
|
-
if ("gen_ai.completion" in attrs) result.completion = String(attrs["gen_ai.completion"]);
|
|
3149
|
-
if ("gen_ai.usage.input_tokens" in attrs) result.input_tokens = Number(attrs["gen_ai.usage.input_tokens"]);
|
|
3150
|
-
if ("gen_ai.usage.output_tokens" in attrs) result.output_tokens = Number(attrs["gen_ai.usage.output_tokens"]);
|
|
3151
|
-
if ("gen_ai.response.model" in attrs) result.model = String(attrs["gen_ai.response.model"]);
|
|
3152
|
-
return [args, result];
|
|
3153
|
-
}
|
|
3154
|
-
extractToolStep(attrs) {
|
|
3155
|
-
const args = {};
|
|
3156
|
-
const result = {};
|
|
3157
|
-
if ("gen_ai.tool.call.id" in attrs) args.call_id = String(attrs["gen_ai.tool.call.id"]);
|
|
3158
|
-
if ("gen_ai.tool.parameters" in attrs) args.parameters = attrs["gen_ai.tool.parameters"];
|
|
3159
|
-
if ("gen_ai.tool.output" in attrs) result.output = attrs["gen_ai.tool.output"];
|
|
3160
|
-
return [args, result];
|
|
3161
|
-
}
|
|
3162
|
-
spanMetadata(span) {
|
|
3163
|
-
const startNs = span.startTime[0] * 1e9 + span.startTime[1];
|
|
3164
|
-
const endNs = span.endTime[0] * 1e9 + span.endTime[1];
|
|
3165
|
-
return { duration_ms: Math.floor((endNs - startNs) / 1e6) };
|
|
3166
|
-
}
|
|
3167
|
-
};
|
|
4187
|
+
return out;
|
|
4188
|
+
});
|
|
4189
|
+
return { resourceSpans: [{ scopeSpans: [{ spans: encoded }] }] };
|
|
4190
|
+
}
|
|
3168
4191
|
|
|
3169
4192
|
// src/adapters/langchain.ts
|
|
3170
4193
|
var LangChainAdapter = class {
|
|
@@ -3232,13 +4255,13 @@ var LangChainAdapter = class {
|
|
|
3232
4255
|
* @param options.model - Model name override.
|
|
3233
4256
|
* @returns A finalized Trace object.
|
|
3234
4257
|
*/
|
|
3235
|
-
buildTrace(
|
|
4258
|
+
buildTrace(options2) {
|
|
3236
4259
|
const builder = new TraceBuilder(this.agentId);
|
|
3237
4260
|
const latencyMs = this.startTime > 0 ? Date.now() - this.startTime : void 0;
|
|
3238
4261
|
let totalTokens;
|
|
3239
4262
|
for (const llm of this.llmCalls) {
|
|
3240
4263
|
const args = {};
|
|
3241
|
-
if (llm.model ??
|
|
4264
|
+
if (llm.model ?? options2?.model) args.model = llm.model ?? options2?.model;
|
|
3242
4265
|
const result = { completion: llm.text };
|
|
3243
4266
|
if (llm.tokens !== void 0) {
|
|
3244
4267
|
result.tokens = llm.tokens;
|
|
@@ -3255,9 +4278,9 @@ var LangChainAdapter = class {
|
|
|
3255
4278
|
builder.setOutput({ message: outputMessage });
|
|
3256
4279
|
builder.setMetadata({
|
|
3257
4280
|
total_tokens: totalTokens,
|
|
3258
|
-
cost_usd:
|
|
4281
|
+
cost_usd: options2?.costUsd,
|
|
3259
4282
|
latency_ms: latencyMs,
|
|
3260
|
-
model:
|
|
4283
|
+
model: options2?.model ?? this.llmCalls[0]?.model
|
|
3261
4284
|
});
|
|
3262
4285
|
return builder.build();
|
|
3263
4286
|
}
|
|
@@ -3272,6 +4295,7 @@ var LangChainAdapter = class {
|
|
|
3272
4295
|
AttestClient,
|
|
3273
4296
|
CONFUSED_USER,
|
|
3274
4297
|
COOPERATIVE_USER,
|
|
4298
|
+
CaptureError,
|
|
3275
4299
|
ContinuousEvalRunner,
|
|
3276
4300
|
ENGINE_VERSION,
|
|
3277
4301
|
ERR_ASSERTION_ERROR,
|
|
@@ -3280,6 +4304,7 @@ var LangChainAdapter = class {
|
|
|
3280
4304
|
ERR_PROVIDER_ERROR,
|
|
3281
4305
|
ERR_SESSION_ERROR,
|
|
3282
4306
|
ERR_TIMEOUT,
|
|
4307
|
+
EgressBlockedError,
|
|
3283
4308
|
EngineManager,
|
|
3284
4309
|
EngineTimeoutError,
|
|
3285
4310
|
ExpectChain,
|
|
@@ -3288,6 +4313,7 @@ var LangChainAdapter = class {
|
|
|
3288
4313
|
GeminiAdapter,
|
|
3289
4314
|
LangChainAdapter,
|
|
3290
4315
|
ManualAdapter,
|
|
4316
|
+
OTLPConversionError,
|
|
3291
4317
|
OTelAdapter,
|
|
3292
4318
|
OUTCOME_FAIL,
|
|
3293
4319
|
OUTCOME_PASS,
|
|
@@ -3298,11 +4324,17 @@ var LangChainAdapter = class {
|
|
|
3298
4324
|
ProtocolDesyncError,
|
|
3299
4325
|
ProtocolDiagnosticBuffer,
|
|
3300
4326
|
ProtocolError,
|
|
4327
|
+
REASON_BUDGET_EXHAUSTED,
|
|
4328
|
+
REASON_JUDGE_ABSTAINED,
|
|
3301
4329
|
REASON_MISSING_EVIDENCE,
|
|
3302
4330
|
REASON_PROVIDER_NOT_CONFIGURED,
|
|
3303
4331
|
REASON_PROVIDER_UNAVAILABLE,
|
|
4332
|
+
REASON_REPLAY_MISS,
|
|
3304
4333
|
REASON_SHORT_CIRCUITED,
|
|
3305
4334
|
REASON_SIMULATION,
|
|
4335
|
+
ReplayError,
|
|
4336
|
+
ReplayMissError,
|
|
4337
|
+
ReplayNotEvaluatedError,
|
|
3306
4338
|
STATUS_HARD_FAIL,
|
|
3307
4339
|
STATUS_NOT_EVALUATED,
|
|
3308
4340
|
STATUS_PASS,
|
|
@@ -3317,6 +4349,7 @@ var LangChainAdapter = class {
|
|
|
3317
4349
|
TIER_3,
|
|
3318
4350
|
TYPE_CONSTRAINT,
|
|
3319
4351
|
TYPE_CONTENT,
|
|
4352
|
+
TYPE_EFFECT,
|
|
3320
4353
|
TYPE_EMBEDDING,
|
|
3321
4354
|
TYPE_LLM_JUDGE,
|
|
3322
4355
|
TYPE_PLUGIN,
|
|
@@ -3326,7 +4359,10 @@ var LangChainAdapter = class {
|
|
|
3326
4359
|
TraceBuilder,
|
|
3327
4360
|
TraceTree,
|
|
3328
4361
|
VERSION,
|
|
4362
|
+
WorldChecks,
|
|
4363
|
+
WorldError,
|
|
3329
4364
|
activeBuilder,
|
|
4365
|
+
addUsage,
|
|
3330
4366
|
agent,
|
|
3331
4367
|
agentId,
|
|
3332
4368
|
assertionId,
|
|
@@ -3335,24 +4371,44 @@ var LangChainAdapter = class {
|
|
|
3335
4371
|
config,
|
|
3336
4372
|
decodeResponse,
|
|
3337
4373
|
delegate,
|
|
4374
|
+
dslCall,
|
|
4375
|
+
egressBlockRequested,
|
|
4376
|
+
emitVitest,
|
|
3338
4377
|
encodeRequest,
|
|
3339
4378
|
executePluginAssertion,
|
|
3340
4379
|
extractId,
|
|
3341
4380
|
extractResult,
|
|
4381
|
+
firstMiss,
|
|
4382
|
+
formatUsage,
|
|
3342
4383
|
getSampleRate,
|
|
4384
|
+
installEgressGuard,
|
|
4385
|
+
isProviderHost,
|
|
3343
4386
|
isSimulationMode,
|
|
3344
4387
|
layerForType,
|
|
4388
|
+
liveLine,
|
|
4389
|
+
loadFixture,
|
|
3345
4390
|
loadLabelsCSV,
|
|
3346
4391
|
loadLabelsJSONL,
|
|
4392
|
+
parseUsage,
|
|
3347
4393
|
previewLine,
|
|
3348
4394
|
promptHash,
|
|
4395
|
+
record,
|
|
3349
4396
|
renderAssertionFailure,
|
|
3350
4397
|
renderDiagnostics,
|
|
3351
4398
|
renderSummary,
|
|
4399
|
+
renderVitest,
|
|
3352
4400
|
requireSimulationCapability,
|
|
3353
4401
|
resetConfig,
|
|
4402
|
+
resolveMode,
|
|
3354
4403
|
simulationEnvEnabled,
|
|
4404
|
+
startReplay,
|
|
4405
|
+
startWorld,
|
|
4406
|
+
sumUsage,
|
|
4407
|
+
testPathFor,
|
|
3355
4408
|
tier,
|
|
3356
|
-
traceId
|
|
4409
|
+
traceId,
|
|
4410
|
+
uninstallEgressGuard,
|
|
4411
|
+
usageSummaryLine,
|
|
4412
|
+
zeroUsage
|
|
3357
4413
|
});
|
|
3358
4414
|
//# sourceMappingURL=index.cjs.map
|