@runtypelabs/cli 2.59.1 → 2.59.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-0x7UKC1U.js +17 -0
- package/dist/chunk-2ySxXtcg.js +418 -0
- package/dist/chunk-3An_BRHv.js +501 -0
- package/dist/chunk-6uTxgdiY.js +16218 -0
- package/dist/chunk-87nRnNRG.js +1363 -0
- package/dist/chunk-B3wo8uRx.js +322 -0
- package/dist/chunk-BGp3ZdBy.js +560 -0
- package/dist/chunk-BJQv9Elp.js +107 -0
- package/dist/chunk-BOpO6xE-.js +6053 -0
- package/dist/chunk-BQXXCL8B.js +102 -0
- package/dist/chunk-BV-8isOd.js +34 -0
- package/dist/chunk-BVMhjGVe.js +826 -0
- package/dist/chunk-BW2fg46T.js +141 -0
- package/dist/chunk-BWJdY5Ag.js +36 -0
- package/dist/chunk-BYJOrLhh.js +302 -0
- package/dist/chunk-BYab8EVL.js +127 -0
- package/dist/chunk-BfEKvp7h.js +82 -0
- package/dist/chunk-BhgLSfH6.js +1123 -0
- package/dist/chunk-BirUxTtH.js +296 -0
- package/dist/chunk-BmB6UVZ9.js +1559 -0
- package/dist/chunk-BnWhIqW1.js +109 -0
- package/dist/chunk-Bpfnfk0t.js +123 -0
- package/dist/chunk-BrhgdnM3.js +108 -0
- package/dist/chunk-Bt0XAgX5.js +327 -0
- package/dist/chunk-Bu0FJr6S.js +84 -0
- package/dist/chunk-BvEoKpN3.js +255 -0
- package/dist/chunk-BxNJgYbE.js +28 -0
- package/dist/chunk-C1OFhd7F.js +6131 -0
- package/dist/chunk-C7e7WwbU.js +331 -0
- package/dist/chunk-CDTYP6bN.js +558 -0
- package/dist/chunk-CGDQoqDC.js +24 -0
- package/dist/chunk-CJraEBVd.js +127 -0
- package/dist/chunk-CQrNhOtv.js +40 -0
- package/dist/chunk-CW6AzBH6.js +128 -0
- package/dist/chunk-CXgL4w-9.js +317 -0
- package/dist/chunk-CZrqoPyf.js +233 -0
- package/dist/chunk-CcFCyqbB.js +83 -0
- package/dist/chunk-CdaIvO-y.js +485 -0
- package/dist/chunk-CehSXtgK.js +624 -0
- package/dist/chunk-Ciqkop65.js +218 -0
- package/dist/chunk-CoAWdSiE.js +43 -0
- package/dist/chunk-Crb4y7t4.js +830 -0
- package/dist/chunk-CrlqIfmb.js +337 -0
- package/dist/chunk-Cs4pwCQ-.js +324 -0
- package/dist/chunk-CwM_9y01.js +519 -0
- package/dist/chunk-CyRbIJg8.js +440 -0
- package/dist/chunk-D0OHsyMy.js +212 -0
- package/dist/chunk-D3y2bURs.js +1347 -0
- package/dist/chunk-D4_mJk03.js +2338 -0
- package/dist/chunk-D6C8rned.js +578 -0
- package/dist/chunk-D9ZbUnks.js +11 -0
- package/dist/chunk-DEPmLshX.js +834 -0
- package/dist/chunk-DFj1UPB2.js +330 -0
- package/dist/chunk-DGdFdZVf.js +197 -0
- package/dist/chunk-DGv7X9S_.js +63 -0
- package/dist/chunk-DJQ47WYZ.js +664 -0
- package/dist/chunk-DJw3O61j.js +1974 -0
- package/dist/chunk-DK4FbWVK.js +100 -0
- package/dist/chunk-DPuofXMp.js +3907 -0
- package/dist/chunk-DSRYhUus.js +993 -0
- package/dist/chunk-DajZ9Eg4.js +299 -0
- package/dist/chunk-Df4dunvU.js +195 -0
- package/dist/chunk-DjNTPhfO.js +481 -0
- package/dist/chunk-Dn8eaJuu.js +1042 -0
- package/dist/chunk-DwDWqSxM.js +637 -0
- package/dist/chunk-Dz7cEtxl.js +84 -0
- package/dist/chunk-E_0bgjrS.js +359 -0
- package/dist/chunk-KPYN2vUp.js +776 -0
- package/dist/chunk-KcqPQJBi.js +86 -0
- package/dist/chunk-N9H0JnoG.js +349 -0
- package/dist/chunk-NONFcXkn.js +728 -0
- package/dist/chunk-PrFjcpmH.js +256 -0
- package/dist/chunk-QSfNYiw9.js +11 -0
- package/dist/chunk-R265bzj9.js +226 -0
- package/dist/chunk-X9V_r6jM.js +166 -0
- package/dist/chunk-ZBtPUtyA.js +55 -0
- package/dist/chunk-ZCTfmOs7.js +4939 -0
- package/dist/chunk-h6oavj0t.js +97 -0
- package/dist/chunk-i7DwkvvD.js +42 -0
- package/dist/chunk-mG-4eZ6S.js +203 -0
- package/dist/chunk-o0ygUbCf.js +313 -0
- package/dist/chunk-ucu5lrDZ.js +5715 -0
- package/dist/index.js +11367 -73789
- package/package.json +5 -5
|
@@ -0,0 +1,3907 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { C as unknown, S as union, _ as preprocess, a as _null, b as string, c as boolean, f as literal, g as partialRecord, h as object, i as _enum, m as number, s as array, u as discriminatedUnion, v as record } from "./chunk-BOpO6xE-.js";
|
|
3
|
+
import { i as number$1, n as jsonPatchOperationSchema, t as EVAL_CASE_PERSONAS } from "./chunk-BJQv9Elp.js";
|
|
4
|
+
import { a as graderConfigSchema, l as judgeEvidenceSchema, o as graderSeveritySchema, r as evalGraderSnapshotSchema, u as judgeVerdictSchema } from "./chunk-Bt0XAgX5.js";
|
|
5
|
+
import { a as harnessCaptureSummarySchema } from "./chunk-Bu0FJr6S.js";
|
|
6
|
+
import { r as createCliClient } from "./chunk-BW2fg46T.js";
|
|
7
|
+
import { n as ensureAuth } from "./chunk-CJraEBVd.js";
|
|
8
|
+
import { a as printJson } from "./chunk-CcFCyqbB.js";
|
|
9
|
+
import { Command } from "commander";
|
|
10
|
+
import chalk from "chalk";
|
|
11
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
12
|
+
import { join, relative, resolve, sep } from "node:path";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
15
|
+
import { Readable, Transform } from "node:stream";
|
|
16
|
+
import { RuntypeApiError } from "@runtypelabs/sdk";
|
|
17
|
+
import { lstat, mkdtemp, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";
|
|
18
|
+
import { Gzip } from "fflate";
|
|
19
|
+
import { pipeline } from "node:stream/promises";
|
|
20
|
+
// ../shared/dist/simulation/reported-usage.mjs
|
|
21
|
+
const amount = number().finite().nonnegative();
|
|
22
|
+
const tokens = amount.int();
|
|
23
|
+
/** Producer estimates for one turn, independent of Runtype's measured usage and billing. */
|
|
24
|
+
const reportedUsageSchema = object({
|
|
25
|
+
costUsd: amount.nullable(),
|
|
26
|
+
completeness: _enum(["complete", "partial"]),
|
|
27
|
+
scope: string().min(1).max(120),
|
|
28
|
+
models: array(object({
|
|
29
|
+
model: string().min(1).max(200),
|
|
30
|
+
canonicalModel: string().min(1).max(200).optional(),
|
|
31
|
+
provider: string().max(100).optional(),
|
|
32
|
+
pricingBasis: _enum([
|
|
33
|
+
"list",
|
|
34
|
+
"managed",
|
|
35
|
+
"unknown"
|
|
36
|
+
]).optional(),
|
|
37
|
+
costUsd: amount.nullable().optional(),
|
|
38
|
+
inputTokens: tokens.optional(),
|
|
39
|
+
outputTokens: tokens.optional(),
|
|
40
|
+
cacheReadInputTokens: tokens.optional(),
|
|
41
|
+
cacheCreationInputTokens: tokens.optional()
|
|
42
|
+
})).max(100).optional()
|
|
43
|
+
});
|
|
44
|
+
/** Known estimates and reporting coverage; a partial sum is never a whole-episode cost. */
|
|
45
|
+
const selfReportedCostSchema = object({
|
|
46
|
+
knownCostUsd: amount.nullable(),
|
|
47
|
+
reportedTurns: tokens,
|
|
48
|
+
totalTurns: tokens,
|
|
49
|
+
completeTurns: tokens,
|
|
50
|
+
unknownEpisodes: tokens.optional(),
|
|
51
|
+
completeness: _enum([
|
|
52
|
+
"complete",
|
|
53
|
+
"partial",
|
|
54
|
+
"unknown"
|
|
55
|
+
])
|
|
56
|
+
});
|
|
57
|
+
const simTerminationSchema = _enum([
|
|
58
|
+
"goal_met",
|
|
59
|
+
"gave_up",
|
|
60
|
+
"max_turns",
|
|
61
|
+
"agent_error",
|
|
62
|
+
"simulator_error",
|
|
63
|
+
"stalled",
|
|
64
|
+
"deadline",
|
|
65
|
+
"target_unavailable",
|
|
66
|
+
"client_limit",
|
|
67
|
+
"recording_miss",
|
|
68
|
+
"spend_cap",
|
|
69
|
+
"engine_error"
|
|
70
|
+
]);
|
|
71
|
+
/** Who the simulated user is to the target's own access controls (#9489 section 4.1). */
|
|
72
|
+
const simPrincipalSchema = object({
|
|
73
|
+
email: string().email().optional(),
|
|
74
|
+
groups: array(string().max(200)).max(50).optional(),
|
|
75
|
+
department: string().max(200).optional()
|
|
76
|
+
}).strict();
|
|
77
|
+
/**
|
|
78
|
+
* Who a simulated user is to the product; the simulator's first rule follows it. `operator` is left out on purpose:
|
|
79
|
+
* Simulations already uses it for client-agent episodes, and someone who runs the product is a `teammate`.
|
|
80
|
+
*/
|
|
81
|
+
const SIM_USER_ROLES = [
|
|
82
|
+
"customer",
|
|
83
|
+
"end_user",
|
|
84
|
+
"teammate",
|
|
85
|
+
"developer"
|
|
86
|
+
];
|
|
87
|
+
const simUserRoleSchema = _enum(SIM_USER_ROLES);
|
|
88
|
+
object({
|
|
89
|
+
goal: string().min(1).max(2e3),
|
|
90
|
+
profile: string().max(2e3).optional(),
|
|
91
|
+
persona: string().max(64).optional(),
|
|
92
|
+
role: simUserRoleSchema.optional(),
|
|
93
|
+
traits: array(string().max(200)).max(20).optional(),
|
|
94
|
+
knowledge: record(string(), string().max(2e3)).optional(),
|
|
95
|
+
successSignal: string().max(1e3).optional(),
|
|
96
|
+
openingMessage: string().max(4e3).optional(),
|
|
97
|
+
maxTurns: number().int().min(1).max(20).optional(),
|
|
98
|
+
principal: simPrincipalSchema.optional()
|
|
99
|
+
}).strict();
|
|
100
|
+
/** One tool call inside an assistant message, `DispatchExecutionMessage`-shaped. */
|
|
101
|
+
const simToolCallSchema = object({
|
|
102
|
+
toolCallId: string(),
|
|
103
|
+
toolName: string(),
|
|
104
|
+
args: record(string(), unknown())
|
|
105
|
+
});
|
|
106
|
+
/** One tool result inside a tool message, `DispatchExecutionMessage`-shaped. */
|
|
107
|
+
const simToolResultSchema = object({
|
|
108
|
+
toolCallId: string(),
|
|
109
|
+
toolName: string(),
|
|
110
|
+
result: unknown()
|
|
111
|
+
});
|
|
112
|
+
/** One message of a simulated conversation; the target sees all of it, the simulated user only the text. */
|
|
113
|
+
const simMessageSchema = object({
|
|
114
|
+
role: _enum([
|
|
115
|
+
"user",
|
|
116
|
+
"assistant",
|
|
117
|
+
"tool"
|
|
118
|
+
]),
|
|
119
|
+
content: string(),
|
|
120
|
+
toolCalls: array(simToolCallSchema).optional(),
|
|
121
|
+
toolResults: array(simToolResultSchema).optional()
|
|
122
|
+
});
|
|
123
|
+
object({
|
|
124
|
+
message: string(),
|
|
125
|
+
status: _enum([
|
|
126
|
+
"continue",
|
|
127
|
+
"goal_met",
|
|
128
|
+
"gave_up"
|
|
129
|
+
]),
|
|
130
|
+
reason: string().nullable().optional(),
|
|
131
|
+
revealed: array(string()).optional()
|
|
132
|
+
});
|
|
133
|
+
// ../shared/dist/chunk-acmbThRp.mjs
|
|
134
|
+
/** Every change kind a scrub report can name; a leaf so the data-source contracts reach it without the scrubber. */
|
|
135
|
+
const METADATA_SCRUB_CHANGE_KIND_NAMES = [
|
|
136
|
+
"url_redirected",
|
|
137
|
+
"url_removed",
|
|
138
|
+
"secret_refused",
|
|
139
|
+
"secret_env_noted",
|
|
140
|
+
"auth_replaced",
|
|
141
|
+
"connection_replaced",
|
|
142
|
+
"unknown_key_kept"
|
|
143
|
+
];
|
|
144
|
+
function sortKeys(value) {
|
|
145
|
+
if (Array.isArray(value)) return value.map(sortKeys);
|
|
146
|
+
if (value && typeof value === "object") {
|
|
147
|
+
const out = {};
|
|
148
|
+
for (const key of Object.keys(value).sort()) {
|
|
149
|
+
const entry = value[key];
|
|
150
|
+
if (entry !== void 0) out[key] = sortKeys(entry);
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
156
|
+
/** JSON with sorted keys and undefined fields dropped, so equal values always serialize alike. */
|
|
157
|
+
function canonicalJson(value) {
|
|
158
|
+
return JSON.stringify(sortKeys(value)) ?? "null";
|
|
159
|
+
}
|
|
160
|
+
/** Most episodes one run may plan, across every round. */
|
|
161
|
+
const MAX_EPISODES_PER_RUN = 1e3;
|
|
162
|
+
/** Most characters in a simulated user's profile. */
|
|
163
|
+
const MAX_PROFILE_CHARS = 2e3;
|
|
164
|
+
/** Most characters in a task goal. */
|
|
165
|
+
const MAX_GOAL_CHARS = 2e3;
|
|
166
|
+
/** Most characters in a task's opening message. */
|
|
167
|
+
const MAX_OPENING_MESSAGE_CHARS = 4e3;
|
|
168
|
+
/** Most characters in a task's success signal. */
|
|
169
|
+
const MAX_SUCCESS_SIGNAL_CHARS = 1e3;
|
|
170
|
+
/** Most records, or quarantine entries, one episode's state may report. */
|
|
171
|
+
const MAX_SIDE_EFFECT_RECORDS = 1e3;
|
|
172
|
+
/** Most characters in a synthetic dataset's schema-only DDL export (6.9). */
|
|
173
|
+
const MAX_DATASET_SCHEMA_DDL_CHARS = 2e6;
|
|
174
|
+
/** Most characters in a synthetic dataset's permission metadata export, serialized as JSON. */
|
|
175
|
+
const MAX_DATASET_PERMISSION_METADATA_CHARS = 4e6;
|
|
176
|
+
/** Most generated rows in one table of a synthetic dataset. */
|
|
177
|
+
const MAX_DATASET_ROWS_PER_TABLE = 5e3;
|
|
178
|
+
/** Most generated rows across a synthetic dataset. */
|
|
179
|
+
const MAX_DATASET_TOTAL_ROWS = 5e4;
|
|
180
|
+
/** Most changes one scrub report may list. */
|
|
181
|
+
const MAX_DATASET_SCRUB_CHANGES = 2e3;
|
|
182
|
+
const productKindSchema = _enum([
|
|
183
|
+
"runtype",
|
|
184
|
+
"external",
|
|
185
|
+
"spec_only"
|
|
186
|
+
]);
|
|
187
|
+
// ../shared/dist/chunk-Cf0i0FDN.mjs
|
|
188
|
+
/** The manifest file at the root of a candidate's source bundle. */
|
|
189
|
+
const CANDIDATE_MANIFEST_FILENAME = "runtype-candidate.json";
|
|
190
|
+
const candidateRuntimeSchema = _enum([
|
|
191
|
+
"node",
|
|
192
|
+
"bun",
|
|
193
|
+
"python"
|
|
194
|
+
]);
|
|
195
|
+
const candidateProtocolSchema = _enum(["a2a", "runtype-stream"]);
|
|
196
|
+
const candidateSessionsSchema = _enum(["multi_turn", "single_turn"]);
|
|
197
|
+
/** `runtype-candidate.json`. Unknown fields are refused, so a typo fails before a run, not during one. */
|
|
198
|
+
const candidateManifestSchema = object({
|
|
199
|
+
$schema: string().max(512).optional(),
|
|
200
|
+
name: string().trim().min(1).max(120),
|
|
201
|
+
framework: object({
|
|
202
|
+
name: string().trim().min(1).max(64),
|
|
203
|
+
version: string().max(64).regex(/^v?\d+(\.\d+){0,3}([-+][0-9A-Za-z.-]+)?$/)
|
|
204
|
+
}).strict(),
|
|
205
|
+
runtime: candidateRuntimeSchema,
|
|
206
|
+
start: string().trim().min(1).max(1e3),
|
|
207
|
+
protocol: candidateProtocolSchema,
|
|
208
|
+
sessions: candidateSessionsSchema,
|
|
209
|
+
licence: string().regex(/^[A-Za-z0-9][A-Za-z0-9.+-]{0,63}$/)
|
|
210
|
+
}).strict();
|
|
211
|
+
Object.values({
|
|
212
|
+
port: "RUNTYPE_CANDIDATE_PORT",
|
|
213
|
+
modelBaseUrl: "RUNTYPE_MODEL_BASE_URL",
|
|
214
|
+
modelApiKey: "RUNTYPE_MODEL_API_KEY",
|
|
215
|
+
model: "RUNTYPE_MODEL",
|
|
216
|
+
mcpUrl: "RUNTYPE_MCP_URL",
|
|
217
|
+
episodeId: "RUNTYPE_EPISODE_ID"
|
|
218
|
+
});
|
|
219
|
+
/** Parses a manifest file's text; the error names the first field that failed. */
|
|
220
|
+
function parseCandidateManifest(text) {
|
|
221
|
+
let json;
|
|
222
|
+
try {
|
|
223
|
+
json = JSON.parse(text);
|
|
224
|
+
} catch {
|
|
225
|
+
return {
|
|
226
|
+
ok: false,
|
|
227
|
+
error: `${CANDIDATE_MANIFEST_FILENAME} is not valid JSON.`
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const parsed = candidateManifestSchema.safeParse(json);
|
|
231
|
+
if (parsed.success) return {
|
|
232
|
+
ok: true,
|
|
233
|
+
manifest: parsed.data
|
|
234
|
+
};
|
|
235
|
+
const issue = parsed.error.issues[0];
|
|
236
|
+
return {
|
|
237
|
+
ok: false,
|
|
238
|
+
error: `${issue?.path.join(".") || "runtype-candidate.json"}: ${issue?.message ?? "invalid"}`
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/** Bounded and credential-redacted output observed at the authenticated candidate MCP gateway. */
|
|
242
|
+
const simulationAccessToolOutcomeSchema = object({
|
|
243
|
+
name: string(),
|
|
244
|
+
input: unknown(),
|
|
245
|
+
output: unknown(),
|
|
246
|
+
isError: boolean(),
|
|
247
|
+
stubbed: boolean(),
|
|
248
|
+
readOnly: boolean(),
|
|
249
|
+
truncated: boolean()
|
|
250
|
+
});
|
|
251
|
+
/** Server-observed MCP facts for one candidate identity; tool text remains untrusted data. */
|
|
252
|
+
const simulationAccessEvidenceSchema = object({
|
|
253
|
+
source: literal("candidate_mcp_gateway"),
|
|
254
|
+
sourceKind: _enum([
|
|
255
|
+
"mcp_target",
|
|
256
|
+
"stand_in",
|
|
257
|
+
"fixture",
|
|
258
|
+
"none"
|
|
259
|
+
]),
|
|
260
|
+
targetId: string().optional(),
|
|
261
|
+
identity: union([object({
|
|
262
|
+
kind: literal("saved_server"),
|
|
263
|
+
savedServerName: string()
|
|
264
|
+
}), object({ kind: literal("source_default") })]),
|
|
265
|
+
identityMatched: boolean(),
|
|
266
|
+
complete: boolean(),
|
|
267
|
+
rawToolCount: number().int().nonnegative(),
|
|
268
|
+
effectiveToolCount: number().int().nonnegative(),
|
|
269
|
+
tools: array(object({
|
|
270
|
+
name: string(),
|
|
271
|
+
description: string(),
|
|
272
|
+
readOnly: boolean(),
|
|
273
|
+
execution: _enum(["live", "stubbed"])
|
|
274
|
+
})),
|
|
275
|
+
observedAt: string(),
|
|
276
|
+
truncated: boolean(),
|
|
277
|
+
toolOutcomes: array(simulationAccessToolOutcomeSchema)
|
|
278
|
+
});
|
|
279
|
+
const isoDate$1 = string().datetime({ offset: true });
|
|
280
|
+
const entityId$2 = string().min(1).max(255);
|
|
281
|
+
const sha256HexSchema = string().regex(/^[a-f0-9]{64}$/);
|
|
282
|
+
_enum([
|
|
283
|
+
"none",
|
|
284
|
+
"live",
|
|
285
|
+
"synthetic",
|
|
286
|
+
"cloned"
|
|
287
|
+
]);
|
|
288
|
+
const syntheticDatasetKindSchema = _enum(["synthetic"]);
|
|
289
|
+
const dataScopeSchema = _enum([
|
|
290
|
+
"run",
|
|
291
|
+
"variant",
|
|
292
|
+
"episode"
|
|
293
|
+
]);
|
|
294
|
+
const syntheticDatasetBuildStatusSchema = _enum([
|
|
295
|
+
"pending",
|
|
296
|
+
"building",
|
|
297
|
+
"ready",
|
|
298
|
+
"failed"
|
|
299
|
+
]);
|
|
300
|
+
const permissionMetadataEngineSchema = _enum(["hasura"]);
|
|
301
|
+
/** A simulated user's identity alias, mapped to an engine grant by the dataset (`alias_grants`). */
|
|
302
|
+
const DATASET_IDENTITY_ALIAS_PATTERN = /^[a-z0-9][a-z0-9_.-]{0,63}$/;
|
|
303
|
+
/** A runner clone hook's name, as the runner config declares it. */
|
|
304
|
+
const CLONE_HOOK_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/;
|
|
305
|
+
/** A table, optionally schema-qualified, as the DDL export names it. */
|
|
306
|
+
const DATASET_TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_$]*(\.[A-Za-z_][A-Za-z0-9_$]*)?$/;
|
|
307
|
+
/** An OCI image reference pinned by digest. */
|
|
308
|
+
const DATASET_IMAGE_REFERENCE_PATTERN = /^[a-z0-9][a-z0-9._/:-]*@sha256:[a-f0-9]{64}$/;
|
|
309
|
+
const grantNameSchema = string().min(1).max(128);
|
|
310
|
+
/** One tool as a grant's `tools/list` returned it; a stand-in serves these names, schemas and annotations. */
|
|
311
|
+
const datasetToolSnapshotSchema = object({
|
|
312
|
+
name: string().min(1).max(128),
|
|
313
|
+
title: string().max(255).optional(),
|
|
314
|
+
description: string().max(4e3).optional(),
|
|
315
|
+
inputSchema: record(string(), unknown()),
|
|
316
|
+
outputSchema: record(string(), unknown()).optional(),
|
|
317
|
+
annotations: record(string(), unknown()).optional()
|
|
318
|
+
});
|
|
319
|
+
/** What one grant (an engine role) saw from `tools/list`; a stand-in's permissions come from these. */
|
|
320
|
+
const datasetGrantToolsSnapshotSchema = object({
|
|
321
|
+
grant: grantNameSchema,
|
|
322
|
+
tools: array(datasetToolSnapshotSchema).max(200)
|
|
323
|
+
});
|
|
324
|
+
/** The MCP server a synthetic dataset serves: the customer's image by digest, or a stand-in built from per-grant snapshots. */
|
|
325
|
+
const datasetMcpServerSchema = discriminatedUnion("kind", [object({
|
|
326
|
+
kind: literal("image"),
|
|
327
|
+
image: string().max(512).regex(DATASET_IMAGE_REFERENCE_PATTERN)
|
|
328
|
+
}), object({
|
|
329
|
+
kind: literal("stand_in"),
|
|
330
|
+
grants: array(datasetGrantToolsSnapshotSchema).min(1).max(50)
|
|
331
|
+
})]);
|
|
332
|
+
/** Row counts the customer supplies (never rows), and the seed that makes a rebuild byte-identical. */
|
|
333
|
+
const datasetSpecInputSchema = object({
|
|
334
|
+
rowCounts: record(string().max(128).regex(DATASET_TABLE_NAME_PATTERN), number().int().min(0).max(MAX_DATASET_ROWS_PER_TABLE)).refine((counts) => Object.keys(counts).length <= 500, { message: `At most 500 tables.` }),
|
|
335
|
+
seed: number().int().min(0).max(4294967295)
|
|
336
|
+
}).refine((dataset) => Object.values(dataset.rowCounts).reduce((sum, count) => sum + count, 0) <= MAX_DATASET_TOTAL_ROWS, {
|
|
337
|
+
message: `At most ${MAX_DATASET_TOTAL_ROWS} rows across the dataset.`,
|
|
338
|
+
path: ["rowCounts"]
|
|
339
|
+
});
|
|
340
|
+
/** The dataset spec as stored (`dataset_spec`), with the generator version it was written for. */
|
|
341
|
+
const datasetSpecSchema = object({
|
|
342
|
+
rowCounts: record(string(), number().int().nonnegative()),
|
|
343
|
+
seed: number().int().nonnegative(),
|
|
344
|
+
generatorVersion: number().int().positive()
|
|
345
|
+
});
|
|
346
|
+
const datasetScrubChangeKindSchema = _enum(METADATA_SCRUB_CHANGE_KIND_NAMES);
|
|
347
|
+
/** One change import made to a permission metadata export, as `scrubPermissionMetadata` reports it. */
|
|
348
|
+
const datasetScrubChangeSchema = object({
|
|
349
|
+
/** JSON Pointer into the metadata object. */
|
|
350
|
+
path: string().max(1024),
|
|
351
|
+
kind: datasetScrubChangeKindSchema,
|
|
352
|
+
/** The original value, redacted: URL hosts masked, secrets reduced to their length. */
|
|
353
|
+
before: string().max(4e3).nullable(),
|
|
354
|
+
/** The value now at `path`; null when it was removed or left as exported. */
|
|
355
|
+
after: unknown(),
|
|
356
|
+
note: string().max(1e3).optional()
|
|
357
|
+
});
|
|
358
|
+
/** Every change the scrubber made, so the customer can review what the dataset runs instead of their rules. */
|
|
359
|
+
const datasetScrubReportSchema = object({
|
|
360
|
+
format: literal("hasura_metadata_v3"),
|
|
361
|
+
/** `refused` when a literal secret was found; a refused export is never stored. */
|
|
362
|
+
status: _enum(["clean", "refused"]),
|
|
363
|
+
changes: array(datasetScrubChangeSchema).max(MAX_DATASET_SCRUB_CHANGES),
|
|
364
|
+
/** Changes per kind, keyed by kind; an open record, so the spec lists no snake_case property names. */
|
|
365
|
+
counts: record(string(), number().int().nonnegative()),
|
|
366
|
+
/** Every role a permission or inherited role names; identity roles and stand-in grants map onto these. */
|
|
367
|
+
roles: array(string().max(255)).max(MAX_DATASET_SCRUB_CHANGES)
|
|
368
|
+
});
|
|
369
|
+
/** The report a stored definition carries: only a clean scrub is ever imported. */
|
|
370
|
+
const cleanDatasetScrubReportSchema = datasetScrubReportSchema.extend({ status: literal("clean") });
|
|
371
|
+
/** A GraphQL engine's permission metadata export, already scrubbed; configuration, not data, but sealed at rest. */
|
|
372
|
+
const datasetPermissionMetadataSchema = object({
|
|
373
|
+
engine: permissionMetadataEngineSchema,
|
|
374
|
+
export: record(string(), unknown()).refine((value) => canonicalJson(value).length <= MAX_DATASET_PERMISSION_METADATA_CHARS, { message: `The metadata export must be at most ${MAX_DATASET_PERMISSION_METADATA_CHARS} characters.` })
|
|
375
|
+
});
|
|
376
|
+
object({
|
|
377
|
+
name: string().min(1).max(255),
|
|
378
|
+
kind: syntheticDatasetKindSchema,
|
|
379
|
+
schemaDdl: string().min(1).max(MAX_DATASET_SCHEMA_DDL_CHARS),
|
|
380
|
+
permissionMetadata: datasetPermissionMetadataSchema,
|
|
381
|
+
scrubReport: cleanDatasetScrubReportSchema,
|
|
382
|
+
dataset: datasetSpecInputSchema,
|
|
383
|
+
mcpServer: datasetMcpServerSchema.nullable().optional(),
|
|
384
|
+
aliasGrants: record(string().regex(DATASET_IDENTITY_ALIAS_PATTERN), grantNameSchema).refine((roles) => Object.keys(roles).length <= 50, { message: `At most 50 identity aliases.` }).optional()
|
|
385
|
+
}).omit({
|
|
386
|
+
scrubReport: true,
|
|
387
|
+
kind: true
|
|
388
|
+
}).extend({ kind: syntheticDatasetKindSchema.default("synthetic") });
|
|
389
|
+
/** A saved synthetic dataset as the API returns it; the DDL and the metadata export never leave the store. */
|
|
390
|
+
const simulationSyntheticDatasetSchema = object({
|
|
391
|
+
id: entityId$2,
|
|
392
|
+
name: string(),
|
|
393
|
+
kind: syntheticDatasetKindSchema,
|
|
394
|
+
permissionEngine: permissionMetadataEngineSchema,
|
|
395
|
+
scrubReport: datasetScrubReportSchema,
|
|
396
|
+
dataset: datasetSpecSchema,
|
|
397
|
+
mcpServer: datasetMcpServerSchema.nullable(),
|
|
398
|
+
aliasGrants: record(string(), string()),
|
|
399
|
+
buildStatus: syntheticDatasetBuildStatusSchema,
|
|
400
|
+
buildError: string().nullable(),
|
|
401
|
+
builtAt: isoDate$1.nullable(),
|
|
402
|
+
definitionHash: sha256HexSchema,
|
|
403
|
+
createdByUserId: entityId$2.nullable(),
|
|
404
|
+
createdAt: isoDate$1,
|
|
405
|
+
updatedAt: isoDate$1
|
|
406
|
+
});
|
|
407
|
+
/** The data a variant runs against (`simulation_variants.data`); none means `none` or `live` by target kind. */
|
|
408
|
+
const variantDataSchema = discriminatedUnion("kind", [object({
|
|
409
|
+
kind: literal("synthetic"),
|
|
410
|
+
syntheticDatasetId: entityId$2,
|
|
411
|
+
scope: dataScopeSchema,
|
|
412
|
+
/** The grant a stand-in serves this variant's client agent; needed when the dataset has more than one. */
|
|
413
|
+
grant: grantNameSchema.optional()
|
|
414
|
+
}), object({
|
|
415
|
+
kind: literal("cloned"),
|
|
416
|
+
hook: string().regex(CLONE_HOOK_NAME_PATTERN),
|
|
417
|
+
scope: dataScopeSchema
|
|
418
|
+
})]);
|
|
419
|
+
string().max(160).regex(/^(none|live|synthetic:[a-f0-9]{64}|cloned:[a-z0-9][a-z0-9_-]{0,63})$/);
|
|
420
|
+
const recordTypeSchema = string().min(1).max(128);
|
|
421
|
+
const fieldPathSchema = string().min(1).max(128).regex(/^[A-Za-z0-9_$-]+(\.[A-Za-z0-9_$-]+)*$/);
|
|
422
|
+
/**
|
|
423
|
+
* A field's expected value: equal to a literal, or containing a substring. Strings may name task data with
|
|
424
|
+
* `{{knowledge.NAME}}`, `{{user.NAME}}` or `{{episode.id}}`; strings compare trimmed and case-insensitively.
|
|
425
|
+
*/
|
|
426
|
+
const sideEffectValueSchema = union([
|
|
427
|
+
string().max(1e3),
|
|
428
|
+
number(),
|
|
429
|
+
boolean(),
|
|
430
|
+
_null(),
|
|
431
|
+
object({ contains: string().min(1).max(1e3) }).strict()
|
|
432
|
+
]);
|
|
433
|
+
/** Field conditions, all of which a record must meet; a dotted path reads a nested field. */
|
|
434
|
+
const sideEffectWhereSchema = record(fieldPathSchema, sideEffectValueSchema).refine((where) => Object.keys(where).length <= 20, { message: `At most 20 conditions.` });
|
|
435
|
+
/** A record a `linked` check expects a link to: by id, by its fields, or both. */
|
|
436
|
+
const sideEffectLinkTargetSchema = object({
|
|
437
|
+
type: recordTypeSchema,
|
|
438
|
+
id: string().min(1).max(256).optional(),
|
|
439
|
+
where: sideEffectWhereSchema.optional()
|
|
440
|
+
});
|
|
441
|
+
const labelSchema = string().min(1).max(120).optional();
|
|
442
|
+
/** One declarative check over the records an episode wrote; deterministic, with no judge. */
|
|
443
|
+
const sideEffectCheckSchema = discriminatedUnion("assert", [
|
|
444
|
+
object({
|
|
445
|
+
assert: literal("exists"),
|
|
446
|
+
type: recordTypeSchema,
|
|
447
|
+
where: sideEffectWhereSchema.optional(),
|
|
448
|
+
label: labelSchema
|
|
449
|
+
}),
|
|
450
|
+
object({
|
|
451
|
+
assert: literal("absent"),
|
|
452
|
+
type: recordTypeSchema,
|
|
453
|
+
where: sideEffectWhereSchema.optional(),
|
|
454
|
+
label: labelSchema
|
|
455
|
+
}),
|
|
456
|
+
object({
|
|
457
|
+
assert: literal("linked"),
|
|
458
|
+
type: recordTypeSchema,
|
|
459
|
+
where: sideEffectWhereSchema.optional(),
|
|
460
|
+
to: array(sideEffectLinkTargetSchema).min(1).max(5),
|
|
461
|
+
label: labelSchema
|
|
462
|
+
}),
|
|
463
|
+
object({
|
|
464
|
+
assert: literal("in_quarantine"),
|
|
465
|
+
where: sideEffectWhereSchema.optional(),
|
|
466
|
+
label: labelSchema
|
|
467
|
+
})
|
|
468
|
+
]);
|
|
469
|
+
const SIDE_EFFECT_ASSERTIONS = [
|
|
470
|
+
"exists",
|
|
471
|
+
"absent",
|
|
472
|
+
"linked",
|
|
473
|
+
"in_quarantine"
|
|
474
|
+
];
|
|
475
|
+
/** A simulation task's grader over the episode's resulting state, not its transcript (6.10). */
|
|
476
|
+
const sideEffectGraderSchema = object({
|
|
477
|
+
kind: literal("side_effect"),
|
|
478
|
+
name: string().min(1).max(120).optional(),
|
|
479
|
+
checks: array(sideEffectCheckSchema).min(1).max(20),
|
|
480
|
+
/**
|
|
481
|
+
* On a synthetic dataset: the record type (the dataset's table) each write tool writes, keyed by tool name. A stand-in
|
|
482
|
+
* keeps no state, so this is how its accepted writes become records; a target's state endpoint names types itself.
|
|
483
|
+
*/
|
|
484
|
+
writes: record(string().min(1).max(128), recordTypeSchema).refine((writes) => Object.keys(writes).length <= 100, { message: `At most 100 write tools.` }).optional(),
|
|
485
|
+
severity: graderSeveritySchema.optional()
|
|
486
|
+
});
|
|
487
|
+
/** One check's outcome on one episode, by its position in the grader's `checks`. */
|
|
488
|
+
const sideEffectCheckResultSchema = object({
|
|
489
|
+
index: number().int().min(0),
|
|
490
|
+
assert: _enum(SIDE_EFFECT_ASSERTIONS),
|
|
491
|
+
passed: boolean()
|
|
492
|
+
});
|
|
493
|
+
/** The side-effect grader as a score row stores it, with each check's result. */
|
|
494
|
+
const sideEffectGraderSnapshotSchema = sideEffectGraderSchema.extend({ results: array(sideEffectCheckResultSchema).optional() });
|
|
495
|
+
const stateIdSchema = union([string().min(1).max(256), number()]).transform(String);
|
|
496
|
+
const stateFieldsSchema = record(string(), unknown());
|
|
497
|
+
/** One record an episode wrote, as a state endpoint or the synthetic dataset reports it. */
|
|
498
|
+
const sideEffectRecordSchema = object({
|
|
499
|
+
type: recordTypeSchema,
|
|
500
|
+
id: stateIdSchema,
|
|
501
|
+
fields: stateFieldsSchema,
|
|
502
|
+
links: array(object({
|
|
503
|
+
type: recordTypeSchema,
|
|
504
|
+
id: stateIdSchema
|
|
505
|
+
})).max(100).default([])
|
|
506
|
+
});
|
|
507
|
+
/** One submission held in quarantine for review. */
|
|
508
|
+
const sideEffectQuarantineEntrySchema = object({
|
|
509
|
+
id: stateIdSchema,
|
|
510
|
+
fields: stateFieldsSchema
|
|
511
|
+
});
|
|
512
|
+
object({
|
|
513
|
+
episode: string().min(1).max(255),
|
|
514
|
+
records: array(sideEffectRecordSchema).max(MAX_SIDE_EFFECT_RECORDS),
|
|
515
|
+
quarantine: array(sideEffectQuarantineEntrySchema).max(MAX_SIDE_EFFECT_RECORDS)
|
|
516
|
+
});
|
|
517
|
+
const toolNameSchema = string().min(1).max(128);
|
|
518
|
+
const argumentPathSchema = string().min(1).max(128).regex(/^[A-Za-z0-9_$-]+(\.[A-Za-z0-9_$-]+)*$/);
|
|
519
|
+
const allowedCountSchema = number().int().nonnegative().max(1e3);
|
|
520
|
+
/** One deterministic check over the tool calls an episode made; no judge reads it. */
|
|
521
|
+
const toolProcessCheckSchema = discriminatedUnion("assert", [
|
|
522
|
+
object({
|
|
523
|
+
assert: literal("no_wrong_tool"),
|
|
524
|
+
/** Calls the expected-tool rule may label wrong before the check fails. */
|
|
525
|
+
max: allowedCountSchema.optional()
|
|
526
|
+
}),
|
|
527
|
+
object({
|
|
528
|
+
assert: literal("argument_format"),
|
|
529
|
+
tools: array(toolNameSchema).min(1).max(20),
|
|
530
|
+
argument: argumentPathSchema,
|
|
531
|
+
/** `uuid`: the value is a UUID string. `uuid_keyed`: an object whose every key is `<uuid>_<suffix>`. */
|
|
532
|
+
format: _enum(["uuid", "uuid_keyed"]),
|
|
533
|
+
/** For `uuid_keyed`, the suffixes a key may end in, such as filter operators. */
|
|
534
|
+
keySuffixes: array(string().min(1).max(32).regex(/^[A-Za-z0-9]+$/)).max(20).optional(),
|
|
535
|
+
/** Count only calls that also ended in an error, for a surface that may accept the other form. */
|
|
536
|
+
onlyFailed: boolean().optional(),
|
|
537
|
+
max: allowedCountSchema.optional()
|
|
538
|
+
}),
|
|
539
|
+
object({
|
|
540
|
+
assert: literal("no_repeat_call"),
|
|
541
|
+
tool: toolNameSchema,
|
|
542
|
+
/** The arguments that make two calls the same fetch; absent compares every argument. */
|
|
543
|
+
keyArguments: array(argumentPathSchema).min(1).max(8).optional(),
|
|
544
|
+
/** Calls allowed per key before the check fails; defaults to 1. */
|
|
545
|
+
max: number().int().positive().max(1e3).optional()
|
|
546
|
+
}),
|
|
547
|
+
object({
|
|
548
|
+
assert: literal("budget"),
|
|
549
|
+
maxTurns: number().int().positive().max(1e3).optional(),
|
|
550
|
+
maxTokens: number().int().positive().optional()
|
|
551
|
+
}),
|
|
552
|
+
object({
|
|
553
|
+
assert: literal("must_call"),
|
|
554
|
+
/** A completed call to any one of these counts; a designed rename counts under its shown or its server name. */
|
|
555
|
+
tools: array(toolNameSchema).min(1).max(20),
|
|
556
|
+
/** Completed calls the listed tools must add up to; defaults to 1. */
|
|
557
|
+
minCalls: number().int().positive().max(1e3).optional()
|
|
558
|
+
})
|
|
559
|
+
]);
|
|
560
|
+
/**
|
|
561
|
+
* A Simulations-only grader over how an episode used its tools: wrong tool choices, argument formats, repeated
|
|
562
|
+
* fetches, a turn or token budget and a required call. Deterministic; eval suites refuse it.
|
|
563
|
+
*/
|
|
564
|
+
const toolProcessGraderSchema = object({
|
|
565
|
+
kind: literal("tool_process"),
|
|
566
|
+
name: string().min(1).max(120),
|
|
567
|
+
check: toolProcessCheckSchema,
|
|
568
|
+
severity: graderSeveritySchema.optional()
|
|
569
|
+
}).refine((grader) => grader.check.assert !== "budget" || grader.check.maxTurns !== void 0 || grader.check.maxTokens !== void 0, {
|
|
570
|
+
message: "A budget check needs maxTurns or maxTokens.",
|
|
571
|
+
path: ["check"]
|
|
572
|
+
});
|
|
573
|
+
new RegExp(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, "i");
|
|
574
|
+
const variantPlacementSchema = _enum([
|
|
575
|
+
"runtype",
|
|
576
|
+
"runtype_sandbox",
|
|
577
|
+
"external_endpoint",
|
|
578
|
+
"vendor",
|
|
579
|
+
"private_runner"
|
|
580
|
+
]);
|
|
581
|
+
const variantFidelitySchema = _enum([
|
|
582
|
+
"runtype",
|
|
583
|
+
"real_code",
|
|
584
|
+
"endpoint",
|
|
585
|
+
"vendor_hosted",
|
|
586
|
+
"simulated_agent"
|
|
587
|
+
]);
|
|
588
|
+
const episodeRanOnSchema = _enum([
|
|
589
|
+
"runtype_worker",
|
|
590
|
+
"runtype_durable",
|
|
591
|
+
"sandbox",
|
|
592
|
+
"external_endpoint",
|
|
593
|
+
"private_runner",
|
|
594
|
+
"vendor"
|
|
595
|
+
]);
|
|
596
|
+
const isoDate = string().datetime({ offset: true });
|
|
597
|
+
const entityId$1 = string().min(1).max(255);
|
|
598
|
+
const shortText = (max) => string().max(max);
|
|
599
|
+
const simulationSlugSchema = string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/);
|
|
600
|
+
const simulationVariantKeySchema = string().regex(/^[a-z0-9][a-z0-9_-]{0,47}$/);
|
|
601
|
+
const simulationModeSchema = _enum(["benchmark", "user_testing"]);
|
|
602
|
+
const simulationPresetSchema = _enum(["agent_experience"]);
|
|
603
|
+
const targetKindSchema = _enum([
|
|
604
|
+
"runtype_surface",
|
|
605
|
+
"runtype_capability",
|
|
606
|
+
"external_agent",
|
|
607
|
+
"external_mcp",
|
|
608
|
+
"external_api",
|
|
609
|
+
"website",
|
|
610
|
+
"cli",
|
|
611
|
+
"skill",
|
|
612
|
+
"sandbox_candidate",
|
|
613
|
+
"vendor_hosted"
|
|
614
|
+
]);
|
|
615
|
+
const targetVerificationStatusSchema = _enum([
|
|
616
|
+
"not_required",
|
|
617
|
+
"unverified",
|
|
618
|
+
"pending",
|
|
619
|
+
"verified"
|
|
620
|
+
]);
|
|
621
|
+
const targetVerificationMethodSchema = _enum(["well_known", "dns_txt"]);
|
|
622
|
+
/** An ownership challenge (4.8): the token, where to publish it, and the last check's outcome. */
|
|
623
|
+
const simulationTargetVerificationSchema = object({
|
|
624
|
+
/** The host the challenge covers; changing the target to another host clears the verification. */
|
|
625
|
+
host: string(),
|
|
626
|
+
token: string(),
|
|
627
|
+
/** The URL that may serve `token` on a line of its own. */
|
|
628
|
+
wellKnownUrl: string(),
|
|
629
|
+
/** The DNS name whose TXT record may hold `token` instead. */
|
|
630
|
+
dnsRecordName: string(),
|
|
631
|
+
startedAt: isoDate,
|
|
632
|
+
checkedAt: isoDate.nullable(),
|
|
633
|
+
verifiedAt: isoDate.nullable(),
|
|
634
|
+
method: targetVerificationMethodSchema.nullable(),
|
|
635
|
+
/** Why the last check did not verify; null after a success or before any check. */
|
|
636
|
+
lastError: string().nullable()
|
|
637
|
+
});
|
|
638
|
+
const SECRET_REFERENCE = /^\{\{secret:[^{}]+\}\}$/;
|
|
639
|
+
const CREDENTIAL_HEADER_NAME = /auth|cookie|api[-_]?key|token|secret|passw|session|credential/i;
|
|
640
|
+
const CREDENTIAL_HEADER_VALUE = /^(bearer|basic|token)\s/i;
|
|
641
|
+
/** Whether a target header carries a literal credential: a credential-looking name or value that is not a `{{secret:NAME}}` reference. */
|
|
642
|
+
function isLiteralCredentialHeader(name, value) {
|
|
643
|
+
if (SECRET_REFERENCE.test(value.trim())) return false;
|
|
644
|
+
return CREDENTIAL_HEADER_NAME.test(name) || CREDENTIAL_HEADER_VALUE.test(value.trim());
|
|
645
|
+
}
|
|
646
|
+
const CREDENTIAL_QUERY_NAME = /^key$|auth|api[-_]?key|token|secret|passw|session|credential|signature|sig$/i;
|
|
647
|
+
/** Whether a target URL carries a credential: userinfo, or a credential-looking query parameter. */
|
|
648
|
+
function urlCarriesCredential(url) {
|
|
649
|
+
let parsed;
|
|
650
|
+
try {
|
|
651
|
+
parsed = new URL(url);
|
|
652
|
+
} catch {
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
if (parsed.username || parsed.password) return true;
|
|
656
|
+
return [...parsed.searchParams.keys()].some((name) => CREDENTIAL_QUERY_NAME.test(name));
|
|
657
|
+
}
|
|
658
|
+
/** A target URL; a credential never rides in it, only in a saved connection or a `{{secret:NAME}}` header. */
|
|
659
|
+
const credentialFreeUrl = string().url().refine((url) => !urlCarriesCredential(url), { message: "A target URL may not carry a credential. Connect the server as a saved connection (savedServerName) instead." });
|
|
660
|
+
/** Target headers; a credential is never stored literally, only as a `{{secret:NAME}}` reference or a saved connection. */
|
|
661
|
+
const secretReferenceHeaders = record(string().min(1).max(128), string().max(2048)).refine((headers) => Object.entries(headers).every(([name, value]) => !isLiteralCredentialHeader(name, value)), { message: "A target header may not carry a literal credential. Connect the server as a saved connection (savedServerName) instead." });
|
|
662
|
+
const runtypeSurfaceTargetConfigSchema = object({
|
|
663
|
+
kind: literal("runtype_surface"),
|
|
664
|
+
productId: entityId$1,
|
|
665
|
+
surfaceId: entityId$1,
|
|
666
|
+
surfaceType: string().min(1).max(64)
|
|
667
|
+
});
|
|
668
|
+
const runtypeCapabilityTargetConfigSchema = object({
|
|
669
|
+
kind: literal("runtype_capability"),
|
|
670
|
+
productId: entityId$1.optional(),
|
|
671
|
+
capabilityId: entityId$1.optional(),
|
|
672
|
+
flowId: entityId$1.optional(),
|
|
673
|
+
agentId: entityId$1.optional()
|
|
674
|
+
});
|
|
675
|
+
const externalAgentTargetConfigSchema = object({
|
|
676
|
+
kind: literal("external_agent"),
|
|
677
|
+
agentId: entityId$1
|
|
678
|
+
});
|
|
679
|
+
/** Largest `grantsToTools.grants`, as UTF-8 bytes of JSON; admission copies it into the run plan, a 2 MB value. */
|
|
680
|
+
const MAX_GRANTS_TO_TOOLS_JSON_BYTES = 64e3;
|
|
681
|
+
/**
|
|
682
|
+
* Which tools each grant should see in `tools/list`, as a `grants-to-tools.json` document carries it; other keys of
|
|
683
|
+
* the document are ignored. Simulated users name their grants in `identity.expectedGrants`, and the permission matrix
|
|
684
|
+
* (6.11) compares what each one saw and called with the union of its grants' tools.
|
|
685
|
+
*/
|
|
686
|
+
const grantsToToolsSchema = object({
|
|
687
|
+
version: literal(1).optional(),
|
|
688
|
+
grants: record(string().trim().min(1).max(64), array(string().min(1).max(128)).max(500)).refine((grants) => Object.keys(grants).length > 0, { message: "Name at least one grant and the tools it should see." }).refine((grants) => Object.keys(grants).length <= 50, { message: `Name at most 50 grants.` }).refine((grants) => new TextEncoder().encode(JSON.stringify(grants)).byteLength <= MAX_GRANTS_TO_TOOLS_JSON_BYTES, { message: `The grants-to-tools map may be at most ${MAX_GRANTS_TO_TOOLS_JSON_BYTES} bytes of JSON.` })
|
|
689
|
+
});
|
|
690
|
+
/**
|
|
691
|
+
* Where a target reports what an episode wrote (6.10): `GET <url>?episode=<episodeId>` with the resolved secret as a
|
|
692
|
+
* bearer token. It must be on the target's own origin.
|
|
693
|
+
*/
|
|
694
|
+
const stateEndpointSchema = object({
|
|
695
|
+
url: credentialFreeUrl,
|
|
696
|
+
secretRef: string().max(110).regex(/^\{\{secret:([A-Z][A-Z0-9_]{0,98}[A-Z0-9])\}\}$/, { message: "secretRef must be a {{secret:NAME}} reference to a managed secret, never a literal value." })
|
|
697
|
+
});
|
|
698
|
+
const externalMcpTargetConfigSchema = object({
|
|
699
|
+
kind: literal("external_mcp"),
|
|
700
|
+
url: credentialFreeUrl,
|
|
701
|
+
transport: _enum(["streamable_http", "sse"]).optional(),
|
|
702
|
+
headers: secretReferenceHeaders.optional(),
|
|
703
|
+
savedServerName: string().min(1).max(128).optional(),
|
|
704
|
+
allowedTools: array(string().min(1).max(128)).max(200).optional(),
|
|
705
|
+
/** The expected result of the permission matrix: the tools each grant should see. */
|
|
706
|
+
grantsToTools: grantsToToolsSchema.optional(),
|
|
707
|
+
/** Read after each episode by side-effect graders when the variant has no synthetic dataset. */
|
|
708
|
+
stateEndpoint: stateEndpointSchema.optional()
|
|
709
|
+
});
|
|
710
|
+
const externalApiTargetConfigSchema = object({
|
|
711
|
+
kind: literal("external_api"),
|
|
712
|
+
openapiUrl: credentialFreeUrl.optional(),
|
|
713
|
+
document: unknown().optional(),
|
|
714
|
+
baseUrl: credentialFreeUrl,
|
|
715
|
+
headers: secretReferenceHeaders.optional(),
|
|
716
|
+
operations: array(string().min(1).max(256)).max(50).optional()
|
|
717
|
+
});
|
|
718
|
+
const websiteTargetConfigSchema = object({
|
|
719
|
+
kind: literal("website"),
|
|
720
|
+
url: credentialFreeUrl
|
|
721
|
+
});
|
|
722
|
+
const cliTargetConfigSchema = object({
|
|
723
|
+
kind: literal("cli"),
|
|
724
|
+
package: union([object({
|
|
725
|
+
registry: _enum(["npm", "pip"]),
|
|
726
|
+
name: string().min(1).max(214),
|
|
727
|
+
version: string().min(1).max(64)
|
|
728
|
+
}), object({
|
|
729
|
+
url: credentialFreeUrl,
|
|
730
|
+
sha256: string().regex(/^[a-f0-9]{64}$/)
|
|
731
|
+
})]),
|
|
732
|
+
command: string().min(1).max(128),
|
|
733
|
+
secretEnv: record(string().min(1).max(128), string().max(256)).optional()
|
|
734
|
+
});
|
|
735
|
+
const skillTargetConfigSchema = object({
|
|
736
|
+
kind: literal("skill"),
|
|
737
|
+
skillId: entityId$1.optional(),
|
|
738
|
+
bundleUrl: credentialFreeUrl.optional()
|
|
739
|
+
});
|
|
740
|
+
/** A candidate's source bundle with its dependency lock, run as real code in a Runtype sandbox (6.9). */
|
|
741
|
+
const sandboxCandidateTargetConfigSchema = object({
|
|
742
|
+
kind: literal("sandbox_candidate"),
|
|
743
|
+
bundle: object({
|
|
744
|
+
sha256: string().regex(/^[a-f0-9]{64}$/),
|
|
745
|
+
/** Where the bundle is stored; opaque to the contract, and never a URL that carries a credential. */
|
|
746
|
+
ref: string().min(1).max(512).refine((ref) => !urlCarriesCredential(ref), { message: "A bundle reference may not carry a credential." })
|
|
747
|
+
}),
|
|
748
|
+
/** The bundle's `runtype-candidate.json`, read at upload so admission never unpacks the bundle. */
|
|
749
|
+
manifest: candidateManifestSchema,
|
|
750
|
+
/** An `external_mcp` target whose tools the candidate gets when its variant sets no data. */
|
|
751
|
+
mcpTargetId: entityId$1.optional()
|
|
752
|
+
});
|
|
753
|
+
/** A vendor's hosted agent, driven over A2A, pinned to a version where the vendor allows it. */
|
|
754
|
+
const vendorHostedTargetConfigSchema = object({
|
|
755
|
+
kind: literal("vendor_hosted"),
|
|
756
|
+
vendor: string().min(1).max(64),
|
|
757
|
+
protocol: literal("a2a"),
|
|
758
|
+
url: credentialFreeUrl,
|
|
759
|
+
headers: secretReferenceHeaders.optional(),
|
|
760
|
+
version: string().min(1).max(64).optional()
|
|
761
|
+
});
|
|
762
|
+
/** One member per target kind; secrets are references only. */
|
|
763
|
+
const targetConfigSchema = discriminatedUnion("kind", [
|
|
764
|
+
runtypeSurfaceTargetConfigSchema,
|
|
765
|
+
runtypeCapabilityTargetConfigSchema,
|
|
766
|
+
externalAgentTargetConfigSchema,
|
|
767
|
+
externalMcpTargetConfigSchema,
|
|
768
|
+
externalApiTargetConfigSchema,
|
|
769
|
+
websiteTargetConfigSchema,
|
|
770
|
+
cliTargetConfigSchema,
|
|
771
|
+
skillTargetConfigSchema,
|
|
772
|
+
sandboxCandidateTargetConfigSchema,
|
|
773
|
+
vendorHostedTargetConfigSchema
|
|
774
|
+
]);
|
|
775
|
+
/** Whether the operator host records a target's tool answers for replay, and for how long it keeps them (D1). */
|
|
776
|
+
const targetRecordingSchema = object({
|
|
777
|
+
enabled: boolean(),
|
|
778
|
+
retentionDays: number().int().min(1).max(365)
|
|
779
|
+
});
|
|
780
|
+
const exchangeRecordingModeSchema = _enum([
|
|
781
|
+
"off",
|
|
782
|
+
"shape-only",
|
|
783
|
+
"full"
|
|
784
|
+
]);
|
|
785
|
+
/** One target's recording policy as admission froze it into the run plan. */
|
|
786
|
+
const frozenTargetRecordingSchema = object({
|
|
787
|
+
mode: exchangeRecordingModeSchema,
|
|
788
|
+
retentionDays: number().int().min(0).max(365)
|
|
789
|
+
});
|
|
790
|
+
/** A saved target, as the API returns it. */
|
|
791
|
+
const simulationTargetSchema = object({
|
|
792
|
+
id: entityId$1,
|
|
793
|
+
name: string().min(1).max(255),
|
|
794
|
+
kind: targetKindSchema,
|
|
795
|
+
config: targetConfigSchema,
|
|
796
|
+
productId: entityId$1.nullable(),
|
|
797
|
+
surfaceId: entityId$1.nullable(),
|
|
798
|
+
flowId: entityId$1.nullable(),
|
|
799
|
+
agentId: entityId$1.nullable(),
|
|
800
|
+
originHost: string().nullable(),
|
|
801
|
+
verificationStatus: targetVerificationStatusSchema,
|
|
802
|
+
/** The ownership challenge once one is started; absent on a target built in memory. */
|
|
803
|
+
verification: simulationTargetVerificationSchema.nullable().optional(),
|
|
804
|
+
/** Additive: absent on targets saved before recording existed, which record nothing. */
|
|
805
|
+
recording: targetRecordingSchema.optional(),
|
|
806
|
+
createdByUserId: entityId$1.nullable(),
|
|
807
|
+
createdAt: isoDate,
|
|
808
|
+
updatedAt: isoDate
|
|
809
|
+
});
|
|
810
|
+
const variantRoleSchema = _enum([
|
|
811
|
+
"baseline",
|
|
812
|
+
"candidate",
|
|
813
|
+
"comparison"
|
|
814
|
+
]);
|
|
815
|
+
const variantSourceSchema = _enum([
|
|
816
|
+
"manual",
|
|
817
|
+
"grid",
|
|
818
|
+
"optimizer"
|
|
819
|
+
]);
|
|
820
|
+
/** A capability id, or `*` for every capability the variant reaches. */
|
|
821
|
+
const capabilityRef = union([literal("*"), entityId$1]);
|
|
822
|
+
const modelVariantChangeSchema = object({
|
|
823
|
+
kind: literal("model"),
|
|
824
|
+
capabilityId: capabilityRef,
|
|
825
|
+
stepId: string().min(1).max(255).optional(),
|
|
826
|
+
model: string().min(1).max(255)
|
|
827
|
+
});
|
|
828
|
+
const samplingVariantChangeSchema = object({
|
|
829
|
+
kind: literal("sampling"),
|
|
830
|
+
capabilityId: capabilityRef,
|
|
831
|
+
temperature: number().min(0).max(2).optional(),
|
|
832
|
+
maxTokens: number().int().positive().max(1e6).optional(),
|
|
833
|
+
reasoning: union([boolean(), record(string(), unknown())]).optional()
|
|
834
|
+
});
|
|
835
|
+
const instructionsVariantChangeSchema = object({
|
|
836
|
+
kind: literal("instructions"),
|
|
837
|
+
capabilityId: entityId$1,
|
|
838
|
+
mode: _enum(["replace", "append"]),
|
|
839
|
+
text: string().min(1).max(2e4)
|
|
840
|
+
});
|
|
841
|
+
const loopVariantChangeSchema = object({
|
|
842
|
+
kind: literal("loop"),
|
|
843
|
+
capabilityId: entityId$1,
|
|
844
|
+
maxTurns: number().int().positive().max(200).optional(),
|
|
845
|
+
maxCost: number().positive().optional(),
|
|
846
|
+
enableReflection: boolean().optional(),
|
|
847
|
+
toolSearch: boolean().optional()
|
|
848
|
+
});
|
|
849
|
+
const toolsVariantChangeSchema = object({
|
|
850
|
+
kind: literal("tools"),
|
|
851
|
+
capabilityId: entityId$1,
|
|
852
|
+
add: array(entityId$1).max(50).optional(),
|
|
853
|
+
remove: array(entityId$1).max(50).optional()
|
|
854
|
+
});
|
|
855
|
+
const flowStepsVariantChangeSchema = object({
|
|
856
|
+
kind: literal("flowSteps"),
|
|
857
|
+
capabilityId: entityId$1,
|
|
858
|
+
ops: array(jsonPatchOperationSchema).min(1).max(100)
|
|
859
|
+
});
|
|
860
|
+
const surfaceRoutingVariantChangeSchema = object({
|
|
861
|
+
kind: literal("surfaceRouting"),
|
|
862
|
+
surfaceId: entityId$1,
|
|
863
|
+
routerModel: string().min(1).max(255).optional(),
|
|
864
|
+
routerPrompt: string().max(2e4).optional(),
|
|
865
|
+
enabledCapabilityIds: array(entityId$1).max(100).optional(),
|
|
866
|
+
orchestratorCapabilityId: entityId$1.nullable().optional()
|
|
867
|
+
});
|
|
868
|
+
const frameworkVariantChangeSchema = object({
|
|
869
|
+
kind: literal("framework"),
|
|
870
|
+
capabilityId: entityId$1,
|
|
871
|
+
agentId: entityId$1
|
|
872
|
+
});
|
|
873
|
+
const clientModelVariantChangeSchema = object({
|
|
874
|
+
kind: literal("clientModel"),
|
|
875
|
+
model: string().min(1).max(255)
|
|
876
|
+
});
|
|
877
|
+
/** A JSON Schema property as a tool lists it; open beyond the fields an overlay edits. */
|
|
878
|
+
const toolSchemaPropertySchema = object({
|
|
879
|
+
type: string().optional(),
|
|
880
|
+
description: string().optional(),
|
|
881
|
+
enum: array(union([
|
|
882
|
+
string(),
|
|
883
|
+
number(),
|
|
884
|
+
boolean()
|
|
885
|
+
])).optional(),
|
|
886
|
+
default: unknown().optional(),
|
|
887
|
+
examples: array(unknown()).optional()
|
|
888
|
+
}).passthrough();
|
|
889
|
+
const errorMatchSchema = union([object({ code: string().min(1).max(128) }), object({ pattern: string().min(1).max(512) })]);
|
|
890
|
+
/** One deterministic step of a response mapping; paths are dot paths into the tool's answer. */
|
|
891
|
+
const shapeOpSchema = discriminatedUnion("kind", [
|
|
892
|
+
object({
|
|
893
|
+
kind: literal("pick"),
|
|
894
|
+
paths: array(string().min(1)).min(1).max(64)
|
|
895
|
+
}),
|
|
896
|
+
object({
|
|
897
|
+
kind: literal("rename"),
|
|
898
|
+
from: string().min(1),
|
|
899
|
+
to: string().min(1)
|
|
900
|
+
}),
|
|
901
|
+
object({
|
|
902
|
+
kind: literal("flatten"),
|
|
903
|
+
path: string().min(1)
|
|
904
|
+
}),
|
|
905
|
+
object({
|
|
906
|
+
kind: literal("drop"),
|
|
907
|
+
paths: array(string().min(1)).min(1).max(64)
|
|
908
|
+
}),
|
|
909
|
+
object({
|
|
910
|
+
kind: literal("truncate"),
|
|
911
|
+
path: string().min(1),
|
|
912
|
+
maxItems: number().int().positive().max(1e4).optional(),
|
|
913
|
+
maxChars: number().int().positive().max(1e6).optional()
|
|
914
|
+
}),
|
|
915
|
+
object({
|
|
916
|
+
kind: literal("count"),
|
|
917
|
+
path: string().min(1),
|
|
918
|
+
as: string().min(1)
|
|
919
|
+
}),
|
|
920
|
+
object({
|
|
921
|
+
kind: literal("map"),
|
|
922
|
+
path: string().min(1),
|
|
923
|
+
pick: array(string().min(1)).min(1).max(64).optional(),
|
|
924
|
+
rename: record(string(), string()).optional()
|
|
925
|
+
})
|
|
926
|
+
]);
|
|
927
|
+
/** A read a shape script needs, answered from a recording (or a live read-only call) before the script runs. */
|
|
928
|
+
const shapeLookupSchema = object({
|
|
929
|
+
as: string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/).max(64),
|
|
930
|
+
tool: string().min(1),
|
|
931
|
+
arguments: record(string(), unknown())
|
|
932
|
+
});
|
|
933
|
+
/**
|
|
934
|
+
* A response mapping: a list of shape steps, or a script run in a sandboxed Dynamic Worker (D4). A script is a
|
|
935
|
+
* transform body (`return ...`) over `result`, `args`, `profile` and `lookups`, each lookup keyed by its `as`.
|
|
936
|
+
*/
|
|
937
|
+
const shapeTransformSchema = discriminatedUnion("kind", [object({
|
|
938
|
+
kind: literal("ops"),
|
|
939
|
+
ops: array(shapeOpSchema).min(1).max(64)
|
|
940
|
+
}), object({
|
|
941
|
+
kind: literal("script"),
|
|
942
|
+
code: string().min(1).max(32768),
|
|
943
|
+
lookups: array(shapeLookupSchema).max(4).optional()
|
|
944
|
+
})]);
|
|
945
|
+
/** The designed error an `error_design` operation returns; text fields are templates over the real error. */
|
|
946
|
+
const errorEnvelopeSchema = object({
|
|
947
|
+
class: _enum([
|
|
948
|
+
"transient",
|
|
949
|
+
"permanent",
|
|
950
|
+
"userInput",
|
|
951
|
+
"notFound"
|
|
952
|
+
]),
|
|
953
|
+
message: string().min(1).max(2e3),
|
|
954
|
+
reason: string().max(2e3).optional(),
|
|
955
|
+
recoverySteps: array(string().min(1).max(500)).max(8).default([]),
|
|
956
|
+
retryAfterSeconds: number().int().nonnegative().optional(),
|
|
957
|
+
suggestionsFrom: object({
|
|
958
|
+
tool: string().min(1),
|
|
959
|
+
arguments: record(string(), unknown())
|
|
960
|
+
}).optional()
|
|
961
|
+
});
|
|
962
|
+
/** The definition an `add_tool` operation lists, as `tools/list` would show it. */
|
|
963
|
+
const addToolDefinitionSchema = object({
|
|
964
|
+
name: string().min(1).max(128),
|
|
965
|
+
description: string().max(4e3),
|
|
966
|
+
inputSchema: record(string(), unknown()),
|
|
967
|
+
outputSchema: record(string(), unknown()).optional(),
|
|
968
|
+
annotations: record(string(), unknown()).optional()
|
|
969
|
+
});
|
|
970
|
+
/** How an added tool answers: other tools through the ladder, or the mini-agent grounded on recordings (D4, D5). */
|
|
971
|
+
const addToolImplementationSchema = discriminatedUnion("kind", [object({
|
|
972
|
+
kind: literal("composite"),
|
|
973
|
+
steps: array(object({
|
|
974
|
+
tool: string().min(1),
|
|
975
|
+
arguments: record(string(), unknown())
|
|
976
|
+
})).min(1).max(8),
|
|
977
|
+
shape: shapeTransformSchema.optional()
|
|
978
|
+
}), object({
|
|
979
|
+
kind: literal("synthesized"),
|
|
980
|
+
grounding: array(string().min(1)).min(1).max(8),
|
|
981
|
+
worldFacts: array(string().min(1).max(2e3)).max(32).optional()
|
|
982
|
+
})]);
|
|
983
|
+
/** One operation of a `tool_surface` overlay ("Agent experience", "Tool-surface overlays"). */
|
|
984
|
+
const toolSurfaceOpSchema = discriminatedUnion("op", [
|
|
985
|
+
object({
|
|
986
|
+
op: literal("rename"),
|
|
987
|
+
tool: string().min(1),
|
|
988
|
+
name: string().min(1).max(128)
|
|
989
|
+
}),
|
|
990
|
+
object({
|
|
991
|
+
op: literal("describe"),
|
|
992
|
+
tool: string().min(1),
|
|
993
|
+
description: string().max(4e3)
|
|
994
|
+
}),
|
|
995
|
+
object({
|
|
996
|
+
op: literal("schema"),
|
|
997
|
+
tool: string().min(1),
|
|
998
|
+
properties: record(string(), toolSchemaPropertySchema).optional(),
|
|
999
|
+
required: array(string()).optional()
|
|
1000
|
+
}),
|
|
1001
|
+
object({
|
|
1002
|
+
op: literal("hide"),
|
|
1003
|
+
tool: string().min(1)
|
|
1004
|
+
}),
|
|
1005
|
+
object({
|
|
1006
|
+
op: literal("preset"),
|
|
1007
|
+
tool: string().min(1),
|
|
1008
|
+
name: string().min(1).max(128),
|
|
1009
|
+
description: string().max(4e3).optional(),
|
|
1010
|
+
fixed: record(string(), unknown())
|
|
1011
|
+
}),
|
|
1012
|
+
object({
|
|
1013
|
+
op: literal("error_rewrite"),
|
|
1014
|
+
tool: string().min(1),
|
|
1015
|
+
match: errorMatchSchema,
|
|
1016
|
+
message: string().min(1).max(2e3)
|
|
1017
|
+
}),
|
|
1018
|
+
object({
|
|
1019
|
+
op: literal("shape"),
|
|
1020
|
+
tool: string().min(1),
|
|
1021
|
+
transform: shapeTransformSchema
|
|
1022
|
+
}),
|
|
1023
|
+
object({
|
|
1024
|
+
op: literal("paginate"),
|
|
1025
|
+
tool: string().min(1),
|
|
1026
|
+
style: literal("cursor"),
|
|
1027
|
+
limitField: string().min(1),
|
|
1028
|
+
itemsPath: string().min(1),
|
|
1029
|
+
defaultLimit: number().int().positive().max(1e3)
|
|
1030
|
+
}),
|
|
1031
|
+
object({
|
|
1032
|
+
op: literal("error_design"),
|
|
1033
|
+
tool: string().min(1),
|
|
1034
|
+
match: errorMatchSchema,
|
|
1035
|
+
envelope: errorEnvelopeSchema
|
|
1036
|
+
}),
|
|
1037
|
+
object({
|
|
1038
|
+
op: literal("accept_identifier"),
|
|
1039
|
+
tool: string().min(1),
|
|
1040
|
+
field: string().min(1),
|
|
1041
|
+
resolveVia: object({
|
|
1042
|
+
tool: string().min(1),
|
|
1043
|
+
queryField: string().min(1),
|
|
1044
|
+
idField: string().min(1),
|
|
1045
|
+
labelFields: array(string().min(1)).min(1).max(8)
|
|
1046
|
+
}),
|
|
1047
|
+
thresholds: object({
|
|
1048
|
+
single: number().min(0).max(1).default(.9),
|
|
1049
|
+
candidate: number().min(0).max(1).default(.5),
|
|
1050
|
+
maxOptions: number().int().positive().max(20).default(5)
|
|
1051
|
+
}).default({
|
|
1052
|
+
single: .9,
|
|
1053
|
+
candidate: .5,
|
|
1054
|
+
maxOptions: 5
|
|
1055
|
+
})
|
|
1056
|
+
}),
|
|
1057
|
+
object({
|
|
1058
|
+
op: literal("add_tool"),
|
|
1059
|
+
tool: string().min(1),
|
|
1060
|
+
definition: addToolDefinitionSchema,
|
|
1061
|
+
implementation: addToolImplementationSchema
|
|
1062
|
+
}).refine((value) => value.tool === value.definition.name, {
|
|
1063
|
+
message: "tool must equal definition.name",
|
|
1064
|
+
path: ["tool"]
|
|
1065
|
+
})
|
|
1066
|
+
]);
|
|
1067
|
+
const TOOL_SURFACE_OPS = [
|
|
1068
|
+
"rename",
|
|
1069
|
+
"describe",
|
|
1070
|
+
"schema",
|
|
1071
|
+
"hide",
|
|
1072
|
+
"preset",
|
|
1073
|
+
"error_rewrite",
|
|
1074
|
+
"shape",
|
|
1075
|
+
"paginate",
|
|
1076
|
+
"error_design",
|
|
1077
|
+
"accept_identifier",
|
|
1078
|
+
"add_tool"
|
|
1079
|
+
];
|
|
1080
|
+
/** Operations that change what a tool answers, not only what it lists; the rest are request-side (D4). */
|
|
1081
|
+
const FUTURE_STATE_OPS = [
|
|
1082
|
+
"shape",
|
|
1083
|
+
"paginate",
|
|
1084
|
+
"error_design",
|
|
1085
|
+
"accept_identifier",
|
|
1086
|
+
"add_tool"
|
|
1087
|
+
];
|
|
1088
|
+
new Set(FUTURE_STATE_OPS);
|
|
1089
|
+
/** Where a proposed overlay came from ("Overlay proposals"): never sent to a target, kept so a person can check it. */
|
|
1090
|
+
const toolSurfaceProvenanceSchema = object({
|
|
1091
|
+
proposer: _enum(["rules", "model"]),
|
|
1092
|
+
runId: entityId$1,
|
|
1093
|
+
episodeIds: array(entityId$1).max(50),
|
|
1094
|
+
scenarioIds: array(entityId$1).max(50),
|
|
1095
|
+
rows: array(object({
|
|
1096
|
+
table: _enum([
|
|
1097
|
+
"tools",
|
|
1098
|
+
"errors",
|
|
1099
|
+
"confusion",
|
|
1100
|
+
"never_discovered"
|
|
1101
|
+
]),
|
|
1102
|
+
tool: string().max(128),
|
|
1103
|
+
detail: string().max(500)
|
|
1104
|
+
})).max(50),
|
|
1105
|
+
reasons: array(object({
|
|
1106
|
+
tool: string().max(128),
|
|
1107
|
+
op: _enum(TOOL_SURFACE_OPS),
|
|
1108
|
+
text: string().max(1e3),
|
|
1109
|
+
/** The tool-design checklist row the operation satisfies (D7). */
|
|
1110
|
+
checklistRow: number().int().positive().optional(),
|
|
1111
|
+
/** The trace signal that motivated the operation, such as "high retry rate". */
|
|
1112
|
+
signal: string().max(200).optional()
|
|
1113
|
+
})).max(50),
|
|
1114
|
+
/** The touched tools as the server listed them during the run, which the diff and the patch read. */
|
|
1115
|
+
tools: array(object({
|
|
1116
|
+
name: string().max(128),
|
|
1117
|
+
description: string().max(4e3),
|
|
1118
|
+
inputSchema: record(string(), unknown())
|
|
1119
|
+
})).max(50),
|
|
1120
|
+
/** The server's error text each rewrite replaces, by `tool:match`. */
|
|
1121
|
+
errors: record(string().max(300), string().max(500)).optional()
|
|
1122
|
+
});
|
|
1123
|
+
const toolSurfaceVariantChangeSchema = object({
|
|
1124
|
+
kind: literal("tool_surface"),
|
|
1125
|
+
capabilityId: entityId$1.optional(),
|
|
1126
|
+
targetId: entityId$1.optional(),
|
|
1127
|
+
/** A short name for the overlay, such as "Clearer descriptions and errors". */
|
|
1128
|
+
label: string().min(1).max(120).optional(),
|
|
1129
|
+
ops: array(toolSurfaceOpSchema).min(1).max(200),
|
|
1130
|
+
provenance: toolSurfaceProvenanceSchema.optional()
|
|
1131
|
+
});
|
|
1132
|
+
const helpTextVariantChangeSchema = object({
|
|
1133
|
+
kind: literal("help_text"),
|
|
1134
|
+
targetId: entityId$1.optional(),
|
|
1135
|
+
commandPath: array(string().min(1).max(64)).max(8),
|
|
1136
|
+
helpText: string().min(1).max(2e4)
|
|
1137
|
+
});
|
|
1138
|
+
const aliasVariantChangeSchema = object({
|
|
1139
|
+
kind: literal("alias"),
|
|
1140
|
+
targetId: entityId$1.optional(),
|
|
1141
|
+
commandPath: array(string().min(1).max(64)).min(1).max(8),
|
|
1142
|
+
realCommandPath: array(string().min(1).max(64)).min(1).max(8),
|
|
1143
|
+
fixedArgs: array(string().max(256)).max(32).optional()
|
|
1144
|
+
});
|
|
1145
|
+
/** Every change kind a variant may carry (2.1); the last three are overlays ("Agent experience"). */
|
|
1146
|
+
const variantChangeSchema = discriminatedUnion("kind", [
|
|
1147
|
+
modelVariantChangeSchema,
|
|
1148
|
+
samplingVariantChangeSchema,
|
|
1149
|
+
instructionsVariantChangeSchema,
|
|
1150
|
+
loopVariantChangeSchema,
|
|
1151
|
+
toolsVariantChangeSchema,
|
|
1152
|
+
flowStepsVariantChangeSchema,
|
|
1153
|
+
surfaceRoutingVariantChangeSchema,
|
|
1154
|
+
frameworkVariantChangeSchema,
|
|
1155
|
+
clientModelVariantChangeSchema,
|
|
1156
|
+
toolSurfaceVariantChangeSchema,
|
|
1157
|
+
helpTextVariantChangeSchema,
|
|
1158
|
+
aliasVariantChangeSchema
|
|
1159
|
+
]);
|
|
1160
|
+
const overlayKindSchema = _enum([
|
|
1161
|
+
"proxy",
|
|
1162
|
+
"materialized",
|
|
1163
|
+
"shim"
|
|
1164
|
+
]);
|
|
1165
|
+
const toolPolicySchema = _enum([
|
|
1166
|
+
"live",
|
|
1167
|
+
"recorded",
|
|
1168
|
+
"simulated"
|
|
1169
|
+
]);
|
|
1170
|
+
/** A policy per tool name, with `*` for every tool not named. */
|
|
1171
|
+
const variantToolPolicySchema = record(string().min(1), toolPolicySchema);
|
|
1172
|
+
/** Replay matching: argument keys that may differ between a call and a recording without changing the answer. */
|
|
1173
|
+
const variantReplaySchema = object({
|
|
1174
|
+
answerNeutralKeys: array(string().min(1)).max(32).default([]),
|
|
1175
|
+
/** False turns rung 6 off for a `simulated` design: a call no recording answers gets the stub, never a model call. */
|
|
1176
|
+
synthesize: boolean().optional()
|
|
1177
|
+
});
|
|
1178
|
+
/** A saved variant, as the API returns it. */
|
|
1179
|
+
const simulationVariantSchema = object({
|
|
1180
|
+
id: entityId$1,
|
|
1181
|
+
simulationId: entityId$1,
|
|
1182
|
+
runId: entityId$1.nullable(),
|
|
1183
|
+
key: simulationVariantKeySchema,
|
|
1184
|
+
label: string().min(1).max(255),
|
|
1185
|
+
role: variantRoleSchema,
|
|
1186
|
+
targetId: entityId$1,
|
|
1187
|
+
changes: array(variantChangeSchema),
|
|
1188
|
+
changesHash: string(),
|
|
1189
|
+
clientModel: string().nullable(),
|
|
1190
|
+
source: variantSourceSchema,
|
|
1191
|
+
parentVariantId: entityId$1.nullable(),
|
|
1192
|
+
/** Where the variant's target code runs (6.1). */
|
|
1193
|
+
placement: variantPlacementSchema,
|
|
1194
|
+
fidelity: variantFidelitySchema,
|
|
1195
|
+
/** The data the variant runs against; null runs in the target kind's world (`TARGET_KIND_WORLDS`). */
|
|
1196
|
+
data: variantDataSchema.nullable(),
|
|
1197
|
+
/** Additive: absent means every tool is `live`. */
|
|
1198
|
+
toolPolicy: variantToolPolicySchema.optional(),
|
|
1199
|
+
replay: variantReplaySchema.optional(),
|
|
1200
|
+
archivedAt: isoDate.nullable(),
|
|
1201
|
+
createdAt: isoDate
|
|
1202
|
+
});
|
|
1203
|
+
const simulationDefinitionSourceSchema = _enum([
|
|
1204
|
+
"manual",
|
|
1205
|
+
"generated",
|
|
1206
|
+
"from_conversation",
|
|
1207
|
+
"agent",
|
|
1208
|
+
"authored"
|
|
1209
|
+
]);
|
|
1210
|
+
/** A reference to what a generated user or task was drawn from. */
|
|
1211
|
+
const simulationSourceRefSchema = object({
|
|
1212
|
+
kind: _enum([
|
|
1213
|
+
"spec",
|
|
1214
|
+
"capability",
|
|
1215
|
+
"surface",
|
|
1216
|
+
"conversation",
|
|
1217
|
+
"execution",
|
|
1218
|
+
"tools_list",
|
|
1219
|
+
"help",
|
|
1220
|
+
"homepage"
|
|
1221
|
+
]),
|
|
1222
|
+
id: string().max(255).optional(),
|
|
1223
|
+
label: string().max(255).optional()
|
|
1224
|
+
});
|
|
1225
|
+
/** A simulated user's preset: an eval-case persona, or `adversarial_submitter` (6.12), which only Simulations has. */
|
|
1226
|
+
const SIMULATED_USER_PRESETS = [
|
|
1227
|
+
EVAL_CASE_PERSONAS[0].id,
|
|
1228
|
+
...EVAL_CASE_PERSONAS.slice(1).map((persona) => persona.id),
|
|
1229
|
+
"adversarial_submitter"
|
|
1230
|
+
];
|
|
1231
|
+
const simulatedUserPresetSchema = _enum(SIMULATED_USER_PRESETS);
|
|
1232
|
+
const simulatedUserRelationshipSchema = _enum(SIM_USER_ROLES);
|
|
1233
|
+
/** What a simulated user carries about themselves (email, groups, department); an eval case calls it `principal`. */
|
|
1234
|
+
const simulatedUserAttributesSchema = simPrincipalSchema;
|
|
1235
|
+
/**
|
|
1236
|
+
* A simulated user's own sign-in on an external MCP target: a saved MCP connection of the organization, used in place
|
|
1237
|
+
* of the target's connection for this user's episodes, and the grants that member is expected to hold.
|
|
1238
|
+
*/
|
|
1239
|
+
const simulatedUserIdentitySchema = object({
|
|
1240
|
+
savedServerName: string().trim().min(1).max(128),
|
|
1241
|
+
expectedGrants: array(string().trim().min(1).max(64)).max(20)
|
|
1242
|
+
});
|
|
1243
|
+
/** What a person authors about a simulated user. */
|
|
1244
|
+
const simulatedUserDefinitionSchema = object({
|
|
1245
|
+
slug: simulationSlugSchema,
|
|
1246
|
+
name: string().min(1).max(255),
|
|
1247
|
+
profile: string().max(MAX_PROFILE_CHARS),
|
|
1248
|
+
preset: simulatedUserPresetSchema.nullable().optional(),
|
|
1249
|
+
relationship: simulatedUserRelationshipSchema.nullable().optional(),
|
|
1250
|
+
traits: array(shortText(200)).max(20).optional(),
|
|
1251
|
+
knowledge: record(string().min(1).max(128), shortText(2e3)).optional(),
|
|
1252
|
+
attributes: simulatedUserAttributesSchema.nullable().optional(),
|
|
1253
|
+
identity: simulatedUserIdentitySchema.nullable().optional(),
|
|
1254
|
+
productId: entityId$1.nullable().optional(),
|
|
1255
|
+
source: simulationDefinitionSourceSchema.optional(),
|
|
1256
|
+
sourceRefs: array(simulationSourceRefSchema).max(20).optional()
|
|
1257
|
+
});
|
|
1258
|
+
/** A saved simulated user. Never call it a persona: Persona is the chat widget. */
|
|
1259
|
+
const simulatedUserSchema = object({
|
|
1260
|
+
id: entityId$1,
|
|
1261
|
+
slug: simulationSlugSchema,
|
|
1262
|
+
name: string(),
|
|
1263
|
+
profile: string(),
|
|
1264
|
+
preset: string().nullable(),
|
|
1265
|
+
relationship: simulatedUserRelationshipSchema.nullable(),
|
|
1266
|
+
traits: array(string()),
|
|
1267
|
+
knowledge: record(string(), string()),
|
|
1268
|
+
attributes: simulatedUserAttributesSchema.nullable(),
|
|
1269
|
+
identity: simulatedUserIdentitySchema.nullable(),
|
|
1270
|
+
productId: entityId$1.nullable(),
|
|
1271
|
+
source: simulationDefinitionSourceSchema,
|
|
1272
|
+
sourceRefs: array(simulationSourceRefSchema),
|
|
1273
|
+
createdByUserId: entityId$1.nullable(),
|
|
1274
|
+
createdAt: isoDate,
|
|
1275
|
+
updatedAt: isoDate
|
|
1276
|
+
});
|
|
1277
|
+
const episodeShapeSchema = _enum([
|
|
1278
|
+
"conversation",
|
|
1279
|
+
"tool_use",
|
|
1280
|
+
"any"
|
|
1281
|
+
]);
|
|
1282
|
+
/** A task's expected-tool hint; a hint always wins over the tool-choice judge. */
|
|
1283
|
+
const expectedToolsSchema = object({
|
|
1284
|
+
tools: array(string().min(1).max(128)).min(1).max(50),
|
|
1285
|
+
ordered: boolean().optional(),
|
|
1286
|
+
avoid: array(string().min(1).max(128)).max(50).optional()
|
|
1287
|
+
});
|
|
1288
|
+
/**
|
|
1289
|
+
* The injection grader (6.12, read side): a task that carries it fails an episode whose agent calls a tool in
|
|
1290
|
+
* `expectedTools.avoid`, or puts the episode's canary in a tool call's arguments or a reply. Simulations only.
|
|
1291
|
+
*/
|
|
1292
|
+
const injectionGraderSchema = object({
|
|
1293
|
+
kind: literal("injection"),
|
|
1294
|
+
/** Text added to tool results the client agent reads, with `{{episode.canary}}` filled in; absent plants none. */
|
|
1295
|
+
plant: object({
|
|
1296
|
+
/** The tool whose results carry it; absent means every read-only tool. */
|
|
1297
|
+
tool: string().min(1).max(128).optional(),
|
|
1298
|
+
text: string().min(1).max(2e3)
|
|
1299
|
+
}).optional(),
|
|
1300
|
+
/** Tools whose arguments may carry the planted text, such as a submission tool; a canary there is no leak. */
|
|
1301
|
+
passThroughTools: array(string().min(1).max(128)).max(20).optional()
|
|
1302
|
+
});
|
|
1303
|
+
const injectionCanaryPlaceSchema = _enum(["tool_args", "reply"]);
|
|
1304
|
+
/** What an injection grader found on one episode; names only, so it survives `metrics` retention. */
|
|
1305
|
+
const injectionFindingsSchema = object({
|
|
1306
|
+
canaryIn: array(injectionCanaryPlaceSchema),
|
|
1307
|
+
avoidedTools: array(string())
|
|
1308
|
+
});
|
|
1309
|
+
/** The injection grader as a score row stores it: its config, with what it found stamped at grade time. */
|
|
1310
|
+
const injectionGraderSnapshotSchema = injectionGraderSchema.extend({ findings: injectionFindingsSchema.optional() });
|
|
1311
|
+
/** A task's grader: an eval grader, or a Simulations-only injection, side-effect or tool-process grader. */
|
|
1312
|
+
const simulationTaskGraderSchema = union([
|
|
1313
|
+
graderConfigSchema,
|
|
1314
|
+
injectionGraderSchema,
|
|
1315
|
+
sideEffectGraderSchema,
|
|
1316
|
+
toolProcessGraderSchema
|
|
1317
|
+
]);
|
|
1318
|
+
/** A grader as an episode's score row stores it. */
|
|
1319
|
+
const simulationGraderSnapshotSchema = union([
|
|
1320
|
+
evalGraderSnapshotSchema,
|
|
1321
|
+
injectionGraderSnapshotSchema,
|
|
1322
|
+
sideEffectGraderSnapshotSchema,
|
|
1323
|
+
toolProcessGraderSchema
|
|
1324
|
+
]);
|
|
1325
|
+
/** What a person authors about a task. */
|
|
1326
|
+
const simulationTaskDefinitionSchema = object({
|
|
1327
|
+
slug: simulationSlugSchema,
|
|
1328
|
+
name: string().min(1).max(255),
|
|
1329
|
+
goal: string().min(1).max(MAX_GOAL_CHARS),
|
|
1330
|
+
openingMessage: string().max(MAX_OPENING_MESSAGE_CHARS).nullable().optional(),
|
|
1331
|
+
successSignal: string().max(MAX_SUCCESS_SIGNAL_CHARS).nullable().optional(),
|
|
1332
|
+
knowledge: record(string().min(1).max(128), shortText(2e3)).optional(),
|
|
1333
|
+
maxTurns: number().int().min(1).max(20).nullable().optional(),
|
|
1334
|
+
graders: array(simulationTaskGraderSchema).max(20).optional(),
|
|
1335
|
+
category: string().max(64).nullable().optional(),
|
|
1336
|
+
tags: array(string().min(1).max(64)).max(20).optional(),
|
|
1337
|
+
episodeShape: episodeShapeSchema.optional(),
|
|
1338
|
+
expectedTools: expectedToolsSchema.nullable().optional(),
|
|
1339
|
+
productId: entityId$1.nullable().optional(),
|
|
1340
|
+
source: simulationDefinitionSourceSchema.optional(),
|
|
1341
|
+
sourceRefs: array(simulationSourceRefSchema).max(20).optional()
|
|
1342
|
+
});
|
|
1343
|
+
/** A saved task. */
|
|
1344
|
+
const simulationTaskSchema = object({
|
|
1345
|
+
id: entityId$1,
|
|
1346
|
+
slug: simulationSlugSchema,
|
|
1347
|
+
name: string(),
|
|
1348
|
+
goal: string(),
|
|
1349
|
+
openingMessage: string().nullable(),
|
|
1350
|
+
successSignal: string().nullable(),
|
|
1351
|
+
knowledge: record(string(), string()),
|
|
1352
|
+
maxTurns: number().int().nullable(),
|
|
1353
|
+
graders: array(simulationTaskGraderSchema),
|
|
1354
|
+
category: string().nullable(),
|
|
1355
|
+
tags: array(string()),
|
|
1356
|
+
episodeShape: episodeShapeSchema,
|
|
1357
|
+
expectedTools: expectedToolsSchema.nullable(),
|
|
1358
|
+
productId: entityId$1.nullable(),
|
|
1359
|
+
source: simulationDefinitionSourceSchema,
|
|
1360
|
+
sourceRefs: array(simulationSourceRefSchema),
|
|
1361
|
+
createdByUserId: entityId$1.nullable(),
|
|
1362
|
+
createdAt: isoDate,
|
|
1363
|
+
updatedAt: isoDate
|
|
1364
|
+
});
|
|
1365
|
+
/** One simulated user paired with one task, inside one simulation. */
|
|
1366
|
+
const simulationScenarioSchema = object({
|
|
1367
|
+
id: entityId$1,
|
|
1368
|
+
simulationId: entityId$1,
|
|
1369
|
+
simulatedUserId: entityId$1,
|
|
1370
|
+
taskId: entityId$1,
|
|
1371
|
+
weight: number().int().min(1).max(100),
|
|
1372
|
+
enabled: boolean()
|
|
1373
|
+
});
|
|
1374
|
+
/** Who acknowledged that a run may call tools with real side effects (0.4). */
|
|
1375
|
+
const liveToolsAcknowledgementSchema = object({
|
|
1376
|
+
tools: array(string().min(1).max(255)).max(200),
|
|
1377
|
+
actorUserId: entityId$1,
|
|
1378
|
+
at: isoDate
|
|
1379
|
+
});
|
|
1380
|
+
const SEARCH_STRATEGIES = [
|
|
1381
|
+
"none",
|
|
1382
|
+
"grid",
|
|
1383
|
+
"successive_halving"
|
|
1384
|
+
];
|
|
1385
|
+
const SIMULATION_OBJECTIVES = ["success_then_cost", "success_then_latency"];
|
|
1386
|
+
const BASELINE_SOURCES = ["published", "draft"];
|
|
1387
|
+
const simulationDataRetentionSchema = _enum(["transcripts", "metrics"]);
|
|
1388
|
+
/** A simulation's settings as stored; every field is optional and `resolveSimulationSettings` fills the rest. */
|
|
1389
|
+
const simulationSettingsSchema = object({
|
|
1390
|
+
trials: number().int().min(1).max(10).optional(),
|
|
1391
|
+
maxTurns: number().int().min(1).max(20).optional(),
|
|
1392
|
+
simulatedUserModel: string().min(1).max(255).optional(),
|
|
1393
|
+
judge: object({ model: string().min(1).max(255).optional() }).optional(),
|
|
1394
|
+
clientModel: string().min(1).max(255).optional(),
|
|
1395
|
+
/** Above `MAX_CONCURRENCY` only a run with no live upstream accepts it; admission refuses it for any other. */
|
|
1396
|
+
concurrency: number().int().min(1).max(24).optional(),
|
|
1397
|
+
budget: object({
|
|
1398
|
+
maxSpendUsd: number().positive().max(1e4).optional(),
|
|
1399
|
+
maxEpisodes: number().int().positive().max(MAX_EPISODES_PER_RUN).optional()
|
|
1400
|
+
}).optional(),
|
|
1401
|
+
search: object({
|
|
1402
|
+
strategy: _enum(SEARCH_STRATEGIES),
|
|
1403
|
+
eta: number().int().min(2).max(4).optional(),
|
|
1404
|
+
maxRounds: number().int().min(1).max(3).optional()
|
|
1405
|
+
}).optional(),
|
|
1406
|
+
objective: _enum(SIMULATION_OBJECTIVES).optional(),
|
|
1407
|
+
baselineSource: _enum(BASELINE_SOURCES).optional(),
|
|
1408
|
+
liveToolsAcknowledgement: liveToolsAcknowledgementSchema.nullable().optional(),
|
|
1409
|
+
regression: object({
|
|
1410
|
+
passRateDropPoints: number().min(0).max(100).optional(),
|
|
1411
|
+
costPerSuccessRisePct: number().min(0).max(1e3).optional(),
|
|
1412
|
+
latencyRisePct: number().min(0).max(1e3).optional()
|
|
1413
|
+
}).optional(),
|
|
1414
|
+
preset: simulationPresetSchema.nullable().optional(),
|
|
1415
|
+
dataRetention: simulationDataRetentionSchema.optional(),
|
|
1416
|
+
transcriptRetentionDays: number().int().min(1).max(30).optional(),
|
|
1417
|
+
/** The largest share of synthesized tool answers a future design may carry and still be "Recommended" (D6). */
|
|
1418
|
+
synthesizedShareThreshold: number().min(0).max(1).optional()
|
|
1419
|
+
});
|
|
1420
|
+
/** A saved simulation. Every simulation belongs to a product (the product rule). */
|
|
1421
|
+
const simulationSchema = object({
|
|
1422
|
+
id: entityId$1,
|
|
1423
|
+
productId: entityId$1,
|
|
1424
|
+
slug: simulationSlugSchema,
|
|
1425
|
+
name: string(),
|
|
1426
|
+
description: string().nullable(),
|
|
1427
|
+
mode: simulationModeSchema,
|
|
1428
|
+
settings: simulationSettingsSchema,
|
|
1429
|
+
baselineVariantId: entityId$1.nullable(),
|
|
1430
|
+
enabled: boolean(),
|
|
1431
|
+
createdByUserId: entityId$1.nullable(),
|
|
1432
|
+
createdAt: isoDate,
|
|
1433
|
+
updatedAt: isoDate
|
|
1434
|
+
});
|
|
1435
|
+
const simulationRunStatusSchema = _enum([
|
|
1436
|
+
"queued",
|
|
1437
|
+
"running",
|
|
1438
|
+
"completed",
|
|
1439
|
+
"failed",
|
|
1440
|
+
"cancelled",
|
|
1441
|
+
"budget_exhausted"
|
|
1442
|
+
]);
|
|
1443
|
+
const simulationRunExecutorSchema = _enum(["durable_object", "inline"]);
|
|
1444
|
+
const simulationRunTriggerSchema = _enum([
|
|
1445
|
+
"manual",
|
|
1446
|
+
"schedule",
|
|
1447
|
+
"api",
|
|
1448
|
+
"cli",
|
|
1449
|
+
"mcp",
|
|
1450
|
+
"optimizer"
|
|
1451
|
+
]);
|
|
1452
|
+
const simulationEpisodeStatusSchema = _enum([
|
|
1453
|
+
"queued",
|
|
1454
|
+
"running",
|
|
1455
|
+
"completed",
|
|
1456
|
+
"failed",
|
|
1457
|
+
"skipped",
|
|
1458
|
+
"cancelled"
|
|
1459
|
+
]);
|
|
1460
|
+
const episodeLeftOutReasonSchema = _enum([
|
|
1461
|
+
"simulated_user_broke",
|
|
1462
|
+
"spend_cap",
|
|
1463
|
+
"engine_error"
|
|
1464
|
+
]);
|
|
1465
|
+
_enum(["didnt_run"]);
|
|
1466
|
+
const isolationLevelSchema = _enum([
|
|
1467
|
+
"guaranteed",
|
|
1468
|
+
"verified",
|
|
1469
|
+
"unverifiable",
|
|
1470
|
+
"live_tools"
|
|
1471
|
+
]);
|
|
1472
|
+
const layerKindSchema = _enum([
|
|
1473
|
+
"surface",
|
|
1474
|
+
"capability",
|
|
1475
|
+
"implementation",
|
|
1476
|
+
"tool",
|
|
1477
|
+
"model",
|
|
1478
|
+
"framework",
|
|
1479
|
+
"clientModel"
|
|
1480
|
+
]);
|
|
1481
|
+
/**
|
|
1482
|
+
* A tool call's outcome class (6.11): `ok`, `denied` (a permission refusal), `not_found` (an unknown-tool refusal),
|
|
1483
|
+
* `invalid` (its arguments were refused) or `failed` (any other error, transport included).
|
|
1484
|
+
*/
|
|
1485
|
+
const TOOL_CALL_OUTCOMES = [
|
|
1486
|
+
"ok",
|
|
1487
|
+
"denied",
|
|
1488
|
+
"not_found",
|
|
1489
|
+
"invalid",
|
|
1490
|
+
"failed"
|
|
1491
|
+
];
|
|
1492
|
+
const servedBySchema = _enum([
|
|
1493
|
+
"live",
|
|
1494
|
+
"recorded",
|
|
1495
|
+
"transformed",
|
|
1496
|
+
"composite",
|
|
1497
|
+
"synthesized",
|
|
1498
|
+
"stub",
|
|
1499
|
+
"rejected",
|
|
1500
|
+
"fault"
|
|
1501
|
+
]);
|
|
1502
|
+
/** Calls per ladder rung; a rung with no calls is absent. */
|
|
1503
|
+
const servedByCountsSchema = partialRecord(servedBySchema, number().int().nonnegative());
|
|
1504
|
+
/** One tool call as the tool layer records it ("Agent experience", "Per-call facts in the tool layer"). */
|
|
1505
|
+
const toolCallFactsSchema = object({
|
|
1506
|
+
callId: string().optional(),
|
|
1507
|
+
callIndex: number().int().nonnegative(),
|
|
1508
|
+
shownName: string(),
|
|
1509
|
+
serverTool: string().nullable(),
|
|
1510
|
+
argumentErrors: array(object({
|
|
1511
|
+
path: string(),
|
|
1512
|
+
code: string()
|
|
1513
|
+
})),
|
|
1514
|
+
errorKey: string().nullable(),
|
|
1515
|
+
errorOrigin: _enum([
|
|
1516
|
+
"server",
|
|
1517
|
+
"overlay",
|
|
1518
|
+
"transport"
|
|
1519
|
+
]).nullable(),
|
|
1520
|
+
retryOf: number().int().nonnegative().nullable(),
|
|
1521
|
+
intendedTool: string().nullable(),
|
|
1522
|
+
intendedSource: _enum(["hint", "judge"]).nullable(),
|
|
1523
|
+
durationMs: number().nonnegative().nullable(),
|
|
1524
|
+
/** Answered with a fixed reply and never sent: a write tool on a run without the live-tools acknowledgement. */
|
|
1525
|
+
stubbed: boolean().optional(),
|
|
1526
|
+
/** The server annotated the tool `readOnlyHint: true`; the expected-tool rule never calls such a call wrong. */
|
|
1527
|
+
readOnly: boolean().optional(),
|
|
1528
|
+
/** The normalized error text behind `errorKey`, at most 200 characters. */
|
|
1529
|
+
errorText: string().max(200).nullable().optional(),
|
|
1530
|
+
/** The JSON-RPC error code or HTTP status the failure carried, when it carried one. */
|
|
1531
|
+
errorCode: string().max(64).nullable().optional(),
|
|
1532
|
+
/** A later call in the episode succeeded on this tool or on the intended one; set by tool-choice grading. */
|
|
1533
|
+
recoveredLater: boolean().optional(),
|
|
1534
|
+
/** How the call ended, classified while its error text was at hand, so `metrics` retention keeps it. */
|
|
1535
|
+
outcome: _enum(TOOL_CALL_OUTCOMES).optional(),
|
|
1536
|
+
/** The target reported the call without saying whether it failed; it is neither a success nor a failure. */
|
|
1537
|
+
resultUnknown: boolean().optional(),
|
|
1538
|
+
/** The ladder rung that answered; absent on calls recorded before the ladder. */
|
|
1539
|
+
servedBy: servedBySchema.optional(),
|
|
1540
|
+
/** The recorded exchange this call replayed, or the row it wrote. */
|
|
1541
|
+
exchangeId: string().nullable().optional(),
|
|
1542
|
+
/** The answer came from a recording rather than a fresh upstream call. */
|
|
1543
|
+
replayed: boolean().optional(),
|
|
1544
|
+
/** A future-state operation failed on a live write's answer, so the server's answer was returned unshaped. */
|
|
1545
|
+
designFailed: boolean().optional(),
|
|
1546
|
+
/** The future-state operation that wrote this call's error envelope; absent when the error is the upstream's own. */
|
|
1547
|
+
designedBy: _enum(FUTURE_STATE_OPS).optional(),
|
|
1548
|
+
/** The synthesizer's answer failed its contract twice, so the call was answered with a stub (D5). */
|
|
1549
|
+
synthesisFailed: boolean().optional(),
|
|
1550
|
+
/** A recorded read answer that shows the values of the episode's own earlier accepted recorded writes (D3). */
|
|
1551
|
+
writeOverlay: boolean().optional(),
|
|
1552
|
+
/** No rung could answer the call under its policy, so it got a stub; a policy stub of a write never sets it. */
|
|
1553
|
+
unanswered: boolean().optional(),
|
|
1554
|
+
/** Why the synthesizer answered as it did, on a synthesized call; the review panel and the judge citation read it. */
|
|
1555
|
+
rationale: string().max(500).optional(),
|
|
1556
|
+
/** The frozen row of the suggestions the mini-agent filled into this call's designed error (D4); `exchangeId` stays the answer's. */
|
|
1557
|
+
suggestionsExchangeId: string().nullable().optional()
|
|
1558
|
+
});
|
|
1559
|
+
/** What one target turn passed through, per layer; `toolCalls` is empty for targets with no per-call view. */
|
|
1560
|
+
const turnLayerFactsSchema = object({
|
|
1561
|
+
harnessCapture: harnessCaptureSummarySchema.optional(),
|
|
1562
|
+
reportedUsage: reportedUsageSchema.optional(),
|
|
1563
|
+
turnIndex: number().int().nonnegative(),
|
|
1564
|
+
surface: object({
|
|
1565
|
+
surfaceId: string().nullable(),
|
|
1566
|
+
surfaceType: string().nullable(),
|
|
1567
|
+
routedCapabilityId: string().nullable(),
|
|
1568
|
+
routerModel: string().nullable(),
|
|
1569
|
+
routingMs: number().nullable()
|
|
1570
|
+
}).nullable(),
|
|
1571
|
+
capabilityId: string().nullable(),
|
|
1572
|
+
implementation: object({
|
|
1573
|
+
kind: _enum([
|
|
1574
|
+
"agent",
|
|
1575
|
+
"flow",
|
|
1576
|
+
"external"
|
|
1577
|
+
]),
|
|
1578
|
+
id: string().nullable(),
|
|
1579
|
+
/** The target's display name, for an `external` id the product cannot name, such as a sandbox candidate's. */
|
|
1580
|
+
name: string().optional()
|
|
1581
|
+
}).nullable(),
|
|
1582
|
+
tools: array(object({
|
|
1583
|
+
name: string(),
|
|
1584
|
+
calls: number().int(),
|
|
1585
|
+
errors: number().int()
|
|
1586
|
+
})),
|
|
1587
|
+
models: array(object({
|
|
1588
|
+
model: string(),
|
|
1589
|
+
calls: number().int(),
|
|
1590
|
+
inputTokens: number().int(),
|
|
1591
|
+
outputTokens: number().int(),
|
|
1592
|
+
costUsd: number().nullable()
|
|
1593
|
+
})),
|
|
1594
|
+
framework: string().nullable(),
|
|
1595
|
+
clientModel: string().nullable(),
|
|
1596
|
+
latencyMs: number().nonnegative(),
|
|
1597
|
+
errored: boolean(),
|
|
1598
|
+
toolCalls: array(toolCallFactsSchema),
|
|
1599
|
+
accessEvidence: simulationAccessEvidenceSchema.optional(),
|
|
1600
|
+
/** Calls per ladder rung in this turn; absent on turns recorded before the ladder. */
|
|
1601
|
+
servedBy: servedByCountsSchema.optional(),
|
|
1602
|
+
/** The synthesizer's model calls this turn; overhead like the simulated user's, never the agent's own tokens. */
|
|
1603
|
+
synthesis: object({
|
|
1604
|
+
calls: number().int().nonnegative(),
|
|
1605
|
+
inputTokens: number().int().nonnegative(),
|
|
1606
|
+
outputTokens: number().int().nonnegative(),
|
|
1607
|
+
costUsd: number().nonnegative()
|
|
1608
|
+
}).optional()
|
|
1609
|
+
});
|
|
1610
|
+
const egressRefusalReasonSchema = _enum([
|
|
1611
|
+
"internal_range",
|
|
1612
|
+
"not_allowed",
|
|
1613
|
+
"egress_off",
|
|
1614
|
+
"misaddressed"
|
|
1615
|
+
]);
|
|
1616
|
+
/**
|
|
1617
|
+
* A sandbox candidate's refused egress over one episode. Only `egress_proxy` is the proxy's own record; from
|
|
1618
|
+
* `candidate_output` a refusal the candidate swallowed is missing, and `unread` means no record could be read.
|
|
1619
|
+
*/
|
|
1620
|
+
const episodeEgressSchema = object({
|
|
1621
|
+
source: _enum([
|
|
1622
|
+
"egress_proxy",
|
|
1623
|
+
"candidate_output",
|
|
1624
|
+
"unread"
|
|
1625
|
+
]),
|
|
1626
|
+
/** One row per host and reason; `reason` is null and `count` 1 when only the candidate's output named the host. */
|
|
1627
|
+
refusals: array(object({
|
|
1628
|
+
host: string(),
|
|
1629
|
+
reason: egressRefusalReasonSchema.nullable(),
|
|
1630
|
+
count: number()
|
|
1631
|
+
})),
|
|
1632
|
+
/** Refusals the proxy counted but did not keep, past its per-container limit. */
|
|
1633
|
+
overflow: number()
|
|
1634
|
+
});
|
|
1635
|
+
/** One episode's measurements; target and client-agent cost are what a success costs, the rest is overhead. */
|
|
1636
|
+
const episodeMetricsSchema = object({
|
|
1637
|
+
selfReportedCost: selfReportedCostSchema.optional(),
|
|
1638
|
+
/** A checkpointed target turn could not be safely replayed; retained without error text. */
|
|
1639
|
+
interruptedMidTurn: literal(true).optional(),
|
|
1640
|
+
/** The episode ended `agent_error` because the client model's provider stalled after its retries; unscored. */
|
|
1641
|
+
clientModelStall: literal(true).optional(),
|
|
1642
|
+
/**
|
|
1643
|
+
* The read a `recording_miss` episode stopped on, as the server tool and arguments a recording must hold for a
|
|
1644
|
+
* repair to replay it. `arguments` is absent under the `metrics` data policy.
|
|
1645
|
+
*/
|
|
1646
|
+
recordingMiss: object({
|
|
1647
|
+
tool: string(),
|
|
1648
|
+
arguments: record(string(), unknown()).optional()
|
|
1649
|
+
}).optional(),
|
|
1650
|
+
/** Grading could not produce an episode score; retained without grader error text. */
|
|
1651
|
+
scoringFailed: literal(true).optional(),
|
|
1652
|
+
/** Scoring failed because no goal-judge attempt returned a valid verdict: the judge, not the episode, gave no verdict. */
|
|
1653
|
+
goalJudgeModelError: literal(true).optional(),
|
|
1654
|
+
turns: number().int().nonnegative(),
|
|
1655
|
+
targetCostUsd: number().nullable(),
|
|
1656
|
+
clientAgentCostUsd: number().nullable(),
|
|
1657
|
+
simulatorCostUsd: number(),
|
|
1658
|
+
/** The synthesizer's share of `simulatorCostUsd`; absent when no answer was synthesized. */
|
|
1659
|
+
synthesizerCostUsd: number().nonnegative().optional(),
|
|
1660
|
+
judgeCostUsd: number(),
|
|
1661
|
+
inputTokens: number().int().nonnegative(),
|
|
1662
|
+
outputTokens: number().int().nonnegative(),
|
|
1663
|
+
totalLatencyMs: number().nonnegative(),
|
|
1664
|
+
medianTurnLatencyMs: number().nullable(),
|
|
1665
|
+
toolCalls: number().int().nonnegative(),
|
|
1666
|
+
toolErrors: number().int().nonnegative(),
|
|
1667
|
+
errors: number().int().nonnegative(),
|
|
1668
|
+
unpricedModels: array(string()),
|
|
1669
|
+
/** Additive: present only for a sandbox candidate whose container started. Evidence, never scored. */
|
|
1670
|
+
egress: episodeEgressSchema.optional(),
|
|
1671
|
+
/** False when an engine failure measured nothing: every number here is a placeholder, only `egress` holds. */
|
|
1672
|
+
measured: literal(false).optional()
|
|
1673
|
+
});
|
|
1674
|
+
/** One row of a report layer table: one key under one variant. */
|
|
1675
|
+
const layerStatSchema = object({
|
|
1676
|
+
layer: layerKindSchema,
|
|
1677
|
+
key: string(),
|
|
1678
|
+
/** A display name for `key` the report knows, such as a sandbox candidate target's; absent otherwise. */
|
|
1679
|
+
label: string().optional(),
|
|
1680
|
+
variantId: string(),
|
|
1681
|
+
episodes: number().int().nonnegative(),
|
|
1682
|
+
turns: number().int().nonnegative(),
|
|
1683
|
+
calls: number().int().nonnegative(),
|
|
1684
|
+
errors: number().int().nonnegative(),
|
|
1685
|
+
passedEpisodes: number().int().nonnegative(),
|
|
1686
|
+
validEpisodes: number().int().nonnegative(),
|
|
1687
|
+
costUsd: number().nullable(),
|
|
1688
|
+
p50LatencyMs: number().nullable()
|
|
1689
|
+
});
|
|
1690
|
+
/** The dry-run estimate (`estimateRun`); admission compares `costUsd.expected` with the cap. */
|
|
1691
|
+
const runEstimateSchema = object({
|
|
1692
|
+
episodes: number().int().nonnegative(),
|
|
1693
|
+
expectedTurnsPerEpisode: number().nonnegative(),
|
|
1694
|
+
costUsd: object({
|
|
1695
|
+
low: number(),
|
|
1696
|
+
expected: number(),
|
|
1697
|
+
high: number()
|
|
1698
|
+
}),
|
|
1699
|
+
breakdown: object({
|
|
1700
|
+
targetUsd: number(),
|
|
1701
|
+
simulatorUsd: number(),
|
|
1702
|
+
judgeUsd: number(),
|
|
1703
|
+
clientAgentUsd: number(),
|
|
1704
|
+
/** The mini-agent's calls for tools a `simulated` design answers without a recording (D5); absent with none. */
|
|
1705
|
+
synthesizerUsd: number().optional()
|
|
1706
|
+
}),
|
|
1707
|
+
basis: _enum([
|
|
1708
|
+
"history",
|
|
1709
|
+
"catalog",
|
|
1710
|
+
"mixed"
|
|
1711
|
+
]),
|
|
1712
|
+
/** Why a `catalog` estimate read no run costs: too few priced runs yet, or every variant changes what users reach. */
|
|
1713
|
+
catalogReason: _enum(["no_history", "changed"]).optional(),
|
|
1714
|
+
unpricedModels: array(string()),
|
|
1715
|
+
maxSpendUsd: number(),
|
|
1716
|
+
overBudget: boolean()
|
|
1717
|
+
});
|
|
1718
|
+
/** One episode an inline run has started and not finished, with the agent turns it has completed so far. */
|
|
1719
|
+
const simulationActiveEpisodeSchema = object({
|
|
1720
|
+
episodeId: string(),
|
|
1721
|
+
variantId: string(),
|
|
1722
|
+
variantKey: string(),
|
|
1723
|
+
scenarioId: string(),
|
|
1724
|
+
trial: number().int().nonnegative(),
|
|
1725
|
+
turnsCompleted: number().int().nonnegative()
|
|
1726
|
+
});
|
|
1727
|
+
/** A saved run with its counters. */
|
|
1728
|
+
const simulationRunSchema = object({
|
|
1729
|
+
id: entityId$1,
|
|
1730
|
+
simulationId: entityId$1,
|
|
1731
|
+
trigger: simulationRunTriggerSchema,
|
|
1732
|
+
scheduleId: string().nullable(),
|
|
1733
|
+
actorUserId: entityId$1.nullable(),
|
|
1734
|
+
status: simulationRunStatusSchema,
|
|
1735
|
+
/** `inline` runs only while a client calls `advanceSimulationRun`, as the dashboard does while the run is open. */
|
|
1736
|
+
executor: simulationRunExecutorSchema,
|
|
1737
|
+
planFingerprint: string().nullable(),
|
|
1738
|
+
estimate: runEstimateSchema.nullable(),
|
|
1739
|
+
maxSpendUsd: number(),
|
|
1740
|
+
spentUsd: number(),
|
|
1741
|
+
episodesPlanned: number().int(),
|
|
1742
|
+
episodesCompleted: number().int(),
|
|
1743
|
+
episodesFailed: number().int(),
|
|
1744
|
+
episodesSkipped: number().int(),
|
|
1745
|
+
round: number().int(),
|
|
1746
|
+
startedAt: isoDate.nullable(),
|
|
1747
|
+
completedAt: isoDate.nullable(),
|
|
1748
|
+
error: string().nullable(),
|
|
1749
|
+
createdAt: isoDate,
|
|
1750
|
+
/**
|
|
1751
|
+
* An active inline run's started, unfinished episodes, from its saved turn checkpoints; absent for a run the
|
|
1752
|
+
* Durable Object executes and for a run that has finished.
|
|
1753
|
+
*/
|
|
1754
|
+
activeEpisodes: array(simulationActiveEpisodeSchema).optional()
|
|
1755
|
+
});
|
|
1756
|
+
/** A saved episode, without its transcript. */
|
|
1757
|
+
const simulationEpisodeSchema = object({
|
|
1758
|
+
id: entityId$1,
|
|
1759
|
+
runId: entityId$1,
|
|
1760
|
+
variantId: entityId$1,
|
|
1761
|
+
scenarioId: entityId$1.nullable(),
|
|
1762
|
+
simulatedUserId: entityId$1.nullable(),
|
|
1763
|
+
taskId: entityId$1.nullable(),
|
|
1764
|
+
trialIndex: number().int().nonnegative(),
|
|
1765
|
+
round: number().int().nonnegative(),
|
|
1766
|
+
status: simulationEpisodeStatusSchema,
|
|
1767
|
+
termination: simTerminationSchema.nullable(),
|
|
1768
|
+
turns: number().int().nonnegative(),
|
|
1769
|
+
passed: boolean().nullable(),
|
|
1770
|
+
score: number().nullable(),
|
|
1771
|
+
/** Why the episode is left out of pass rates; null when it counts. */
|
|
1772
|
+
leftOutReason: episodeLeftOutReasonSchema.nullable(),
|
|
1773
|
+
isolation: isolationLevelSchema.nullable(),
|
|
1774
|
+
overlay: overlayKindSchema.nullable(),
|
|
1775
|
+
/** ADR 0031 D8's placement, the variant's fidelity and the world it ran in; null on episodes written before them. */
|
|
1776
|
+
ranOn: episodeRanOnSchema.nullable(),
|
|
1777
|
+
fidelity: variantFidelitySchema.nullable(),
|
|
1778
|
+
worldKey: string().nullable(),
|
|
1779
|
+
metrics: episodeMetricsSchema.nullable(),
|
|
1780
|
+
/** Calls per ladder rung over every turn (D3), for the served-by filter; absent when no call was stamped. */
|
|
1781
|
+
servedBy: servedByCountsSchema.optional(),
|
|
1782
|
+
executionIds: array(string()),
|
|
1783
|
+
conversationId: string().nullable(),
|
|
1784
|
+
startedAt: isoDate.nullable(),
|
|
1785
|
+
completedAt: isoDate.nullable(),
|
|
1786
|
+
error: string().nullable()
|
|
1787
|
+
});
|
|
1788
|
+
/** A redacted episode transcript with the simulated-user facts it was produced under. */
|
|
1789
|
+
const simulationEpisodeTranscriptSchema = object({
|
|
1790
|
+
messages: array(simMessageSchema),
|
|
1791
|
+
revealedKeys: record(string(), number().int()),
|
|
1792
|
+
simulatedUserModel: string().nullable(),
|
|
1793
|
+
simulatedUserPromptVersion: number().int().nullable(),
|
|
1794
|
+
judgePromptVersion: number().int().nullable(),
|
|
1795
|
+
expiresAt: isoDate.nullable()
|
|
1796
|
+
});
|
|
1797
|
+
/** One grader's outcome on one episode; mirrors `eval_case_scores`. */
|
|
1798
|
+
const simulationEpisodeScoreSchema = object({
|
|
1799
|
+
id: entityId$1,
|
|
1800
|
+
episodeId: entityId$1,
|
|
1801
|
+
runId: entityId$1,
|
|
1802
|
+
graderIndex: number().int().nonnegative(),
|
|
1803
|
+
grader: simulationGraderSnapshotSchema,
|
|
1804
|
+
passed: boolean(),
|
|
1805
|
+
score: number().nullable(),
|
|
1806
|
+
reasoning: string().nullable(),
|
|
1807
|
+
verdict: judgeVerdictSchema.nullable(),
|
|
1808
|
+
evidence: array(judgeEvidenceSchema).nullable(),
|
|
1809
|
+
outputExcerpt: string().nullable(),
|
|
1810
|
+
error: string().nullable(),
|
|
1811
|
+
costUsd: number().nullable(),
|
|
1812
|
+
durationMs: number().int().nullable(),
|
|
1813
|
+
judgeModel: string().nullable(),
|
|
1814
|
+
humanVerdict: _enum(["agree", "disagree"]).nullable()
|
|
1815
|
+
});
|
|
1816
|
+
const simulationVerdictSchema = _enum([
|
|
1817
|
+
"better",
|
|
1818
|
+
"worse",
|
|
1819
|
+
"no_clear_difference"
|
|
1820
|
+
]);
|
|
1821
|
+
/** A point estimate with its 95% interval. */
|
|
1822
|
+
const estimateIntervalSchema = object({
|
|
1823
|
+
estimate: number(),
|
|
1824
|
+
low: number(),
|
|
1825
|
+
high: number()
|
|
1826
|
+
});
|
|
1827
|
+
/** One count per termination, in `SIM_TERMINATIONS` order; a list, because the values are snake_case. */
|
|
1828
|
+
const terminationCountsSchema = array(object({
|
|
1829
|
+
termination: simTerminationSchema,
|
|
1830
|
+
count: number().int()
|
|
1831
|
+
}));
|
|
1832
|
+
/** One variant's row in the report's variant table. */
|
|
1833
|
+
const reportVariantSummarySchema = object({
|
|
1834
|
+
selfReportedCost: selfReportedCostSchema.optional(),
|
|
1835
|
+
variantId: string(),
|
|
1836
|
+
key: string(),
|
|
1837
|
+
label: string(),
|
|
1838
|
+
role: variantRoleSchema,
|
|
1839
|
+
clientModel: string().nullable(),
|
|
1840
|
+
episodes: number().int(),
|
|
1841
|
+
validEpisodes: number().int(),
|
|
1842
|
+
passedEpisodes: number().int(),
|
|
1843
|
+
simulatedUserBrokeEpisodes: number().int(),
|
|
1844
|
+
targetUnavailableEpisodes: number().int().optional(),
|
|
1845
|
+
clientLimitEpisodes: number().int().optional(),
|
|
1846
|
+
/** Episodes that ended `recording_miss`: a `recorded` policy met a call with no recording. Unscored. */
|
|
1847
|
+
recordingMissEpisodes: number().int().nonnegative().optional(),
|
|
1848
|
+
/** Recorded read answers the read-your-writes overlay patched (`ToolCallFacts.writeOverlay`); absent when none. */
|
|
1849
|
+
writeOverlayAnswers: number().int().nonnegative().optional(),
|
|
1850
|
+
/** Episodes that ended `agent_error` on a client-model provider stall (`metrics.clientModelStall`). Unscored. */
|
|
1851
|
+
clientModelStallEpisodes: number().int().nonnegative().optional(),
|
|
1852
|
+
scoringFailedEpisodes: number().int().optional(),
|
|
1853
|
+
/** Episodes that ended `spend_cap`: the variant's share of the run budget refused its model calls. Unscored. */
|
|
1854
|
+
spendCapEpisodes: number().int().optional(),
|
|
1855
|
+
/** Episodes that ended `engine_error`: the simulation engine failed, not the target. Unscored. */
|
|
1856
|
+
engineErrorEpisodes: number().int().optional(),
|
|
1857
|
+
/** Scored episodes interrupted mid-turn before any turn finished (`didnt_run`). Unscored; absent when none. */
|
|
1858
|
+
didntRunEpisodes: number().int().nonnegative().optional(),
|
|
1859
|
+
passRate: estimateIntervalSchema.nullable(),
|
|
1860
|
+
passK: object({
|
|
1861
|
+
k: number().int(),
|
|
1862
|
+
rate: number().nullable(),
|
|
1863
|
+
evaluableScenarios: number().int(),
|
|
1864
|
+
scenarios: number().int()
|
|
1865
|
+
}).nullable(),
|
|
1866
|
+
terminations: terminationCountsSchema,
|
|
1867
|
+
medianTurns: number().nullable(),
|
|
1868
|
+
p50LatencyMs: number().nullable(),
|
|
1869
|
+
costPerEpisodeUsd: number().nullable(),
|
|
1870
|
+
costPerSuccessUsd: number().nullable(),
|
|
1871
|
+
overheadUsd: number(),
|
|
1872
|
+
inputTokens: number().int(),
|
|
1873
|
+
outputTokens: number().int(),
|
|
1874
|
+
errors: number().int(),
|
|
1875
|
+
/** How the variant reached its target when an overlay or the tool proxy was involved (6.1). */
|
|
1876
|
+
fidelity: string().optional(),
|
|
1877
|
+
/** Additive: the variant carries a tool-surface overlay in its plan, whether or not any episode called a tool. */
|
|
1878
|
+
toolSurfaceOverlay: literal(true).optional(),
|
|
1879
|
+
/** Additive: side-effect grader verdicts over the variant's scored episodes; absent when no task has one (6.10). */
|
|
1880
|
+
sideEffects: object({
|
|
1881
|
+
passed: number().int(),
|
|
1882
|
+
failed: number().int()
|
|
1883
|
+
}).optional(),
|
|
1884
|
+
/** Absent checks over the conversations that ran; `notRun` counts the ones with an absent check that didn't run. */
|
|
1885
|
+
writeGuards: object({
|
|
1886
|
+
passed: number().int(),
|
|
1887
|
+
failed: number().int(),
|
|
1888
|
+
notRun: number().int()
|
|
1889
|
+
}).optional(),
|
|
1890
|
+
/** The exists, linked and in_quarantine checks over every scored conversation, including ones that didn't run. */
|
|
1891
|
+
expectedWrites: object({
|
|
1892
|
+
passed: number().int(),
|
|
1893
|
+
failed: number().int()
|
|
1894
|
+
}).optional(),
|
|
1895
|
+
/** Additive: blocked hosts a sandbox candidate tried, over the episodes that recorded egress. Evidence only. */
|
|
1896
|
+
egress: object({
|
|
1897
|
+
episodes: number().int(),
|
|
1898
|
+
/** Episodes whose evidence is not the egress proxy's own record; `verified` is true when there are none. */
|
|
1899
|
+
unverifiedEpisodes: number().int(),
|
|
1900
|
+
verified: boolean(),
|
|
1901
|
+
refusals: number().int(),
|
|
1902
|
+
overflow: number().int(),
|
|
1903
|
+
hosts: array(object({
|
|
1904
|
+
host: string(),
|
|
1905
|
+
reasons: array(egressRefusalReasonSchema),
|
|
1906
|
+
count: number().int()
|
|
1907
|
+
}))
|
|
1908
|
+
}).optional()
|
|
1909
|
+
});
|
|
1910
|
+
/** A variant against the baseline over (scenario, trial) pairs. */
|
|
1911
|
+
const reportComparisonSchema = object({
|
|
1912
|
+
variantId: string(),
|
|
1913
|
+
baselineVariantId: string(),
|
|
1914
|
+
pairs: number().int(),
|
|
1915
|
+
passRateDifference: estimateIntervalSchema.nullable(),
|
|
1916
|
+
costRatio: estimateIntervalSchema.nullable(),
|
|
1917
|
+
/**
|
|
1918
|
+
* Mean cost per success over the priced pairs, against the baseline's, by the same paired bootstrap as `costRatio`.
|
|
1919
|
+
* Null when either side has no success or no priced pair, or when too few resamples hold a success on both sides.
|
|
1920
|
+
*/
|
|
1921
|
+
costPerSuccessRatio: estimateIntervalSchema.nullable().optional(),
|
|
1922
|
+
latencyRatio: estimateIntervalSchema.nullable(),
|
|
1923
|
+
verdict: simulationVerdictSchema,
|
|
1924
|
+
hint: string().nullable()
|
|
1925
|
+
});
|
|
1926
|
+
/** One variant × simulated user × task cell of the heatmap. */
|
|
1927
|
+
const reportCellSchema = object({
|
|
1928
|
+
variantId: string(),
|
|
1929
|
+
simulatedUserId: string(),
|
|
1930
|
+
taskId: string(),
|
|
1931
|
+
episodes: number().int(),
|
|
1932
|
+
validEpisodes: number().int(),
|
|
1933
|
+
passedEpisodes: number().int(),
|
|
1934
|
+
passRate: number().nullable(),
|
|
1935
|
+
flaky: boolean()
|
|
1936
|
+
});
|
|
1937
|
+
/** A cluster of failing episodes with what they have in common. */
|
|
1938
|
+
const reportFindingSchema = object({
|
|
1939
|
+
/** `left_out` clusters episodes the report left out; its key is a `ReportLeftOutReason`. */
|
|
1940
|
+
kind: _enum([
|
|
1941
|
+
"termination",
|
|
1942
|
+
"grader",
|
|
1943
|
+
"tool",
|
|
1944
|
+
"flaky",
|
|
1945
|
+
"left_out"
|
|
1946
|
+
]),
|
|
1947
|
+
key: string(),
|
|
1948
|
+
title: string(),
|
|
1949
|
+
variantIds: array(string()),
|
|
1950
|
+
episodeIds: array(string()),
|
|
1951
|
+
count: number().int()
|
|
1952
|
+
});
|
|
1953
|
+
/**
|
|
1954
|
+
* A worse number than the previous comparable report. `permission_leak` compares with zero instead: the variant's
|
|
1955
|
+
* target listed or served a tool outside a simulated user's grants, so the run fails and the variant is not recommended.
|
|
1956
|
+
*/
|
|
1957
|
+
const reportRegressionSchema = object({
|
|
1958
|
+
kind: _enum([
|
|
1959
|
+
"pass_rate",
|
|
1960
|
+
"cost_per_success",
|
|
1961
|
+
"latency",
|
|
1962
|
+
"permission_leak"
|
|
1963
|
+
]),
|
|
1964
|
+
variantId: string(),
|
|
1965
|
+
previous: number(),
|
|
1966
|
+
current: number(),
|
|
1967
|
+
interval: estimateIntervalSchema.nullable(),
|
|
1968
|
+
message: string()
|
|
1969
|
+
});
|
|
1970
|
+
/** One tool design view over a set of operator episodes: the whole run, or one client model or variant. */
|
|
1971
|
+
const toolDesignViewSchema = object({
|
|
1972
|
+
episodes: number().int(),
|
|
1973
|
+
rows: array(object({
|
|
1974
|
+
tool: string(),
|
|
1975
|
+
intended: number().int(),
|
|
1976
|
+
rightWhenNeeded: number().int(),
|
|
1977
|
+
chosenWrongly: number().int(),
|
|
1978
|
+
/** Episodes with a wrong choice of this tool; absent on reports built before it was counted. */
|
|
1979
|
+
chosenWronglyEpisodes: number().int().optional(),
|
|
1980
|
+
/** Calls naming this tool, including ones the proxy refused; absent on reports built before it was counted. */
|
|
1981
|
+
calls: number().int().optional(),
|
|
1982
|
+
/** Of those, the calls rejected for their arguments. */
|
|
1983
|
+
invalidArgumentCalls: number().int().optional(),
|
|
1984
|
+
confusedWith: array(object({
|
|
1985
|
+
tool: string(),
|
|
1986
|
+
count: number().int()
|
|
1987
|
+
})),
|
|
1988
|
+
argumentErrors: array(object({
|
|
1989
|
+
path: string(),
|
|
1990
|
+
count: number().int()
|
|
1991
|
+
})),
|
|
1992
|
+
erroredEpisodes: number().int(),
|
|
1993
|
+
recoveredEpisodes: number().int(),
|
|
1994
|
+
medianTurnsToSuccess: number().nullable(),
|
|
1995
|
+
medianTokensToSuccess: number().nullable(),
|
|
1996
|
+
rightLow: boolean(),
|
|
1997
|
+
wrongHigh: boolean(),
|
|
1998
|
+
recoveryLow: boolean()
|
|
1999
|
+
})),
|
|
2000
|
+
quietTools: number().int(),
|
|
2001
|
+
toolCount: number().int(),
|
|
2002
|
+
neverDiscovered: array(object({
|
|
2003
|
+
tool: string(),
|
|
2004
|
+
neededBy: array(string()),
|
|
2005
|
+
episodes: number().int()
|
|
2006
|
+
})),
|
|
2007
|
+
errors: array(object({
|
|
2008
|
+
errorKey: string(),
|
|
2009
|
+
text: string(),
|
|
2010
|
+
tool: string(),
|
|
2011
|
+
/** `design`: a future design's own answer (an error envelope it built), never the server's text. */
|
|
2012
|
+
origin: _enum([
|
|
2013
|
+
"server",
|
|
2014
|
+
"overlay",
|
|
2015
|
+
"transport",
|
|
2016
|
+
"design"
|
|
2017
|
+
]),
|
|
2018
|
+
times: number().int(),
|
|
2019
|
+
episodes: number().int(),
|
|
2020
|
+
recoveredEpisodes: number().int(),
|
|
2021
|
+
/** The conversations that hit it; absent on reports built before they were listed. */
|
|
2022
|
+
episodeIds: array(string()).optional()
|
|
2023
|
+
})),
|
|
2024
|
+
wrongChoicesByClient: array(object({
|
|
2025
|
+
clientModel: string(),
|
|
2026
|
+
count: number().int()
|
|
2027
|
+
})),
|
|
2028
|
+
stubbedTools: array(object({
|
|
2029
|
+
tool: string(),
|
|
2030
|
+
calls: number().int()
|
|
2031
|
+
})).optional(),
|
|
2032
|
+
/** Tool answers per ladder rung; absent on reports built before the ladder. */
|
|
2033
|
+
byServedBy: servedByCountsSchema.optional()
|
|
2034
|
+
});
|
|
2035
|
+
/** One overlay's pooled verdict: each overlay variant paired with the no-overlay variant under the same client model. */
|
|
2036
|
+
const overlayVerdictSchema = object({
|
|
2037
|
+
overlayKey: string(),
|
|
2038
|
+
label: string(),
|
|
2039
|
+
variantIds: array(string()),
|
|
2040
|
+
referenceVariantIds: array(string()),
|
|
2041
|
+
clientModels: array(string()),
|
|
2042
|
+
pairs: number().int(),
|
|
2043
|
+
passRate: estimateIntervalSchema.nullable(),
|
|
2044
|
+
referencePassRate: estimateIntervalSchema.nullable(),
|
|
2045
|
+
passRateDifference: estimateIntervalSchema.nullable(),
|
|
2046
|
+
verdict: simulationVerdictSchema,
|
|
2047
|
+
hint: string().nullable(),
|
|
2048
|
+
/** The best `better` overlay whose result stands on real answers, as the investment card judges; at most one. */
|
|
2049
|
+
recommended: boolean(),
|
|
2050
|
+
/** Injection and write-guard checks its own conversations failed; absent when none. Never recommended then. */
|
|
2051
|
+
safetyFailures: number().int().positive().optional(),
|
|
2052
|
+
/** Permission leaks on the targets its variants ran against, each target counted once; absent when none. */
|
|
2053
|
+
permissionLeaks: number().int().positive().optional(),
|
|
2054
|
+
/** Answers that weren't real decided the pairs that separate it, so its win doesn't hold without them. */
|
|
2055
|
+
heldBySynthesis: literal(true).optional()
|
|
2056
|
+
});
|
|
2057
|
+
/** The tool design view ("Agent experience", "The tool design view"); additive, so older reports render without it. */
|
|
2058
|
+
const toolDesignSectionSchema = toolDesignViewSchema.extend({
|
|
2059
|
+
byClientModel: array(object({
|
|
2060
|
+
clientModel: string(),
|
|
2061
|
+
view: toolDesignViewSchema
|
|
2062
|
+
})).optional(),
|
|
2063
|
+
byVariant: array(object({
|
|
2064
|
+
variantId: string(),
|
|
2065
|
+
variantKey: string(),
|
|
2066
|
+
clientModel: string().nullable(),
|
|
2067
|
+
view: toolDesignViewSchema
|
|
2068
|
+
})).optional(),
|
|
2069
|
+
overlays: array(overlayVerdictSchema).optional(),
|
|
2070
|
+
/** Age in days of the oldest recorded answer the run replayed, as of its exchange cutoff; absent with no replay. */
|
|
2071
|
+
recordedAnswerAgeDays: number().nonnegative().nullable().optional(),
|
|
2072
|
+
/** One row per scored conversation, for the "How each agent did" grid; absent on reports built before it. */
|
|
2073
|
+
conversations: array(object({
|
|
2074
|
+
episodeId: string(),
|
|
2075
|
+
variantId: string(),
|
|
2076
|
+
taskId: string().nullable(),
|
|
2077
|
+
clientModel: string(),
|
|
2078
|
+
trial: number().int().nonnegative(),
|
|
2079
|
+
outcome: _enum([
|
|
2080
|
+
"done",
|
|
2081
|
+
"doneAfterError",
|
|
2082
|
+
"wrongTool",
|
|
2083
|
+
"invalidArguments",
|
|
2084
|
+
"stuckOnError",
|
|
2085
|
+
"gaveUp"
|
|
2086
|
+
]),
|
|
2087
|
+
/** The tool the outcome is about: the one that errored, or the one picked wrongly. */
|
|
2088
|
+
tool: string().optional(),
|
|
2089
|
+
/** For a wrong tool: the tool the task needed. */
|
|
2090
|
+
intendedTool: string().optional(),
|
|
2091
|
+
/** For invalid arguments: the first argument path the server rejected. */
|
|
2092
|
+
argumentPath: string().optional(),
|
|
2093
|
+
/** For an error: its key in `errors`. */
|
|
2094
|
+
errorKey: string().optional()
|
|
2095
|
+
})).optional(),
|
|
2096
|
+
/** Per tool: synthesized answers, contract failures, and how reviewers judged the frozen answers (D5). */
|
|
2097
|
+
synthesis: array(object({
|
|
2098
|
+
tool: string(),
|
|
2099
|
+
answers: number().int().nonnegative(),
|
|
2100
|
+
failed: number().int().nonnegative(),
|
|
2101
|
+
reviewed: number().int().nonnegative(),
|
|
2102
|
+
plausible: number().int().nonnegative()
|
|
2103
|
+
})).optional()
|
|
2104
|
+
});
|
|
2105
|
+
/** One variant's placement row (6.1): what produced its numbers, and which world they come from. */
|
|
2106
|
+
const reportPlacementSchema = object({
|
|
2107
|
+
variantId: string(),
|
|
2108
|
+
placement: variantPlacementSchema,
|
|
2109
|
+
fidelity: variantFidelitySchema,
|
|
2110
|
+
label: string(),
|
|
2111
|
+
worldKey: string(),
|
|
2112
|
+
recommendable: boolean()
|
|
2113
|
+
});
|
|
2114
|
+
const permissionCellSchema = _enum([
|
|
2115
|
+
"Listed",
|
|
2116
|
+
"Hidden",
|
|
2117
|
+
"Called",
|
|
2118
|
+
"Denied",
|
|
2119
|
+
"Leak"
|
|
2120
|
+
]);
|
|
2121
|
+
/** One simulated user's row: what its connection listed and what its calls got, one cell per tool column. */
|
|
2122
|
+
const permissionMatrixRowSchema = object({
|
|
2123
|
+
simulatedUserId: string(),
|
|
2124
|
+
name: string(),
|
|
2125
|
+
/** The connection that listed: the user's identity, or the target's own (`target`, null when it has none). */
|
|
2126
|
+
source: _enum(["identity", "target"]),
|
|
2127
|
+
savedServerName: string().nullable(),
|
|
2128
|
+
grants: array(string()),
|
|
2129
|
+
/** The union of the grants' tools; null when the row has no expectation, so it can show no `Leak`. */
|
|
2130
|
+
expectedTools: array(string()).nullable(),
|
|
2131
|
+
/** Grants the target's map does not name; any one leaves the row without an expectation. */
|
|
2132
|
+
unknownGrants: array(string()),
|
|
2133
|
+
/** Aligned with the target's `tools`. */
|
|
2134
|
+
cells: array(permissionCellSchema),
|
|
2135
|
+
leaks: array(object({
|
|
2136
|
+
tool: string(),
|
|
2137
|
+
reason: _enum(["listed", "called"])
|
|
2138
|
+
})),
|
|
2139
|
+
/** The listing kept fewer names than `tools/list` returned, so an unlisted tool may have been listed. */
|
|
2140
|
+
listingTruncated: boolean()
|
|
2141
|
+
});
|
|
2142
|
+
/** One external MCP target's matrix: rows are simulated users, columns are tools. */
|
|
2143
|
+
const permissionMatrixTargetSchema = object({
|
|
2144
|
+
targetId: string(),
|
|
2145
|
+
/** The variants that ran against the target on its own server; a leak blocks each of them. */
|
|
2146
|
+
variantIds: array(string()),
|
|
2147
|
+
/** The target carried a `grantsToTools` map; without one the matrix shows observations only. */
|
|
2148
|
+
expectation: boolean(),
|
|
2149
|
+
tools: array(string()),
|
|
2150
|
+
/** Tool columns left out past the column cap; never one with a `Leak`, `Called` or `Denied` cell. */
|
|
2151
|
+
omittedTools: number().int(),
|
|
2152
|
+
rows: array(permissionMatrixRowSchema),
|
|
2153
|
+
leaks: number().int(),
|
|
2154
|
+
/** Rows whose listing kept fewer names than `tools/list` returned, so a leak there cannot be ruled out. */
|
|
2155
|
+
truncatedRows: number().int().optional(),
|
|
2156
|
+
/** Rows with an expectation, so checked for leaks; the others show observations only. */
|
|
2157
|
+
checkedRows: number().int().optional()
|
|
2158
|
+
});
|
|
2159
|
+
/** The permission matrix section; additive, so older reports render without it. */
|
|
2160
|
+
const permissionMatrixSectionSchema = object({
|
|
2161
|
+
targets: array(permissionMatrixTargetSchema),
|
|
2162
|
+
leaks: number().int()
|
|
2163
|
+
});
|
|
2164
|
+
/** `SimulationReportBody`, version 1 ("Report model (reference)"). */
|
|
2165
|
+
/**
|
|
2166
|
+
* One variant's injection verdicts (6.12, read side). `episodes` and `failed` cover only conversations that ran;
|
|
2167
|
+
* `notRun` counts the scored conversations that didn't.
|
|
2168
|
+
*/
|
|
2169
|
+
const reportInjectionVerdictSchema = object({
|
|
2170
|
+
variantId: string(),
|
|
2171
|
+
episodes: number().int(),
|
|
2172
|
+
failed: number().int(),
|
|
2173
|
+
notRun: number().int().optional(),
|
|
2174
|
+
canaryLeaks: number().int(),
|
|
2175
|
+
avoidedToolCalls: number().int(),
|
|
2176
|
+
/** The avoided tools its episodes called, by name. */
|
|
2177
|
+
avoidedTools: array(string())
|
|
2178
|
+
});
|
|
2179
|
+
/** One operation in the investment card, with its own delta when it ran alone. */
|
|
2180
|
+
const investmentOperationRowSchema = object({
|
|
2181
|
+
op: _enum(TOOL_SURFACE_OPS),
|
|
2182
|
+
tool: string(),
|
|
2183
|
+
checklistRow: number().int().positive().nullable(),
|
|
2184
|
+
signal: string().nullable(),
|
|
2185
|
+
delta: object({
|
|
2186
|
+
successPoints: number().nullable(),
|
|
2187
|
+
low: number().nullable(),
|
|
2188
|
+
high: number().nullable()
|
|
2189
|
+
}).nullable()
|
|
2190
|
+
});
|
|
2191
|
+
/** "If you ship this design": a future design against its baseline, with the evidence mix behind it (D6). */
|
|
2192
|
+
const investmentSectionSchema = object({
|
|
2193
|
+
variantKey: string(),
|
|
2194
|
+
baselineKey: string(),
|
|
2195
|
+
/** The design's variants, one per client model, pooled into one paired verdict; absent on older reports. */
|
|
2196
|
+
variantIds: array(string()).optional(),
|
|
2197
|
+
baselineVariantIds: array(string()).optional(),
|
|
2198
|
+
/** Paired episodes, and the pairs whose outcomes differ: the episodes that separate the designs. */
|
|
2199
|
+
pairs: number().int().nonnegative().optional(),
|
|
2200
|
+
separating: number().int().nonnegative().optional(),
|
|
2201
|
+
/** Pairs left out because the design or baseline episode ended `recording_miss`; absent when none were. */
|
|
2202
|
+
recordingMissPairsDropped: number().int().positive().optional(),
|
|
2203
|
+
/** Pairs left out because either side stalled on its client model and neither missed; absent when none were. */
|
|
2204
|
+
clientModelStallPairsDropped: number().int().positive().optional(),
|
|
2205
|
+
headline: _enum(["recommended", "indicative"]),
|
|
2206
|
+
reason: string().nullable(),
|
|
2207
|
+
/** Injection and write-guard checks the design's own conversations failed; absent when none. Never recommended then. */
|
|
2208
|
+
safetyFailures: number().int().positive().optional(),
|
|
2209
|
+
/** Permission leaks on the targets its variants ran against, each target counted once; never recommended then. */
|
|
2210
|
+
permissionLeaks: number().int().positive().optional(),
|
|
2211
|
+
success: object({
|
|
2212
|
+
baseline: number(),
|
|
2213
|
+
candidate: number(),
|
|
2214
|
+
delta: number(),
|
|
2215
|
+
low: number(),
|
|
2216
|
+
high: number()
|
|
2217
|
+
}),
|
|
2218
|
+
tokensPerSuccessDeltaPct: number().nullable(),
|
|
2219
|
+
turnsToDone: object({
|
|
2220
|
+
baseline: number().nullable(),
|
|
2221
|
+
candidate: number().nullable()
|
|
2222
|
+
}),
|
|
2223
|
+
servedBy: servedByCountsSchema,
|
|
2224
|
+
synthesizedShare: number().min(0).max(1),
|
|
2225
|
+
synthesizedShareThreshold: number().min(0).max(1),
|
|
2226
|
+
decidedBySynthesis: object({
|
|
2227
|
+
episodes: number().int().nonnegative(),
|
|
2228
|
+
decisive: number().int().nonnegative(),
|
|
2229
|
+
/** The flagged design episodes, at most 20, so the report can open them. */
|
|
2230
|
+
episodeIds: array(string()).max(20).optional(),
|
|
2231
|
+
/** Flagged because the judge's reasoning cited a synthesized answer, not only because one came last. */
|
|
2232
|
+
cited: number().int().nonnegative().optional(),
|
|
2233
|
+
/** Flagged because a stub (a stubbed write's `accepted`, a failed synthesis) was the last tool answer. */
|
|
2234
|
+
stubbed: number().int().nonnegative().optional()
|
|
2235
|
+
}),
|
|
2236
|
+
recordedAnswerAgeDays: number().nonnegative().nullable(),
|
|
2237
|
+
toolsHashChanged: boolean(),
|
|
2238
|
+
operations: array(investmentOperationRowSchema)
|
|
2239
|
+
});
|
|
2240
|
+
/** Evidence quality and the scope of the pass rate; absent on reports created before this check. */
|
|
2241
|
+
const comparisonQualitySchema = object({
|
|
2242
|
+
status: _enum(["usable", "diagnostic"]),
|
|
2243
|
+
reasons: array(_enum([
|
|
2244
|
+
"partial_run",
|
|
2245
|
+
"incomplete_grading",
|
|
2246
|
+
"interrupted_execution",
|
|
2247
|
+
"unscored_episodes",
|
|
2248
|
+
"recording_coverage"
|
|
2249
|
+
])),
|
|
2250
|
+
missingGradeEpisodes: number().int().nonnegative(),
|
|
2251
|
+
interruptedEpisodes: number().int().nonnegative(),
|
|
2252
|
+
/** Unscored finished episodes other than `recording_miss`, client-model stalls and `didnt_run`, which have their own counts. */
|
|
2253
|
+
unscoredEpisodes: number().int().nonnegative(),
|
|
2254
|
+
/**
|
|
2255
|
+
* Episodes that ended `recording_miss`. Each leaves the paired comparison with its partner; past
|
|
2256
|
+
* `RECORDING_MISS_SHARE_CAP` of finished episodes or of an overlay's pooled pairs (misses and client-model stalls
|
|
2257
|
+
* together), or with a comparison left without pairs, the reason is `recording_coverage`. Absent when none missed.
|
|
2258
|
+
*/
|
|
2259
|
+
recordingMissEpisodes: number().int().nonnegative().optional(),
|
|
2260
|
+
/** Baseline/variant pairs left out of the paired comparisons because either side ended `recording_miss`. */
|
|
2261
|
+
recordingMissPairsDropped: number().int().nonnegative().optional(),
|
|
2262
|
+
/** Episodes that ended `agent_error` on a client-model provider stall; they drop pairs like `recording_miss`. */
|
|
2263
|
+
clientModelStallEpisodes: number().int().nonnegative().optional(),
|
|
2264
|
+
/** Pairs left out because either side stalled on its client model and neither ended `recording_miss`. */
|
|
2265
|
+
clientModelStallPairsDropped: number().int().nonnegative().optional(),
|
|
2266
|
+
outcomeScope: _enum([
|
|
2267
|
+
"task_completion",
|
|
2268
|
+
"safety",
|
|
2269
|
+
"mixed"
|
|
2270
|
+
]),
|
|
2271
|
+
safetyEpisodes: number().int().nonnegative(),
|
|
2272
|
+
safetyPassedEpisodes: number().int().nonnegative(),
|
|
2273
|
+
safetyPassedWithoutGoalMetEpisodes: number().int().nonnegative()
|
|
2274
|
+
});
|
|
2275
|
+
/** Which way a difference against the baseline is clear; null when it is too close to call. */
|
|
2276
|
+
const reportVerdictClearSchema = _enum(["better", "worse"]).nullable();
|
|
2277
|
+
/** A ratio against the baseline with its 95% range; clear when the range excludes 1. */
|
|
2278
|
+
const reportVerdictRatioSchema = object({
|
|
2279
|
+
estimate: number(),
|
|
2280
|
+
low: number(),
|
|
2281
|
+
high: number(),
|
|
2282
|
+
clear: reportVerdictClearSchema
|
|
2283
|
+
});
|
|
2284
|
+
/** One version in the verdict: its tasks done, safety, cost and speed, and the summary its card leads with. */
|
|
2285
|
+
const reportVerdictVersionSchema = object({
|
|
2286
|
+
variantId: string(),
|
|
2287
|
+
label: string(),
|
|
2288
|
+
role: _enum(["baseline", "candidate"]),
|
|
2289
|
+
/** Conversations that reached the goal and passed their checks, of those that count toward the pass rate. */
|
|
2290
|
+
done: number().int().nonnegative(),
|
|
2291
|
+
counted: number().int().nonnegative(),
|
|
2292
|
+
/** Conversations interrupted before the agent's first reply; they count against no one. */
|
|
2293
|
+
didntRun: number().int().nonnegative(),
|
|
2294
|
+
conversations: number().int().nonnegative(),
|
|
2295
|
+
rate: number().nullable(),
|
|
2296
|
+
low: number().nullable(),
|
|
2297
|
+
high: number().nullable(),
|
|
2298
|
+
costPerSuccessUsd: number().nullable(),
|
|
2299
|
+
costPerEpisodeUsd: number().nullable(),
|
|
2300
|
+
p50LatencyMs: number().nullable(),
|
|
2301
|
+
inputTokens: number().int(),
|
|
2302
|
+
modelCalls: number().int().nullable(),
|
|
2303
|
+
toolCalls: number().int().nullable(),
|
|
2304
|
+
/** Injection checks and write checks (guards, or the legacy side-effect totals). */
|
|
2305
|
+
safetyPassed: number().int().nonnegative(),
|
|
2306
|
+
safetyTotal: number().int().nonnegative(),
|
|
2307
|
+
/** Permission-matrix leaks on the targets this version reached; counted apart from `safetyTotal`. */
|
|
2308
|
+
permissionLeaks: number().int().nonnegative(),
|
|
2309
|
+
/** Cost per conversation against the baseline; null for the baseline. */
|
|
2310
|
+
cost: reportVerdictRatioSchema.nullable(),
|
|
2311
|
+
/** Cost per success against the baseline; null for the baseline or when either side has no success. */
|
|
2312
|
+
costPerSuccess: reportVerdictRatioSchema.nullable(),
|
|
2313
|
+
/** Time per turn against the baseline; null for the baseline. */
|
|
2314
|
+
latency: reportVerdictRatioSchema.nullable(),
|
|
2315
|
+
quality: reportVerdictClearSchema,
|
|
2316
|
+
/** False when the report would never recommend it: a comparison target, a placement that allows none, or a leak. */
|
|
2317
|
+
recommendable: boolean(),
|
|
2318
|
+
summary: string()
|
|
2319
|
+
});
|
|
2320
|
+
/** The verdict a benchmark of three or more versions leads with (dashboard, Markdown and MCP). */
|
|
2321
|
+
const reportVerdictSchema = object({
|
|
2322
|
+
baselineVariantId: string(),
|
|
2323
|
+
headline: string(),
|
|
2324
|
+
sentence: string(),
|
|
2325
|
+
chips: array(object({
|
|
2326
|
+
key: _enum([
|
|
2327
|
+
"cost",
|
|
2328
|
+
"speed",
|
|
2329
|
+
"quality"
|
|
2330
|
+
]),
|
|
2331
|
+
label: string(),
|
|
2332
|
+
clear: boolean()
|
|
2333
|
+
})),
|
|
2334
|
+
recommendation: object({
|
|
2335
|
+
/** The candidate to switch to or consider; null to keep the baseline. */
|
|
2336
|
+
variantId: string().nullable(),
|
|
2337
|
+
title: string(),
|
|
2338
|
+
detail: string().nullable(),
|
|
2339
|
+
sampleNote: string().nullable()
|
|
2340
|
+
}),
|
|
2341
|
+
/** The baseline first, then the candidates by tasks done and cost. */
|
|
2342
|
+
versions: array(reportVerdictVersionSchema),
|
|
2343
|
+
didntRun: object({
|
|
2344
|
+
total: number().int().positive(),
|
|
2345
|
+
sentence: string(),
|
|
2346
|
+
episodeIds: array(string()),
|
|
2347
|
+
byVersion: array(object({
|
|
2348
|
+
variantId: string(),
|
|
2349
|
+
label: string(),
|
|
2350
|
+
count: number().int().positive()
|
|
2351
|
+
}))
|
|
2352
|
+
}).nullable(),
|
|
2353
|
+
/** Facts behind the costliest candidate's gap: its input tokens and model calls against the baseline's. */
|
|
2354
|
+
gap: object({
|
|
2355
|
+
variantId: string(),
|
|
2356
|
+
label: string(),
|
|
2357
|
+
tokenRatio: number(),
|
|
2358
|
+
modelCalls: number().int().nullable(),
|
|
2359
|
+
baselineModelCalls: number().int().nullable(),
|
|
2360
|
+
inputTokens: number().int(),
|
|
2361
|
+
baselineInputTokens: number().int(),
|
|
2362
|
+
/** The fewest and most tool calls any version made; null when a version has no tool rows. */
|
|
2363
|
+
toolCallRange: object({
|
|
2364
|
+
min: number().int(),
|
|
2365
|
+
max: number().int()
|
|
2366
|
+
}).nullable(),
|
|
2367
|
+
title: string(),
|
|
2368
|
+
detail: string()
|
|
2369
|
+
}).nullable()
|
|
2370
|
+
});
|
|
2371
|
+
const reportHarnessCaptureSchema = object({
|
|
2372
|
+
totalTurns: number().int().nonnegative(),
|
|
2373
|
+
omittedTurns: number().int().nonnegative(),
|
|
2374
|
+
turns: array(object({
|
|
2375
|
+
episodeId: string(),
|
|
2376
|
+
variantId: string(),
|
|
2377
|
+
turnIndex: number().int().nonnegative(),
|
|
2378
|
+
capture: harnessCaptureSummarySchema
|
|
2379
|
+
})).max(200)
|
|
2380
|
+
});
|
|
2381
|
+
const simulationReportBodySchema = object({
|
|
2382
|
+
harnessCapture: reportHarnessCaptureSchema.optional(),
|
|
2383
|
+
comparisonQuality: comparisonQualitySchema.optional(),
|
|
2384
|
+
version: literal(1),
|
|
2385
|
+
run: object({
|
|
2386
|
+
runId: string(),
|
|
2387
|
+
simulationId: string(),
|
|
2388
|
+
productId: string(),
|
|
2389
|
+
status: simulationRunStatusSchema,
|
|
2390
|
+
mode: simulationModeSchema,
|
|
2391
|
+
preset: simulationPresetSchema.nullable(),
|
|
2392
|
+
planFingerprint: string(),
|
|
2393
|
+
trials: number().int(),
|
|
2394
|
+
episodesPlanned: number().int(),
|
|
2395
|
+
episodesFinished: number().int(),
|
|
2396
|
+
partial: boolean()
|
|
2397
|
+
}),
|
|
2398
|
+
caveats: object({
|
|
2399
|
+
simulatedUserModel: string(),
|
|
2400
|
+
simulatedUserPromptVersion: number().int(),
|
|
2401
|
+
judgeModel: string().nullable(),
|
|
2402
|
+
judgePromptVersion: number().int().nullable(),
|
|
2403
|
+
gradingVersion: number().int().positive().optional(),
|
|
2404
|
+
episodes: number().int(),
|
|
2405
|
+
validEpisodes: number().int(),
|
|
2406
|
+
simulatedUserBrokeEpisodes: number().int(),
|
|
2407
|
+
targetUnavailableEpisodes: number().int().optional(),
|
|
2408
|
+
clientLimitEpisodes: number().int().optional(),
|
|
2409
|
+
recordingMissEpisodes: number().int().nonnegative().optional(),
|
|
2410
|
+
/** Pairs left out of the paired comparisons because either side ended `recording_miss`. */
|
|
2411
|
+
recordingMissPairsDropped: number().int().nonnegative().optional(),
|
|
2412
|
+
clientModelStallEpisodes: number().int().nonnegative().optional(),
|
|
2413
|
+
/** Pairs left out because either side stalled on its client model and neither ended `recording_miss`. */
|
|
2414
|
+
clientModelStallPairsDropped: number().int().nonnegative().optional(),
|
|
2415
|
+
scoringFailedEpisodes: number().int().optional(),
|
|
2416
|
+
spendCapEpisodes: number().int().optional(),
|
|
2417
|
+
engineErrorEpisodes: number().int().optional(),
|
|
2418
|
+
didntRunEpisodes: number().int().nonnegative().optional(),
|
|
2419
|
+
isolation: array(isolationLevelSchema),
|
|
2420
|
+
unpricedModels: array(string()),
|
|
2421
|
+
partial: boolean(),
|
|
2422
|
+
notes: array(string())
|
|
2423
|
+
}),
|
|
2424
|
+
variants: array(reportVariantSummarySchema),
|
|
2425
|
+
comparisons: array(reportComparisonSchema),
|
|
2426
|
+
cells: array(reportCellSchema),
|
|
2427
|
+
layers: array(layerStatSchema),
|
|
2428
|
+
terminations: array(object({
|
|
2429
|
+
variantId: string(),
|
|
2430
|
+
termination: simTerminationSchema,
|
|
2431
|
+
count: number().int()
|
|
2432
|
+
})),
|
|
2433
|
+
findings: array(reportFindingSchema),
|
|
2434
|
+
recommendation: object({
|
|
2435
|
+
variantId: string(),
|
|
2436
|
+
reason: string()
|
|
2437
|
+
}).nullable(),
|
|
2438
|
+
regressions: array(reportRegressionSchema),
|
|
2439
|
+
search: object({ rounds: array(object({
|
|
2440
|
+
round: number().int(),
|
|
2441
|
+
variantIds: array(string()),
|
|
2442
|
+
episodes: number().int(),
|
|
2443
|
+
survivors: array(string())
|
|
2444
|
+
})) }).nullable(),
|
|
2445
|
+
toolDesign: toolDesignSectionSchema.optional(),
|
|
2446
|
+
/** Additive: present on runs whose plan froze placements, so older reports recompute unchanged. */
|
|
2447
|
+
placements: array(reportPlacementSchema).optional(),
|
|
2448
|
+
/** Additive: present when a target had identities or a `grantsToTools` map (6.17 addition 4). */
|
|
2449
|
+
permissionMatrix: permissionMatrixSectionSchema.optional(),
|
|
2450
|
+
/** False when nothing about the run's targets or simulated users could make permissions apply; absent on older reports. */
|
|
2451
|
+
permissionsApplicable: boolean().optional(),
|
|
2452
|
+
/** Additive: present when a run's tasks carry the injection grader, one row per variant. */
|
|
2453
|
+
injection: array(reportInjectionVerdictSchema).optional(),
|
|
2454
|
+
/** Additive: present when a run compared a future design against its baseline (D6). */
|
|
2455
|
+
investment: investmentSectionSchema.optional(),
|
|
2456
|
+
/** Null when the report has no verdict (not a benchmark, fewer than three versions, a diagnostic run); absent on older reports. */
|
|
2457
|
+
verdict: reportVerdictSchema.nullable().optional()
|
|
2458
|
+
});
|
|
2459
|
+
const isPlainObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2460
|
+
/** Moves `from` to `to` on a copy of `value` when only the old key is set. */
|
|
2461
|
+
function moveKey(value, from, to) {
|
|
2462
|
+
if (!(from in value)) return value;
|
|
2463
|
+
const { [from]: moved, ...rest } = value;
|
|
2464
|
+
return to in rest ? rest : {
|
|
2465
|
+
...rest,
|
|
2466
|
+
[to]: moved
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
/** Rewrites a report body as reports built before the renames hold it: simulator counts and search rungs. */
|
|
2470
|
+
function upgradeLegacyReportBody(value) {
|
|
2471
|
+
if (!isPlainObject(value)) return value;
|
|
2472
|
+
const body = { ...value };
|
|
2473
|
+
if (Array.isArray(body.variants)) body.variants = body.variants.map((variant) => isPlainObject(variant) ? moveKey(variant, "simulatorInvalidEpisodes", "simulatedUserBrokeEpisodes") : variant);
|
|
2474
|
+
if (isPlainObject(body.caveats)) body.caveats = moveKey(moveKey(moveKey(body.caveats, "simulatorModel", "simulatedUserModel"), "simulatorPromptVersion", "simulatedUserPromptVersion"), "simulatorInvalidEpisodes", "simulatedUserBrokeEpisodes");
|
|
2475
|
+
if (isPlainObject(body.search)) {
|
|
2476
|
+
const search = moveKey(body.search, "rungs", "rounds");
|
|
2477
|
+
body.search = Array.isArray(search.rounds) ? {
|
|
2478
|
+
...search,
|
|
2479
|
+
rounds: search.rounds.map((round) => isPlainObject(round) ? moveKey(round, "rung", "round") : round)
|
|
2480
|
+
} : search;
|
|
2481
|
+
}
|
|
2482
|
+
if (Array.isArray(body.placements)) body.placements = body.placements.map((row) => isPlainObject(row) && row.placement === "customer_runner" ? {
|
|
2483
|
+
...row,
|
|
2484
|
+
placement: "private_runner"
|
|
2485
|
+
} : row);
|
|
2486
|
+
return body;
|
|
2487
|
+
}
|
|
2488
|
+
preprocess(upgradeLegacyReportBody, simulationReportBodySchema);
|
|
2489
|
+
/** The one-line summary stored beside the body, for lists, Home and email subjects. */
|
|
2490
|
+
const simulationReportHeadlineSchema = object({
|
|
2491
|
+
sentence: string(),
|
|
2492
|
+
recommendedVariantId: string().nullable(),
|
|
2493
|
+
baselinePassRate: number().nullable(),
|
|
2494
|
+
bestPassRate: number().nullable(),
|
|
2495
|
+
costPerSuccessUsd: number().nullable(),
|
|
2496
|
+
regression: boolean()
|
|
2497
|
+
});
|
|
2498
|
+
/** A saved report. */
|
|
2499
|
+
const simulationReportSchema = object({
|
|
2500
|
+
id: entityId$1,
|
|
2501
|
+
runId: entityId$1,
|
|
2502
|
+
simulationId: entityId$1,
|
|
2503
|
+
productId: entityId$1,
|
|
2504
|
+
reportVersion: number().int(),
|
|
2505
|
+
headline: simulationReportHeadlineSchema,
|
|
2506
|
+
body: simulationReportBodySchema,
|
|
2507
|
+
previousReportId: entityId$1.nullable(),
|
|
2508
|
+
regression: boolean(),
|
|
2509
|
+
createdAt: isoDate
|
|
2510
|
+
});
|
|
2511
|
+
const synthesizerContractSchema = object({
|
|
2512
|
+
source: _enum([
|
|
2513
|
+
"add_tool",
|
|
2514
|
+
"declared",
|
|
2515
|
+
"observed",
|
|
2516
|
+
"none"
|
|
2517
|
+
]),
|
|
2518
|
+
/** The JSON Schema a successful answer must validate against; null when no schema is known. */
|
|
2519
|
+
schema: record(string(), unknown()).nullable()
|
|
2520
|
+
});
|
|
2521
|
+
/** One recorded exchange a synthesizer grounds on, by id and the saved connection it was recorded under. */
|
|
2522
|
+
const synthesizerGroundingExchangeSchema = object({
|
|
2523
|
+
id: string().min(1),
|
|
2524
|
+
grant: string().nullable()
|
|
2525
|
+
});
|
|
2526
|
+
object({
|
|
2527
|
+
variantId: string().min(1),
|
|
2528
|
+
/** The server tool a call reaches after the overlay, or the name of a synthesized `add_tool`. */
|
|
2529
|
+
tool: string().min(1),
|
|
2530
|
+
added: boolean(),
|
|
2531
|
+
readOnly: boolean(),
|
|
2532
|
+
definition: object({
|
|
2533
|
+
name: string(),
|
|
2534
|
+
description: string(),
|
|
2535
|
+
inputSchema: record(string(), unknown())
|
|
2536
|
+
}),
|
|
2537
|
+
contract: synthesizerContractSchema,
|
|
2538
|
+
grounding: object({
|
|
2539
|
+
exchanges: array(synthesizerGroundingExchangeSchema).max(40),
|
|
2540
|
+
/** The server tools the exchanges come from: the tool itself, or a synthesized `add_tool`'s grounding tools. */
|
|
2541
|
+
tools: array(string().min(1)).max(8),
|
|
2542
|
+
worldFacts: array(string().min(1).max(2e3)).max(32),
|
|
2543
|
+
/** The run's recorded schema reads, so a real collection, field or option id is known (D5); absent on older plans. */
|
|
2544
|
+
schemaExchanges: array(synthesizerGroundingExchangeSchema).max(64).optional(),
|
|
2545
|
+
linkedListExchanges: array(synthesizerGroundingExchangeSchema).max(32).optional()
|
|
2546
|
+
}),
|
|
2547
|
+
/** Tools the variant lists, so a designed error's `recoverySteps` can only name a real one. */
|
|
2548
|
+
listedTools: array(string()).max(400),
|
|
2549
|
+
promptVersion: number().int().positive()
|
|
2550
|
+
});
|
|
2551
|
+
object({ plausible: boolean().nullable() });
|
|
2552
|
+
object({
|
|
2553
|
+
exchangeId: string(),
|
|
2554
|
+
plausible: boolean().nullable()
|
|
2555
|
+
});
|
|
2556
|
+
const resynthesizeToolName = string().min(1).max(255);
|
|
2557
|
+
object({ tools: union([resynthesizeToolName, array(resynthesizeToolName).min(1).max(64)]).optional().describe("A server tool whose frozen answers to discard; repeat for several. Omit to discard every tool's.") });
|
|
2558
|
+
object({ discarded: number().int().nonnegative() });
|
|
2559
|
+
/** A tool's synthesized answers and how reviewers judged them, for the profile and the report caveats. */
|
|
2560
|
+
const synthesisReviewStatsSchema = object({
|
|
2561
|
+
answers: number().int().nonnegative(),
|
|
2562
|
+
reviewed: number().int().nonnegative(),
|
|
2563
|
+
plausible: number().int().nonnegative()
|
|
2564
|
+
});
|
|
2565
|
+
const exchangeWorldSchema = string().regex(/^(live|synthetic:[A-Za-z0-9_-]+)$/);
|
|
2566
|
+
const exchangeSourceSchema = _enum([
|
|
2567
|
+
"simulation",
|
|
2568
|
+
"transcript_backfill",
|
|
2569
|
+
"tool_executions",
|
|
2570
|
+
"uploaded"
|
|
2571
|
+
]);
|
|
2572
|
+
/**
|
|
2573
|
+
* One recorded `tools/call` of one target. `arguments` and `result` are null on a shape-only row, which a run with
|
|
2574
|
+
* `dataRetention: 'metrics'` or a declined recording consent writes.
|
|
2575
|
+
*/
|
|
2576
|
+
const simulationToolExchangeSchema = object({
|
|
2577
|
+
id: string(),
|
|
2578
|
+
targetId: string(),
|
|
2579
|
+
grant: string().nullable(),
|
|
2580
|
+
serverTool: string(),
|
|
2581
|
+
toolsHash: string().nullable(),
|
|
2582
|
+
argsFingerprint: string(),
|
|
2583
|
+
arguments: record(string(), unknown()).nullable(),
|
|
2584
|
+
result: unknown().nullable(),
|
|
2585
|
+
isError: boolean(),
|
|
2586
|
+
errorCode: string().nullable(),
|
|
2587
|
+
errorKey: string().nullable(),
|
|
2588
|
+
durationMs: number().nonnegative(),
|
|
2589
|
+
world: exchangeWorldSchema,
|
|
2590
|
+
source: exchangeSourceSchema,
|
|
2591
|
+
servedBy: servedBySchema,
|
|
2592
|
+
variantHash: string().nullable(),
|
|
2593
|
+
ledgerHash: string().nullable(),
|
|
2594
|
+
/** The synthesizer prompt version a frozen synthesized row was written under; null on a recording. */
|
|
2595
|
+
promptVersion: number().int().nullable().optional(),
|
|
2596
|
+
/** Why the synthesizer answered as it did; null on a recording. */
|
|
2597
|
+
rationale: string().nullable().optional(),
|
|
2598
|
+
/** A reviewer's verdict on a synthesized answer; null until reviewed. */
|
|
2599
|
+
plausible: boolean().nullable().optional(),
|
|
2600
|
+
shapeOnly: boolean(),
|
|
2601
|
+
runId: string().nullable(),
|
|
2602
|
+
episodeId: string().nullable(),
|
|
2603
|
+
callIndex: number().int().nonnegative().nullable(),
|
|
2604
|
+
createdAt: string(),
|
|
2605
|
+
expiresAt: string().nullable()
|
|
2606
|
+
});
|
|
2607
|
+
/** One exchange a person uploads for a tool their server answered outside a simulation. */
|
|
2608
|
+
const uploadedExchangeSchema = object({
|
|
2609
|
+
tool: string().min(1),
|
|
2610
|
+
arguments: record(string(), unknown()),
|
|
2611
|
+
result: unknown(),
|
|
2612
|
+
isError: boolean().optional()
|
|
2613
|
+
});
|
|
2614
|
+
union([object({ exchanges: array(uploadedExchangeSchema).min(1).max(500) }), object({
|
|
2615
|
+
backfill: _enum(["transcripts", "tool_executions"]),
|
|
2616
|
+
runId: string().min(1).optional()
|
|
2617
|
+
})]);
|
|
2618
|
+
object({ recorded: number().int().nonnegative() });
|
|
2619
|
+
object({
|
|
2620
|
+
tool: string().optional(),
|
|
2621
|
+
/** `true` lists only published recordings that kept their payload: no shape-only and no frozen synthesized row. */
|
|
2622
|
+
published: _enum(["true", "false"]).optional(),
|
|
2623
|
+
limit: number$1().int().min(1).max(200).default(50),
|
|
2624
|
+
cursor: string().optional()
|
|
2625
|
+
});
|
|
2626
|
+
object({
|
|
2627
|
+
exchanges: array(simulationToolExchangeSchema),
|
|
2628
|
+
nextCursor: string().nullable()
|
|
2629
|
+
});
|
|
2630
|
+
object({ deleted: number().int().nonnegative() });
|
|
2631
|
+
/** What a tool's answers held at one field path: whether it was always set, its types and its first distinct values. */
|
|
2632
|
+
const toolProfileFieldSchema = object({
|
|
2633
|
+
alwaysSet: boolean(),
|
|
2634
|
+
types: array(string()),
|
|
2635
|
+
values: array(unknown()).max(25),
|
|
2636
|
+
valuesTruncated: boolean()
|
|
2637
|
+
});
|
|
2638
|
+
/** One error a tool returned, keyed by its normalized error key in the profile's catalog. */
|
|
2639
|
+
const toolProfileErrorSchema = object({
|
|
2640
|
+
count: number().int(),
|
|
2641
|
+
exemplarExchangeId: string().nullable(),
|
|
2642
|
+
isError: boolean(),
|
|
2643
|
+
code: string().nullable(),
|
|
2644
|
+
recoveredShare: number().min(0).max(1).nullable()
|
|
2645
|
+
});
|
|
2646
|
+
const shapeFindingKindSchema = _enum([
|
|
2647
|
+
"raw_nested_payload",
|
|
2648
|
+
"two_paging_styles",
|
|
2649
|
+
"success_shaped_failure",
|
|
2650
|
+
"error_without_class",
|
|
2651
|
+
"error_without_next_call",
|
|
2652
|
+
"plain_string_beside_json",
|
|
2653
|
+
"timeout_without_async"
|
|
2654
|
+
]);
|
|
2655
|
+
const toolProfileShapeFindingSchema = object({
|
|
2656
|
+
kind: shapeFindingKindSchema,
|
|
2657
|
+
checklistRow: number().int().positive(),
|
|
2658
|
+
exemplarExchangeId: string().nullable(),
|
|
2659
|
+
/** The one-line evidence: what the recorded answers show, and the fix the row asks for. */
|
|
2660
|
+
detail: string().max(500),
|
|
2661
|
+
/** Every recorded exchange the finding rests on, the exemplar first; empty on profiles built before WP-F8. */
|
|
2662
|
+
exchangeIds: array(string()).max(8).default([]),
|
|
2663
|
+
/** The future-state operation that answers the finding, when one maps deterministically ("Add to design", D7). */
|
|
2664
|
+
suggestedOp: toolSurfaceOpSchema.optional()
|
|
2665
|
+
});
|
|
2666
|
+
const adoptionStateSchema = _enum([
|
|
2667
|
+
"adopted",
|
|
2668
|
+
"partial",
|
|
2669
|
+
"not_seen"
|
|
2670
|
+
]);
|
|
2671
|
+
const toolProfileAdoptionSchema = object({
|
|
2672
|
+
briefId: string(),
|
|
2673
|
+
exportedAt: string(),
|
|
2674
|
+
/** The simulation, run and design the brief was exported from; absent on entries written before export recorded them. */
|
|
2675
|
+
simulationId: string().optional(),
|
|
2676
|
+
runId: string().optional(),
|
|
2677
|
+
variantKey: string().optional(),
|
|
2678
|
+
/** When live answers were last compared with the brief; null until a live run after the export. */
|
|
2679
|
+
observedAt: string().nullable().optional(),
|
|
2680
|
+
operations: array(object({
|
|
2681
|
+
op: _enum(TOOL_SURFACE_OPS),
|
|
2682
|
+
state: adoptionStateSchema,
|
|
2683
|
+
detail: string().max(500).nullable()
|
|
2684
|
+
}))
|
|
2685
|
+
});
|
|
2686
|
+
object({
|
|
2687
|
+
targetId: string(),
|
|
2688
|
+
tool: string(),
|
|
2689
|
+
exchangesSeen: number().int().nonnegative(),
|
|
2690
|
+
updatedAt: string(),
|
|
2691
|
+
declaredOutputSchema: record(string(), unknown()).nullable(),
|
|
2692
|
+
observedOutputSchema: unknown().nullable(),
|
|
2693
|
+
samples: array(string()).max(8),
|
|
2694
|
+
errors: record(string(), toolProfileErrorSchema),
|
|
2695
|
+
fields: record(string(), toolProfileFieldSchema),
|
|
2696
|
+
latencyMs: object({
|
|
2697
|
+
p50: number().nullable(),
|
|
2698
|
+
p95: number().nullable()
|
|
2699
|
+
}),
|
|
2700
|
+
writesObserved: boolean(),
|
|
2701
|
+
shapeFindings: array(toolProfileShapeFindingSchema),
|
|
2702
|
+
adoption: array(toolProfileAdoptionSchema).default([]),
|
|
2703
|
+
recordedAnswerAgeDays: number().nonnegative().nullable(),
|
|
2704
|
+
latestToolsHash: string().nullable(),
|
|
2705
|
+
/** The tool's frozen synthesized answers across every design, and how reviewers judged them (D5). */
|
|
2706
|
+
synthesis: synthesisReviewStatsSchema.optional()
|
|
2707
|
+
});
|
|
2708
|
+
object({
|
|
2709
|
+
tool: string().min(1),
|
|
2710
|
+
exchangeId: string().min(1),
|
|
2711
|
+
synthesize: boolean().default(false)
|
|
2712
|
+
});
|
|
2713
|
+
object({
|
|
2714
|
+
before: unknown(),
|
|
2715
|
+
after: unknown(),
|
|
2716
|
+
servedBy: servedBySchema,
|
|
2717
|
+
rationale: string().nullable(),
|
|
2718
|
+
exchangeId: string()
|
|
2719
|
+
});
|
|
2720
|
+
/** The predicted change in task success a brief carried when it was exported, from the run's investment card. */
|
|
2721
|
+
const designBriefPredictionSchema = object({
|
|
2722
|
+
baseline: number(),
|
|
2723
|
+
candidate: number(),
|
|
2724
|
+
delta: number(),
|
|
2725
|
+
low: number(),
|
|
2726
|
+
high: number(),
|
|
2727
|
+
headline: _enum(["recommended", "indicative"])
|
|
2728
|
+
});
|
|
2729
|
+
object({
|
|
2730
|
+
briefId: string(),
|
|
2731
|
+
exportedAt: string(),
|
|
2732
|
+
/** The brief as Markdown: headline, investment card, operations, before and after definitions, caveats. */
|
|
2733
|
+
markdown: string()
|
|
2734
|
+
});
|
|
2735
|
+
/** One later run's baseline, for the trend a brief's prediction is read against. */
|
|
2736
|
+
const designBriefTrendPointSchema = object({
|
|
2737
|
+
runId: string(),
|
|
2738
|
+
completedAt: string(),
|
|
2739
|
+
passRate: number().nullable(),
|
|
2740
|
+
/** The baseline's tool answers came from the server, not from recordings, so the run can show adoption. */
|
|
2741
|
+
live: boolean()
|
|
2742
|
+
});
|
|
2743
|
+
/** One exported brief: what it predicted, which operations live answers now show, and the baseline since. */
|
|
2744
|
+
const simulationDesignBriefSchema = object({
|
|
2745
|
+
briefId: string(),
|
|
2746
|
+
exportedAt: string(),
|
|
2747
|
+
runId: string(),
|
|
2748
|
+
variantKey: string(),
|
|
2749
|
+
targetId: string(),
|
|
2750
|
+
prediction: designBriefPredictionSchema.nullable(),
|
|
2751
|
+
operations: array(object({
|
|
2752
|
+
op: _enum(TOOL_SURFACE_OPS),
|
|
2753
|
+
tool: string(),
|
|
2754
|
+
state: adoptionStateSchema,
|
|
2755
|
+
detail: string().max(500).nullable()
|
|
2756
|
+
})),
|
|
2757
|
+
observedAt: string().nullable(),
|
|
2758
|
+
/** The baseline's pass rate when the brief was exported, then each completed run since, oldest first. */
|
|
2759
|
+
baselineAtExport: number().nullable(),
|
|
2760
|
+
trend: array(designBriefTrendPointSchema)
|
|
2761
|
+
});
|
|
2762
|
+
object({ briefs: array(simulationDesignBriefSchema) });
|
|
2763
|
+
const entityId = string().min(1).max(255);
|
|
2764
|
+
/** Every list route's page envelope: `data` plus a keyset cursor. */
|
|
2765
|
+
function simulationPageSchema(item) {
|
|
2766
|
+
return object({
|
|
2767
|
+
data: array(item),
|
|
2768
|
+
pagination: object({
|
|
2769
|
+
nextCursor: string().nullable(),
|
|
2770
|
+
hasMore: boolean(),
|
|
2771
|
+
limit: number().int()
|
|
2772
|
+
})
|
|
2773
|
+
});
|
|
2774
|
+
}
|
|
2775
|
+
/** Cursor and page size shared by every list route. */
|
|
2776
|
+
const simulationListQuerySchema = object({
|
|
2777
|
+
cursor: string().max(512).optional(),
|
|
2778
|
+
limit: number$1().int().min(1).max(100).optional()
|
|
2779
|
+
});
|
|
2780
|
+
/** The latest run a list row carries, so the sidebar and roll-up read one request. */
|
|
2781
|
+
const simulationLatestRunSummarySchema = object({
|
|
2782
|
+
runId: entityId,
|
|
2783
|
+
status: simulationRunStatusSchema,
|
|
2784
|
+
episodesPlanned: number().int(),
|
|
2785
|
+
episodesCompleted: number().int(),
|
|
2786
|
+
episodesFailed: number().int(),
|
|
2787
|
+
episodesSkipped: number().int(),
|
|
2788
|
+
spentUsd: number(),
|
|
2789
|
+
createdAt: string(),
|
|
2790
|
+
completedAt: string().nullable(),
|
|
2791
|
+
headline: simulationReportHeadlineSchema.nullable()
|
|
2792
|
+
});
|
|
2793
|
+
const simulationListItemSchema = simulationSchema.extend({
|
|
2794
|
+
latestRun: simulationLatestRunSummarySchema.nullable(),
|
|
2795
|
+
/** Additive: live versions and the distinct simulated users and tasks its enabled pairings use. */
|
|
2796
|
+
shape: object({
|
|
2797
|
+
versions: number().int(),
|
|
2798
|
+
simulatedUsers: number().int(),
|
|
2799
|
+
tasks: number().int()
|
|
2800
|
+
}).optional()
|
|
2801
|
+
});
|
|
2802
|
+
simulationListQuerySchema.extend({
|
|
2803
|
+
productId: entityId.optional(),
|
|
2804
|
+
mode: simulationModeSchema.optional()
|
|
2805
|
+
});
|
|
2806
|
+
simulationPageSchema(simulationListItemSchema);
|
|
2807
|
+
/** A variant as it is added; the server computes `changesHash`. */
|
|
2808
|
+
const createSimulationVariantRequestSchema = object({
|
|
2809
|
+
key: simulationVariantKeySchema,
|
|
2810
|
+
label: string().min(1).max(255),
|
|
2811
|
+
role: variantRoleSchema,
|
|
2812
|
+
targetId: entityId,
|
|
2813
|
+
changes: array(variantChangeSchema).max(50).optional(),
|
|
2814
|
+
clientModel: string().min(1).max(255).nullable().optional(),
|
|
2815
|
+
parentVariantId: entityId.nullable().optional(),
|
|
2816
|
+
source: variantSourceSchema.optional(),
|
|
2817
|
+
/** Defaults by target kind (`TARGET_KIND_PLACEMENTS`); fixed once the variant exists. */
|
|
2818
|
+
placement: variantPlacementSchema.optional(),
|
|
2819
|
+
fidelity: variantFidelitySchema.optional(),
|
|
2820
|
+
data: variantDataSchema.nullable().optional(),
|
|
2821
|
+
/** Which upstream answers each tool (D3); absent means every tool answers live. */
|
|
2822
|
+
toolPolicy: variantToolPolicySchema.optional(),
|
|
2823
|
+
replay: variantReplaySchema.optional()
|
|
2824
|
+
});
|
|
2825
|
+
object({
|
|
2826
|
+
productId: entityId,
|
|
2827
|
+
slug: simulationSlugSchema.optional(),
|
|
2828
|
+
name: string().min(1).max(255),
|
|
2829
|
+
description: string().max(2e3).nullable().optional(),
|
|
2830
|
+
mode: simulationModeSchema.optional(),
|
|
2831
|
+
settings: simulationSettingsSchema.optional(),
|
|
2832
|
+
enabled: boolean().optional(),
|
|
2833
|
+
variants: array(createSimulationVariantRequestSchema).max(12).optional()
|
|
2834
|
+
});
|
|
2835
|
+
object({
|
|
2836
|
+
name: string().min(1).max(255).optional(),
|
|
2837
|
+
description: string().max(2e3).nullable().optional(),
|
|
2838
|
+
mode: simulationModeSchema.optional(),
|
|
2839
|
+
settings: simulationSettingsSchema.optional(),
|
|
2840
|
+
enabled: boolean().optional(),
|
|
2841
|
+
baselineVariantId: entityId.nullable().optional()
|
|
2842
|
+
});
|
|
2843
|
+
object({
|
|
2844
|
+
simulation: simulationSchema,
|
|
2845
|
+
variants: array(simulationVariantSchema),
|
|
2846
|
+
scenarios: array(simulationScenarioSchema),
|
|
2847
|
+
latestRun: simulationLatestRunSummarySchema.nullable()
|
|
2848
|
+
});
|
|
2849
|
+
object({ scenarios: array(object({
|
|
2850
|
+
simulatedUserId: entityId,
|
|
2851
|
+
taskId: entityId,
|
|
2852
|
+
weight: number().int().min(1).max(100).optional(),
|
|
2853
|
+
enabled: boolean().optional()
|
|
2854
|
+
})).max(50) });
|
|
2855
|
+
object({ scenarios: array(simulationScenarioSchema) });
|
|
2856
|
+
object({ variants: array(simulationVariantSchema) });
|
|
2857
|
+
object({ variants: array(createSimulationVariantRequestSchema).min(1).max(12) });
|
|
2858
|
+
object({ variants: array(simulationVariantSchema) });
|
|
2859
|
+
object({
|
|
2860
|
+
label: string().min(1).max(255).optional(),
|
|
2861
|
+
role: variantRoleSchema.optional(),
|
|
2862
|
+
changes: array(variantChangeSchema).max(50).optional(),
|
|
2863
|
+
clientModel: string().min(1).max(255).nullable().optional(),
|
|
2864
|
+
/** Null clears the policy, so every tool answers live again. */
|
|
2865
|
+
toolPolicy: variantToolPolicySchema.nullable().optional(),
|
|
2866
|
+
replay: variantReplaySchema.nullable().optional(),
|
|
2867
|
+
/** Check the saved design as run admission would, on the target's recordings, before it is saved (D4). */
|
|
2868
|
+
validate: boolean().optional()
|
|
2869
|
+
});
|
|
2870
|
+
object({ variant: simulationVariantSchema });
|
|
2871
|
+
object({
|
|
2872
|
+
name: string().min(1).max(255),
|
|
2873
|
+
config: targetConfigSchema
|
|
2874
|
+
});
|
|
2875
|
+
object({
|
|
2876
|
+
name: string().min(1).max(255).optional(),
|
|
2877
|
+
config: targetConfigSchema.optional(),
|
|
2878
|
+
/** Whether the operator host records this target's tool answers, and for how long. */
|
|
2879
|
+
recording: targetRecordingSchema.optional()
|
|
2880
|
+
});
|
|
2881
|
+
object({ target: simulationTargetSchema });
|
|
2882
|
+
/** What one ownership check found at each place the token may be published. */
|
|
2883
|
+
const simulationTargetVerificationCheckSchema = object({
|
|
2884
|
+
verified: boolean(),
|
|
2885
|
+
method: targetVerificationMethodSchema.nullable(),
|
|
2886
|
+
wellKnown: object({
|
|
2887
|
+
found: boolean(),
|
|
2888
|
+
detail: string()
|
|
2889
|
+
}),
|
|
2890
|
+
dnsTxt: object({
|
|
2891
|
+
found: boolean(),
|
|
2892
|
+
detail: string()
|
|
2893
|
+
})
|
|
2894
|
+
});
|
|
2895
|
+
object({
|
|
2896
|
+
target: simulationTargetSchema,
|
|
2897
|
+
check: simulationTargetVerificationCheckSchema
|
|
2898
|
+
});
|
|
2899
|
+
object({ config: targetConfigSchema });
|
|
2900
|
+
/** One tool a target lists, as a client agent would see it; `readOnly` is the server's `readOnlyHint`. */
|
|
2901
|
+
const probedSimulationToolSchema = object({
|
|
2902
|
+
name: string(),
|
|
2903
|
+
description: string(),
|
|
2904
|
+
readOnly: boolean(),
|
|
2905
|
+
inputSchema: record(string(), unknown()).optional()
|
|
2906
|
+
});
|
|
2907
|
+
object({
|
|
2908
|
+
tools: array(probedSimulationToolSchema),
|
|
2909
|
+
total: number().int(),
|
|
2910
|
+
/** The server's `serverInfo.name` from `initialize`, when it sent one. */
|
|
2911
|
+
serverName: string().nullable().optional()
|
|
2912
|
+
});
|
|
2913
|
+
/** A tool description longer than this is cut before it reaches the model. */
|
|
2914
|
+
const MAX_DRAFT_TOOL_DESCRIPTION_CHARS = 1e3;
|
|
2915
|
+
/** The most characters one tool's `inputSchema` may take as JSON; send none for a larger one. */
|
|
2916
|
+
const MAX_DRAFT_TOOL_INPUT_SCHEMA_CHARS = 16e3;
|
|
2917
|
+
/** The most characters every `inputSchema` of one request may take together, as JSON. */
|
|
2918
|
+
const MAX_DRAFT_TOOL_INPUT_SCHEMAS_TOTAL_CHARS = 1e6;
|
|
2919
|
+
const jsonChars = (value) => JSON.stringify(value)?.length ?? 0;
|
|
2920
|
+
/** The tools a draft reads, as a target listed them: bounded per tool and in total. */
|
|
2921
|
+
const draftSimulationToolsSchema = array(object({
|
|
2922
|
+
name: string().min(1).max(128),
|
|
2923
|
+
description: string().max(MAX_DRAFT_TOOL_DESCRIPTION_CHARS),
|
|
2924
|
+
readOnly: boolean().optional(),
|
|
2925
|
+
/** The tool's input JSON Schema as the target listed it; the drafting agent reads its parameters. */
|
|
2926
|
+
inputSchema: record(string(), unknown()).refine((schema) => jsonChars(schema) <= MAX_DRAFT_TOOL_INPUT_SCHEMA_CHARS, { message: `An inputSchema may take at most ${MAX_DRAFT_TOOL_INPUT_SCHEMA_CHARS} characters as JSON.` }).optional()
|
|
2927
|
+
})).min(1).max(1e3).refine((tools) => tools.reduce((sum, tool) => sum + (tool.inputSchema ? jsonChars(tool.inputSchema) : 0), 0) <= MAX_DRAFT_TOOL_INPUT_SCHEMAS_TOTAL_CHARS, { message: `The inputSchemas of one request may take at most ${MAX_DRAFT_TOOL_INPUT_SCHEMAS_TOTAL_CHARS} characters together.` });
|
|
2928
|
+
/** Harder tasks than a simulation already runs: each chains two or more tool calls, around the tools agents used least. */
|
|
2929
|
+
const draftHarderTasksSchema = object({
|
|
2930
|
+
/** Tools agents called least in a report; when any is listed, every task uses at least one of them. */
|
|
2931
|
+
focusTools: array(string().min(1).max(128)).max(12),
|
|
2932
|
+
/** The titles of the tasks the simulation already has; a task repeating one is turned back. */
|
|
2933
|
+
existingTitles: array(string().min(1).max(255)).max(200)
|
|
2934
|
+
});
|
|
2935
|
+
object({
|
|
2936
|
+
targetName: string().min(1).max(255),
|
|
2937
|
+
tools: draftSimulationToolsSchema,
|
|
2938
|
+
count: number().int().min(1).max(12).optional(),
|
|
2939
|
+
harder: draftHarderTasksSchema.optional()
|
|
2940
|
+
});
|
|
2941
|
+
/** One drafted task; every `expectedTools` name is one the request listed. */
|
|
2942
|
+
const draftedSimulationTaskSchema = object({
|
|
2943
|
+
title: string(),
|
|
2944
|
+
goal: string(),
|
|
2945
|
+
openingMessage: string(),
|
|
2946
|
+
expectedTools: array(string()).min(1),
|
|
2947
|
+
category: string().nullable()
|
|
2948
|
+
});
|
|
2949
|
+
object({
|
|
2950
|
+
tasks: array(draftedSimulationTaskSchema),
|
|
2951
|
+
/** Why fewer tasks than asked came back, or what was dropped. */
|
|
2952
|
+
notes: array(string()),
|
|
2953
|
+
/** The drafting agent's run, to see how the tasks were written; null when no agent run backs them. */
|
|
2954
|
+
executionId: string().nullable().optional()
|
|
2955
|
+
});
|
|
2956
|
+
/** What a draft is written from. The server loads a Runtype product itself; it never trusts product data sent in. */
|
|
2957
|
+
const simulationDraftSourceSchema = discriminatedUnion("kind", [object({
|
|
2958
|
+
kind: literal("runtype"),
|
|
2959
|
+
productId: entityId,
|
|
2960
|
+
/** Narrow the draft to these capabilities of the product; all enabled ones when unset. */
|
|
2961
|
+
capabilityIds: array(entityId).min(1).max(100).optional(),
|
|
2962
|
+
/** Narrow the draft to these surfaces of the product; all of them when unset. */
|
|
2963
|
+
surfaceIds: array(entityId).min(1).max(100).optional()
|
|
2964
|
+
}), object({
|
|
2965
|
+
kind: literal("mcp"),
|
|
2966
|
+
targetName: string().min(1).max(255),
|
|
2967
|
+
/** The server's URL. Set, it lets the draft read the brand's homepage and look up real ids with read-only tools. */
|
|
2968
|
+
url: string().url().max(2048).optional(),
|
|
2969
|
+
/** The saved MCP connection whose credential the read-only lookups carry. */
|
|
2970
|
+
savedServerName: string().min(1).max(128).optional(),
|
|
2971
|
+
tools: draftSimulationToolsSchema
|
|
2972
|
+
})]);
|
|
2973
|
+
object({
|
|
2974
|
+
source: simulationDraftSourceSchema,
|
|
2975
|
+
/** The preset the simulation will use; unset or null drafts from the spec (`user_testing` or `benchmark`). */
|
|
2976
|
+
preset: simulationPresetSchema.nullable().optional(),
|
|
2977
|
+
/** Simulated users to write: 2 for `agent_experience`, 3 otherwise, when unset. */
|
|
2978
|
+
userCount: number().int().min(1).max(4).optional(),
|
|
2979
|
+
/** Tasks to write; 6 when unset. */
|
|
2980
|
+
taskCount: number().int().min(1).max(12).optional(),
|
|
2981
|
+
/** The person's own line about who uses the product and what they try to do. */
|
|
2982
|
+
brief: string().max(2e3).optional(),
|
|
2983
|
+
/** A saved simulation to draft for: records Activity on it, so the dashboard shows the drafting. Nothing is saved. */
|
|
2984
|
+
simulationId: string().min(1).optional()
|
|
2985
|
+
});
|
|
2986
|
+
/** One drafted simulated user, ready to save with `createSimulatedUser` once a person keeps it. */
|
|
2987
|
+
const draftedSimulatedUserSchema = object({
|
|
2988
|
+
name: string(),
|
|
2989
|
+
profile: string(),
|
|
2990
|
+
relationship: simulatedUserRelationshipSchema,
|
|
2991
|
+
traits: array(string()),
|
|
2992
|
+
knowledge: record(string(), string()),
|
|
2993
|
+
/** The product surface this user reaches it through; the target's name for an MCP server. */
|
|
2994
|
+
surfaceName: string().nullable(),
|
|
2995
|
+
/** A client agent drives the product for this user (an MCP or API surface, or an MCP server). */
|
|
2996
|
+
toolUse: boolean(),
|
|
2997
|
+
source: literal("agent"),
|
|
2998
|
+
/** What the user was drawn from, the drafting run first. */
|
|
2999
|
+
sourceRefs: array(simulationSourceRefSchema)
|
|
3000
|
+
});
|
|
3001
|
+
/** One drafted task of a full draft, ready to save with `createSimulationTask` once a person keeps it. */
|
|
3002
|
+
const simulationDraftTaskSchema = draftedSimulationTaskSchema.extend({
|
|
3003
|
+
/** Exact tool names the source lists, in order; empty for a Runtype product task, which names its capability. */
|
|
3004
|
+
expectedTools: array(string()),
|
|
3005
|
+
successSignal: string().nullable(),
|
|
3006
|
+
/** The product capability the task exercises; null for an MCP server. */
|
|
3007
|
+
capabilityName: string().nullable(),
|
|
3008
|
+
/** Names of drafted users who would bring this task; pair it with each when saving scenarios. */
|
|
3009
|
+
suitableFor: array(string()),
|
|
3010
|
+
source: literal("agent"),
|
|
3011
|
+
sourceRefs: array(simulationSourceRefSchema)
|
|
3012
|
+
});
|
|
3013
|
+
object({
|
|
3014
|
+
users: array(draftedSimulatedUserSchema),
|
|
3015
|
+
tasks: array(simulationDraftTaskSchema),
|
|
3016
|
+
/** What was dropped, turned back or missing, in plain sentences. */
|
|
3017
|
+
notes: array(string()),
|
|
3018
|
+
/** The drafting agent's run, which opens like any run. */
|
|
3019
|
+
executionId: string(),
|
|
3020
|
+
context: object({
|
|
3021
|
+
/** The brand homepage the draft tried to read, when the source is an MCP server on the brand's own domain. */
|
|
3022
|
+
homepage: object({
|
|
3023
|
+
url: string(),
|
|
3024
|
+
fetched: boolean(),
|
|
3025
|
+
/** The run that read it. */
|
|
3026
|
+
executionId: string().nullable()
|
|
3027
|
+
}).nullable(),
|
|
3028
|
+
/** Read-only tool calls the agent made to look up real ids and names. */
|
|
3029
|
+
lookups: number().int()
|
|
3030
|
+
})
|
|
3031
|
+
});
|
|
3032
|
+
simulationListQuerySchema.extend({
|
|
3033
|
+
productId: entityId.optional(),
|
|
3034
|
+
/** Comma-separated target ids, so a page reads the targets its variants name in one request. */
|
|
3035
|
+
ids: string().max(4096).optional()
|
|
3036
|
+
});
|
|
3037
|
+
simulationPageSchema(simulationTargetSchema);
|
|
3038
|
+
simulatedUserDefinitionSchema.partial();
|
|
3039
|
+
object({ simulatedUser: simulatedUserSchema });
|
|
3040
|
+
simulationListQuerySchema.extend({ productId: entityId.optional() });
|
|
3041
|
+
simulationPageSchema(simulatedUserSchema);
|
|
3042
|
+
simulationTaskDefinitionSchema.partial();
|
|
3043
|
+
object({ task: simulationTaskSchema });
|
|
3044
|
+
simulationListQuerySchema.extend({ productId: entityId.optional() });
|
|
3045
|
+
simulationPageSchema(simulationTaskSchema);
|
|
3046
|
+
object({
|
|
3047
|
+
dryRun: boolean().optional(),
|
|
3048
|
+
maxSpendUsd: number().positive().max(1e4).optional(),
|
|
3049
|
+
allowPartial: boolean().optional(),
|
|
3050
|
+
variantIds: array(entityId).min(1).max(12).optional(),
|
|
3051
|
+
/** Run only these enabled scenarios, for example the ones an overlay proposal was grounded in. */
|
|
3052
|
+
scenarioIds: array(entityId).min(1).max(50).optional(),
|
|
3053
|
+
trials: number().int().min(1).max(10).optional(),
|
|
3054
|
+
trigger: simulationRunTriggerSchema.optional(),
|
|
3055
|
+
/** The variant the others are compared against: one of the run's variants, over the simulation's own baseline. */
|
|
3056
|
+
baselineVariantId: entityId.optional(),
|
|
3057
|
+
/**
|
|
3058
|
+
* A recording run: these server tools answer live on every variant, so the recorder keeps their real answers for
|
|
3059
|
+
* later replay. Every variant must be today's version (no overlay), and every target must record in `full`.
|
|
3060
|
+
*/
|
|
3061
|
+
recordTools: array(string().min(1).max(255)).min(1).max(64).optional()
|
|
3062
|
+
});
|
|
3063
|
+
const estimateSimulationRunResponseSchema = object({
|
|
3064
|
+
dryRun: literal(true),
|
|
3065
|
+
estimate: runEstimateSchema,
|
|
3066
|
+
/** How each variant target would record this run, keyed by target id: the kill switch, then the target (D1). */
|
|
3067
|
+
recording: record(string(), frozenTargetRecordingSchema).optional()
|
|
3068
|
+
});
|
|
3069
|
+
const startSimulationRunResponseSchema = object({
|
|
3070
|
+
dryRun: literal(false),
|
|
3071
|
+
runId: entityId,
|
|
3072
|
+
status: simulationRunStatusSchema,
|
|
3073
|
+
estimate: runEstimateSchema
|
|
3074
|
+
});
|
|
3075
|
+
discriminatedUnion("dryRun", [estimateSimulationRunResponseSchema, startSimulationRunResponseSchema]);
|
|
3076
|
+
simulationPageSchema(simulationRunSchema);
|
|
3077
|
+
const simulationRunResponseSchema = object({
|
|
3078
|
+
run: simulationRunSchema,
|
|
3079
|
+
reportId: entityId.nullable()
|
|
3080
|
+
});
|
|
3081
|
+
object({
|
|
3082
|
+
/** Comma-separated server tools; the response then counts this run's live calls to each that the recorder kept. */
|
|
3083
|
+
recordedCallsFor: string().max(4096).optional() });
|
|
3084
|
+
simulationRunResponseSchema.extend({
|
|
3085
|
+
/** Per tool named in `recordedCallsFor`: this run's live calls the recorder kept, 0 for a tool with none. */
|
|
3086
|
+
recordedCalls: record(string(), number().int()).optional() });
|
|
3087
|
+
object({ run: simulationRunSchema });
|
|
3088
|
+
simulationRunResponseSchema.extend({ repair: object({
|
|
3089
|
+
/** Episodes whose AI judge failed, graded again from their stored transcripts. */
|
|
3090
|
+
regrading: number().int(),
|
|
3091
|
+
/** Episodes the platform kept from running, and recording misses since recorded, played again under fresh ids. */
|
|
3092
|
+
replaying: number().int()
|
|
3093
|
+
}) });
|
|
3094
|
+
simulationRunResponseSchema.extend({ advance: object({
|
|
3095
|
+
/** Episodes this call finished; one it parked at the time budget resumes on the next call. */
|
|
3096
|
+
episodesFinished: number().int(),
|
|
3097
|
+
/** Another advance call holds the run's lease, so this one did nothing. */
|
|
3098
|
+
busy: boolean()
|
|
3099
|
+
}) });
|
|
3100
|
+
simulationListQuerySchema.extend({
|
|
3101
|
+
variantId: entityId.optional(),
|
|
3102
|
+
simulatedUserId: entityId.optional(),
|
|
3103
|
+
taskId: entityId.optional(),
|
|
3104
|
+
status: simulationEpisodeStatusSchema.optional(),
|
|
3105
|
+
passed: _enum(["true", "false"]).optional()
|
|
3106
|
+
});
|
|
3107
|
+
simulationPageSchema(simulationEpisodeSchema);
|
|
3108
|
+
object({
|
|
3109
|
+
episode: simulationEpisodeSchema,
|
|
3110
|
+
transcript: simulationEpisodeTranscriptSchema.nullable(),
|
|
3111
|
+
scores: array(simulationEpisodeScoreSchema),
|
|
3112
|
+
layerStats: array(turnLayerFactsSchema),
|
|
3113
|
+
/** Each synthesized answer's review by its exchange id: plausible, implausible, or null when unreviewed (D5). */
|
|
3114
|
+
synthesisReviews: record(string(), boolean().nullable()).optional()
|
|
3115
|
+
});
|
|
3116
|
+
object({ format: _enum([
|
|
3117
|
+
"json",
|
|
3118
|
+
"csv",
|
|
3119
|
+
"md"
|
|
3120
|
+
]).optional() });
|
|
3121
|
+
object({ report: simulationReportSchema });
|
|
3122
|
+
simulationListQuerySchema.extend({
|
|
3123
|
+
productId: entityId.optional(),
|
|
3124
|
+
simulationId: entityId.optional(),
|
|
3125
|
+
regressionsOnly: _enum(["true", "false"]).optional()
|
|
3126
|
+
});
|
|
3127
|
+
simulationPageSchema(simulationReportSchema.omit({ body: true }));
|
|
3128
|
+
/** Which proposer writes the designs: the deterministic rules (rung 0) or one model call over the rubric (rung 1, D7). */
|
|
3129
|
+
const VARIANT_PROPOSERS = ["rules", "model"];
|
|
3130
|
+
object({
|
|
3131
|
+
runId: entityId,
|
|
3132
|
+
/** `model` falls back to `rules` when the model call fails or proposes nothing that passes admission. */
|
|
3133
|
+
proposer: _enum(VARIANT_PROPOSERS).default("rules")
|
|
3134
|
+
});
|
|
3135
|
+
object({
|
|
3136
|
+
variants: array(simulationVariantSchema),
|
|
3137
|
+
/** Why nothing, or less than asked, was proposed. */
|
|
3138
|
+
notes: array(string()),
|
|
3139
|
+
/** The rung that wrote these proposals: `rules` after a model fallback too. */
|
|
3140
|
+
proposer: _enum(VARIANT_PROPOSERS).optional()
|
|
3141
|
+
});
|
|
3142
|
+
object({
|
|
3143
|
+
url: string().url().max(2048),
|
|
3144
|
+
kind: _enum([
|
|
3145
|
+
"mcp",
|
|
3146
|
+
"openapi",
|
|
3147
|
+
"a2a"
|
|
3148
|
+
]),
|
|
3149
|
+
name: string().min(1).max(255).optional(),
|
|
3150
|
+
savedServerName: string().min(1).max(128).optional()
|
|
3151
|
+
});
|
|
3152
|
+
object({
|
|
3153
|
+
product: object({
|
|
3154
|
+
id: entityId,
|
|
3155
|
+
name: string(),
|
|
3156
|
+
kind: productKindSchema
|
|
3157
|
+
}),
|
|
3158
|
+
target: simulationTargetSchema,
|
|
3159
|
+
serverName: string().nullable(),
|
|
3160
|
+
tools: array(probedSimulationToolSchema)
|
|
3161
|
+
});
|
|
3162
|
+
object({
|
|
3163
|
+
url: string(),
|
|
3164
|
+
host: string(),
|
|
3165
|
+
oauthHinted: boolean(),
|
|
3166
|
+
suggestedServerName: string()
|
|
3167
|
+
});
|
|
3168
|
+
object({
|
|
3169
|
+
error: string(),
|
|
3170
|
+
code: _enum([
|
|
3171
|
+
"SIMULATION_NOT_FOUND",
|
|
3172
|
+
"SIMULATION_SLUG_TAKEN",
|
|
3173
|
+
"SIMULATION_VARIANT_KEY_TAKEN",
|
|
3174
|
+
"SIMULATION_INVALID_REQUEST",
|
|
3175
|
+
"SIMULATION_PRODUCT_NOT_FOUND",
|
|
3176
|
+
"SIMULATION_TARGET_NOT_SUPPORTED",
|
|
3177
|
+
"SIMULATION_TARGET_AUTH_REQUIRED",
|
|
3178
|
+
"SIMULATION_LIVE_TOOLS_NOT_ACKNOWLEDGED",
|
|
3179
|
+
"SIMULATION_ESTIMATE_OVER_BUDGET",
|
|
3180
|
+
"SIMULATION_TOO_MANY_RUNNING",
|
|
3181
|
+
"SIMULATION_LIMIT_EXCEEDED",
|
|
3182
|
+
"SIMULATION_OVERLAY_INVALID",
|
|
3183
|
+
"SIMULATION_OVERLAY_NOT_OWN_TARGET",
|
|
3184
|
+
"SIMULATION_RUN_NOT_CANCELLABLE",
|
|
3185
|
+
"SIMULATION_RUN_NOT_REPAIRABLE",
|
|
3186
|
+
"SIMULATION_VARIANT_INVALID",
|
|
3187
|
+
"SIMULATION_IDENTITY_INVALID",
|
|
3188
|
+
"SIMULATION_FORBIDDEN",
|
|
3189
|
+
"SIMULATION_QUOTA_EXCEEDED",
|
|
3190
|
+
"SIMULATION_REPORT_NOT_READY",
|
|
3191
|
+
"SIMULATION_DRAFT_FAILED",
|
|
3192
|
+
"SIMULATION_RATE_LIMITED",
|
|
3193
|
+
"SIMULATION_SYNTHETIC_DATASET_DDL_INVALID",
|
|
3194
|
+
"SIMULATION_SYNTHETIC_DATASET_SCRUB_REFUSED",
|
|
3195
|
+
"SIMULATION_SYNTHETIC_DATASET_BUILD_FAILED",
|
|
3196
|
+
"SIMULATION_SYNTHETIC_DATASET_SOURCE_UNMATCHED",
|
|
3197
|
+
"SIMULATIONS_DISABLED",
|
|
3198
|
+
"SIMULATION_WORLD_MISMATCH",
|
|
3199
|
+
"SIMULATION_FUTURE_OP_INVALID",
|
|
3200
|
+
"SIMULATION_SYNTH_UNGROUNDED",
|
|
3201
|
+
"SIMULATION_RECORDING_BLOCKED"
|
|
3202
|
+
]),
|
|
3203
|
+
details: unknown().optional()
|
|
3204
|
+
});
|
|
3205
|
+
object({ deleted: literal(true) }), object({ deleted: literal(true) }), object({ deleted: literal(true) }), object({ deleted: literal(true) });
|
|
3206
|
+
object({
|
|
3207
|
+
line: number().int(),
|
|
3208
|
+
reason: _enum([
|
|
3209
|
+
"DDL_SYNTAX",
|
|
3210
|
+
"DDL_UNSUPPORTED",
|
|
3211
|
+
"DDL_INVALID"
|
|
3212
|
+
])
|
|
3213
|
+
});
|
|
3214
|
+
object({ report: datasetScrubReportSchema });
|
|
3215
|
+
const simulationSyntheticDatasetResponseSchema = object({ syntheticDataset: simulationSyntheticDatasetSchema });
|
|
3216
|
+
simulationSyntheticDatasetResponseSchema.extend({
|
|
3217
|
+
/** What the DDL parse found: tables, and statements skipped because they cannot change which rows are valid. */
|
|
3218
|
+
ddl: object({
|
|
3219
|
+
tables: number().int(),
|
|
3220
|
+
skipped: array(object({
|
|
3221
|
+
line: number().int(),
|
|
3222
|
+
summary: string()
|
|
3223
|
+
}))
|
|
3224
|
+
}) });
|
|
3225
|
+
simulationPageSchema(simulationSyntheticDatasetSchema);
|
|
3226
|
+
simulationSyntheticDatasetResponseSchema.extend({
|
|
3227
|
+
outcome: _enum([
|
|
3228
|
+
"built",
|
|
3229
|
+
"reused",
|
|
3230
|
+
"in_progress"
|
|
3231
|
+
]),
|
|
3232
|
+
/** Set when this call built: model-written cells, deterministic fallbacks, and how many model calls. */
|
|
3233
|
+
stats: object({
|
|
3234
|
+
tables: number().int(),
|
|
3235
|
+
rows: number().int(),
|
|
3236
|
+
modelCalls: number().int(),
|
|
3237
|
+
modelCells: number().int(),
|
|
3238
|
+
fallbackCells: number().int()
|
|
3239
|
+
}).nullable()
|
|
3240
|
+
});
|
|
3241
|
+
object({ deleted: literal(true) });
|
|
3242
|
+
// src/lib/candidate-bundle.ts
|
|
3243
|
+
/** The largest compressed bundle the API accepts (`POST /v1/simulations/candidate-bundles`). */
|
|
3244
|
+
const CANDIDATE_BUNDLE_MAX_BYTES = 262144e3;
|
|
3245
|
+
/** The API refuses a manifest larger than this. */
|
|
3246
|
+
const CANDIDATE_MANIFEST_MAX_BYTES = 16384;
|
|
3247
|
+
var CandidateBundleError = class extends Error {
|
|
3248
|
+
code;
|
|
3249
|
+
constructor(code, message) {
|
|
3250
|
+
super(message);
|
|
3251
|
+
this.code = code;
|
|
3252
|
+
this.name = "CandidateBundleError";
|
|
3253
|
+
}
|
|
3254
|
+
};
|
|
3255
|
+
const ARCHIVE_SUFFIXES = [".tgz", ".tar.gz"];
|
|
3256
|
+
/** Classifies `<path>` as a candidate directory to pack or a gzipped tar to upload as is. */
|
|
3257
|
+
async function resolveCandidateBundleInput(path) {
|
|
3258
|
+
const absolute = resolve(path);
|
|
3259
|
+
const info = await stat(absolute).catch(() => null);
|
|
3260
|
+
if (!info) throw new CandidateBundleError("NOT_FOUND", `${path} does not exist.`);
|
|
3261
|
+
if (info.isDirectory()) return {
|
|
3262
|
+
kind: "directory",
|
|
3263
|
+
path: absolute
|
|
3264
|
+
};
|
|
3265
|
+
const lower = absolute.toLowerCase();
|
|
3266
|
+
if (!info.isFile() || !ARCHIVE_SUFFIXES.some((suffix) => lower.endsWith(suffix))) throw new CandidateBundleError("UNSUPPORTED_INPUT", `${path} is neither a directory nor a .tgz / .tar.gz file.`);
|
|
3267
|
+
const handle = await open(absolute, "r");
|
|
3268
|
+
try {
|
|
3269
|
+
const magic = Buffer.alloc(2);
|
|
3270
|
+
await handle.read(magic, 0, 2, 0);
|
|
3271
|
+
if (magic[0] !== 31 || magic[1] !== 139) throw new CandidateBundleError("UNSUPPORTED_INPUT", `${path} is not gzip-compressed.`);
|
|
3272
|
+
} finally {
|
|
3273
|
+
await handle.close();
|
|
3274
|
+
}
|
|
3275
|
+
return {
|
|
3276
|
+
kind: "archive",
|
|
3277
|
+
path: absolute
|
|
3278
|
+
};
|
|
3279
|
+
}
|
|
3280
|
+
/** Reads and validates the directory's `runtype-candidate.json` with the same parser the API uses. */
|
|
3281
|
+
async function readCandidateManifest(dir) {
|
|
3282
|
+
const file = join(dir, CANDIDATE_MANIFEST_FILENAME);
|
|
3283
|
+
const info = await lstat(file).catch(() => null);
|
|
3284
|
+
if (!info) throw new CandidateBundleError("MANIFEST_MISSING", `No ${CANDIDATE_MANIFEST_FILENAME} at the root of ${dir}. A candidate bundle needs its manifest at its root.`);
|
|
3285
|
+
if (!info.isFile()) throw new CandidateBundleError("MANIFEST_INVALID", `${CANDIDATE_MANIFEST_FILENAME} must be a regular file, not a symlink or directory.`);
|
|
3286
|
+
if (info.size > 16384) throw new CandidateBundleError("MANIFEST_INVALID", `${CANDIDATE_MANIFEST_FILENAME} is over ${CANDIDATE_MANIFEST_MAX_BYTES / 1024} KB.`);
|
|
3287
|
+
const parsed = parseCandidateManifest(await readFile(file, "utf8"));
|
|
3288
|
+
if (!parsed.ok) throw new CandidateBundleError("MANIFEST_INVALID", `${CANDIDATE_MANIFEST_FILENAME} is invalid: ${parsed.error}`);
|
|
3289
|
+
return parsed.manifest;
|
|
3290
|
+
}
|
|
3291
|
+
/** True for paths a bundle never carries: `.git`, `.DS_Store` and `node_modules/.cache`, at any depth. */
|
|
3292
|
+
function isExcludedCandidatePath(posixPath) {
|
|
3293
|
+
const segments = posixPath.split("/");
|
|
3294
|
+
return segments.some((segment, index) => segment === ".git" || segment === ".DS_Store" || segment === ".cache" && segments[index - 1] === "node_modules");
|
|
3295
|
+
}
|
|
3296
|
+
function compareBytes(a, b) {
|
|
3297
|
+
return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"));
|
|
3298
|
+
}
|
|
3299
|
+
/** Every entry under `dir` in archive order: the manifest first, then the rest sorted by path bytes. */
|
|
3300
|
+
async function listCandidateEntries(dir, options = {}) {
|
|
3301
|
+
const skip = new Set((options.skipAbsolute ?? []).map((path) => resolve(path)));
|
|
3302
|
+
const entries = [];
|
|
3303
|
+
const walk = async (current) => {
|
|
3304
|
+
const names = await readdir(current);
|
|
3305
|
+
for (const name of names) {
|
|
3306
|
+
const absolute = join(current, name);
|
|
3307
|
+
const path = relative(dir, absolute).split(sep).join("/");
|
|
3308
|
+
if (isExcludedCandidatePath(path) || skip.has(absolute)) continue;
|
|
3309
|
+
const info = await lstat(absolute);
|
|
3310
|
+
const executable = (info.mode & 73) !== 0;
|
|
3311
|
+
if (info.isSymbolicLink()) {
|
|
3312
|
+
const linkTarget = await readlink(absolute);
|
|
3313
|
+
entries.push({
|
|
3314
|
+
path,
|
|
3315
|
+
type: "symlink",
|
|
3316
|
+
absolute,
|
|
3317
|
+
size: 0,
|
|
3318
|
+
executable,
|
|
3319
|
+
linkTarget
|
|
3320
|
+
});
|
|
3321
|
+
} else if (info.isDirectory()) {
|
|
3322
|
+
entries.push({
|
|
3323
|
+
path,
|
|
3324
|
+
type: "directory",
|
|
3325
|
+
absolute,
|
|
3326
|
+
size: 0,
|
|
3327
|
+
executable
|
|
3328
|
+
});
|
|
3329
|
+
await walk(absolute);
|
|
3330
|
+
} else if (info.isFile()) entries.push({
|
|
3331
|
+
path,
|
|
3332
|
+
type: "file",
|
|
3333
|
+
absolute,
|
|
3334
|
+
size: info.size,
|
|
3335
|
+
executable
|
|
3336
|
+
});
|
|
3337
|
+
}
|
|
3338
|
+
};
|
|
3339
|
+
await walk(dir);
|
|
3340
|
+
const archivePath = (entry) => entry.type === "directory" ? `${entry.path}/` : entry.path;
|
|
3341
|
+
entries.sort((a, b) => {
|
|
3342
|
+
if (a.path === "runtype-candidate.json") return -1;
|
|
3343
|
+
if (b.path === "runtype-candidate.json") return 1;
|
|
3344
|
+
return compareBytes(archivePath(a), archivePath(b));
|
|
3345
|
+
});
|
|
3346
|
+
return entries;
|
|
3347
|
+
}
|
|
3348
|
+
const BLOCK = 512;
|
|
3349
|
+
const USTAR_NAME_BYTES = 100;
|
|
3350
|
+
const MAX_OCTAL_SIZE = 8589934591;
|
|
3351
|
+
const FIXED_MTIME = 0;
|
|
3352
|
+
const PAX_HEADER_NAME = "././@PaxHeader";
|
|
3353
|
+
const encoder = new TextEncoder();
|
|
3354
|
+
function writeText(header, offset, length, value) {
|
|
3355
|
+
header.set(encoder.encode(value).subarray(0, length), offset);
|
|
3356
|
+
}
|
|
3357
|
+
function writeOctal(header, offset, length, value) {
|
|
3358
|
+
writeText(header, offset, length - 1, value.toString(8).padStart(length - 1, "0"));
|
|
3359
|
+
}
|
|
3360
|
+
function fitsUstar(value) {
|
|
3361
|
+
return /^[\x20-\x7e]*$/.test(value) && value.length <= USTAR_NAME_BYTES;
|
|
3362
|
+
}
|
|
3363
|
+
function ustarFallback(value) {
|
|
3364
|
+
return value.replace(/[^\x20-\x7e]/g, "_").slice(0, USTAR_NAME_BYTES);
|
|
3365
|
+
}
|
|
3366
|
+
function ustarHeader(fields) {
|
|
3367
|
+
const header = new Uint8Array(BLOCK);
|
|
3368
|
+
writeText(header, 0, USTAR_NAME_BYTES, fields.name);
|
|
3369
|
+
writeOctal(header, 100, 8, fields.mode);
|
|
3370
|
+
writeOctal(header, 108, 8, 0);
|
|
3371
|
+
writeOctal(header, 116, 8, 0);
|
|
3372
|
+
writeOctal(header, 124, 12, fields.size);
|
|
3373
|
+
writeOctal(header, 136, 12, FIXED_MTIME);
|
|
3374
|
+
header.fill(32, 148, 156);
|
|
3375
|
+
header[156] = fields.typeflag.charCodeAt(0);
|
|
3376
|
+
writeText(header, 157, USTAR_NAME_BYTES, fields.linkname ?? "");
|
|
3377
|
+
writeText(header, 257, 6, "ustar");
|
|
3378
|
+
writeText(header, 263, 2, "00");
|
|
3379
|
+
writeOctal(header, 329, 8, 0);
|
|
3380
|
+
writeOctal(header, 337, 8, 0);
|
|
3381
|
+
writeText(header, 148, 7, `${header.reduce((sum, byte) => sum + byte, 0).toString(8).padStart(6, "0")}\0`);
|
|
3382
|
+
return header;
|
|
3383
|
+
}
|
|
3384
|
+
/** One pax extended-header record: its decimal length counts the length digits themselves. */
|
|
3385
|
+
function paxRecord(key, value) {
|
|
3386
|
+
const body = ` ${key}=${value}\n`;
|
|
3387
|
+
const bodyBytes = Buffer.byteLength(body, "utf8");
|
|
3388
|
+
let length = bodyBytes + String(bodyBytes).length;
|
|
3389
|
+
while (length !== bodyBytes + String(length).length) length = bodyBytes + String(length).length;
|
|
3390
|
+
return `${length}${body}`;
|
|
3391
|
+
}
|
|
3392
|
+
function padding(size) {
|
|
3393
|
+
return new Uint8Array((BLOCK - size % BLOCK) % BLOCK);
|
|
3394
|
+
}
|
|
3395
|
+
function entryMode(entry) {
|
|
3396
|
+
if (entry.type === "symlink") return 511;
|
|
3397
|
+
return entry.type === "directory" || entry.executable ? 493 : 420;
|
|
3398
|
+
}
|
|
3399
|
+
/** The uncompressed tar stream for `entries`: ustar headers, a pax header for long or non-ASCII names. */
|
|
3400
|
+
async function* tarCandidateEntries(entries) {
|
|
3401
|
+
for (const entry of entries) {
|
|
3402
|
+
const name = entry.type === "directory" ? `${entry.path}/` : entry.path;
|
|
3403
|
+
const records = [];
|
|
3404
|
+
if (!fitsUstar(name)) records.push(paxRecord("path", name));
|
|
3405
|
+
if (entry.linkTarget !== void 0 && !fitsUstar(entry.linkTarget)) records.push(paxRecord("linkpath", entry.linkTarget));
|
|
3406
|
+
if (entry.size > MAX_OCTAL_SIZE) records.push(paxRecord("size", String(entry.size)));
|
|
3407
|
+
if (records.length > 0) {
|
|
3408
|
+
const data = encoder.encode(records.join(""));
|
|
3409
|
+
yield ustarHeader({
|
|
3410
|
+
name: PAX_HEADER_NAME,
|
|
3411
|
+
mode: 420,
|
|
3412
|
+
size: data.byteLength,
|
|
3413
|
+
typeflag: "x"
|
|
3414
|
+
});
|
|
3415
|
+
yield data;
|
|
3416
|
+
yield padding(data.byteLength);
|
|
3417
|
+
}
|
|
3418
|
+
const typeflag = entry.type === "file" ? "0" : entry.type === "directory" ? "5" : "2";
|
|
3419
|
+
yield ustarHeader({
|
|
3420
|
+
name: ustarFallback(name),
|
|
3421
|
+
mode: entryMode(entry),
|
|
3422
|
+
size: entry.size > MAX_OCTAL_SIZE ? 0 : entry.size,
|
|
3423
|
+
typeflag,
|
|
3424
|
+
linkname: entry.linkTarget === void 0 ? void 0 : ustarFallback(entry.linkTarget)
|
|
3425
|
+
});
|
|
3426
|
+
if (entry.type !== "file") continue;
|
|
3427
|
+
let read = 0;
|
|
3428
|
+
for await (const chunk of createReadStream(entry.absolute)) {
|
|
3429
|
+
const bytes = chunk;
|
|
3430
|
+
read += bytes.byteLength;
|
|
3431
|
+
if (read > entry.size) break;
|
|
3432
|
+
yield bytes;
|
|
3433
|
+
}
|
|
3434
|
+
if (read !== entry.size) throw new CandidateBundleError("CHANGED_DURING_PACK", `${entry.path} changed while it was being packed; run the push again.`);
|
|
3435
|
+
yield padding(entry.size);
|
|
3436
|
+
}
|
|
3437
|
+
yield new Uint8Array(BLOCK * 2);
|
|
3438
|
+
}
|
|
3439
|
+
const GZIP_INPUT_CHUNK = 65536;
|
|
3440
|
+
/** Gzip with a zero mtime over fixed-size input chunks, so equal bytes in give equal bytes out on any machine. */
|
|
3441
|
+
function deterministicGzip() {
|
|
3442
|
+
const gzip = new Gzip({
|
|
3443
|
+
level: 6,
|
|
3444
|
+
mtime: FIXED_MTIME
|
|
3445
|
+
});
|
|
3446
|
+
let block = new Uint8Array(GZIP_INPUT_CHUNK);
|
|
3447
|
+
let filled = 0;
|
|
3448
|
+
const transform = new Transform({
|
|
3449
|
+
transform(chunk, _encoding, callback) {
|
|
3450
|
+
try {
|
|
3451
|
+
let offset = 0;
|
|
3452
|
+
while (offset < chunk.byteLength) {
|
|
3453
|
+
const count = Math.min(GZIP_INPUT_CHUNK - filled, chunk.byteLength - offset);
|
|
3454
|
+
block.set(chunk.subarray(offset, offset + count), filled);
|
|
3455
|
+
filled += count;
|
|
3456
|
+
offset += count;
|
|
3457
|
+
if (filled === GZIP_INPUT_CHUNK) {
|
|
3458
|
+
gzip.push(block);
|
|
3459
|
+
block = new Uint8Array(GZIP_INPUT_CHUNK);
|
|
3460
|
+
filled = 0;
|
|
3461
|
+
}
|
|
3462
|
+
}
|
|
3463
|
+
callback();
|
|
3464
|
+
} catch (error) {
|
|
3465
|
+
callback(error);
|
|
3466
|
+
}
|
|
3467
|
+
},
|
|
3468
|
+
flush(callback) {
|
|
3469
|
+
try {
|
|
3470
|
+
gzip.push(block.subarray(0, filled), true);
|
|
3471
|
+
callback();
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
callback(error);
|
|
3474
|
+
}
|
|
3475
|
+
}
|
|
3476
|
+
});
|
|
3477
|
+
gzip.ondata = (data) => {
|
|
3478
|
+
transform.push(data);
|
|
3479
|
+
};
|
|
3480
|
+
return transform;
|
|
3481
|
+
}
|
|
3482
|
+
function tooLarge() {
|
|
3483
|
+
return new CandidateBundleError("TOO_LARGE", `The bundle is over ${CANDIDATE_BUNDLE_MAX_BYTES / 1048576} MB compressed, the API's limit.`);
|
|
3484
|
+
}
|
|
3485
|
+
/** Hashes and counts the bytes passing through, failing once they pass `maxBytes`. */
|
|
3486
|
+
function digestAndCount(maxBytes) {
|
|
3487
|
+
const hash = createHash("sha256");
|
|
3488
|
+
let bytes = 0;
|
|
3489
|
+
return {
|
|
3490
|
+
transform: new Transform({ transform(chunk, _encoding, callback) {
|
|
3491
|
+
bytes += chunk.byteLength;
|
|
3492
|
+
if (bytes > maxBytes) {
|
|
3493
|
+
callback(tooLarge());
|
|
3494
|
+
return;
|
|
3495
|
+
}
|
|
3496
|
+
hash.update(chunk);
|
|
3497
|
+
callback(null, chunk);
|
|
3498
|
+
} }),
|
|
3499
|
+
result: () => ({
|
|
3500
|
+
sha256: hash.digest("hex"),
|
|
3501
|
+
bytes
|
|
3502
|
+
})
|
|
3503
|
+
};
|
|
3504
|
+
}
|
|
3505
|
+
/**
|
|
3506
|
+
* Packs `dir` into a reproducible `.tar.gz` at `outFile`, hashing it as it is written: the manifest first, entries
|
|
3507
|
+
* sorted, mtime 0, uid/gid 0 with no owner names, normalized modes, symlinks kept as symlinks.
|
|
3508
|
+
*/
|
|
3509
|
+
async function packCandidateDirectory(dir, outFile, options = {}) {
|
|
3510
|
+
const manifest = await readCandidateManifest(dir);
|
|
3511
|
+
const entries = await listCandidateEntries(dir, { skipAbsolute: [outFile, ...options.skip ?? []] });
|
|
3512
|
+
const measured = digestAndCount(options.maxBytes ?? 262144e3);
|
|
3513
|
+
await pipeline(Readable.from(tarCandidateEntries(entries)), deterministicGzip(), measured.transform, createWriteStream(outFile));
|
|
3514
|
+
return {
|
|
3515
|
+
...measured.result(),
|
|
3516
|
+
manifest,
|
|
3517
|
+
entries: entries.length
|
|
3518
|
+
};
|
|
3519
|
+
}
|
|
3520
|
+
/** Streams an existing archive through SHA-256, refusing one over `maxBytes` before reading it. */
|
|
3521
|
+
async function digestCandidateArchive(file, options = {}) {
|
|
3522
|
+
const maxBytes = options.maxBytes ?? 262144e3;
|
|
3523
|
+
if ((await stat(file)).size > maxBytes) throw tooLarge();
|
|
3524
|
+
const hash = createHash("sha256");
|
|
3525
|
+
let bytes = 0;
|
|
3526
|
+
for await (const chunk of createReadStream(file)) {
|
|
3527
|
+
bytes += chunk.byteLength;
|
|
3528
|
+
hash.update(chunk);
|
|
3529
|
+
}
|
|
3530
|
+
if (bytes > maxBytes) throw tooLarge();
|
|
3531
|
+
return {
|
|
3532
|
+
sha256: hash.digest("hex"),
|
|
3533
|
+
bytes
|
|
3534
|
+
};
|
|
3535
|
+
}
|
|
3536
|
+
var ExchangesFileError = class extends Error {
|
|
3537
|
+
code = "INVALID_INPUT";
|
|
3538
|
+
constructor(message) {
|
|
3539
|
+
super(message);
|
|
3540
|
+
this.name = "ExchangesFileError";
|
|
3541
|
+
}
|
|
3542
|
+
};
|
|
3543
|
+
function isRecord(value) {
|
|
3544
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3545
|
+
}
|
|
3546
|
+
function toExchange(value, where) {
|
|
3547
|
+
if (!isRecord(value)) throw new ExchangesFileError(`${where}: expected an object.`);
|
|
3548
|
+
const { tool, arguments: args, result, isError } = value;
|
|
3549
|
+
if (typeof tool !== "string" || tool.length === 0) throw new ExchangesFileError(`${where}: "tool" must be a non-empty string.`);
|
|
3550
|
+
if (!isRecord(args)) throw new ExchangesFileError(`${where}: "arguments" must be an object.`);
|
|
3551
|
+
if (!("result" in value)) throw new ExchangesFileError(`${where}: "result" is missing.`);
|
|
3552
|
+
if (isError !== void 0 && typeof isError !== "boolean") throw new ExchangesFileError(`${where}: "isError" must be true or false.`);
|
|
3553
|
+
return {
|
|
3554
|
+
tool,
|
|
3555
|
+
arguments: args,
|
|
3556
|
+
result,
|
|
3557
|
+
...isError === void 0 ? {} : { isError }
|
|
3558
|
+
};
|
|
3559
|
+
}
|
|
3560
|
+
/** Reads a JSON array, or JSON lines, of `{ tool, arguments, result, isError? }`. */
|
|
3561
|
+
function parseExchangesFile(text) {
|
|
3562
|
+
const trimmed = text.trim();
|
|
3563
|
+
if (trimmed.length === 0) throw new ExchangesFileError("The file holds no exchanges.");
|
|
3564
|
+
if (trimmed.startsWith("[")) {
|
|
3565
|
+
let parsed;
|
|
3566
|
+
try {
|
|
3567
|
+
parsed = JSON.parse(trimmed);
|
|
3568
|
+
} catch (error) {
|
|
3569
|
+
throw new ExchangesFileError(`The file is not valid JSON: ${error.message}`);
|
|
3570
|
+
}
|
|
3571
|
+
if (!Array.isArray(parsed) || parsed.length === 0) throw new ExchangesFileError("The JSON array holds no exchanges.");
|
|
3572
|
+
return parsed.map((entry, index) => toExchange(entry, `Entry ${index + 1}`));
|
|
3573
|
+
}
|
|
3574
|
+
const exchanges = [];
|
|
3575
|
+
for (const [index, line] of text.split("\n").entries()) {
|
|
3576
|
+
if (line.trim().length === 0) continue;
|
|
3577
|
+
let parsed;
|
|
3578
|
+
try {
|
|
3579
|
+
parsed = JSON.parse(line);
|
|
3580
|
+
} catch (error) {
|
|
3581
|
+
throw new ExchangesFileError(`Line ${index + 1} is not valid JSON: ${error.message}`);
|
|
3582
|
+
}
|
|
3583
|
+
exchanges.push(toExchange(parsed, `Line ${index + 1}`));
|
|
3584
|
+
}
|
|
3585
|
+
return exchanges;
|
|
3586
|
+
}
|
|
3587
|
+
/** Uploads in requests of at most 500 and returns how many the API recorded. */
|
|
3588
|
+
async function uploadExchangesInBatches(client, targetId, exchanges) {
|
|
3589
|
+
let recorded = 0;
|
|
3590
|
+
for (let start = 0; start < exchanges.length; start += 500) {
|
|
3591
|
+
const batch = exchanges.slice(start, start + 500);
|
|
3592
|
+
const response = await client.simulations.uploadExchanges(targetId, { exchanges: batch });
|
|
3593
|
+
recorded += response.recorded;
|
|
3594
|
+
}
|
|
3595
|
+
return recorded;
|
|
3596
|
+
}
|
|
3597
|
+
function apiErrorCode$1(error) {
|
|
3598
|
+
const data = error.data;
|
|
3599
|
+
if (!isRecord(data)) return void 0;
|
|
3600
|
+
return typeof data.code === "string" ? data.code : void 0;
|
|
3601
|
+
}
|
|
3602
|
+
/** Maps an API failure to the code and message the command prints. */
|
|
3603
|
+
function describeSimulationApiError(error) {
|
|
3604
|
+
if (error instanceof ExchangesFileError) return {
|
|
3605
|
+
code: error.code,
|
|
3606
|
+
message: error.message
|
|
3607
|
+
};
|
|
3608
|
+
if (!(error instanceof RuntypeApiError)) return {
|
|
3609
|
+
code: "EXECUTION_FAILED",
|
|
3610
|
+
message: error instanceof Error ? error.message : String(error)
|
|
3611
|
+
};
|
|
3612
|
+
const code = apiErrorCode$1(error);
|
|
3613
|
+
if (error.statusCode === 401) return {
|
|
3614
|
+
code: "AUTH_REQUIRED",
|
|
3615
|
+
message: "Not authenticated: run `runtype login` or set RUNTYPE_API_KEY."
|
|
3616
|
+
};
|
|
3617
|
+
if (code === "SIMULATIONS_DISABLED") return {
|
|
3618
|
+
code,
|
|
3619
|
+
message: "Simulations are not available on this API. They run on staging: pass --api-url https://api.runtype-staging.com with a staging key."
|
|
3620
|
+
};
|
|
3621
|
+
return {
|
|
3622
|
+
code: code ?? "EXECUTION_FAILED",
|
|
3623
|
+
message: error.message
|
|
3624
|
+
};
|
|
3625
|
+
}
|
|
3626
|
+
function fail(error, json, what) {
|
|
3627
|
+
const { code, message } = describeSimulationApiError(error);
|
|
3628
|
+
if (json) printJson({
|
|
3629
|
+
success: false,
|
|
3630
|
+
error: {
|
|
3631
|
+
code,
|
|
3632
|
+
message
|
|
3633
|
+
}
|
|
3634
|
+
});
|
|
3635
|
+
else {
|
|
3636
|
+
console.error(chalk.red(what));
|
|
3637
|
+
console.error(chalk.red(message));
|
|
3638
|
+
}
|
|
3639
|
+
process.exit(1);
|
|
3640
|
+
}
|
|
3641
|
+
async function authedClient() {
|
|
3642
|
+
const apiKey = await ensureAuth();
|
|
3643
|
+
return apiKey ? createCliClient(apiKey) : null;
|
|
3644
|
+
}
|
|
3645
|
+
/** Adds `exchanges upload` and `report` to the `simulations` command. */
|
|
3646
|
+
function registerToolMemoryCommands(simulations) {
|
|
3647
|
+
simulations.command("exchanges").description("A target's recorded tool exchanges: what its server answered").command("upload <target> <file>").description("Record calls your server answered outside a simulation, from a JSON array or JSON lines of { tool, arguments, result, isError? }").option("--json", "Output as JSON").action(async (target, file, options) => {
|
|
3648
|
+
let recorded;
|
|
3649
|
+
let sent;
|
|
3650
|
+
try {
|
|
3651
|
+
const parsed = parseExchangesFile(await readFile(file, "utf8"));
|
|
3652
|
+
const client = await authedClient();
|
|
3653
|
+
if (!client) process.exit(1);
|
|
3654
|
+
sent = parsed.length;
|
|
3655
|
+
recorded = await uploadExchangesInBatches(client, target, parsed);
|
|
3656
|
+
} catch (error) {
|
|
3657
|
+
fail(error, options.json, "Failed to upload exchanges");
|
|
3658
|
+
}
|
|
3659
|
+
if (options.json) {
|
|
3660
|
+
printJson({
|
|
3661
|
+
success: true,
|
|
3662
|
+
targetId: target,
|
|
3663
|
+
sent,
|
|
3664
|
+
recorded
|
|
3665
|
+
});
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3668
|
+
console.log(chalk.green(`✓ Recorded ${recorded} of ${sent} exchanges for ${target}`));
|
|
3669
|
+
});
|
|
3670
|
+
simulations.command("report <runId>").description("Print a finished run's report as Markdown, or with --brief export its future design as a design brief").option("--brief", "Export the design brief (records it, so later live runs track adoption)").option("--json", "Output as JSON").action(async (runId, options) => {
|
|
3671
|
+
try {
|
|
3672
|
+
const client = await authedClient();
|
|
3673
|
+
if (!client) process.exit(1);
|
|
3674
|
+
const path = `/simulations/runs/${encodeURIComponent(runId)}/report`;
|
|
3675
|
+
if (options.brief) {
|
|
3676
|
+
const brief = await client.simulations.exportDesignBrief(runId);
|
|
3677
|
+
if (options.json) printJson(brief);
|
|
3678
|
+
else console.log(brief.markdown);
|
|
3679
|
+
return;
|
|
3680
|
+
}
|
|
3681
|
+
if (options.json) {
|
|
3682
|
+
printJson(await client.get(path));
|
|
3683
|
+
return;
|
|
3684
|
+
}
|
|
3685
|
+
console.log(await client.get(path, { format: "md" }));
|
|
3686
|
+
} catch (error) {
|
|
3687
|
+
fail(error, options.json, options.brief ? "Failed to export the design brief" : "Failed to read the report");
|
|
3688
|
+
}
|
|
3689
|
+
});
|
|
3690
|
+
}
|
|
3691
|
+
// src/commands/simulations.ts
|
|
3692
|
+
const ACTION_SCOPE = {
|
|
3693
|
+
upload: "SIMULATIONS:WRITE",
|
|
3694
|
+
"read-target": "SIMULATIONS:READ",
|
|
3695
|
+
"write-target": "SIMULATIONS:WRITE",
|
|
3696
|
+
conformance: "SIMULATIONS:RUN"
|
|
3697
|
+
};
|
|
3698
|
+
var CandidatePushError = class extends Error {
|
|
3699
|
+
code;
|
|
3700
|
+
constructor(code, message) {
|
|
3701
|
+
super(message);
|
|
3702
|
+
this.code = code;
|
|
3703
|
+
this.name = "CandidatePushError";
|
|
3704
|
+
}
|
|
3705
|
+
};
|
|
3706
|
+
function apiErrorCode(error) {
|
|
3707
|
+
const data = error.data;
|
|
3708
|
+
if (typeof data !== "object" || data === null || !("code" in data)) return void 0;
|
|
3709
|
+
return typeof data.code === "string" ? data.code : void 0;
|
|
3710
|
+
}
|
|
3711
|
+
/** Maps an API failure from one step of a push to a message that says what to do about it. */
|
|
3712
|
+
function describeCandidateApiError(error, action) {
|
|
3713
|
+
if (!(error instanceof RuntypeApiError)) return new CandidatePushError("EXECUTION_FAILED", error instanceof Error ? error.message : String(error));
|
|
3714
|
+
const code = apiErrorCode(error);
|
|
3715
|
+
switch (error.statusCode) {
|
|
3716
|
+
case 401: return new CandidatePushError("AUTH_REQUIRED", "Not authenticated: run `runtype login` or set RUNTYPE_API_KEY.");
|
|
3717
|
+
case 403: return new CandidatePushError("FORBIDDEN", `${error.message} This step needs an API key with ${ACTION_SCOPE[action]}.`);
|
|
3718
|
+
case 404:
|
|
3719
|
+
if (code === "SIMULATIONS_DISABLED") return new CandidatePushError("SIMULATIONS_DISABLED", "Simulations are not available on this API. They run on staging: pass --api-url https://api.runtype-staging.com with a staging key.");
|
|
3720
|
+
return new CandidatePushError("NOT_FOUND", `Not found: ${error.message}`);
|
|
3721
|
+
case 411: return new CandidatePushError("LENGTH_REQUIRED", "The API refused the upload because it carried no Content-Length.");
|
|
3722
|
+
case 413: return new CandidatePushError("TOO_LARGE", "The API refused the bundle: it is over the 250 MB compressed limit.");
|
|
3723
|
+
case 422: return new CandidatePushError("INVALID_BUNDLE", action === "upload" ? `The API rejected the bundle: ${error.message}` : `The API rejected the request: ${error.message}`);
|
|
3724
|
+
case 503: return new CandidatePushError("STORAGE_NOT_CONFIGURED", "Bundle storage is not configured on this API, so it cannot take candidate bundles.");
|
|
3725
|
+
default: return new CandidatePushError(code ?? "EXECUTION_FAILED", error.message);
|
|
3726
|
+
}
|
|
3727
|
+
}
|
|
3728
|
+
/** Streams the archive to the API with its digest and an explicit Content-Length. */
|
|
3729
|
+
async function uploadCandidateBundleFile(client, file, digest) {
|
|
3730
|
+
const body = Readable.toWeb(createReadStream(file));
|
|
3731
|
+
return await (await client.requestStream(`/simulations/candidate-bundles?digest=${digest.sha256}`, {
|
|
3732
|
+
method: "POST",
|
|
3733
|
+
headers: {
|
|
3734
|
+
"Content-Type": "application/gzip",
|
|
3735
|
+
"Content-Length": String(digest.bytes)
|
|
3736
|
+
},
|
|
3737
|
+
body,
|
|
3738
|
+
duplex: "half"
|
|
3739
|
+
})).json();
|
|
3740
|
+
}
|
|
3741
|
+
async function pointTargetAtBundle(client, uploaded, options) {
|
|
3742
|
+
const mcp = options.mcpTarget ? { mcpTargetId: options.mcpTarget } : {};
|
|
3743
|
+
if (options.createTarget !== void 0) {
|
|
3744
|
+
const config = {
|
|
3745
|
+
kind: "sandbox_candidate",
|
|
3746
|
+
bundle: uploaded.bundle,
|
|
3747
|
+
manifest: uploaded.manifest,
|
|
3748
|
+
...mcp
|
|
3749
|
+
};
|
|
3750
|
+
return {
|
|
3751
|
+
target: (await client.post("/simulations/targets", {
|
|
3752
|
+
name: options.createTarget,
|
|
3753
|
+
config
|
|
3754
|
+
}).catch((error) => {
|
|
3755
|
+
throw describeCandidateApiError(error, "write-target");
|
|
3756
|
+
})).target,
|
|
3757
|
+
action: "created"
|
|
3758
|
+
};
|
|
3759
|
+
}
|
|
3760
|
+
if (options.target === void 0) return null;
|
|
3761
|
+
const targetPath = `/simulations/targets/${encodeURIComponent(options.target)}`;
|
|
3762
|
+
const existing = await client.get(targetPath).catch((error) => {
|
|
3763
|
+
throw describeCandidateApiError(error, "read-target");
|
|
3764
|
+
});
|
|
3765
|
+
if (existing.target.config.kind !== "sandbox_candidate") throw new CandidatePushError("INVALID_TARGET", `Target ${options.target} is a ${existing.target.config.kind} target, not a sandbox_candidate target.`);
|
|
3766
|
+
const config = {
|
|
3767
|
+
...existing.target.config,
|
|
3768
|
+
bundle: uploaded.bundle,
|
|
3769
|
+
manifest: uploaded.manifest,
|
|
3770
|
+
...mcp
|
|
3771
|
+
};
|
|
3772
|
+
return {
|
|
3773
|
+
target: (await client.patch(targetPath, { config }).catch((error) => {
|
|
3774
|
+
throw describeCandidateApiError(error, "write-target");
|
|
3775
|
+
})).target,
|
|
3776
|
+
action: "updated"
|
|
3777
|
+
};
|
|
3778
|
+
}
|
|
3779
|
+
function formatSize(bytes) {
|
|
3780
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
3781
|
+
if (bytes < 1048576) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
3782
|
+
return `${(bytes / 1048576).toFixed(1)} MB`;
|
|
3783
|
+
}
|
|
3784
|
+
const STEP_MARK = {
|
|
3785
|
+
passed: chalk.green("✓"),
|
|
3786
|
+
failed: chalk.red("✗"),
|
|
3787
|
+
skipped: chalk.gray("-")
|
|
3788
|
+
};
|
|
3789
|
+
function printConformance(result) {
|
|
3790
|
+
const verdict = result.passed ? chalk.green("passed") : chalk.red("failed");
|
|
3791
|
+
console.log(`\n${chalk.cyan("Conformance:")} ${verdict}`);
|
|
3792
|
+
for (const step of result.steps) {
|
|
3793
|
+
const detail = step.detail ? chalk.gray(` ${step.detail}`) : "";
|
|
3794
|
+
const status = step.status === "passed" ? "" : ` ${step.status}`;
|
|
3795
|
+
console.log(` ${STEP_MARK[step.status]} ${step.step}${status}${detail}`);
|
|
3796
|
+
}
|
|
3797
|
+
if (result.blockedHosts.length > 0) console.log(chalk.red(` Blocked hosts: ${result.blockedHosts.join(", ")}`));
|
|
3798
|
+
}
|
|
3799
|
+
function printPushSummary(uploaded, change, savedArchive) {
|
|
3800
|
+
console.log(chalk.green(uploaded.reused ? "✓ Candidate bundle already stored (reused)" : "✓ Candidate bundle uploaded"));
|
|
3801
|
+
const { manifest } = uploaded;
|
|
3802
|
+
console.log(` ${chalk.gray("Digest:")} ${uploaded.bundle.sha256}`);
|
|
3803
|
+
console.log(` ${chalk.gray("Size:")} ${formatSize(uploaded.compressedBytes)}`);
|
|
3804
|
+
console.log(` ${chalk.gray("Manifest:")} ${manifest.name} (${manifest.framework.name} ${manifest.framework.version}, ${manifest.runtime}, ${manifest.protocol}, ${manifest.sessions})`);
|
|
3805
|
+
if (savedArchive) console.log(` ${chalk.gray("Archive:")} ${savedArchive}`);
|
|
3806
|
+
if (change) {
|
|
3807
|
+
console.log(` ${chalk.gray("Target:")} ${change.target.id} (${change.action})`);
|
|
3808
|
+
return;
|
|
3809
|
+
}
|
|
3810
|
+
console.log(`\n${chalk.cyan("Bundle for a sandbox_candidate target config:")}`);
|
|
3811
|
+
console.log(JSON.stringify({ bundle: uploaded.bundle }, null, 2));
|
|
3812
|
+
}
|
|
3813
|
+
function validatePushOptions(options) {
|
|
3814
|
+
if (options.target !== void 0 && options.createTarget !== void 0) throw new CandidatePushError("INVALID_INPUT", "Pass either --target or --create-target, not both.");
|
|
3815
|
+
const hasTarget = options.target !== void 0 || options.createTarget !== void 0;
|
|
3816
|
+
if (options.mcpTarget !== void 0 && !hasTarget) throw new CandidatePushError("INVALID_INPUT", "--mcp-target needs --target or --create-target.");
|
|
3817
|
+
if (options.conformance && !hasTarget) throw new CandidatePushError("INVALID_INPUT", "--conformance needs --target or --create-target: the check runs against a target.");
|
|
3818
|
+
}
|
|
3819
|
+
async function runPush(path, options) {
|
|
3820
|
+
validatePushOptions(options);
|
|
3821
|
+
const input = await resolveCandidateBundleInput(path);
|
|
3822
|
+
if (input.kind === "archive" && options.out) throw new CandidatePushError("INVALID_INPUT", "--out applies only when packing a directory.");
|
|
3823
|
+
let archive = input.path;
|
|
3824
|
+
let scratch;
|
|
3825
|
+
let digest;
|
|
3826
|
+
try {
|
|
3827
|
+
if (input.kind === "directory") {
|
|
3828
|
+
await readCandidateManifest(input.path);
|
|
3829
|
+
if (options.out) {
|
|
3830
|
+
const saved = resolve(options.out);
|
|
3831
|
+
const partial = `${saved}.partial-${process.pid}-${Date.now()}`;
|
|
3832
|
+
try {
|
|
3833
|
+
digest = await packCandidateDirectory(input.path, partial, { skip: [saved] });
|
|
3834
|
+
await rename(partial, saved);
|
|
3835
|
+
} catch (error) {
|
|
3836
|
+
await rm(partial, { force: true });
|
|
3837
|
+
throw error;
|
|
3838
|
+
}
|
|
3839
|
+
archive = saved;
|
|
3840
|
+
} else {
|
|
3841
|
+
scratch = await mkdtemp(join(tmpdir(), "runtype-candidate-"));
|
|
3842
|
+
archive = join(scratch, "candidate.tgz");
|
|
3843
|
+
digest = await packCandidateDirectory(input.path, archive);
|
|
3844
|
+
}
|
|
3845
|
+
} else digest = await digestCandidateArchive(input.path);
|
|
3846
|
+
const apiKey = await ensureAuth();
|
|
3847
|
+
if (!apiKey) return false;
|
|
3848
|
+
const client = createCliClient(apiKey);
|
|
3849
|
+
const uploaded = await uploadCandidateBundleFile(client, archive, digest).catch((error) => {
|
|
3850
|
+
throw describeCandidateApiError(error, "upload");
|
|
3851
|
+
});
|
|
3852
|
+
const change = await pointTargetAtBundle(client, uploaded, options);
|
|
3853
|
+
const conformance = options.conformance && change ? await client.post(`/simulations/targets/${encodeURIComponent(change.target.id)}/conformance`, {}).catch((error) => {
|
|
3854
|
+
throw describeCandidateApiError(error, "conformance");
|
|
3855
|
+
}) : void 0;
|
|
3856
|
+
const savedArchive = input.kind === "directory" && options.out ? archive : void 0;
|
|
3857
|
+
if (options.json) printJson({
|
|
3858
|
+
...uploaded,
|
|
3859
|
+
...savedArchive ? { archive: savedArchive } : {},
|
|
3860
|
+
...change ? { target: {
|
|
3861
|
+
id: change.target.id,
|
|
3862
|
+
name: change.target.name,
|
|
3863
|
+
action: change.action
|
|
3864
|
+
} } : {},
|
|
3865
|
+
...conformance ? { conformance } : {}
|
|
3866
|
+
});
|
|
3867
|
+
else {
|
|
3868
|
+
printPushSummary(uploaded, change, savedArchive);
|
|
3869
|
+
if (conformance) printConformance(conformance);
|
|
3870
|
+
}
|
|
3871
|
+
return conformance ? conformance.passed : true;
|
|
3872
|
+
} finally {
|
|
3873
|
+
if (scratch) await rm(scratch, {
|
|
3874
|
+
recursive: true,
|
|
3875
|
+
force: true
|
|
3876
|
+
});
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
function reportFailure(error, json) {
|
|
3880
|
+
const code = error instanceof CandidatePushError || error instanceof CandidateBundleError ? error.code : "EXECUTION_FAILED";
|
|
3881
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3882
|
+
if (json) printJson({
|
|
3883
|
+
success: false,
|
|
3884
|
+
error: {
|
|
3885
|
+
code,
|
|
3886
|
+
message
|
|
3887
|
+
}
|
|
3888
|
+
});
|
|
3889
|
+
else {
|
|
3890
|
+
console.error(chalk.red("Failed to push candidate bundle"));
|
|
3891
|
+
console.error(chalk.red(message));
|
|
3892
|
+
}
|
|
3893
|
+
}
|
|
3894
|
+
const simulationsCommand = new Command("simulations").description("Simulations: sandbox candidates, recorded tool exchanges and reports");
|
|
3895
|
+
registerToolMemoryCommands(simulationsCommand);
|
|
3896
|
+
simulationsCommand.command("candidate").description("Sandbox candidates: an agent run as real code in a Runtype sandbox").command("push <path>").description("Upload a candidate bundle: a directory with runtype-candidate.json at its root (packed reproducibly) or a .tgz / .tar.gz").option("--target <id>", "Point this sandbox_candidate target at the new bundle, keeping its other config").option("--create-target <name>", "Create a sandbox_candidate target for the new bundle").option("--mcp-target <id>", "With --target or --create-target: the external_mcp target whose tools episodes get (config.mcpTargetId)").option("--conformance", "Run the conformance check on the target after the push").option("--out <file>", "Also save the packed archive to this file (directory input only)").option("--json", "Output as JSON").action(async (path, options) => {
|
|
3897
|
+
let passed;
|
|
3898
|
+
try {
|
|
3899
|
+
passed = await runPush(path, options);
|
|
3900
|
+
} catch (error) {
|
|
3901
|
+
reportFailure(error, options.json);
|
|
3902
|
+
process.exit(1);
|
|
3903
|
+
}
|
|
3904
|
+
if (!passed) process.exit(1);
|
|
3905
|
+
});
|
|
3906
|
+
export { simulationsCommand };
|
|
3907
|
+
|