@mcpjam/inspector 3.12.3 → 3.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/assets/OAuthDesktopReturnNotice-B2D5moE-.css +2 -0
- package/dist/client/assets/{OAuthDesktopReturnNotice-C5D90OSm.js → OAuthDesktopReturnNotice-CTGRhDuo.js} +2 -2
- package/dist/client/assets/{app-bootstrap-C1Jn01kP.js → app-bootstrap-B7TKDg4p.js} +338 -338
- package/dist/client/assets/{button-RD2Dghjp.js → button-BgMj96d7.js} +2 -2
- package/dist/client/assets/{chunk-YOKDWASO-DTTU1wSE.js → chunk-YOKDWASO-DiKLTu3U.js} +5 -5
- package/dist/client/assets/{dead-clicks-autocapture-CDkbT21a.js → dead-clicks-autocapture-B2D_6BXS.js} +2 -2
- package/dist/client/assets/{error-reporting-Dg5b1JwS.js → error-reporting-6uZvG5HZ.js} +4 -4
- package/dist/client/assets/{error-reporting-_SEhX6IL.js → error-reporting-xR8v_8my.js} +1 -1
- package/dist/client/assets/{exception-autocapture-CtxHwPtn.js → exception-autocapture-Xx2wbAOf.js} +2 -2
- package/dist/client/assets/{failed-request-tracker-kUaJ-sVM.js → failed-request-tracker-BH4w1sBR.js} +2 -2
- package/dist/client/assets/{highlighted-body-KPVGNVTW-CBC2hHHb.js → highlighted-body-KPVGNVTW-DIgOE9D7.js} +2 -2
- package/dist/client/assets/{index-yOjxp36p.js → index-Cu4ySOft.js} +3 -3
- package/dist/client/assets/{jszip.min-B3Qii_az.js → jszip.min-C24J_3Y_.js} +2 -2
- package/dist/client/assets/{mermaid-HWGCJPDP-BzT3-9sO.js → mermaid-HWGCJPDP-BTzLBCkg.js} +1 -1
- package/dist/client/assets/{posthog-recorder-B06ULcLl.js → posthog-recorder-CEtP2mh8.js} +2 -2
- package/dist/client/assets/{preload-helper-DvWwGI6x.js → preload-helper-Cfn3cu1N.js} +2 -2
- package/dist/client/assets/{rolldown-runtime-DVMSi0Yr.js → rolldown-runtime-BPNN4GNu.js} +1 -1
- package/dist/client/assets/{surveys-D941hgn3.js → surveys-CWniVjxJ.js} +2 -2
- package/dist/client/assets/{trace-timeline-Bk3hAXNq.js → trace-timeline-D-nhBNaR.js} +1 -1
- package/dist/client/assets/{trace-timeline-DVZqlQSW.js → trace-timeline-D9qZRjVv.js} +2 -2
- package/dist/client/assets/{web-vitals-DZUBt4dL.js → web-vitals-zTwn9Gi6.js} +2 -2
- package/dist/client/index.html +7 -7
- package/dist/server/{chunk-WGJHSGJP.js → chunk-5T6DFO27.js} +2 -2
- package/dist/server/harness-install-cli.js +1 -1
- package/dist/server/index.js +1181 -251
- package/package.json +2 -2
- package/dist/client/assets/OAuthDesktopReturnNotice-C95FI7VH.css +0 -2
package/dist/server/index.js
CHANGED
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
supportsOwnershipProof,
|
|
36
36
|
terminateOwnedProcess,
|
|
37
37
|
terminateOwnedProcessGroup
|
|
38
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-5T6DFO27.js";
|
|
39
39
|
import {
|
|
40
40
|
ALLOWED_HOSTS,
|
|
41
41
|
BROWSER_CONSENT_HEADER,
|
|
@@ -665,6 +665,7 @@ __export(canonical_exports, {
|
|
|
665
665
|
CanonicalJsonError: () => CanonicalJsonError,
|
|
666
666
|
canonicalDigest: () => canonicalDigest,
|
|
667
667
|
canonicalJson: () => canonicalJson,
|
|
668
|
+
canonicalJsonBounded: () => canonicalJsonBounded,
|
|
668
669
|
sha256Hex: () => sha256Hex
|
|
669
670
|
});
|
|
670
671
|
|
|
@@ -1068,7 +1069,7 @@ var sha256 = /* @__PURE__ */ createHasher(
|
|
|
1068
1069
|
/* @__PURE__ */ oidNist(1)
|
|
1069
1070
|
);
|
|
1070
1071
|
|
|
1071
|
-
// ../evaluators/dist/chunk-
|
|
1072
|
+
// ../evaluators/dist/chunk-3KMKYC6O.js
|
|
1072
1073
|
var CanonicalJsonError = class extends Error {
|
|
1073
1074
|
constructor(message) {
|
|
1074
1075
|
super(message);
|
|
@@ -1137,6 +1138,118 @@ function write(value, ancestors) {
|
|
|
1137
1138
|
ancestors.delete(value);
|
|
1138
1139
|
}
|
|
1139
1140
|
}
|
|
1141
|
+
var OVER_BUDGET = Symbol("canonicalJsonBounded.overBudget");
|
|
1142
|
+
var BoundedOutput = class {
|
|
1143
|
+
constructor(maxChars) {
|
|
1144
|
+
this.maxChars = maxChars;
|
|
1145
|
+
}
|
|
1146
|
+
parts = [];
|
|
1147
|
+
length = 0;
|
|
1148
|
+
/** Characters still available before the budget is exceeded. */
|
|
1149
|
+
remaining() {
|
|
1150
|
+
return this.maxChars - this.length;
|
|
1151
|
+
}
|
|
1152
|
+
push(text2) {
|
|
1153
|
+
if (text2.length > this.remaining()) throw OVER_BUDGET;
|
|
1154
|
+
this.parts.push(text2);
|
|
1155
|
+
this.length += text2.length;
|
|
1156
|
+
}
|
|
1157
|
+
/**
|
|
1158
|
+
* `JSON.stringify(text)` is at least `text.length + 2` characters, so a
|
|
1159
|
+
* string that cannot fit is refused BEFORE it is escaped. Escaping a
|
|
1160
|
+
* multi-megabyte argument only to throw the result away is the work the
|
|
1161
|
+
* budget exists to avoid.
|
|
1162
|
+
*/
|
|
1163
|
+
pushString(text2, suffix = "") {
|
|
1164
|
+
if (text2.length + 2 + suffix.length > this.remaining()) throw OVER_BUDGET;
|
|
1165
|
+
this.push(JSON.stringify(text2) + suffix);
|
|
1166
|
+
}
|
|
1167
|
+
toString() {
|
|
1168
|
+
return this.parts.join("");
|
|
1169
|
+
}
|
|
1170
|
+
};
|
|
1171
|
+
function canonicalJsonBounded(value, maxChars) {
|
|
1172
|
+
if (value === void 0) {
|
|
1173
|
+
return {
|
|
1174
|
+
ok: false,
|
|
1175
|
+
reason: "uncanonical",
|
|
1176
|
+
message: "Cannot canonicalize `undefined` at the root"
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
const out = new BoundedOutput(maxChars);
|
|
1180
|
+
try {
|
|
1181
|
+
writeBounded(value, /* @__PURE__ */ new Set(), out);
|
|
1182
|
+
} catch (error) {
|
|
1183
|
+
if (error === OVER_BUDGET) return { ok: false, reason: "overBudget" };
|
|
1184
|
+
return {
|
|
1185
|
+
ok: false,
|
|
1186
|
+
reason: "uncanonical",
|
|
1187
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
return { ok: true, json: out.toString() };
|
|
1191
|
+
}
|
|
1192
|
+
function writeBounded(value, ancestors, out) {
|
|
1193
|
+
if (value === null) {
|
|
1194
|
+
out.push("null");
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
switch (typeof value) {
|
|
1198
|
+
case "boolean":
|
|
1199
|
+
out.push(value ? "true" : "false");
|
|
1200
|
+
return;
|
|
1201
|
+
case "number":
|
|
1202
|
+
out.push(canonicalizeNumber(value));
|
|
1203
|
+
return;
|
|
1204
|
+
case "string":
|
|
1205
|
+
out.pushString(value);
|
|
1206
|
+
return;
|
|
1207
|
+
case "object":
|
|
1208
|
+
break;
|
|
1209
|
+
default:
|
|
1210
|
+
throw new CanonicalJsonError(
|
|
1211
|
+
`Cannot canonicalize value of type ${typeof value}`
|
|
1212
|
+
);
|
|
1213
|
+
}
|
|
1214
|
+
if (ancestors.has(value)) {
|
|
1215
|
+
throw new CanonicalJsonError("Cannot canonicalize a circular structure");
|
|
1216
|
+
}
|
|
1217
|
+
ancestors.add(value);
|
|
1218
|
+
try {
|
|
1219
|
+
if (Array.isArray(value)) {
|
|
1220
|
+
out.push("[");
|
|
1221
|
+
const length = value.length;
|
|
1222
|
+
for (let index = 0; index < length; index += 1) {
|
|
1223
|
+
if (index > 0) out.push(",");
|
|
1224
|
+
const entry3 = value[index];
|
|
1225
|
+
if (entry3 === void 0) out.push("null");
|
|
1226
|
+
else writeBounded(entry3, ancestors, out);
|
|
1227
|
+
}
|
|
1228
|
+
out.push("]");
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
const prototype = Object.getPrototypeOf(value);
|
|
1232
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
1233
|
+
throw new CanonicalJsonError(
|
|
1234
|
+
"Cannot canonicalize a non-plain object (only plain objects, arrays and JSON primitives are part of the contract)"
|
|
1235
|
+
);
|
|
1236
|
+
}
|
|
1237
|
+
const record5 = value;
|
|
1238
|
+
out.push("{");
|
|
1239
|
+
let first = true;
|
|
1240
|
+
for (const key of Object.keys(record5).sort()) {
|
|
1241
|
+
const entry3 = record5[key];
|
|
1242
|
+
if (entry3 === void 0) continue;
|
|
1243
|
+
if (!first) out.push(",");
|
|
1244
|
+
first = false;
|
|
1245
|
+
out.pushString(key, ":");
|
|
1246
|
+
writeBounded(entry3, ancestors, out);
|
|
1247
|
+
}
|
|
1248
|
+
out.push("}");
|
|
1249
|
+
} finally {
|
|
1250
|
+
ancestors.delete(value);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1140
1253
|
function sha256Hex(text2) {
|
|
1141
1254
|
return bytesToHex(sha256(utf8ToBytes(text2)));
|
|
1142
1255
|
}
|
|
@@ -1164,7 +1277,8 @@ __export(types_exports2, {
|
|
|
1164
1277
|
requiresRenderObservations: () => requiresRenderObservations
|
|
1165
1278
|
});
|
|
1166
1279
|
|
|
1167
|
-
// ../evaluators/dist/chunk-
|
|
1280
|
+
// ../evaluators/dist/chunk-YDCS7DHF.js
|
|
1281
|
+
import { RE2JS } from "re2js";
|
|
1168
1282
|
var MAX_RESPONSE_CLOSE_TO_CHARS = 1e5;
|
|
1169
1283
|
var MAX_RESPONSE_CLOSE_TO_CELLS = 4e6;
|
|
1170
1284
|
var RESPONSE_CLOSE_TO_IMPLEMENTATION = {
|
|
@@ -1218,8 +1332,283 @@ function normalizedResponseDistance(message, reference, options) {
|
|
|
1218
1332
|
}
|
|
1219
1333
|
return previous[columns.length] / denominator;
|
|
1220
1334
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1335
|
+
var MAX_MATCH_PATTERNS = 8;
|
|
1336
|
+
var MAX_MATCH_PATTERN_CHARS = 512;
|
|
1337
|
+
var MIN_MATCH_PATH_CHARS = 2;
|
|
1338
|
+
var MAX_MATCH_PATH_CHARS = 257;
|
|
1339
|
+
var MATCH_PATH_PATTERN = /^\/(?:[^/~]|~[01])+$/;
|
|
1340
|
+
var MAX_MATCH_SUBJECT_CHARS = 1e5;
|
|
1341
|
+
var MAX_DISPLAYED_PATTERN_CHARS = 120;
|
|
1342
|
+
var MATCH_PATTERN_FLAGS = [
|
|
1343
|
+
"i",
|
|
1344
|
+
"m",
|
|
1345
|
+
"s",
|
|
1346
|
+
"im",
|
|
1347
|
+
"is",
|
|
1348
|
+
"ms",
|
|
1349
|
+
"ims"
|
|
1350
|
+
];
|
|
1351
|
+
var TOOL_INPUT_MATCHES_IMPLEMENTATION = {
|
|
1352
|
+
implementationVersion: 1,
|
|
1353
|
+
engine: "re2js",
|
|
1354
|
+
encoding: "canonical-json-bounded",
|
|
1355
|
+
maxSubjectChars: MAX_MATCH_SUBJECT_CHARS
|
|
1356
|
+
};
|
|
1357
|
+
var TOOL_RESULT_MATCHES_IMPLEMENTATION = {
|
|
1358
|
+
implementationVersion: 1,
|
|
1359
|
+
engine: "re2js",
|
|
1360
|
+
encoding: "canonical-json-bounded",
|
|
1361
|
+
maxSubjectChars: MAX_MATCH_SUBJECT_CHARS,
|
|
1362
|
+
subject: "text,structuredContent,json",
|
|
1363
|
+
pathSource: "structuredContent,json"
|
|
1364
|
+
};
|
|
1365
|
+
function matchFlagBits(flags) {
|
|
1366
|
+
let bits = 0;
|
|
1367
|
+
if (flags?.includes("i")) bits |= RE2JS.CASE_INSENSITIVE;
|
|
1368
|
+
if (flags?.includes("m")) bits |= RE2JS.MULTILINE;
|
|
1369
|
+
if (flags?.includes("s")) bits |= RE2JS.DOTALL;
|
|
1370
|
+
return bits;
|
|
1371
|
+
}
|
|
1372
|
+
function isMatchPatternFlags(flags) {
|
|
1373
|
+
return flags === void 0 || MATCH_PATTERN_FLAGS.includes(flags);
|
|
1374
|
+
}
|
|
1375
|
+
function compileMatchPattern(pattern, flags) {
|
|
1376
|
+
try {
|
|
1377
|
+
const compiled = RE2JS.compile(
|
|
1378
|
+
RE2JS.translateRegExp(pattern),
|
|
1379
|
+
matchFlagBits(flags)
|
|
1380
|
+
);
|
|
1381
|
+
return { ok: true, compiled };
|
|
1382
|
+
} catch (error) {
|
|
1383
|
+
return {
|
|
1384
|
+
ok: false,
|
|
1385
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1386
|
+
};
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
function matchPatternError(pattern, flags) {
|
|
1390
|
+
const result = compileMatchPattern(pattern, flags);
|
|
1391
|
+
return result.ok ? void 0 : result.message;
|
|
1392
|
+
}
|
|
1393
|
+
function compileMatchPatterns(patterns, flags) {
|
|
1394
|
+
const compiled = [];
|
|
1395
|
+
for (let index = 0; index < patterns.length; index += 1) {
|
|
1396
|
+
const result = compileMatchPattern(patterns[index], flags);
|
|
1397
|
+
if (!result.ok) return { ok: false, index, message: result.message };
|
|
1398
|
+
compiled.push(result.compiled);
|
|
1399
|
+
}
|
|
1400
|
+
return { ok: true, compiled };
|
|
1401
|
+
}
|
|
1402
|
+
function matchBoundsError(min, max, unit) {
|
|
1403
|
+
if (min === 0 && max === void 0) {
|
|
1404
|
+
return {
|
|
1405
|
+
path: "min",
|
|
1406
|
+
message: `min: 0 needs a max; on its own it passes every transcript (use max: 0 for "no ${unit} matches")`
|
|
1407
|
+
};
|
|
1408
|
+
}
|
|
1409
|
+
if (max !== void 0 && max < (min ?? 1)) {
|
|
1410
|
+
return {
|
|
1411
|
+
path: "max",
|
|
1412
|
+
message: `max must be at least min (${min ?? 1}${min === void 0 ? ", the default" : ""})`
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1415
|
+
return void 0;
|
|
1416
|
+
}
|
|
1417
|
+
function matchPathError(path8) {
|
|
1418
|
+
if (typeof path8 !== "string") return "path must be a string";
|
|
1419
|
+
if (path8.length === 0) {
|
|
1420
|
+
return 'path "" is the whole value; omit path instead';
|
|
1421
|
+
}
|
|
1422
|
+
if (path8.length < MIN_MATCH_PATH_CHARS || path8.length > MAX_MATCH_PATH_CHARS) {
|
|
1423
|
+
return `path must be ${MIN_MATCH_PATH_CHARS} to ${MAX_MATCH_PATH_CHARS} characters: "/" and the key, e.g. "/elements"`;
|
|
1424
|
+
}
|
|
1425
|
+
if (!path8.startsWith("/")) {
|
|
1426
|
+
return 'path must be a JSON Pointer that starts with "/", e.g. "/elements"';
|
|
1427
|
+
}
|
|
1428
|
+
if (path8.indexOf("/", 1) !== -1) {
|
|
1429
|
+
return 'path must name exactly one top-level key; nested paths are not supported yet (write a "/" inside a key as ~1)';
|
|
1430
|
+
}
|
|
1431
|
+
if (!MATCH_PATH_PATTERN.test(path8)) {
|
|
1432
|
+
return 'path has an invalid escape: "~" must be followed by 0 or 1';
|
|
1433
|
+
}
|
|
1434
|
+
return void 0;
|
|
1435
|
+
}
|
|
1436
|
+
function parseMatchPath(path8) {
|
|
1437
|
+
const error = matchPathError(path8);
|
|
1438
|
+
if (error !== void 0) return { ok: false, message: error };
|
|
1439
|
+
return {
|
|
1440
|
+
ok: true,
|
|
1441
|
+
key: path8.slice(1).replace(/~1/g, "/").replace(/~0/g, "~")
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
function matchRuleConfigError(kind, rule) {
|
|
1445
|
+
const toolNameRequired = kind === "toolInputMatches";
|
|
1446
|
+
if ((toolNameRequired || rule.toolName !== void 0) && (typeof rule.toolName !== "string" || rule.toolName.length === 0)) {
|
|
1447
|
+
return toolNameRequired ? `${kind} requires a non-empty toolName` : `${kind} toolName, when set, must be non-empty`;
|
|
1448
|
+
}
|
|
1449
|
+
const patterns = rule.patterns;
|
|
1450
|
+
if (!Array.isArray(patterns) || patterns.length < 1 || patterns.length > MAX_MATCH_PATTERNS) {
|
|
1451
|
+
return `${kind} requires 1 to ${MAX_MATCH_PATTERNS} patterns`;
|
|
1452
|
+
}
|
|
1453
|
+
for (const pattern of patterns) {
|
|
1454
|
+
if (typeof pattern !== "string" || pattern.length < 1 || pattern.length > MAX_MATCH_PATTERN_CHARS) {
|
|
1455
|
+
return `every ${kind} pattern must be 1 to ${MAX_MATCH_PATTERN_CHARS} characters`;
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
if (!isMatchPatternFlags(rule.flags)) {
|
|
1459
|
+
return `${kind} flags must be one of ${MATCH_PATTERN_FLAGS.join(", ")}`;
|
|
1460
|
+
}
|
|
1461
|
+
if (rule.path !== void 0) {
|
|
1462
|
+
const pathError = matchPathError(rule.path);
|
|
1463
|
+
if (pathError !== void 0) return `${kind} ${pathError}`;
|
|
1464
|
+
}
|
|
1465
|
+
for (const key of ["min", "max"]) {
|
|
1466
|
+
const count5 = rule[key];
|
|
1467
|
+
if (count5 !== void 0 && (typeof count5 !== "number" || !Number.isInteger(count5) || count5 < 0)) {
|
|
1468
|
+
return `${kind} ${key} must be a non-negative integer`;
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
const bounds = matchBoundsError(
|
|
1472
|
+
rule.min,
|
|
1473
|
+
rule.max,
|
|
1474
|
+
kind === "toolInputMatches" ? "call" : "result"
|
|
1475
|
+
);
|
|
1476
|
+
return bounds?.message;
|
|
1477
|
+
}
|
|
1478
|
+
function encodeJson(value, maxChars) {
|
|
1479
|
+
const encoded = canonicalJsonBounded(value, maxChars);
|
|
1480
|
+
return encoded.ok ? { kind: "subject", text: encoded.json } : { kind: "unreadable", reason: encoded.reason };
|
|
1481
|
+
}
|
|
1482
|
+
function encodeMatchValue(value, maxChars = MAX_MATCH_SUBJECT_CHARS) {
|
|
1483
|
+
if (typeof value === "string") {
|
|
1484
|
+
return value.length > maxChars ? { kind: "unreadable", reason: "overBudget" } : { kind: "subject", text: value };
|
|
1485
|
+
}
|
|
1486
|
+
return encodeJson(value, maxChars);
|
|
1487
|
+
}
|
|
1488
|
+
function ownValue(container, key) {
|
|
1489
|
+
if (!container || typeof container !== "object" || Array.isArray(container) || !Object.prototype.hasOwnProperty.call(container, key)) {
|
|
1490
|
+
return void 0;
|
|
1491
|
+
}
|
|
1492
|
+
return container[key];
|
|
1493
|
+
}
|
|
1494
|
+
function encodeCallSubject(args, key, maxChars = MAX_MATCH_SUBJECT_CHARS) {
|
|
1495
|
+
const value = args ?? {};
|
|
1496
|
+
if (key === void 0) return encodeJson(value, maxChars);
|
|
1497
|
+
const at2 = ownValue(value, key);
|
|
1498
|
+
return at2 === void 0 ? { kind: "missing" } : encodeMatchValue(at2, maxChars);
|
|
1499
|
+
}
|
|
1500
|
+
function resultValueAt(result, key) {
|
|
1501
|
+
return ownValue(
|
|
1502
|
+
result.structuredContent !== void 0 ? result.structuredContent : result.json,
|
|
1503
|
+
key
|
|
1504
|
+
);
|
|
1505
|
+
}
|
|
1506
|
+
function encodeResultSubject(result, key, maxChars = MAX_MATCH_SUBJECT_CHARS) {
|
|
1507
|
+
if (key !== void 0) {
|
|
1508
|
+
const at2 = resultValueAt(result, key);
|
|
1509
|
+
return at2 === void 0 ? { kind: "missing" } : encodeMatchValue(at2, maxChars);
|
|
1510
|
+
}
|
|
1511
|
+
if (result.truncated === true) {
|
|
1512
|
+
return { kind: "unreadable", reason: "truncated" };
|
|
1513
|
+
}
|
|
1514
|
+
const parts = [];
|
|
1515
|
+
let used = 0;
|
|
1516
|
+
const append = (encode) => {
|
|
1517
|
+
const separator = parts.length === 0 ? 0 : 1;
|
|
1518
|
+
const budget = maxChars - used - separator;
|
|
1519
|
+
if (budget < 0) return { kind: "unreadable", reason: "overBudget" };
|
|
1520
|
+
const part = encode(budget);
|
|
1521
|
+
if (part.kind !== "subject") return part;
|
|
1522
|
+
parts.push(part.text);
|
|
1523
|
+
used += separator + part.text.length;
|
|
1524
|
+
return void 0;
|
|
1525
|
+
};
|
|
1526
|
+
const text2 = result.text;
|
|
1527
|
+
if (typeof text2 === "string") {
|
|
1528
|
+
const refused2 = append((budget) => encodeMatchValue(text2, budget));
|
|
1529
|
+
if (refused2) return refused2;
|
|
1530
|
+
}
|
|
1531
|
+
for (const payload of [result.structuredContent, result.json]) {
|
|
1532
|
+
if (payload === void 0) continue;
|
|
1533
|
+
const refused2 = append((budget) => encodeJson(payload, budget));
|
|
1534
|
+
if (refused2) return refused2;
|
|
1535
|
+
}
|
|
1536
|
+
return { kind: "subject", text: parts.join("\n") };
|
|
1537
|
+
}
|
|
1538
|
+
function matchVerdict(tally, bounds, unseen = 0) {
|
|
1539
|
+
const { matched } = tally;
|
|
1540
|
+
const unknown = tally.unreadable + unseen;
|
|
1541
|
+
const max = bounds.max ?? Number.POSITIVE_INFINITY;
|
|
1542
|
+
if (matched >= bounds.min && matched + unknown <= max) return "pass";
|
|
1543
|
+
if (matched + unknown < bounds.min || matched > max) return "fail";
|
|
1544
|
+
return "unscored";
|
|
1545
|
+
}
|
|
1546
|
+
function countOf(count5, unit) {
|
|
1547
|
+
return `${count5} ${unit}${count5 === 1 ? "" : "s"}`;
|
|
1548
|
+
}
|
|
1549
|
+
function describeMatchExpectation(bounds, unit) {
|
|
1550
|
+
const { min, max } = bounds;
|
|
1551
|
+
if (max === void 0) return `at least ${min} matching ${unit}(s)`;
|
|
1552
|
+
if (max === 0) return `no matching ${unit}`;
|
|
1553
|
+
if (min === max) return `exactly ${min} matching ${unit}(s)`;
|
|
1554
|
+
if (min === 0) return `at most ${max} matching ${unit}(s)`;
|
|
1555
|
+
return `${min} to ${max} matching ${unit}s`;
|
|
1556
|
+
}
|
|
1557
|
+
function allPatterns(patternCount) {
|
|
1558
|
+
return patternCount === 1 ? "the pattern" : `all ${patternCount} patterns`;
|
|
1559
|
+
}
|
|
1560
|
+
function expectedClause(bounds, unit) {
|
|
1561
|
+
return `expected ${describeMatchExpectation(bounds, unit)} (min/max count matching ${unit}s, not all ${unit}s)`;
|
|
1562
|
+
}
|
|
1563
|
+
function describeInputMatch(input) {
|
|
1564
|
+
const { toolName: toolName3, key, tally, bounds, verdict } = input;
|
|
1565
|
+
const all = allPatterns(input.patternCount);
|
|
1566
|
+
const where = key === void 0 ? "its arguments" : `its "${key}" argument`;
|
|
1567
|
+
const expected = expectedClause(bounds, "call");
|
|
1568
|
+
const detail = `Patterns: ${input.patternsShown}`;
|
|
1569
|
+
const got = input.samplesShown ? `. Got: ${input.samplesShown}` : "";
|
|
1570
|
+
const missing = key !== void 0 && tally.missing > 0 ? `, ${countOf(tally.missing, "call")} had no "${key}" argument` : "";
|
|
1571
|
+
const unreadable = tally.unreadable > 0 ? `, ${tally.unreadable} could not be read (over ${MAX_MATCH_SUBJECT_CHARS} characters or not serializable)` : "";
|
|
1572
|
+
if (tally.units === 0) {
|
|
1573
|
+
return verdict === "pass" ? `"${toolName3}" was not called, so no call matched; ${expected}. ${detail}` : `"${toolName3}" was never called; ${expected}. ${detail}`;
|
|
1574
|
+
}
|
|
1575
|
+
const matched = tally.matched === 0 ? `none matched ${all} in ${where}` : `${tally.matched} matched ${all} in ${where}`;
|
|
1576
|
+
const tooMany4 = verdict === "fail" && bounds.max !== void 0 && tally.matched > bounds.max ? `, allowed at most ${bounds.max}` : "";
|
|
1577
|
+
const head = `"${toolName3}" called ${tally.units}\xD7, ${matched}${tooMany4}${missing}${unreadable}`;
|
|
1578
|
+
if (verdict === "pass") return `${head}; ${expected}. ${detail}`;
|
|
1579
|
+
if (verdict === "unscored") {
|
|
1580
|
+
return `${head}; ${expected}, and the unreadable call(s) could decide it either way. ${detail}`;
|
|
1581
|
+
}
|
|
1582
|
+
return `${head}; ${expected}. ${detail}${got}`;
|
|
1583
|
+
}
|
|
1584
|
+
function describeResultMatch(input) {
|
|
1585
|
+
const { toolName: toolName3, key, tally, bounds, verdict, gap } = input;
|
|
1586
|
+
const all = allPatterns(input.patternCount);
|
|
1587
|
+
const where = key === void 0 ? "their content" : `their "${key}" field`;
|
|
1588
|
+
const expected = expectedClause(bounds, "result");
|
|
1589
|
+
const detail = `Patterns: ${input.patternsShown}`;
|
|
1590
|
+
const got = input.samplesShown ? `. Got: ${input.samplesShown}` : "";
|
|
1591
|
+
const missing = key !== void 0 && tally.missing > 0 ? `, ${countOf(tally.missing, "result")} had no "${key}" field` : "";
|
|
1592
|
+
const unreadable = tally.unreadable > 0 ? `, ${tally.unreadable} could not be read (truncated for storage, over ${MAX_MATCH_SUBJECT_CHARS} characters, or not serializable)` : "";
|
|
1593
|
+
if (tally.units === 0) {
|
|
1594
|
+
if (verdict === "unscored") {
|
|
1595
|
+
return `${gap ?? "no results were read"}; cannot count matching results; ${expected}. ${detail}`;
|
|
1596
|
+
}
|
|
1597
|
+
const none = toolName3 === void 0 ? "no tool returned a result" : `"${toolName3}" returned no results`;
|
|
1598
|
+
return verdict === "pass" ? `${none}, so no result matched; ${expected}. ${detail}` : `${none}; ${expected}. ${detail}`;
|
|
1599
|
+
}
|
|
1600
|
+
const source = toolName3 === void 0 ? `tools returned ${countOf(tally.units, "result")}` : `"${toolName3}" returned ${countOf(tally.units, "result")}`;
|
|
1601
|
+
const matched = tally.matched === 0 ? `none matched ${all} in ${where}` : `${tally.matched} matched ${all} in ${where}`;
|
|
1602
|
+
const tooMany4 = verdict === "fail" && bounds.max !== void 0 && tally.matched > bounds.max ? `, allowed at most ${bounds.max}` : "";
|
|
1603
|
+
const head = `${source}, ${matched}${tooMany4}${missing}${unreadable}`;
|
|
1604
|
+
if (verdict === "pass") return `${head}; ${expected}. ${detail}`;
|
|
1605
|
+
if (verdict === "unscored") {
|
|
1606
|
+
return gap !== void 0 ? `${head}; ${expected}, but ${gap}; results that were not read could decide it. ${detail}` : `${head}; ${expected}, and the unreadable result(s) could decide it either way. ${detail}`;
|
|
1607
|
+
}
|
|
1608
|
+
return `${head}; ${expected}. ${detail}${got}`;
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
// ../evaluators/dist/chunk-HCLNEORA.js
|
|
1223
1612
|
import { z } from "zod";
|
|
1224
1613
|
var TURN_SCOPABLE_PREDICATE_KINDS = [
|
|
1225
1614
|
"toolCalledWith",
|
|
@@ -1239,7 +1628,8 @@ var TURN_SCOPABLE_PREDICATE_KINDS = [
|
|
|
1239
1628
|
// Read only the turn's own calls, which `buildTurnTranscript` carries.
|
|
1240
1629
|
"toolCallCountUnder",
|
|
1241
1630
|
"toolCalledBefore",
|
|
1242
|
-
"noRepeatedIdenticalCall"
|
|
1631
|
+
"noRepeatedIdenticalCall",
|
|
1632
|
+
"toolInputMatches"
|
|
1243
1633
|
];
|
|
1244
1634
|
function isTurnScopablePredicateKind(kind) {
|
|
1245
1635
|
return TURN_SCOPABLE_PREDICATE_KINDS.includes(kind);
|
|
@@ -1293,6 +1683,40 @@ var checkPolicyShape = {
|
|
|
1293
1683
|
role: z.enum(["gating", "advisory", "required"]).optional(),
|
|
1294
1684
|
severity: z.literal("warn").optional()
|
|
1295
1685
|
};
|
|
1686
|
+
var patternMatchShape = {
|
|
1687
|
+
// Authored order is kept — it is part of the criterion id.
|
|
1688
|
+
patterns: z.array(z.string().min(1).max(MAX_MATCH_PATTERN_CHARS)).min(1).max(MAX_MATCH_PATTERNS),
|
|
1689
|
+
flags: z.enum(MATCH_PATTERN_FLAGS).optional(),
|
|
1690
|
+
// A one-key JSON Pointer. The pattern is what the published JSON Schema
|
|
1691
|
+
// carries; `matchPathError` says which part of it an author got wrong.
|
|
1692
|
+
path: z.string().min(MIN_MATCH_PATH_CHARS).max(MAX_MATCH_PATH_CHARS).regex(MATCH_PATH_PATTERN, {
|
|
1693
|
+
message: 'path must be a one-key JSON Pointer such as "/elements" ("~1" writes "/" and "~0" writes "~" inside the key)'
|
|
1694
|
+
}).optional(),
|
|
1695
|
+
min: z.number().int().nonnegative().optional(),
|
|
1696
|
+
max: z.number().int().nonnegative().optional()
|
|
1697
|
+
};
|
|
1698
|
+
function refinePatternMatch(unit) {
|
|
1699
|
+
return (value, ctx) => {
|
|
1700
|
+
value.patterns.forEach((pattern, index) => {
|
|
1701
|
+
const error = matchPatternError(pattern, value.flags);
|
|
1702
|
+
if (error !== void 0) {
|
|
1703
|
+
ctx.addIssue({
|
|
1704
|
+
code: z.ZodIssueCode.custom,
|
|
1705
|
+
path: ["patterns", index],
|
|
1706
|
+
message: `pattern does not compile in re2js: ${error}`
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
});
|
|
1710
|
+
const bounds = matchBoundsError(value.min, value.max, unit);
|
|
1711
|
+
if (bounds) {
|
|
1712
|
+
ctx.addIssue({
|
|
1713
|
+
code: z.ZodIssueCode.custom,
|
|
1714
|
+
path: [bounds.path],
|
|
1715
|
+
message: bounds.message
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
};
|
|
1719
|
+
}
|
|
1296
1720
|
var predicateUnion = z.discriminatedUnion("type", [
|
|
1297
1721
|
z.object({
|
|
1298
1722
|
type: z.literal("toolDescriptionsPresent"),
|
|
@@ -1452,6 +1876,20 @@ var predicateUnion = z.discriminatedUnion("type", [
|
|
|
1452
1876
|
toolName: z.string().min(1).optional(),
|
|
1453
1877
|
...checkPolicyShape
|
|
1454
1878
|
}),
|
|
1879
|
+
z.object({
|
|
1880
|
+
type: z.literal("toolInputMatches"),
|
|
1881
|
+
toolName: z.string().min(1),
|
|
1882
|
+
...patternMatchShape,
|
|
1883
|
+
...checkPolicyShape
|
|
1884
|
+
}).superRefine(refinePatternMatch("call")),
|
|
1885
|
+
z.object({
|
|
1886
|
+
type: z.literal("toolResultMatches"),
|
|
1887
|
+
// Omitted = every tool's results, the same scope `toolResultContains`
|
|
1888
|
+
// reads.
|
|
1889
|
+
toolName: z.string().min(1).optional(),
|
|
1890
|
+
...patternMatchShape,
|
|
1891
|
+
...checkPolicyShape
|
|
1892
|
+
}).superRefine(refinePatternMatch("result")),
|
|
1455
1893
|
z.object({
|
|
1456
1894
|
type: z.literal("noRepeatedIdenticalCall"),
|
|
1457
1895
|
toolName: z.string().min(1).optional(),
|
|
@@ -1538,7 +1976,7 @@ __export(derive_exports, {
|
|
|
1538
1976
|
skippedScoreResult: () => skippedScoreResult
|
|
1539
1977
|
});
|
|
1540
1978
|
|
|
1541
|
-
// ../evaluators/dist/chunk-
|
|
1979
|
+
// ../evaluators/dist/chunk-KAI5M4KO.js
|
|
1542
1980
|
function resolveScoreDefinition(definition) {
|
|
1543
1981
|
const fallback = isRequiredRole(definition.role) ? "fail" : "ignore";
|
|
1544
1982
|
const { onError, onSkipped, ...rest } = definition;
|
|
@@ -1789,7 +2227,7 @@ __export(adapters_exports, {
|
|
|
1789
2227
|
toolMatchScoreDefinition: () => toolMatchScoreDefinition
|
|
1790
2228
|
});
|
|
1791
2229
|
|
|
1792
|
-
// ../evaluators/dist/chunk-
|
|
2230
|
+
// ../evaluators/dist/chunk-5E4WULER.js
|
|
1793
2231
|
var TOOL_MATCH_SCORER_ID = "tool-match";
|
|
1794
2232
|
var LEGACY_TEST_SCORER_ID = "legacy:test";
|
|
1795
2233
|
var LEGACY_TEST_VERSION = "1";
|
|
@@ -1803,10 +2241,18 @@ function predicateScoreDefinition(predicate, options) {
|
|
|
1803
2241
|
scorerId: explicit || generatedPredicateScorerId(predicate, options.ordinal),
|
|
1804
2242
|
idSource: explicit ? "explicit" : "generated",
|
|
1805
2243
|
scorerVersion: PREDICATES_VERSION,
|
|
2244
|
+
// Kinds whose verdict depends on more than the authored rule — an engine,
|
|
2245
|
+
// an encoding, a budget — digest that implementation beside the rule.
|
|
1806
2246
|
implementationHash: canonicalDigest(
|
|
1807
2247
|
predicate.type === "responseCloseTo" ? {
|
|
1808
2248
|
...RESPONSE_CLOSE_TO_IMPLEMENTATION,
|
|
1809
2249
|
rule: stripCheckPolicy(predicate)
|
|
2250
|
+
} : predicate.type === "toolInputMatches" ? {
|
|
2251
|
+
...TOOL_INPUT_MATCHES_IMPLEMENTATION,
|
|
2252
|
+
rule: stripCheckPolicy(predicate)
|
|
2253
|
+
} : predicate.type === "toolResultMatches" ? {
|
|
2254
|
+
...TOOL_RESULT_MATCHES_IMPLEMENTATION,
|
|
2255
|
+
rule: stripCheckPolicy(predicate)
|
|
1810
2256
|
} : stripCheckPolicy(predicate)
|
|
1811
2257
|
),
|
|
1812
2258
|
label: predicate.type,
|
|
@@ -1953,7 +2399,7 @@ __export(evaluator_derive_exports, {
|
|
|
1953
2399
|
toScoreRawOutcome: () => toScoreRawOutcome
|
|
1954
2400
|
});
|
|
1955
2401
|
|
|
1956
|
-
// ../evaluators/dist/chunk-
|
|
2402
|
+
// ../evaluators/dist/chunk-QVKDBSXS.js
|
|
1957
2403
|
function toEvaluatorResult(score2) {
|
|
1958
2404
|
return {
|
|
1959
2405
|
schemaVersion: EVALUATOR_RESULT_SCHEMA_VERSION,
|
|
@@ -2106,7 +2552,7 @@ var Semaphore = class {
|
|
|
2106
2552
|
var DEFAULT_SCORER_TIMEOUT_MS = 6e4;
|
|
2107
2553
|
var DEFAULT_SCORER_CONCURRENCY = 4;
|
|
2108
2554
|
|
|
2109
|
-
// ../evaluators/dist/chunk-
|
|
2555
|
+
// ../evaluators/dist/chunk-NRZVABEW.js
|
|
2110
2556
|
var ScorerTimeoutError = class extends Error {
|
|
2111
2557
|
constructor(timeoutMs) {
|
|
2112
2558
|
super(`scorer timed out after ${timeoutMs}ms`);
|
|
@@ -2215,7 +2661,7 @@ function scoresPassed(scores, definitions2) {
|
|
|
2215
2661
|
return true;
|
|
2216
2662
|
}
|
|
2217
2663
|
|
|
2218
|
-
// ../evaluators/dist/chunk-
|
|
2664
|
+
// ../evaluators/dist/chunk-ZQI2XLYE.js
|
|
2219
2665
|
function isCanonical(evaluator) {
|
|
2220
2666
|
return typeof evaluator.evaluate === "function";
|
|
2221
2667
|
}
|
|
@@ -2605,7 +3051,7 @@ function validateAgainstSchema(schema, instance) {
|
|
|
2605
3051
|
}
|
|
2606
3052
|
}
|
|
2607
3053
|
|
|
2608
|
-
// ../evaluators/dist/chunk-
|
|
3054
|
+
// ../evaluators/dist/chunk-JKPDKMTE.js
|
|
2609
3055
|
var MAX_VALUE_CHARS = 200;
|
|
2610
3056
|
var MAX_ERROR_MSG_CHARS = 200;
|
|
2611
3057
|
var MAX_ITEMS_SHOWN = 3;
|
|
@@ -2878,6 +3324,50 @@ function hasContinuation(payload) {
|
|
|
2878
3324
|
function estimatedTokens(bytes2) {
|
|
2879
3325
|
return `\u2248${Math.ceil(bytes2 / 4).toLocaleString()} tokens (estimate)`;
|
|
2880
3326
|
}
|
|
3327
|
+
function preparePatternMatch(predicate) {
|
|
3328
|
+
const configError = matchRuleConfigError(
|
|
3329
|
+
predicate.type,
|
|
3330
|
+
predicate
|
|
3331
|
+
);
|
|
3332
|
+
if (configError !== void 0) return { ok: false, reason: configError };
|
|
3333
|
+
const compiled = compileMatchPatterns(predicate.patterns, predicate.flags);
|
|
3334
|
+
if (!compiled.ok) {
|
|
3335
|
+
return {
|
|
3336
|
+
ok: false,
|
|
3337
|
+
reason: `${predicate.type} pattern ${compiled.index + 1} does not compile in re2js: ${truncate2(scrubText(compiled.message), MAX_VALUE_CHARS)}`
|
|
3338
|
+
};
|
|
3339
|
+
}
|
|
3340
|
+
let key;
|
|
3341
|
+
if (predicate.path !== void 0) {
|
|
3342
|
+
const parsed = parseMatchPath(predicate.path);
|
|
3343
|
+
if (!parsed.ok) return { ok: false, reason: parsed.message };
|
|
3344
|
+
key = parsed.key;
|
|
3345
|
+
}
|
|
3346
|
+
const flags = predicate.flags ?? "";
|
|
3347
|
+
const patternsShown = predicate.patterns.slice(0, MAX_ITEMS_SHOWN).map(
|
|
3348
|
+
(pattern) => `/${truncate2(
|
|
3349
|
+
scrubText(pattern),
|
|
3350
|
+
MAX_DISPLAYED_PATTERN_CHARS
|
|
3351
|
+
)}/${flags}`
|
|
3352
|
+
).join(", ") + (predicate.patterns.length > MAX_ITEMS_SHOWN ? `, +${predicate.patterns.length - MAX_ITEMS_SHOWN} more` : "");
|
|
3353
|
+
return {
|
|
3354
|
+
ok: true,
|
|
3355
|
+
compiled: compiled.compiled,
|
|
3356
|
+
key,
|
|
3357
|
+
bounds: { min: predicate.min ?? 1, max: predicate.max },
|
|
3358
|
+
patternsShown
|
|
3359
|
+
};
|
|
3360
|
+
}
|
|
3361
|
+
function matchesEvery(compiled, subject) {
|
|
3362
|
+
return compiled.every((pattern) => pattern.test(subject));
|
|
3363
|
+
}
|
|
3364
|
+
function samplesOnShortfall(verdict, tally, bounds, unmatched, sample) {
|
|
3365
|
+
const tooMany4 = bounds.max !== void 0 && tally.matched > bounds.max;
|
|
3366
|
+
if (verdict !== "fail" || tooMany4 || unmatched.length === 0) {
|
|
3367
|
+
return void 0;
|
|
3368
|
+
}
|
|
3369
|
+
return `[${unmatched.slice(0, MAX_ITEMS_SHOWN).map(sample).join(", ")}${unmatched.length > MAX_ITEMS_SHOWN ? `, +${unmatched.length - MAX_ITEMS_SHOWN} more` : ""}]`;
|
|
3370
|
+
}
|
|
2881
3371
|
function evaluatePredicate(transcript, predicate) {
|
|
2882
3372
|
switch (predicate.type) {
|
|
2883
3373
|
case "toolDescriptionsPresent":
|
|
@@ -3869,6 +4359,151 @@ function evaluatePredicate(transcript, predicate) {
|
|
|
3869
4359
|
inspected === 0 ? `no full page from ${scopeLabel(predicate.toolName)} to inspect` : `all ${inspected} full page(s) carried continuation metadata`
|
|
3870
4360
|
);
|
|
3871
4361
|
}
|
|
4362
|
+
case "toolInputMatches": {
|
|
4363
|
+
const prepared = preparePatternMatch(predicate);
|
|
4364
|
+
if (!prepared.ok) return evidenceError(predicate, prepared.reason);
|
|
4365
|
+
const { compiled, key, bounds, patternsShown } = prepared;
|
|
4366
|
+
const calls = callsTo(transcript, predicate.toolName);
|
|
4367
|
+
const tally = {
|
|
4368
|
+
units: calls.length,
|
|
4369
|
+
matched: 0,
|
|
4370
|
+
unreadable: 0,
|
|
4371
|
+
missing: 0
|
|
4372
|
+
};
|
|
4373
|
+
const unmatched = [];
|
|
4374
|
+
for (const call of calls) {
|
|
4375
|
+
const subject = encodeCallSubject(call.arguments, key);
|
|
4376
|
+
if (subject.kind === "missing") {
|
|
4377
|
+
tally.missing += 1;
|
|
4378
|
+
unmatched.push(call);
|
|
4379
|
+
} else if (subject.kind === "unreadable") {
|
|
4380
|
+
tally.unreadable += 1;
|
|
4381
|
+
} else if (matchesEvery(compiled, subject.text)) {
|
|
4382
|
+
tally.matched += 1;
|
|
4383
|
+
} else {
|
|
4384
|
+
unmatched.push(call);
|
|
4385
|
+
}
|
|
4386
|
+
}
|
|
4387
|
+
const verdict = matchVerdict(tally, bounds);
|
|
4388
|
+
const sample = (call) => {
|
|
4389
|
+
const args = call.arguments ?? {};
|
|
4390
|
+
if (key === void 0) return brief(args);
|
|
4391
|
+
return Object.prototype.hasOwnProperty.call(args, key) ? brief({ [key]: args[key] }) : `(no "${key}")`;
|
|
4392
|
+
};
|
|
4393
|
+
const reason = describeInputMatch({
|
|
4394
|
+
toolName: predicate.toolName,
|
|
4395
|
+
key,
|
|
4396
|
+
patternCount: predicate.patterns.length,
|
|
4397
|
+
tally,
|
|
4398
|
+
bounds,
|
|
4399
|
+
verdict,
|
|
4400
|
+
patternsShown,
|
|
4401
|
+
samplesShown: samplesOnShortfall(
|
|
4402
|
+
verdict,
|
|
4403
|
+
tally,
|
|
4404
|
+
bounds,
|
|
4405
|
+
unmatched,
|
|
4406
|
+
sample
|
|
4407
|
+
)
|
|
4408
|
+
});
|
|
4409
|
+
if (verdict === "pass") return pass(predicate, reason);
|
|
4410
|
+
if (verdict === "fail") return fail(predicate, reason);
|
|
4411
|
+
return evidenceError(predicate, reason);
|
|
4412
|
+
}
|
|
4413
|
+
case "toolResultMatches": {
|
|
4414
|
+
const prepared = preparePatternMatch(predicate);
|
|
4415
|
+
if (!prepared.ok) return evidenceError(predicate, prepared.reason);
|
|
4416
|
+
const { compiled, key, bounds, patternsShown } = prepared;
|
|
4417
|
+
const scope = resultScope(transcript, predicate.toolName);
|
|
4418
|
+
const describe = (tally2, verdict2, extra = {}) => describeResultMatch({
|
|
4419
|
+
toolName: predicate.toolName,
|
|
4420
|
+
key,
|
|
4421
|
+
patternCount: predicate.patterns.length,
|
|
4422
|
+
tally: tally2,
|
|
4423
|
+
bounds,
|
|
4424
|
+
verdict: verdict2,
|
|
4425
|
+
patternsShown,
|
|
4426
|
+
...extra
|
|
4427
|
+
});
|
|
4428
|
+
if (scope.length === 0) {
|
|
4429
|
+
const empty = emptyScopeIsScored(
|
|
4430
|
+
transcript,
|
|
4431
|
+
"toolResults",
|
|
4432
|
+
predicate.toolName
|
|
4433
|
+
);
|
|
4434
|
+
const none = {
|
|
4435
|
+
units: 0,
|
|
4436
|
+
matched: 0,
|
|
4437
|
+
unreadable: 0,
|
|
4438
|
+
missing: 0
|
|
4439
|
+
};
|
|
4440
|
+
if (!empty.scored) {
|
|
4441
|
+
return evidenceError(
|
|
4442
|
+
predicate,
|
|
4443
|
+
describe(none, "unscored", { gap: empty.reason })
|
|
4444
|
+
);
|
|
4445
|
+
}
|
|
4446
|
+
const verdict2 = matchVerdict(none, bounds);
|
|
4447
|
+
return verdict2 === "pass" ? pass(predicate, describe(none, verdict2)) : fail(predicate, describe(none, verdict2));
|
|
4448
|
+
}
|
|
4449
|
+
const tally = {
|
|
4450
|
+
units: scope.length,
|
|
4451
|
+
matched: 0,
|
|
4452
|
+
unreadable: 0,
|
|
4453
|
+
missing: 0
|
|
4454
|
+
};
|
|
4455
|
+
const unmatched = [];
|
|
4456
|
+
for (const result of scope) {
|
|
4457
|
+
const subject = encodeResultSubject(result, key);
|
|
4458
|
+
if (subject.kind === "missing") {
|
|
4459
|
+
tally.missing += 1;
|
|
4460
|
+
unmatched.push(result);
|
|
4461
|
+
} else if (subject.kind === "unreadable") {
|
|
4462
|
+
tally.unreadable += 1;
|
|
4463
|
+
} else if (matchesEvery(compiled, subject.text)) {
|
|
4464
|
+
tally.matched += 1;
|
|
4465
|
+
} else {
|
|
4466
|
+
unmatched.push(result);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
const gap = incompleteScopeReason(
|
|
4470
|
+
transcript,
|
|
4471
|
+
"toolResults",
|
|
4472
|
+
scope.length
|
|
4473
|
+
);
|
|
4474
|
+
const verdict = matchVerdict(
|
|
4475
|
+
tally,
|
|
4476
|
+
bounds,
|
|
4477
|
+
gap === void 0 ? 0 : Number.POSITIVE_INFINITY
|
|
4478
|
+
);
|
|
4479
|
+
const sample = (result) => {
|
|
4480
|
+
const prefix = predicate.toolName === void 0 ? `"${result.toolName}" ` : "";
|
|
4481
|
+
if (key !== void 0) {
|
|
4482
|
+
const value = resultValueAt(result, key);
|
|
4483
|
+
return value === void 0 ? `${prefix}(no "${key}")` : `${prefix}${scrubText(brief({ [key]: value }))}`;
|
|
4484
|
+
}
|
|
4485
|
+
const payload = result.structuredContent !== void 0 ? result.structuredContent : result.json;
|
|
4486
|
+
if (payload !== void 0) {
|
|
4487
|
+
return `${prefix}${scrubText(brief(payload))}`;
|
|
4488
|
+
}
|
|
4489
|
+
return `${prefix}${JSON.stringify(
|
|
4490
|
+
truncate2(scrubText(result.text ?? ""), MAX_VALUE_CHARS)
|
|
4491
|
+
)}`;
|
|
4492
|
+
};
|
|
4493
|
+
const reason = describe(tally, verdict, {
|
|
4494
|
+
...verdict === "unscored" && gap !== void 0 ? { gap } : {},
|
|
4495
|
+
samplesShown: samplesOnShortfall(
|
|
4496
|
+
verdict,
|
|
4497
|
+
tally,
|
|
4498
|
+
bounds,
|
|
4499
|
+
unmatched,
|
|
4500
|
+
sample
|
|
4501
|
+
)
|
|
4502
|
+
});
|
|
4503
|
+
if (verdict === "pass") return pass(predicate, reason);
|
|
4504
|
+
if (verdict === "fail") return fail(predicate, reason);
|
|
4505
|
+
return evidenceError(predicate, reason);
|
|
4506
|
+
}
|
|
3872
4507
|
default: {
|
|
3873
4508
|
const exhaustive = predicate;
|
|
3874
4509
|
return {
|
|
@@ -4076,7 +4711,7 @@ __export(predicate_scorer_exports, {
|
|
|
4076
4711
|
predicateScorer: () => predicateScorer
|
|
4077
4712
|
});
|
|
4078
4713
|
|
|
4079
|
-
// ../evaluators/dist/chunk-
|
|
4714
|
+
// ../evaluators/dist/chunk-RXIKS7LO.js
|
|
4080
4715
|
function assertLocallyEvaluable(predicate) {
|
|
4081
4716
|
if (!requiresRenderObservations(predicate.type)) return;
|
|
4082
4717
|
throw new Error(
|
|
@@ -4105,7 +4740,7 @@ function predicateScorer(predicate, options) {
|
|
|
4105
4740
|
rationale: "predicate evaluator returned no verdict"
|
|
4106
4741
|
};
|
|
4107
4742
|
}
|
|
4108
|
-
if (predicate.type === "responseCloseTo" && result.status === "error")
|
|
4743
|
+
if ((predicate.type === "responseCloseTo" || predicate.type === "toolInputMatches" || predicate.type === "toolResultMatches") && result.status === "error")
|
|
4109
4744
|
throw new Error(result.reason);
|
|
4110
4745
|
return {
|
|
4111
4746
|
kind: "scored",
|
|
@@ -4154,7 +4789,7 @@ function toEvaluatorRawOutcome(outcome) {
|
|
|
4154
4789
|
};
|
|
4155
4790
|
}
|
|
4156
4791
|
|
|
4157
|
-
// ../evaluators/dist/chunk-
|
|
4792
|
+
// ../evaluators/dist/chunk-3VGEMHWF.js
|
|
4158
4793
|
function assertLocallyEvaluable2(rule) {
|
|
4159
4794
|
if (!requiresRenderObservations(rule.type)) return;
|
|
4160
4795
|
throw new Error(
|
|
@@ -22753,6 +23388,10 @@ var ASSERTION_STAGE = {
|
|
|
22753
23388
|
// ── Tool call: was the call itself well formed ──────────────────────────
|
|
22754
23389
|
argumentsMatchToolSchema: "call",
|
|
22755
23390
|
noRepeatedIdenticalCall: "call",
|
|
23391
|
+
// What went INTO the call, so a wrong argument files as `argumentMismatch`.
|
|
23392
|
+
// A never-called tool on its own also files here; pair it with a
|
|
23393
|
+
// `toolCalledWith` (which files at `selection`, earlier) to attribute that.
|
|
23394
|
+
toolInputMatches: "call",
|
|
22756
23395
|
// ── Response: what the server answered with ─────────────────────────────
|
|
22757
23396
|
//
|
|
22758
23397
|
// `noToolErrors` MOVED HERE in analyzer 11, from `userValue` where it had
|
|
@@ -22766,6 +23405,8 @@ var ASSERTION_STAGE = {
|
|
|
22766
23405
|
noToolErrors: "response",
|
|
22767
23406
|
toolLatencyUnder: "response",
|
|
22768
23407
|
toolResultContains: "response",
|
|
23408
|
+
// What came OUT of the call, like the other `toolResult*` checks.
|
|
23409
|
+
toolResultMatches: "response",
|
|
22769
23410
|
toolResultMatchesSchema: "response",
|
|
22770
23411
|
toolResultSizeUnder: "response",
|
|
22771
23412
|
toolErrorNamesInput: "response",
|
|
@@ -27361,7 +28002,7 @@ var noopPosthog = {
|
|
|
27361
28002
|
var posthog = isTelemetryDisabled ? noopPosthog : new PostHog("phc_dTOPniyUNU2kD8Jx8yHMXSqiZHM8I91uWopTMX6EBE9", {
|
|
27362
28003
|
host: "https://us.i.posthog.com"
|
|
27363
28004
|
});
|
|
27364
|
-
var SDK_VERSION = "8.
|
|
28005
|
+
var SDK_VERSION = "8.18.0";
|
|
27365
28006
|
var SDK_RELEASE = `@mcpjam/sdk@${SDK_VERSION}`;
|
|
27366
28007
|
var CAPTURED_ERROR_SYMBOL = Symbol.for("@mcpjam/sdk/captured-eval-error");
|
|
27367
28008
|
init_internal();
|
|
@@ -27372,7 +28013,7 @@ init_eval_tool_execution();
|
|
|
27372
28013
|
init_internal();
|
|
27373
28014
|
init_internal();
|
|
27374
28015
|
function readSdkVersion() {
|
|
27375
|
-
return "8.
|
|
28016
|
+
return "8.18.0";
|
|
27376
28017
|
}
|
|
27377
28018
|
var DEFAULT_PLATFORM_USER_AGENT = `mcpjam-sdk/${readSdkVersion()}`;
|
|
27378
28019
|
init_HostRunner();
|
|
@@ -85378,7 +86019,7 @@ var NEGATIVE_TEST_MODE_DETAILS = {
|
|
|
85378
86019
|
}
|
|
85379
86020
|
};
|
|
85380
86021
|
function readSdkVersion2() {
|
|
85381
|
-
return "8.
|
|
86022
|
+
return "8.18.0";
|
|
85382
86023
|
}
|
|
85383
86024
|
var CONFORMANCE_CHECKER_VERSION2 = readSdkVersion2();
|
|
85384
86025
|
var MUST_POINTS2 = 95;
|
|
@@ -103040,8 +103681,8 @@ function cleanAbandonedLocalDownloads() {
|
|
|
103040
103681
|
if (error.code !== "ESRCH") continue;
|
|
103041
103682
|
}
|
|
103042
103683
|
const path8 = join8(tmpdir(), name15);
|
|
103043
|
-
const
|
|
103044
|
-
if (
|
|
103684
|
+
const stat8 = await lstat2(path8).catch(() => void 0);
|
|
103685
|
+
if (stat8?.isDirectory() && !stat8.isSymbolicLink() && (process.getuid === void 0 || stat8.uid === process.getuid()))
|
|
103045
103686
|
await rm(path8, { recursive: true, force: true });
|
|
103046
103687
|
}
|
|
103047
103688
|
})().catch(() => {
|
|
@@ -105058,9 +105699,9 @@ async function collectProfileEntries(profileDir, currentDir, entries, uncompress
|
|
|
105058
105699
|
const archivePath = relative2(profileDir, absolutePath).split(sep3).join("/");
|
|
105059
105700
|
assertSafeArchivePath(archivePath);
|
|
105060
105701
|
if (isExcludedProfilePath(archivePath)) continue;
|
|
105061
|
-
const
|
|
105062
|
-
if (
|
|
105063
|
-
const header2 = makeTarHeader(archivePath, "directory", 0,
|
|
105702
|
+
const stat8 = await lstat3(absolutePath);
|
|
105703
|
+
if (stat8.isDirectory()) {
|
|
105704
|
+
const header2 = makeTarHeader(archivePath, "directory", 0, stat8.mode);
|
|
105064
105705
|
entries.push(header2);
|
|
105065
105706
|
uncompressedBytes.value += header2.byteLength;
|
|
105066
105707
|
if (uncompressedBytes.value > MAX_BROWSER_PROFILE_UNCOMPRESSED_BYTES) {
|
|
@@ -105076,8 +105717,8 @@ async function collectProfileEntries(profileDir, currentDir, entries, uncompress
|
|
|
105076
105717
|
);
|
|
105077
105718
|
continue;
|
|
105078
105719
|
}
|
|
105079
|
-
if (!
|
|
105080
|
-
if (uncompressedBytes.value +
|
|
105720
|
+
if (!stat8.isFile()) continue;
|
|
105721
|
+
if (uncompressedBytes.value + stat8.size > MAX_BROWSER_PROFILE_UNCOMPRESSED_BYTES) {
|
|
105081
105722
|
throw new Error("browser profile archive exceeds the expanded size limit");
|
|
105082
105723
|
}
|
|
105083
105724
|
const contents = await readFile(absolutePath);
|
|
@@ -105085,7 +105726,7 @@ async function collectProfileEntries(profileDir, currentDir, entries, uncompress
|
|
|
105085
105726
|
archivePath,
|
|
105086
105727
|
"file",
|
|
105087
105728
|
contents.byteLength,
|
|
105088
|
-
|
|
105729
|
+
stat8.mode
|
|
105089
105730
|
);
|
|
105090
105731
|
const padding = Buffer.alloc(
|
|
105091
105732
|
(TAR_BLOCK_BYTES - contents.byteLength % TAR_BLOCK_BYTES) % TAR_BLOCK_BYTES
|
|
@@ -110375,7 +111016,7 @@ function isBlockedEgressHost(hostname3, blockPrivate) {
|
|
|
110375
111016
|
let host = hostname3.trim().toLowerCase();
|
|
110376
111017
|
if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1);
|
|
110377
111018
|
host = host.replace(/\.+$/, "");
|
|
110378
|
-
if (!host) return
|
|
111019
|
+
if (!host) return true;
|
|
110379
111020
|
if (host === "metadata.google.internal" || host === "metadata.goog") {
|
|
110380
111021
|
return true;
|
|
110381
111022
|
}
|
|
@@ -125820,13 +126461,13 @@ async function listFilesRecursive(dirPath, relativeTo = dirPath) {
|
|
|
125820
126461
|
});
|
|
125821
126462
|
} else if (entry3.isFile()) {
|
|
125822
126463
|
try {
|
|
125823
|
-
const
|
|
126464
|
+
const stat8 = await fs.stat(fullPath);
|
|
125824
126465
|
const mimeType = getMimeType(entry3.name);
|
|
125825
126466
|
files.push({
|
|
125826
126467
|
path: relativePath,
|
|
125827
126468
|
name: entry3.name,
|
|
125828
126469
|
type: "file",
|
|
125829
|
-
size:
|
|
126470
|
+
size: stat8.size,
|
|
125830
126471
|
mimeType,
|
|
125831
126472
|
extension: ext || void 0
|
|
125832
126473
|
});
|
|
@@ -126325,8 +126966,8 @@ function formatDisplayPath(fullPath) {
|
|
|
126325
126966
|
}
|
|
126326
126967
|
async function directoryExists2(dirPath) {
|
|
126327
126968
|
try {
|
|
126328
|
-
const
|
|
126329
|
-
return
|
|
126969
|
+
const stat8 = await fs2.stat(dirPath);
|
|
126970
|
+
return stat8.isDirectory();
|
|
126330
126971
|
} catch {
|
|
126331
126972
|
return false;
|
|
126332
126973
|
}
|
|
@@ -126415,10 +127056,10 @@ async function listLocalRuntimeSkillFiles(skillDir) {
|
|
|
126415
127056
|
`"${file.path}" is not readable within the skill directory.`
|
|
126416
127057
|
);
|
|
126417
127058
|
}
|
|
126418
|
-
const
|
|
126419
|
-
if (
|
|
127059
|
+
const stat8 = await fs2.stat(absolute);
|
|
127060
|
+
if (stat8.size > SKILL_FILE_MAX_READ_BYTES) {
|
|
126420
127061
|
throw new Error(
|
|
126421
|
-
`"${file.path}" is too large to read (${
|
|
127062
|
+
`"${file.path}" is too large to read (${stat8.size} bytes).`
|
|
126422
127063
|
);
|
|
126423
127064
|
}
|
|
126424
127065
|
return new Uint8Array(await fs2.readFile(absolute));
|
|
@@ -127544,6 +128185,17 @@ function patchModernClaudeCodeBridgeContent(content) {
|
|
|
127544
128185
|
}
|
|
127545
128186
|
return patched;
|
|
127546
128187
|
}
|
|
128188
|
+
function enforceStrictMcpConfig(content) {
|
|
128189
|
+
const needle = " mcpServers,\n cwd: workdir,";
|
|
128190
|
+
const replacement = " mcpServers,\n strictMcpConfig: true,\n cwd: workdir,";
|
|
128191
|
+
if (content.includes(replacement)) return content;
|
|
128192
|
+
if (!content.includes(needle)) {
|
|
128193
|
+
throw new Error(
|
|
128194
|
+
"Unable to patch Claude Code bridge bootstrap: MCP query options shape changed"
|
|
128195
|
+
);
|
|
128196
|
+
}
|
|
128197
|
+
return content.replace(needle, replacement);
|
|
128198
|
+
}
|
|
127547
128199
|
function patchClaudeCodeBridgeContent(content) {
|
|
127548
128200
|
let patched = content;
|
|
127549
128201
|
if (patched.includes("handleStreamEvent(msg.event,")) {
|
|
@@ -127636,7 +128288,12 @@ function patchClaudeCodeHarnessBootstrap(harness2) {
|
|
|
127636
128288
|
...bootstrap,
|
|
127637
128289
|
files: [
|
|
127638
128290
|
...bootstrap.files.map(
|
|
127639
|
-
(file) => file.path.endsWith("/bridge.mjs") ? {
|
|
128291
|
+
(file) => file.path.endsWith("/bridge.mjs") ? {
|
|
128292
|
+
...file,
|
|
128293
|
+
content: enforceStrictMcpConfig(
|
|
128294
|
+
patchClaudeCodeBridgeContent(file.content)
|
|
128295
|
+
)
|
|
128296
|
+
} : file
|
|
127640
128297
|
),
|
|
127641
128298
|
{
|
|
127642
128299
|
path: `${bootstrap.bootstrapDir}/.npmrc`,
|
|
@@ -130844,9 +131501,6 @@ function parseHarnessToolName(rawToolName, keyToServerId) {
|
|
|
130844
131501
|
const serverId = keyToServerId[key];
|
|
130845
131502
|
return serverId ? { serverId, toolName: tool19 } : { toolName: rawToolName };
|
|
130846
131503
|
}
|
|
130847
|
-
function serializeHarnessMcpJson(json) {
|
|
130848
|
-
return JSON.stringify(json, null, 2);
|
|
130849
|
-
}
|
|
130850
131504
|
var ACP_RESERVED_MCP_SERVER_NAME = "ai-sdk-harness-tools";
|
|
130851
131505
|
function toAcpMcpServers(json) {
|
|
130852
131506
|
const out = {};
|
|
@@ -131117,19 +131771,19 @@ var claudeCodeAdapter = {
|
|
|
131117
131771
|
// tool-approval-request and resumes with the decision), same path as native.
|
|
131118
131772
|
supportsHostExecutedToolApproval: true,
|
|
131119
131773
|
// The CLI's own MCP client connects to the servers from inside the sandbox,
|
|
131120
|
-
// via
|
|
131774
|
+
// via its session MCP configuration — real native function calling. Read from
|
|
131121
131775
|
// the shared declaration so the host editor's promises about which knobs bite
|
|
131122
131776
|
// move with this, instead of being re-asserted by hand on the client.
|
|
131123
131777
|
mcpDelivery: HARNESS_MCP_DELIVERY["claude-code"],
|
|
131124
|
-
// …
|
|
131125
|
-
mcpNativeDelivery: "
|
|
131778
|
+
// …through the SDK session configuration, without writing into the workspace.
|
|
131779
|
+
mcpNativeDelivery: "session-config",
|
|
131126
131780
|
supportsSkills: true,
|
|
131127
131781
|
skillsBaseDir: CLAUDE_CODE_SKILLS_BASE,
|
|
131128
131782
|
// Mirrors `writeClaudeCodeSkills` in `@ai-sdk/harness-claude-code`.
|
|
131129
131783
|
skillsWriteOptions: { trailingNewline: true },
|
|
131130
131784
|
prepareSkills: prepareClaudeCodeSkills,
|
|
131131
131785
|
// No native plugin-unit install: this adapter delivers a plugin's COMPONENTS
|
|
131132
|
-
// (skills param +
|
|
131786
|
+
// (skills param + MCP configuration), which is not the same contract.
|
|
131133
131787
|
supportsPluginBundles: false,
|
|
131134
131788
|
// Claude Code does not emit file-change stream parts.
|
|
131135
131789
|
fileChangeToolName: void 0,
|
|
@@ -131142,15 +131796,10 @@ var claudeCodeAdapter = {
|
|
|
131142
131796
|
modelSupport: modelSupportFor("claude-code"),
|
|
131143
131797
|
supportsModel: supportsModelFor("claude-code"),
|
|
131144
131798
|
parseToolName: parseHarnessToolName,
|
|
131145
|
-
|
|
131146
|
-
await writeTextFile({
|
|
131147
|
-
path: `${sessionWorkDir}/.mcp.json`,
|
|
131148
|
-
content: serializeHarnessMcpJson(mcpJson)
|
|
131149
|
-
});
|
|
131150
|
-
},
|
|
131151
|
-
createHarness({ modelId, auth }) {
|
|
131799
|
+
createHarness({ modelId, auth, mcpJson }) {
|
|
131152
131800
|
const nativeModel = toClaudeCodeModel(modelId);
|
|
131153
131801
|
return createClaudeCodeHarness({
|
|
131802
|
+
mcpServers: mcpJson.mcpServers,
|
|
131154
131803
|
...nativeModel ? { model: nativeModel } : {},
|
|
131155
131804
|
auth,
|
|
131156
131805
|
// Unset, Claude Code defaults to ADAPTIVE thinking, a first-party
|
|
@@ -133122,6 +133771,10 @@ var ASSERTION_STAGE2 = {
|
|
|
133122
133771
|
// ── Tool call: was the call itself well formed ──────────────────────────
|
|
133123
133772
|
argumentsMatchToolSchema: "call",
|
|
133124
133773
|
noRepeatedIdenticalCall: "call",
|
|
133774
|
+
// What went INTO the call, so a wrong argument files as `argumentMismatch`.
|
|
133775
|
+
// A never-called tool on its own also files here; pair it with a
|
|
133776
|
+
// `toolCalledWith` (which files at `selection`, earlier) to attribute that.
|
|
133777
|
+
toolInputMatches: "call",
|
|
133125
133778
|
// ── Response: what the server answered with ─────────────────────────────
|
|
133126
133779
|
//
|
|
133127
133780
|
// `noToolErrors` MOVED HERE in analyzer 11, from `userValue` where it had
|
|
@@ -133135,6 +133788,8 @@ var ASSERTION_STAGE2 = {
|
|
|
133135
133788
|
noToolErrors: "response",
|
|
133136
133789
|
toolLatencyUnder: "response",
|
|
133137
133790
|
toolResultContains: "response",
|
|
133791
|
+
// What came OUT of the call, like the other `toolResult*` checks.
|
|
133792
|
+
toolResultMatches: "response",
|
|
133138
133793
|
toolResultMatchesSchema: "response",
|
|
133139
133794
|
toolResultSizeUnder: "response",
|
|
133140
133795
|
toolErrorNamesInput: "response",
|
|
@@ -143443,6 +144098,18 @@ function isActorEligibleForLocalHarness(actor) {
|
|
|
143443
144098
|
function unavailable(status, message) {
|
|
143444
144099
|
return { available: false, status, message };
|
|
143445
144100
|
}
|
|
144101
|
+
function localHarnessManifestsForDevelopment(manifests = LOCAL_HARNESS_MANIFEST) {
|
|
144102
|
+
const version = process.env.MCPJAM_LOCAL_HARNESS_CONFORMANCE_VERSION?.trim();
|
|
144103
|
+
if (HOSTED_MODE || process.env.ENVIRONMENT !== "dev" || !version) {
|
|
144104
|
+
return manifests;
|
|
144105
|
+
}
|
|
144106
|
+
return Object.fromEntries(
|
|
144107
|
+
Object.entries(manifests).map(([id3, manifest]) => [
|
|
144108
|
+
id3,
|
|
144109
|
+
manifest.lifecycleConformanceVersion || id3 !== "claude-code" ? manifest : { ...manifest, lifecycleConformanceVersion: version }
|
|
144110
|
+
])
|
|
144111
|
+
);
|
|
144112
|
+
}
|
|
143446
144113
|
async function resolveLocalHarnessAvailability(query) {
|
|
143447
144114
|
const hosted = query.hosted ?? HOSTED_MODE;
|
|
143448
144115
|
const enabled = query.killSwitchEnabled ?? LOCAL_HARNESS_ENABLED;
|
|
@@ -143496,7 +144163,7 @@ async function resolveLocalHarnessAvailability(query) {
|
|
|
143496
144163
|
...target.kind === "local-isolated" ? { backend: target.backend } : {},
|
|
143497
144164
|
installedAdapterVersion: query.installedAdapterVersion
|
|
143498
144165
|
},
|
|
143499
|
-
query.manifests ??
|
|
144166
|
+
query.manifests ?? localHarnessManifestsForDevelopment()
|
|
143500
144167
|
);
|
|
143501
144168
|
if (!compatibility.ok) {
|
|
143502
144169
|
return unavailable(compatibility.status, compatibility.message);
|
|
@@ -145096,7 +145763,12 @@ function buildLocalHarnessEnv(opts) {
|
|
|
145096
145763
|
}
|
|
145097
145764
|
env.CI = "1";
|
|
145098
145765
|
env.NO_COLOR = "1";
|
|
145099
|
-
|
|
145766
|
+
validateLocalHarnessScopedEnv(opts.scoped ?? {});
|
|
145767
|
+
Object.assign(env, opts.scoped);
|
|
145768
|
+
return env;
|
|
145769
|
+
}
|
|
145770
|
+
function validateLocalHarnessScopedEnv(scoped) {
|
|
145771
|
+
for (const [name15, value] of Object.entries(scoped)) {
|
|
145100
145772
|
if (SCOPED_NAME_DENYLIST.has(name15.toUpperCase())) {
|
|
145101
145773
|
throw new LocalHarnessEnvError(
|
|
145102
145774
|
`scoped environment entry ${name15} is not allowed: it would redirect executable, library, or config resolution for the child`
|
|
@@ -145112,9 +145784,17 @@ function buildLocalHarnessEnv(opts) {
|
|
|
145112
145784
|
`scoped environment value for ${name15} contains a control character`
|
|
145113
145785
|
);
|
|
145114
145786
|
}
|
|
145115
|
-
env[name15] = value;
|
|
145116
145787
|
}
|
|
145117
|
-
|
|
145788
|
+
}
|
|
145789
|
+
function validateLocalHarnessSecretEnv(scoped) {
|
|
145790
|
+
validateLocalHarnessScopedEnv(scoped);
|
|
145791
|
+
for (const name15 of Object.keys(scoped)) {
|
|
145792
|
+
if (/^(ANTHROPIC_|AI_GATEWAY_|BRIDGE_|CLAUDE_CODE_)/i.test(name15)) {
|
|
145793
|
+
throw new LocalHarnessEnvError(
|
|
145794
|
+
`Project secret ${name15} conflicts with the local harness runtime.`
|
|
145795
|
+
);
|
|
145796
|
+
}
|
|
145797
|
+
}
|
|
145118
145798
|
}
|
|
145119
145799
|
var BRIDGE_SUPPLIED_ENV_ALLOWLIST = [
|
|
145120
145800
|
// Bridge control channel.
|
|
@@ -145886,6 +146566,16 @@ async function removeSessionStateDir(dir) {
|
|
|
145886
146566
|
}
|
|
145887
146567
|
await rm3(target, { recursive: true, force: true });
|
|
145888
146568
|
}
|
|
146569
|
+
function sessionStateDirFor(stateRoot, sessionId) {
|
|
146570
|
+
if (!/^[A-Za-z0-9_-]{1,128}$/.test(sessionId)) {
|
|
146571
|
+
throw new Error(
|
|
146572
|
+
`session id ${JSON.stringify(
|
|
146573
|
+
sessionId
|
|
146574
|
+
)} is not a single safe path segment`
|
|
146575
|
+
);
|
|
146576
|
+
}
|
|
146577
|
+
return join15(stateRoot, "sessions", sessionId);
|
|
146578
|
+
}
|
|
145889
146579
|
|
|
145890
146580
|
// server/utils/harness/local/supervisor.ts
|
|
145891
146581
|
import { spawn as spawn5 } from "child_process";
|
|
@@ -146700,6 +147390,9 @@ var unstopped = /* @__PURE__ */ new Set();
|
|
|
146700
147390
|
function registerLocalHarnessSession(record5) {
|
|
146701
147391
|
sessions2.set(record5.sessionId, record5);
|
|
146702
147392
|
}
|
|
147393
|
+
function getLocalHarnessSession(sessionId) {
|
|
147394
|
+
return sessions2.get(sessionId);
|
|
147395
|
+
}
|
|
146703
147396
|
function forgetLocalHarnessSession(sessionId) {
|
|
146704
147397
|
sessions2.delete(sessionId);
|
|
146705
147398
|
}
|
|
@@ -146788,7 +147481,7 @@ function messageOf(error) {
|
|
|
146788
147481
|
|
|
146789
147482
|
// server/utils/harness/local/local-turn.ts
|
|
146790
147483
|
import { join as join17 } from "path";
|
|
146791
|
-
import { mkdir as mkdir11 } from "fs/promises";
|
|
147484
|
+
import { mkdir as mkdir11, stat as stat7 } from "fs/promises";
|
|
146792
147485
|
import { createRequire as createRequire3 } from "module";
|
|
146793
147486
|
var sharedSupervisor = null;
|
|
146794
147487
|
function localHarnessSupervisor() {
|
|
@@ -146796,6 +147489,7 @@ function localHarnessSupervisor() {
|
|
|
146796
147489
|
return sharedSupervisor;
|
|
146797
147490
|
}
|
|
146798
147491
|
async function prepareLocalHarnessTurn(args) {
|
|
147492
|
+
validateLocalHarnessSecretEnv(args.scopedEnv ?? {});
|
|
146799
147493
|
const verifyStartedAt = Date.now();
|
|
146800
147494
|
const runtimeStatus = await readRuntimeInstallStatus({
|
|
146801
147495
|
harnessId: "claude-code"
|
|
@@ -146854,6 +147548,17 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
146854
147548
|
const args = outer.args;
|
|
146855
147549
|
const runtimeStatus = outer.runtimeStatus;
|
|
146856
147550
|
const verifyStartedAt = outer.verifyStartedAt;
|
|
147551
|
+
const sessionStateDir = sessionStateDirFor(
|
|
147552
|
+
localHarnessStateRoot(),
|
|
147553
|
+
args.sessionId
|
|
147554
|
+
);
|
|
147555
|
+
if (getLocalHarnessSession(args.sessionId)) {
|
|
147556
|
+
return {
|
|
147557
|
+
ok: false,
|
|
147558
|
+
status: "session-still-running",
|
|
147559
|
+
message: "The previous local turn is still stopping. Try again once it has stopped."
|
|
147560
|
+
};
|
|
147561
|
+
}
|
|
146857
147562
|
const availability = await resolveLocalHarnessAvailability({
|
|
146858
147563
|
target: {
|
|
146859
147564
|
kind: "local-native",
|
|
@@ -146911,13 +147616,12 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
146911
147616
|
};
|
|
146912
147617
|
}
|
|
146913
147618
|
const supervisor = localHarnessSupervisor();
|
|
146914
|
-
const sessionStateDir = join17(
|
|
146915
|
-
localHarnessStateRoot(),
|
|
146916
|
-
"sessions",
|
|
146917
|
-
args.sessionId
|
|
146918
|
-
);
|
|
146919
147619
|
let gateway = null;
|
|
146920
147620
|
try {
|
|
147621
|
+
const sessionStateExists = await stat7(sessionStateDir).then((entry3) => entry3.isDirectory()).catch((error) => {
|
|
147622
|
+
if (error.code === "ENOENT") return false;
|
|
147623
|
+
throw error;
|
|
147624
|
+
});
|
|
146921
147625
|
await mkdir11(sessionStateDir, { recursive: true, mode: 448 });
|
|
146922
147626
|
const gatewayStartedAt = Date.now();
|
|
146923
147627
|
try {
|
|
@@ -146955,6 +147659,11 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
146955
147659
|
workspacePath: plan.workspacePath,
|
|
146956
147660
|
workspaceGrantId: plan.target.workspaceGrantId,
|
|
146957
147661
|
sessionStateDir,
|
|
147662
|
+
...args.onSecretEnvDelivered && Object.keys(args.scopedEnv ?? {}).length ? {
|
|
147663
|
+
onBridgeStarted: async () => {
|
|
147664
|
+
args.onSecretEnvDelivered?.();
|
|
147665
|
+
}
|
|
147666
|
+
} : {},
|
|
146958
147667
|
targetKind: "local-native",
|
|
146959
147668
|
bridgePort,
|
|
146960
147669
|
scopedEnv: {
|
|
@@ -146972,6 +147681,7 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
146972
147681
|
releaseRuntime: outer.releaseRuntimeUse,
|
|
146973
147682
|
startedAt: Date.now()
|
|
146974
147683
|
};
|
|
147684
|
+
let treeStopped = false;
|
|
146975
147685
|
const teardownOnce = onceAsync(async () => {
|
|
146976
147686
|
try {
|
|
146977
147687
|
started2.revoke();
|
|
@@ -146980,6 +147690,7 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
146980
147690
|
await revokeLease(broker.runId, args.bearer);
|
|
146981
147691
|
const stop = await supervisor.stopSession(args.sessionId);
|
|
146982
147692
|
if (stop.stopped) {
|
|
147693
|
+
treeStopped = true;
|
|
146983
147694
|
forgetLocalHarnessSessionRecord(sessionRecord);
|
|
146984
147695
|
await outer.releaseRuntimeUse();
|
|
146985
147696
|
}
|
|
@@ -147006,10 +147717,18 @@ async function prepareWithReservedRuntime(outer) {
|
|
|
147006
147717
|
ANTHROPIC_BASE_URL: started2.baseUrl
|
|
147007
147718
|
},
|
|
147008
147719
|
sandboxWorkDir: "project",
|
|
147720
|
+
skillsBaseDir: toAdapterPath(
|
|
147721
|
+
join17(sessionStateDir, "home", ".claude", "skills")
|
|
147722
|
+
),
|
|
147723
|
+
sessionStateExists,
|
|
147009
147724
|
permissionMode,
|
|
147010
147725
|
brokerRunId: broker.runId,
|
|
147011
147726
|
timings: { localRuntimeVerifyMs, localGatewayReadyMs },
|
|
147012
|
-
teardown: teardownOnce
|
|
147727
|
+
teardown: teardownOnce,
|
|
147728
|
+
discardState: onceAsync(async () => {
|
|
147729
|
+
await teardownOnce();
|
|
147730
|
+
if (treeStopped) await removeSessionStateDir(sessionStateDir);
|
|
147731
|
+
})
|
|
147013
147732
|
}
|
|
147014
147733
|
};
|
|
147015
147734
|
} catch (error) {
|
|
@@ -147063,48 +147782,6 @@ function onceAsync(fn) {
|
|
|
147063
147782
|
};
|
|
147064
147783
|
}
|
|
147065
147784
|
|
|
147066
|
-
// server/utils/harness/resolve-sandbox.ts
|
|
147067
|
-
var HarnessSandboxResolutionError = class extends Error {
|
|
147068
|
-
constructor(message, status) {
|
|
147069
|
-
super(message);
|
|
147070
|
-
this.name = "HarnessSandboxResolutionError";
|
|
147071
|
-
this.status = status;
|
|
147072
|
-
}
|
|
147073
|
-
};
|
|
147074
|
-
async function resolveHarnessSandbox(args) {
|
|
147075
|
-
if (!isComputersDataPlaneConfigured()) {
|
|
147076
|
-
throw new HarnessSandboxResolutionError(
|
|
147077
|
-
"this server is not a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md)",
|
|
147078
|
-
503
|
|
147079
|
-
);
|
|
147080
|
-
}
|
|
147081
|
-
const ready = await ensureComputerReady({
|
|
147082
|
-
bearer: args.bearer,
|
|
147083
|
-
projectId: args.projectId,
|
|
147084
|
-
...args.executionScope ? { executionScope: args.executionScope } : {},
|
|
147085
|
-
signal: args.signal,
|
|
147086
|
-
timeoutMs: args.timeoutMs
|
|
147087
|
-
});
|
|
147088
|
-
if (!ready.ok) {
|
|
147089
|
-
throw new HarnessSandboxResolutionError(ready.error, ready.status);
|
|
147090
|
-
}
|
|
147091
|
-
const info = await getComputerSandboxInfo({
|
|
147092
|
-
computerId: ready.value.computerId,
|
|
147093
|
-
signal: args.signal
|
|
147094
|
-
});
|
|
147095
|
-
if (!info.ok) {
|
|
147096
|
-
throw new HarnessSandboxResolutionError(info.error, info.status);
|
|
147097
|
-
}
|
|
147098
|
-
const sandboxId = info.value.providerComputerId;
|
|
147099
|
-
if (!sandboxId) {
|
|
147100
|
-
throw new HarnessSandboxResolutionError(
|
|
147101
|
-
`computer ${ready.value.computerId} has no provider sandbox id yet (status: ${info.value.status})`,
|
|
147102
|
-
503
|
|
147103
|
-
);
|
|
147104
|
-
}
|
|
147105
|
-
return { computerId: ready.value.computerId, sandboxId };
|
|
147106
|
-
}
|
|
147107
|
-
|
|
147108
147785
|
// server/utils/computers/convex-secrets-client.ts
|
|
147109
147786
|
import { ConvexHttpClient as ConvexHttpClient7 } from "convex/browser";
|
|
147110
147787
|
var FN3 = {
|
|
@@ -147197,6 +147874,72 @@ async function convexGetEnvironmentSecretSelection(bearer, args) {
|
|
|
147197
147874
|
return environment?.secretSelection?.secretIds ?? [];
|
|
147198
147875
|
}
|
|
147199
147876
|
|
|
147877
|
+
// server/utils/harness/local/secret-delivery.ts
|
|
147878
|
+
async function assertLocalSecretDelivery(args) {
|
|
147879
|
+
if (args.environmentUnresolvedReason) {
|
|
147880
|
+
throw new Error(
|
|
147881
|
+
"Cannot verify secret delivery for this local run's environment."
|
|
147882
|
+
);
|
|
147883
|
+
}
|
|
147884
|
+
if (!args.environmentId) return;
|
|
147885
|
+
const [bindings, selected] = await Promise.all([
|
|
147886
|
+
convexListProjectSecretBindings(args.bearer, { projectId: args.projectId }),
|
|
147887
|
+
convexGetEnvironmentSecretSelection(args.bearer, {
|
|
147888
|
+
projectId: args.projectId,
|
|
147889
|
+
environmentId: args.environmentId
|
|
147890
|
+
})
|
|
147891
|
+
]);
|
|
147892
|
+
if (bindings.some(
|
|
147893
|
+
(binding) => binding.delivery === "brokered" && selected.includes(binding.secretId)
|
|
147894
|
+
)) {
|
|
147895
|
+
throw new Error(
|
|
147896
|
+
"This environment selects brokered secrets, which require cloud execution. Local execution supports materialized secrets only."
|
|
147897
|
+
);
|
|
147898
|
+
}
|
|
147899
|
+
}
|
|
147900
|
+
|
|
147901
|
+
// server/utils/harness/resolve-sandbox.ts
|
|
147902
|
+
var HarnessSandboxResolutionError = class extends Error {
|
|
147903
|
+
constructor(message, status) {
|
|
147904
|
+
super(message);
|
|
147905
|
+
this.name = "HarnessSandboxResolutionError";
|
|
147906
|
+
this.status = status;
|
|
147907
|
+
}
|
|
147908
|
+
};
|
|
147909
|
+
async function resolveHarnessSandbox(args) {
|
|
147910
|
+
if (!isComputersDataPlaneConfigured()) {
|
|
147911
|
+
throw new HarnessSandboxResolutionError(
|
|
147912
|
+
"this server is not a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md)",
|
|
147913
|
+
503
|
|
147914
|
+
);
|
|
147915
|
+
}
|
|
147916
|
+
const ready = await ensureComputerReady({
|
|
147917
|
+
bearer: args.bearer,
|
|
147918
|
+
projectId: args.projectId,
|
|
147919
|
+
...args.executionScope ? { executionScope: args.executionScope } : {},
|
|
147920
|
+
signal: args.signal,
|
|
147921
|
+
timeoutMs: args.timeoutMs
|
|
147922
|
+
});
|
|
147923
|
+
if (!ready.ok) {
|
|
147924
|
+
throw new HarnessSandboxResolutionError(ready.error, ready.status);
|
|
147925
|
+
}
|
|
147926
|
+
const info = await getComputerSandboxInfo({
|
|
147927
|
+
computerId: ready.value.computerId,
|
|
147928
|
+
signal: args.signal
|
|
147929
|
+
});
|
|
147930
|
+
if (!info.ok) {
|
|
147931
|
+
throw new HarnessSandboxResolutionError(info.error, info.status);
|
|
147932
|
+
}
|
|
147933
|
+
const sandboxId = info.value.providerComputerId;
|
|
147934
|
+
if (!sandboxId) {
|
|
147935
|
+
throw new HarnessSandboxResolutionError(
|
|
147936
|
+
`computer ${ready.value.computerId} has no provider sandbox id yet (status: ${info.value.status})`,
|
|
147937
|
+
503
|
|
147938
|
+
);
|
|
147939
|
+
}
|
|
147940
|
+
return { computerId: ready.value.computerId, sandboxId };
|
|
147941
|
+
}
|
|
147942
|
+
|
|
147200
147943
|
// server/utils/harness/runtime-secrets.ts
|
|
147201
147944
|
async function fetchRuntimeSecrets(bearer, args) {
|
|
147202
147945
|
if (!bearer || !args.projectId || !args.environmentId) {
|
|
@@ -149599,6 +150342,9 @@ function harnessRuntimeFingerprint(parts) {
|
|
|
149599
150342
|
const pluginDimension = pluginVersionsFingerprint(parts.pluginVersions ?? []);
|
|
149600
150343
|
const s = [
|
|
149601
150344
|
String(HARNESS_RUNTIME_COMPAT_VERSION),
|
|
150345
|
+
// Older Claude sessions have workspace .mcp.json files. Start a new lane
|
|
150346
|
+
// when moving to SDK config so stale servers cannot be loaded from disk.
|
|
150347
|
+
...parts.harnessId === "claude-code" ? ["session-mcp-v1"] : [],
|
|
149602
150348
|
(parts.selectedServers ?? []).slice().sort().join(","),
|
|
149603
150349
|
parts.permissionMode,
|
|
149604
150350
|
...pluginDimension ? [pluginDimension] : [],
|
|
@@ -149718,6 +150464,8 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
149718
150464
|
let brokerRunId;
|
|
149719
150465
|
let brokerRevoked = false;
|
|
149720
150466
|
let localTeardown = null;
|
|
150467
|
+
let discardLocalState;
|
|
150468
|
+
let retainLocalState = false;
|
|
149721
150469
|
let reservationHeld = false;
|
|
149722
150470
|
let releaseBoxReservation;
|
|
149723
150471
|
let reservationHeartbeatTimer;
|
|
@@ -149984,6 +150732,14 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
149984
150732
|
const skillsFetch = skillSource.mode !== "live" ? { ok: true, skills: skillSource.skills } : projectId && authHeader ? await fetchRuntimeSkills(authHeader, projectId, executionScope) : { ok: true, skills: [] };
|
|
149985
150733
|
const runtimeSkills2 = skillsFetch.ok ? skillsFetch.skills : null;
|
|
149986
150734
|
const skillsHash = runtimeSkills2 !== null ? skillsFingerprint(runtimeSkills2) : void 0;
|
|
150735
|
+
if (harnessExecutionTarget) {
|
|
150736
|
+
await assertLocalSecretDelivery({
|
|
150737
|
+
bearer: authHeader,
|
|
150738
|
+
projectId,
|
|
150739
|
+
environmentId,
|
|
150740
|
+
environmentUnresolvedReason
|
|
150741
|
+
});
|
|
150742
|
+
}
|
|
149987
150743
|
const runtimeSecrets = runtimeSecretsOverride ?? null;
|
|
149988
150744
|
const secretEnv = runtimeSecrets ? toSecretEnv(runtimeSecrets) : void 0;
|
|
149989
150745
|
const externalAccountCredentialNames = harnessAdapter.modelAccess === "external-account" ? harnessAdapter.externalAccountCredentialEnv : void 0;
|
|
@@ -150167,6 +150923,18 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150167
150923
|
}
|
|
150168
150924
|
}
|
|
150169
150925
|
tClaim = Date.now();
|
|
150926
|
+
const localComputerId = harnessExecutionTarget ? `${harnessExecutionTarget.machineId}:${harnessExecutionTarget.runtimeId}` : void 0;
|
|
150927
|
+
const localEligibility = localComputerId ? getHarnessResumeEligibility({
|
|
150928
|
+
state: continuity?.state ?? null,
|
|
150929
|
+
computerId: localComputerId,
|
|
150930
|
+
sandboxId: continuity?.state?.harnessSessionId ?? ""
|
|
150931
|
+
}) : void 0;
|
|
150932
|
+
if (harnessExecutionTarget && isApprovalResume && !(localEligibility?.resume && continuity?.state?.awaitingApproval)) {
|
|
150933
|
+
throw new Error(
|
|
150934
|
+
"The local session for this approval is no longer available. Start a new turn."
|
|
150935
|
+
);
|
|
150936
|
+
}
|
|
150937
|
+
const localSessionId = harnessExecutionTarget ? localEligibility?.resume ? continuity.state.harnessSessionId : `local-${crypto.randomUUID()}` : void 0;
|
|
150170
150938
|
let localPrepared = null;
|
|
150171
150939
|
if (harnessExecutionTarget) {
|
|
150172
150940
|
const localRunId = crypto.randomUUID();
|
|
@@ -150174,7 +150942,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150174
150942
|
target: harnessExecutionTarget,
|
|
150175
150943
|
harnessId: harnessAdapter.id,
|
|
150176
150944
|
modelId,
|
|
150177
|
-
sessionId:
|
|
150945
|
+
sessionId: localSessionId,
|
|
150178
150946
|
runId: localRunId,
|
|
150179
150947
|
actor: {
|
|
150180
150948
|
isGuest: false,
|
|
@@ -150185,6 +150953,8 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150185
150953
|
projectId,
|
|
150186
150954
|
bearer: authHeader,
|
|
150187
150955
|
requireToolApproval,
|
|
150956
|
+
scopedEnv: sessionSecretEnv,
|
|
150957
|
+
onSecretEnvDelivered,
|
|
150188
150958
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150189
150959
|
});
|
|
150190
150960
|
if (!preparation.ok) {
|
|
@@ -150194,6 +150964,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150194
150964
|
}
|
|
150195
150965
|
localPrepared = preparation.prepared;
|
|
150196
150966
|
localTeardown = preparation.prepared.teardown;
|
|
150967
|
+
discardLocalState = preparation.prepared.discardState;
|
|
150197
150968
|
if (localPrepared.permissionMode !== permissionMode) {
|
|
150198
150969
|
throw new Error(
|
|
150199
150970
|
`Can't run this turn on your machine: the consented permission profile resolves to ${localPrepared.permissionMode} but the agent was configured for ${permissionMode}.`
|
|
@@ -150224,62 +150995,64 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150224
150995
|
};
|
|
150225
150996
|
sandboxId = resolved.sandboxId;
|
|
150226
150997
|
}
|
|
150227
|
-
const computerId = localPrepared !== null ?
|
|
150998
|
+
const computerId = localPrepared !== null ? localComputerId : box.kind === "computer" ? box.computerId : box.sandboxRowId;
|
|
150228
150999
|
const uploadQuotaComputerId = box !== null && box.kind === "computer" ? box.computerId : void 0;
|
|
150229
151000
|
tSandbox = Date.now();
|
|
150230
151001
|
const turnRunId = crypto.randomUUID();
|
|
150231
|
-
|
|
150232
|
-
|
|
150233
|
-
harnessId: harnessAdapter.id,
|
|
150234
|
-
modelId,
|
|
150235
|
-
runId: turnRunId,
|
|
150236
|
-
bearer: authHeader,
|
|
150237
|
-
...abortSignal ? { signal: abortSignal } : {}
|
|
150238
|
-
});
|
|
150239
|
-
if (!reservation.ok) {
|
|
150240
|
-
throw new Error(reservation.error);
|
|
150241
|
-
}
|
|
150242
|
-
reservationHeld = true;
|
|
150243
|
-
releaseBoxReservation = async () => {
|
|
150244
|
-
await releaseHarnessBoxReservation({
|
|
150245
|
-
box,
|
|
150246
|
-
harnessId: harnessAdapter.id,
|
|
150247
|
-
modelId,
|
|
150248
|
-
runId: turnRunId,
|
|
150249
|
-
bearer: authHeader,
|
|
150250
|
-
signal: AbortSignal.timeout(HARNESS_TEARDOWN_TIMEOUT_MS)
|
|
150251
|
-
}).catch(() => {
|
|
150252
|
-
});
|
|
150253
|
-
};
|
|
150254
|
-
reservationHeartbeatTimer = setInterval(() => {
|
|
150255
|
-
if (!reservationHeld || reservationRenewalInFlight) return;
|
|
150256
|
-
reservationRenewalInFlight = true;
|
|
150257
|
-
void renewHarnessBoxReservation({
|
|
151002
|
+
if (box !== null) {
|
|
151003
|
+
const reservation = await reserveHarnessBox({
|
|
150258
151004
|
box,
|
|
150259
151005
|
harnessId: harnessAdapter.id,
|
|
150260
151006
|
modelId,
|
|
150261
151007
|
runId: turnRunId,
|
|
150262
151008
|
bearer: authHeader,
|
|
150263
|
-
signal:
|
|
150264
|
-
}).then((renewed) => {
|
|
150265
|
-
if (!renewed.ok && reservationHeld) {
|
|
150266
|
-
logger.error(
|
|
150267
|
-
"[harness] preparation reservation was lost; aborting the turn"
|
|
150268
|
-
);
|
|
150269
|
-
livenessAbort.abort(new Error("harness box reservation lost"));
|
|
150270
|
-
}
|
|
150271
|
-
}).catch((err) => {
|
|
150272
|
-
if (reservationHeld) {
|
|
150273
|
-
logger.error(
|
|
150274
|
-
"[harness] preparation reservation renewal failed; aborting the turn",
|
|
150275
|
-
err
|
|
150276
|
-
);
|
|
150277
|
-
livenessAbort.abort(new Error("harness box reservation lost"));
|
|
150278
|
-
}
|
|
150279
|
-
}).finally(() => {
|
|
150280
|
-
reservationRenewalInFlight = false;
|
|
151009
|
+
...abortSignal ? { signal: abortSignal } : {}
|
|
150281
151010
|
});
|
|
150282
|
-
|
|
151011
|
+
if (!reservation.ok) {
|
|
151012
|
+
throw new Error(reservation.error);
|
|
151013
|
+
}
|
|
151014
|
+
reservationHeld = true;
|
|
151015
|
+
releaseBoxReservation = async () => {
|
|
151016
|
+
await releaseHarnessBoxReservation({
|
|
151017
|
+
box,
|
|
151018
|
+
harnessId: harnessAdapter.id,
|
|
151019
|
+
modelId,
|
|
151020
|
+
runId: turnRunId,
|
|
151021
|
+
bearer: authHeader,
|
|
151022
|
+
signal: AbortSignal.timeout(HARNESS_TEARDOWN_TIMEOUT_MS)
|
|
151023
|
+
}).catch(() => {
|
|
151024
|
+
});
|
|
151025
|
+
};
|
|
151026
|
+
reservationHeartbeatTimer = setInterval(() => {
|
|
151027
|
+
if (!reservationHeld || reservationRenewalInFlight) return;
|
|
151028
|
+
reservationRenewalInFlight = true;
|
|
151029
|
+
void renewHarnessBoxReservation({
|
|
151030
|
+
box,
|
|
151031
|
+
harnessId: harnessAdapter.id,
|
|
151032
|
+
modelId,
|
|
151033
|
+
runId: turnRunId,
|
|
151034
|
+
bearer: authHeader,
|
|
151035
|
+
signal: AbortSignal.timeout(HARNESS_TEARDOWN_TIMEOUT_MS)
|
|
151036
|
+
}).then((renewed) => {
|
|
151037
|
+
if (!renewed.ok && reservationHeld) {
|
|
151038
|
+
logger.error(
|
|
151039
|
+
"[harness] preparation reservation was lost; aborting the turn"
|
|
151040
|
+
);
|
|
151041
|
+
livenessAbort.abort(new Error("harness box reservation lost"));
|
|
151042
|
+
}
|
|
151043
|
+
}).catch((err) => {
|
|
151044
|
+
if (reservationHeld) {
|
|
151045
|
+
logger.error(
|
|
151046
|
+
"[harness] preparation reservation renewal failed; aborting the turn",
|
|
151047
|
+
err
|
|
151048
|
+
);
|
|
151049
|
+
livenessAbort.abort(new Error("harness box reservation lost"));
|
|
151050
|
+
}
|
|
151051
|
+
}).finally(() => {
|
|
151052
|
+
reservationRenewalInFlight = false;
|
|
151053
|
+
});
|
|
151054
|
+
}, HARNESS_RESERVATION_HEARTBEAT_MS);
|
|
151055
|
+
}
|
|
150283
151056
|
const resolvedHarnessWorkdir = resolveWorkingDirectory(
|
|
150284
151057
|
harnessSandboxBinding?.workdir ?? computerWorkdir
|
|
150285
151058
|
);
|
|
@@ -150359,9 +151132,11 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150359
151132
|
const declared = definition?.needsApproval;
|
|
150360
151133
|
return declared === true || typeof declared === "function";
|
|
150361
151134
|
}).map(([name15]) => name15) : [];
|
|
151135
|
+
const skillsBaseDir = localPrepared?.skillsBaseDir ?? harnessAdapter.skillsBaseDir;
|
|
150362
151136
|
const agent2 = new HarnessAgent({
|
|
150363
151137
|
harness: harnessRuntime,
|
|
150364
151138
|
sandbox,
|
|
151139
|
+
...localPrepared ? { sandboxConfig: { workDir: localPrepared.sandboxWorkDir } } : {},
|
|
150365
151140
|
// Deliver skills via the adapter's own param (host-agnostic: it writes
|
|
150366
151141
|
// them natively at the real $HOME — Claude Code under `.claude/skills`,
|
|
150367
151142
|
// Codex under `.agents/skills`). The per-adapter `prepareSkills` owns the
|
|
@@ -150423,20 +151198,20 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150423
151198
|
session: session2,
|
|
150424
151199
|
skills: deliveredSkills,
|
|
150425
151200
|
skillsHash: skillsHash ?? "",
|
|
150426
|
-
skillsBase:
|
|
151201
|
+
skillsBase: skillsBaseDir,
|
|
150427
151202
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150428
151203
|
}).catch(() => {
|
|
150429
151204
|
});
|
|
150430
151205
|
if (preparedSkills && preparedSkills.payload.length > 0) {
|
|
150431
151206
|
await handOffLegacySkillDirs({
|
|
150432
151207
|
session: session2,
|
|
150433
|
-
skillsBase:
|
|
151208
|
+
skillsBase: skillsBaseDir,
|
|
150434
151209
|
deliveredNames: deliveredSkills.map((s) => s.name),
|
|
150435
151210
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150436
151211
|
});
|
|
150437
151212
|
await preseedAdapterSkills({
|
|
150438
151213
|
session: session2,
|
|
150439
|
-
skillsBase:
|
|
151214
|
+
skillsBase: skillsBaseDir,
|
|
150440
151215
|
payload: preparedSkills.payload,
|
|
150441
151216
|
trailingNewline: harnessAdapter.skillsWriteOptions.trailingNewline,
|
|
150442
151217
|
...abortSignal ? { signal: abortSignal } : {}
|
|
@@ -150446,7 +151221,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150446
151221
|
await materializePinnedSkillFiles({
|
|
150447
151222
|
session: session2,
|
|
150448
151223
|
artifacts: pinnedHarnessSkills,
|
|
150449
|
-
skillsBase:
|
|
151224
|
+
skillsBase: skillsBaseDir,
|
|
150450
151225
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150451
151226
|
}).catch(() => {
|
|
150452
151227
|
});
|
|
@@ -150455,7 +151230,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150455
151230
|
session: session2,
|
|
150456
151231
|
files: capabilitySkillFiles(effectiveCapabilities),
|
|
150457
151232
|
skillNamesById: deliveredSkillNamesById,
|
|
150458
|
-
skillsBase:
|
|
151233
|
+
skillsBase: skillsBaseDir,
|
|
150459
151234
|
...uploadQuotaComputerId ? { computerId: uploadQuotaComputerId } : {},
|
|
150460
151235
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150461
151236
|
}).catch(() => {
|
|
@@ -150474,7 +151249,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150474
151249
|
}
|
|
150475
151250
|
const skillOrigins = deliveredPluginSkillOrigins({
|
|
150476
151251
|
set: effectiveCapabilities,
|
|
150477
|
-
skillsBaseDir
|
|
151252
|
+
skillsBaseDir,
|
|
150478
151253
|
deliveredNameBySkillId: deliveredSkillNamesById
|
|
150479
151254
|
});
|
|
150480
151255
|
if (skillOrigins.length > 0) {
|
|
@@ -150494,7 +151269,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150494
151269
|
session: session2,
|
|
150495
151270
|
files: fileResult.files,
|
|
150496
151271
|
skillNamesById: deliveredSkillNamesById,
|
|
150497
|
-
skillsBase:
|
|
151272
|
+
skillsBase: skillsBaseDir,
|
|
150498
151273
|
...uploadQuotaComputerId ? { computerId: uploadQuotaComputerId } : {},
|
|
150499
151274
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150500
151275
|
}).catch(() => {
|
|
@@ -150509,17 +151284,19 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150509
151284
|
});
|
|
150510
151285
|
}
|
|
150511
151286
|
});
|
|
150512
|
-
const eligibility = getHarnessResumeEligibility({
|
|
151287
|
+
const eligibility = localEligibility ?? getHarnessResumeEligibility({
|
|
150513
151288
|
state: continuity?.state ?? null,
|
|
150514
151289
|
computerId,
|
|
150515
|
-
|
|
150516
|
-
|
|
150517
|
-
|
|
150518
|
-
|
|
150519
|
-
|
|
150520
|
-
|
|
150521
|
-
|
|
150522
|
-
|
|
151290
|
+
sandboxId
|
|
151291
|
+
});
|
|
151292
|
+
const localStateMissing = eligibility.resume && localPrepared?.sessionStateExists === false;
|
|
151293
|
+
if (localStateMissing && isApprovalResume) {
|
|
151294
|
+
throw new Error(
|
|
151295
|
+
"The local session for this approval is no longer available. Start a new turn."
|
|
151296
|
+
);
|
|
151297
|
+
}
|
|
151298
|
+
const resumable = eligibility.resume && !localStateMissing ? continuity?.state ?? void 0 : void 0;
|
|
151299
|
+
let resetReason = localStateMissing ? "resume-failed" : eligibility.reason === "sandbox-replaced" ? "sandbox-replaced" : void 0;
|
|
150523
151300
|
if (eligibility.reason === "legacy-cold-resume") {
|
|
150524
151301
|
logger.warn(
|
|
150525
151302
|
"[harness] resuming a pre-detach sidecar (cold/disk resume; continuity not guaranteed)",
|
|
@@ -150542,6 +151319,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150542
151319
|
});
|
|
150543
151320
|
resumedSession = true;
|
|
150544
151321
|
} catch (resumeErr) {
|
|
151322
|
+
if (localPrepared) throw resumeErr;
|
|
150545
151323
|
logger.warn("[harness] resume failed; starting fresh", {
|
|
150546
151324
|
error: resumeErr instanceof Error ? resumeErr.message : resumeErr
|
|
150547
151325
|
});
|
|
@@ -150549,7 +151327,9 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150549
151327
|
session = await agent2.createSession();
|
|
150550
151328
|
}
|
|
150551
151329
|
} else {
|
|
150552
|
-
session = await agent2.createSession(
|
|
151330
|
+
session = await agent2.createSession(
|
|
151331
|
+
localSessionId ? { sessionId: localSessionId } : void 0
|
|
151332
|
+
);
|
|
150553
151333
|
}
|
|
150554
151334
|
if (resetReason) {
|
|
150555
151335
|
writer.write({
|
|
@@ -150567,7 +151347,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
150567
151347
|
await materializeSkillFrontmatter({
|
|
150568
151348
|
session: sandboxFileSession,
|
|
150569
151349
|
skills: deliveredSkills,
|
|
150570
|
-
skillsBase:
|
|
151350
|
+
skillsBase: skillsBaseDir,
|
|
150571
151351
|
...abortSignal ? { signal: abortSignal } : {}
|
|
150572
151352
|
}).catch(() => {
|
|
150573
151353
|
});
|
|
@@ -151156,7 +151936,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151156
151936
|
// Names already delivered as cloud skills this turn are the adapter's
|
|
151157
151937
|
// own dirs — not adoptions.
|
|
151158
151938
|
managedNames: new Set(deliveredSkills.map((s) => s.name)),
|
|
151159
|
-
skillsBase:
|
|
151939
|
+
skillsBase: skillsBaseDir,
|
|
151160
151940
|
// The turn's `abortSignal` bounded nothing here: this branch only
|
|
151161
151941
|
// runs on a CLEAN success, where that signal is by definition not
|
|
151162
151942
|
// aborted. Compose it with a deadline so a stalled adoption can't
|
|
@@ -151172,7 +151952,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151172
151952
|
await appendManagedSkills({
|
|
151173
151953
|
session: fileSession,
|
|
151174
151954
|
skills: adopted,
|
|
151175
|
-
skillsBase:
|
|
151955
|
+
skillsBase: skillsBaseDir
|
|
151176
151956
|
}).catch(() => {
|
|
151177
151957
|
});
|
|
151178
151958
|
}
|
|
@@ -151194,7 +151974,8 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151194
151974
|
// hanging here is not.
|
|
151195
151975
|
signal: AbortSignal.timeout(HARNESS_TEARDOWN_TIMEOUT_MS)
|
|
151196
151976
|
});
|
|
151197
|
-
|
|
151977
|
+
retainLocalState = ok;
|
|
151978
|
+
if (!ok && !localPrepared) await releaseHarnessLease?.();
|
|
151198
151979
|
} else if (runSucceeded && !aborted && continuity) {
|
|
151199
151980
|
const resumeState = await session.detach();
|
|
151200
151981
|
capturedHarnessCommit = {
|
|
@@ -151216,7 +151997,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151216
151997
|
};
|
|
151217
151998
|
} else {
|
|
151218
151999
|
await session.destroy();
|
|
151219
|
-
if (continuity) await releaseHarnessLease?.();
|
|
152000
|
+
if (continuity && !localPrepared) await releaseHarnessLease?.();
|
|
151220
152001
|
}
|
|
151221
152002
|
} catch (finalizeErr) {
|
|
151222
152003
|
logger.warn(
|
|
@@ -151224,7 +152005,7 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151224
152005
|
{ error: finalizeErr }
|
|
151225
152006
|
);
|
|
151226
152007
|
capturedHarnessCommit = void 0;
|
|
151227
|
-
await releaseHarnessLease?.();
|
|
152008
|
+
if (!localPrepared) await releaseHarnessLease?.();
|
|
151228
152009
|
}
|
|
151229
152010
|
}
|
|
151230
152011
|
} catch (err) {
|
|
@@ -151314,6 +152095,9 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151314
152095
|
runSucceeded ? capturedHarnessCommit : void 0
|
|
151315
152096
|
);
|
|
151316
152097
|
persistOk = !onConversationComplete || persistOutcome === void 0 || persistOutcome.outcome === "saved" || persistOutcome.outcome === "duplicate";
|
|
152098
|
+
if (capturedHarnessCommit && onConversationComplete && persistOk) {
|
|
152099
|
+
retainLocalState = true;
|
|
152100
|
+
}
|
|
151317
152101
|
if (persistOutcome && chatSessionId) {
|
|
151318
152102
|
writePersistReceipt(receiptWriter, persistOutcome, {
|
|
151319
152103
|
chatSessionId,
|
|
@@ -151323,11 +152107,21 @@ async function runHarnessTurn(options, streamSink) {
|
|
|
151323
152107
|
} catch (persistErr) {
|
|
151324
152108
|
logger.error("[harness] onConversationComplete failed", persistErr);
|
|
151325
152109
|
}
|
|
151326
|
-
if (runSucceeded && capturedHarnessCommit && (!onConversationComplete || !persistOk)) {
|
|
152110
|
+
if (runSucceeded && capturedHarnessCommit && !discardLocalState && (!onConversationComplete || !persistOk)) {
|
|
151327
152111
|
await releaseHarnessLease?.();
|
|
151328
152112
|
}
|
|
151329
152113
|
}
|
|
151330
|
-
if (!
|
|
152114
|
+
if (discardLocalState && !retainLocalState) {
|
|
152115
|
+
await Promise.race([
|
|
152116
|
+
discardLocalState().catch((error) => {
|
|
152117
|
+
logger.warn("[harness] local session state cleanup failed", { error });
|
|
152118
|
+
}),
|
|
152119
|
+
new Promise(
|
|
152120
|
+
(resolvePromise) => setTimeout(resolvePromise, HARNESS_TEARDOWN_TIMEOUT_MS).unref?.()
|
|
152121
|
+
)
|
|
152122
|
+
]);
|
|
152123
|
+
}
|
|
152124
|
+
if (!sessionEstablished || discardLocalState && !retainLocalState) {
|
|
151331
152125
|
await releaseHarnessLease?.();
|
|
151332
152126
|
}
|
|
151333
152127
|
try {
|
|
@@ -161939,7 +162733,7 @@ function platformRefusalHint(refusal) {
|
|
|
161939
162733
|
return refusal.canTopUp === false ? `${when} This is a usage limit: topping up credits does not lift it.` : when;
|
|
161940
162734
|
}
|
|
161941
162735
|
function readSdkVersion3() {
|
|
161942
|
-
return "8.
|
|
162736
|
+
return "8.18.0";
|
|
161943
162737
|
}
|
|
161944
162738
|
var DEFAULT_PLATFORM_API_BASE_URL = "https://app.mcpjam.com/api/v1";
|
|
161945
162739
|
function resolvePlatformRequestUrl(spec) {
|
|
@@ -166645,6 +167439,10 @@ var ASSERTION_STAGE3 = {
|
|
|
166645
167439
|
// ── Tool call: was the call itself well formed ──────────────────────────
|
|
166646
167440
|
argumentsMatchToolSchema: "call",
|
|
166647
167441
|
noRepeatedIdenticalCall: "call",
|
|
167442
|
+
// What went INTO the call, so a wrong argument files as `argumentMismatch`.
|
|
167443
|
+
// A never-called tool on its own also files here; pair it with a
|
|
167444
|
+
// `toolCalledWith` (which files at `selection`, earlier) to attribute that.
|
|
167445
|
+
toolInputMatches: "call",
|
|
166648
167446
|
// ── Response: what the server answered with ─────────────────────────────
|
|
166649
167447
|
//
|
|
166650
167448
|
// `noToolErrors` MOVED HERE in analyzer 11, from `userValue` where it had
|
|
@@ -166658,6 +167456,8 @@ var ASSERTION_STAGE3 = {
|
|
|
166658
167456
|
noToolErrors: "response",
|
|
166659
167457
|
toolLatencyUnder: "response",
|
|
166660
167458
|
toolResultContains: "response",
|
|
167459
|
+
// What came OUT of the call, like the other `toolResult*` checks.
|
|
167460
|
+
toolResultMatches: "response",
|
|
166661
167461
|
toolResultMatchesSchema: "response",
|
|
166662
167462
|
toolResultSizeUnder: "response",
|
|
166663
167463
|
toolErrorNamesInput: "response",
|
|
@@ -200366,7 +201166,7 @@ function boolOf(value) {
|
|
|
200366
201166
|
function integrityOf(value) {
|
|
200367
201167
|
return value === "valid" || value === "invalid" ? value : null;
|
|
200368
201168
|
}
|
|
200369
|
-
function
|
|
201169
|
+
function countOf2(value) {
|
|
200370
201170
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
200371
201171
|
}
|
|
200372
201172
|
function scoreContractSide(value) {
|
|
@@ -200374,8 +201174,8 @@ function scoreContractSide(value) {
|
|
|
200374
201174
|
return {
|
|
200375
201175
|
evaluationConfigHash: strOrNull(side.evaluationConfigHash),
|
|
200376
201176
|
scoreIntegrity: integrityOf(side.scoreIntegrity),
|
|
200377
|
-
scoredIterations:
|
|
200378
|
-
quarantinedIterations:
|
|
201177
|
+
scoredIterations: countOf2(side.scoredIterations),
|
|
201178
|
+
quarantinedIterations: countOf2(side.quarantinedIterations)
|
|
200379
201179
|
};
|
|
200380
201180
|
}
|
|
200381
201181
|
function scoreContract(value) {
|
|
@@ -200393,10 +201193,10 @@ function scoreContract(value) {
|
|
|
200393
201193
|
passRate: numericDiff(scorer.passRate),
|
|
200394
201194
|
meanValue: numericDiff(scorer.meanValue),
|
|
200395
201195
|
errorCount: {
|
|
200396
|
-
base:
|
|
201196
|
+
base: countOf2(
|
|
200397
201197
|
isRecord31(scorer.errorCount) ? scorer.errorCount.base : void 0
|
|
200398
201198
|
),
|
|
200399
|
-
compare:
|
|
201199
|
+
compare: countOf2(
|
|
200400
201200
|
isRecord31(scorer.errorCount) ? scorer.errorCount.compare : void 0
|
|
200401
201201
|
)
|
|
200402
201202
|
}
|
|
@@ -200479,15 +201279,15 @@ function runSide(value) {
|
|
|
200479
201279
|
const environment = isRecord31(run3.environment) ? run3.environment : null;
|
|
200480
201280
|
return {
|
|
200481
201281
|
id: str2(run3.id),
|
|
200482
|
-
runNumber:
|
|
201282
|
+
runNumber: countOf2(run3.runNumber),
|
|
200483
201283
|
result: str2(run3.result),
|
|
200484
|
-
createdAt:
|
|
201284
|
+
createdAt: countOf2(run3.createdAt),
|
|
200485
201285
|
completedAt: numOrNull(run3.completedAt),
|
|
200486
201286
|
summary: summary ? {
|
|
200487
|
-
total:
|
|
200488
|
-
passed:
|
|
200489
|
-
failed:
|
|
200490
|
-
passRate:
|
|
201287
|
+
total: countOf2(summary.total),
|
|
201288
|
+
passed: countOf2(summary.passed),
|
|
201289
|
+
failed: countOf2(summary.failed),
|
|
201290
|
+
passRate: countOf2(summary.passRate)
|
|
200491
201291
|
} : null,
|
|
200492
201292
|
...environment ? {
|
|
200493
201293
|
environment: {
|
|
@@ -200517,10 +201317,10 @@ function skills(value) {
|
|
|
200517
201317
|
const compare = isRecord31(value.compare) ? value.compare : {};
|
|
200518
201318
|
const changes = Array.isArray(value.changes) ? value.changes : [];
|
|
200519
201319
|
return {
|
|
200520
|
-
base: { excluded: boolOf(base.excluded), count:
|
|
201320
|
+
base: { excluded: boolOf(base.excluded), count: countOf2(base.count) },
|
|
200521
201321
|
compare: {
|
|
200522
201322
|
excluded: boolOf(compare.excluded),
|
|
200523
|
-
count:
|
|
201323
|
+
count: countOf2(compare.count)
|
|
200524
201324
|
},
|
|
200525
201325
|
changes: changes.filter(isRecord31).map((row2) => {
|
|
200526
201326
|
const baseSide = skillSide(row2.base);
|
|
@@ -200537,7 +201337,7 @@ function skills(value) {
|
|
|
200537
201337
|
...typeof row2.versionDelta === "string" ? { versionDelta: row2.versionDelta } : {}
|
|
200538
201338
|
};
|
|
200539
201339
|
}),
|
|
200540
|
-
unchangedCount:
|
|
201340
|
+
unchangedCount: countOf2(value.unchangedCount)
|
|
200541
201341
|
};
|
|
200542
201342
|
}
|
|
200543
201343
|
function toRunCompareDto(diff, baseline) {
|
|
@@ -202768,6 +203568,34 @@ function evalSnapshotUploadTarget(convexAuthToken, iterationId) {
|
|
|
202768
203568
|
} : void 0;
|
|
202769
203569
|
}
|
|
202770
203570
|
|
|
203571
|
+
// server/utils/upload-receipt.ts
|
|
203572
|
+
async function confirmUploadedObject(args) {
|
|
203573
|
+
if (args.uploadGrantId === void 0) return;
|
|
203574
|
+
if (!args.serviceToken)
|
|
203575
|
+
throw new Error("Upload receipt requires service credentials");
|
|
203576
|
+
const response = await fetch(
|
|
203577
|
+
`${args.convexHttpUrl}/internal/v1/uploads/complete`,
|
|
203578
|
+
{
|
|
203579
|
+
method: "POST",
|
|
203580
|
+
headers: {
|
|
203581
|
+
Authorization: `Bearer ${args.bearer}`,
|
|
203582
|
+
"Content-Type": "application/json",
|
|
203583
|
+
[INSPECTOR_SERVICE_TOKEN_HEADER]: args.serviceToken
|
|
203584
|
+
},
|
|
203585
|
+
body: JSON.stringify({
|
|
203586
|
+
uploadGrantId: args.uploadGrantId,
|
|
203587
|
+
storageId: args.storageId
|
|
203588
|
+
}),
|
|
203589
|
+
redirect: "error",
|
|
203590
|
+
signal: AbortSignal.any([args.signal, AbortSignal.timeout(1e4)])
|
|
203591
|
+
}
|
|
203592
|
+
);
|
|
203593
|
+
const body = await response.json().catch(() => null);
|
|
203594
|
+
if (!response.ok || body?.ok !== true) {
|
|
203595
|
+
throw new Error("Upload receipt was refused");
|
|
203596
|
+
}
|
|
203597
|
+
}
|
|
203598
|
+
|
|
202771
203599
|
// server/utils/storage-destination.ts
|
|
202772
203600
|
function isUsableStorageDestination(value) {
|
|
202773
203601
|
if (typeof value !== "string" || !value) return false;
|
|
@@ -203007,10 +203835,13 @@ async function requestReplayVideoDestination(target, serviceToken3, signal) {
|
|
|
203007
203835
|
`Replay video upload was refused (${response.status})${typeof body?.error === "string" ? `: ${body.error}` : ""}`
|
|
203008
203836
|
);
|
|
203009
203837
|
}
|
|
203010
|
-
if (!isUsableStorageDestination(body?.uploadUrl)) {
|
|
203838
|
+
if (!isUsableStorageDestination(body?.uploadUrl) || body?.uploadGrantId !== void 0 && (typeof body.uploadGrantId !== "string" || !body.uploadGrantId)) {
|
|
203011
203839
|
throw new Error("Replay video upload got no usable destination");
|
|
203012
203840
|
}
|
|
203013
|
-
return
|
|
203841
|
+
return {
|
|
203842
|
+
uploadUrl: body.uploadUrl,
|
|
203843
|
+
uploadGrantId: body.uploadGrantId
|
|
203844
|
+
};
|
|
203014
203845
|
}
|
|
203015
203846
|
async function storeReplayVideo(target, body, contentType, signal) {
|
|
203016
203847
|
const serviceToken3 = getConfiguredInspectorServiceToken();
|
|
@@ -203025,7 +203856,7 @@ async function storeReplayVideo(target, body, contentType, signal) {
|
|
|
203025
203856
|
serviceToken3,
|
|
203026
203857
|
signal
|
|
203027
203858
|
);
|
|
203028
|
-
const response = await fetch(destination, {
|
|
203859
|
+
const response = await fetch(destination.uploadUrl, {
|
|
203029
203860
|
method: "POST",
|
|
203030
203861
|
headers: { "Content-Type": contentType },
|
|
203031
203862
|
body,
|
|
@@ -203041,7 +203872,17 @@ async function storeReplayVideo(target, body, contentType, signal) {
|
|
|
203041
203872
|
} catch (err) {
|
|
203042
203873
|
if (signal.aborted) throw err;
|
|
203043
203874
|
}
|
|
203044
|
-
|
|
203875
|
+
if (typeof stored?.storageId !== "string" || !stored.storageId)
|
|
203876
|
+
return void 0;
|
|
203877
|
+
await confirmUploadedObject({
|
|
203878
|
+
convexHttpUrl: process.env.CONVEX_HTTP_URL,
|
|
203879
|
+
bearer: target.convexAuthToken,
|
|
203880
|
+
serviceToken: serviceToken3,
|
|
203881
|
+
uploadGrantId: destination.uploadGrantId,
|
|
203882
|
+
storageId: stored.storageId,
|
|
203883
|
+
signal
|
|
203884
|
+
});
|
|
203885
|
+
return stored.storageId;
|
|
203045
203886
|
}
|
|
203046
203887
|
async function uploadVideoBlob(target, bytes2, options = {}) {
|
|
203047
203888
|
const contentType = options.contentType || DEFAULT_REPLAY_VIDEO_MIME;
|
|
@@ -203078,7 +203919,9 @@ async function captureMcpAppWidgetSnapshots(params) {
|
|
|
203078
203919
|
const snapshots = await Promise.all(
|
|
203079
203920
|
sources.map(async (source) => {
|
|
203080
203921
|
try {
|
|
203081
|
-
const toolMetadata = params.mcpClientManager.getAllToolsMetadata(
|
|
203922
|
+
const toolMetadata = params.mcpClientManager.getAllToolsMetadata(
|
|
203923
|
+
source.serverId
|
|
203924
|
+
)?.[source.toolName];
|
|
203082
203925
|
if (!isRecord36(toolMetadata) || !isMcpAppTool(toolMetadata)) {
|
|
203083
203926
|
return null;
|
|
203084
203927
|
}
|
|
@@ -203133,13 +203976,16 @@ async function captureMcpAppWidgetSnapshots(params) {
|
|
|
203133
203976
|
}
|
|
203134
203977
|
snapshot2.injectedOpenAiCompat = shouldInjectOpenAiCompat;
|
|
203135
203978
|
} catch (error) {
|
|
203136
|
-
logger.warn(
|
|
203137
|
-
|
|
203138
|
-
|
|
203139
|
-
|
|
203140
|
-
|
|
203141
|
-
|
|
203142
|
-
|
|
203979
|
+
logger.warn(
|
|
203980
|
+
`${LOG_PREFIX} Failed to capture MCP App widget snapshot`,
|
|
203981
|
+
{
|
|
203982
|
+
toolCallId: source.toolCallId,
|
|
203983
|
+
toolName: source.toolName,
|
|
203984
|
+
serverId: source.serverId,
|
|
203985
|
+
resourceUri,
|
|
203986
|
+
error: error instanceof Error ? error.message : String(error)
|
|
203987
|
+
}
|
|
203988
|
+
);
|
|
203143
203989
|
}
|
|
203144
203990
|
return snapshot2;
|
|
203145
203991
|
} catch (error) {
|
|
@@ -204862,9 +205708,21 @@ function environmentServerNames(resolved) {
|
|
|
204862
205708
|
function environmentServerRefsForManager(resolved, manager) {
|
|
204863
205709
|
const serverIds = environmentServerIds(resolved);
|
|
204864
205710
|
const serverNames = environmentServerNames(resolved);
|
|
204865
|
-
|
|
205711
|
+
const refs = serverIds.map(
|
|
204866
205712
|
(serverId, index) => manager.hasServer(serverId) ? serverId : serverNames[index] ?? serverId
|
|
204867
205713
|
);
|
|
205714
|
+
const selection = resolved.environmentRef.serverSelection;
|
|
205715
|
+
if (selection?.mode === "local") {
|
|
205716
|
+
if (resolved.runtimeVenue !== "local") {
|
|
205717
|
+
throw new WebRouteError(
|
|
205718
|
+
409,
|
|
205719
|
+
ErrorCode.CONFLICT,
|
|
205720
|
+
"This environment requires a local runner."
|
|
205721
|
+
);
|
|
205722
|
+
}
|
|
205723
|
+
refs.push(...selection.names);
|
|
205724
|
+
}
|
|
205725
|
+
return [...new Set(refs)];
|
|
204868
205726
|
}
|
|
204869
205727
|
function environmentEffectiveServerIds(resolved) {
|
|
204870
205728
|
return resolved.effectiveServerIds ?? resolved.selectedServerIds;
|
|
@@ -204887,7 +205745,9 @@ async function resolveEnvironmentForLaunch(convexClient6, args) {
|
|
|
204887
205745
|
"This deployment cannot resolve project environments yet. Retry after the backend deploys."
|
|
204888
205746
|
);
|
|
204889
205747
|
}
|
|
204890
|
-
if (args.serverSource && /Object contains extra field [`'"]?serverSource(?:[`'"]|\b)/i.test(
|
|
205748
|
+
if (args.serverSource && /Object contains extra field [`'"]?serverSource(?:[`'"]|\b)/i.test(
|
|
205749
|
+
message
|
|
205750
|
+
)) {
|
|
204891
205751
|
if (!loggedServerSourceDowngrade) {
|
|
204892
205752
|
loggedServerSourceDowngrade = true;
|
|
204893
205753
|
logger.warn(
|
|
@@ -204910,7 +205770,10 @@ async function resolveEnvironmentForLaunch(convexClient6, args) {
|
|
|
204910
205770
|
"Environment not found (it may have been archived). Update the suite's environments and retry."
|
|
204911
205771
|
);
|
|
204912
205772
|
}
|
|
204913
|
-
return
|
|
205773
|
+
return {
|
|
205774
|
+
...resolved,
|
|
205775
|
+
...args.runtimeVenue ? { runtimeVenue: args.runtimeVenue } : {}
|
|
205776
|
+
};
|
|
204914
205777
|
}
|
|
204915
205778
|
function isEnvironmentLaunchConflict(error) {
|
|
204916
205779
|
const data = error?.data;
|
|
@@ -204955,10 +205818,15 @@ function environmentLaunchRejectionError(error) {
|
|
|
204955
205818
|
const status = LAUNCH_REJECTION_STATUS[code];
|
|
204956
205819
|
if (!status) return null;
|
|
204957
205820
|
if (status === 404) {
|
|
204958
|
-
return new WebRouteError(
|
|
204959
|
-
|
|
204960
|
-
|
|
204961
|
-
|
|
205821
|
+
return new WebRouteError(
|
|
205822
|
+
404,
|
|
205823
|
+
ErrorCode.NOT_FOUND,
|
|
205824
|
+
"Environment not found",
|
|
205825
|
+
{
|
|
205826
|
+
code: "ENV_NOT_FOUND",
|
|
205827
|
+
reason: "env_not_found"
|
|
205828
|
+
}
|
|
205829
|
+
);
|
|
204962
205830
|
}
|
|
204963
205831
|
const message = typeof record5.message === "string" && record5.message.trim() ? record5.message : "This eval run could not be started as requested.";
|
|
204964
205832
|
const details = record5.details && typeof record5.details === "object" && !Array.isArray(record5.details) ? record5.details : {};
|
|
@@ -205306,6 +206174,7 @@ var startSuiteRunWithRecorder = async ({
|
|
|
205306
206174
|
namedHostId,
|
|
205307
206175
|
runGroupId,
|
|
205308
206176
|
environmentId,
|
|
206177
|
+
runtimeVenue,
|
|
205309
206178
|
expectedEnvironmentRevision,
|
|
205310
206179
|
expectedEnvironmentHostConfigId,
|
|
205311
206180
|
expectedEnvironmentServerIds,
|
|
@@ -205338,6 +206207,7 @@ var startSuiteRunWithRecorder = async ({
|
|
|
205338
206207
|
...namedHostId ? { namedHostId } : {},
|
|
205339
206208
|
...runGroupId ? { runGroupId } : {},
|
|
205340
206209
|
...environmentId ? { environmentId } : {},
|
|
206210
|
+
...runtimeVenue ? { runtimeVenue } : {},
|
|
205341
206211
|
...expectedEnvironmentRevision !== void 0 ? { expectedEnvironmentRevision } : {},
|
|
205342
206212
|
...expectedEnvironmentHostConfigId !== void 0 ? { expectedEnvironmentHostConfigId } : {},
|
|
205343
206213
|
...expectedEnvironmentServerIds !== void 0 ? { expectedEnvironmentServerIds } : {},
|
|
@@ -207630,6 +208500,7 @@ function createStepExecutionState() {
|
|
|
207630
208500
|
toolCalls: [],
|
|
207631
208501
|
toolCallsByTurn: [],
|
|
207632
208502
|
toolErrors: [],
|
|
208503
|
+
spans: [],
|
|
207633
208504
|
widgetRenderObservations: [],
|
|
207634
208505
|
assertionResults: [],
|
|
207635
208506
|
interactionFailures: [],
|
|
@@ -207649,8 +208520,10 @@ function recordToolCalls(state, turn, calls) {
|
|
|
207649
208520
|
}
|
|
207650
208521
|
function applyOutcome(state, outcome, turn) {
|
|
207651
208522
|
if (outcome.messages?.length) state.messages.push(...outcome.messages);
|
|
207652
|
-
if (outcome.toolCalls?.length)
|
|
208523
|
+
if (outcome.toolCalls?.length)
|
|
208524
|
+
recordToolCalls(state, turn, outcome.toolCalls);
|
|
207653
208525
|
if (outcome.toolErrors?.length) state.toolErrors.push(...outcome.toolErrors);
|
|
208526
|
+
if (outcome.spans?.length) state.spans.push(...outcome.spans);
|
|
207654
208527
|
if (outcome.usage) {
|
|
207655
208528
|
state.usage.inputTokens += outcome.usage.inputTokens ?? 0;
|
|
207656
208529
|
state.usage.outputTokens += outcome.usage.outputTokens ?? 0;
|
|
@@ -207659,8 +208532,16 @@ function applyOutcome(state, outcome, turn) {
|
|
|
207659
208532
|
}
|
|
207660
208533
|
function snapshotTranscript(state) {
|
|
207661
208534
|
const finalAssistantMessage = extractFinalAssistantMessage(state.messages);
|
|
208535
|
+
const evidence = extractTranscriptEvidence({
|
|
208536
|
+
messages: state.messages,
|
|
208537
|
+
spans: state.spans
|
|
208538
|
+
});
|
|
207662
208539
|
return buildIterationTranscript({
|
|
207663
208540
|
toolCalls: state.toolCalls,
|
|
208541
|
+
toolResults: evidence.toolResults,
|
|
208542
|
+
resultsCaptured: evidence.resultsCaptured,
|
|
208543
|
+
toolCallTimings: evidence.toolCallTimings,
|
|
208544
|
+
timingsCaptured: evidence.timingsCaptured,
|
|
207664
208545
|
...finalAssistantMessage !== void 0 ? { finalAssistantMessage } : {},
|
|
207665
208546
|
usage: state.usage.inputTokens || state.usage.outputTokens || state.usage.totalTokens ? state.usage : void 0,
|
|
207666
208547
|
renderObservations: summarizeRenderObservations(
|
|
@@ -207704,7 +208585,11 @@ async function drainAndDriveFollowUps(stepLabel, stepIndex, turn, browser, handl
|
|
|
207704
208585
|
return void 0;
|
|
207705
208586
|
}
|
|
207706
208587
|
remaining -= 1;
|
|
207707
|
-
const outcome = await handlers.onFollowUp({
|
|
208588
|
+
const outcome = await handlers.onFollowUp({
|
|
208589
|
+
text: text2,
|
|
208590
|
+
stepIndex,
|
|
208591
|
+
turnOrdinal: turn
|
|
208592
|
+
});
|
|
207708
208593
|
applyOutcome(state, outcome, turn);
|
|
207709
208594
|
if (outcome.cancelled || outcome.iterationError) return outcome;
|
|
207710
208595
|
}
|
|
@@ -207817,7 +208702,8 @@ async function executeSteps(args) {
|
|
|
207817
208702
|
emitStatus(stepIndex, "running");
|
|
207818
208703
|
const outcome = await handlers.onPrompt({ step, stepIndex, turnOrdinal });
|
|
207819
208704
|
applyOutcome(state, outcome, turnOrdinal);
|
|
207820
|
-
if (outcome.cancelled)
|
|
208705
|
+
if (outcome.cancelled)
|
|
208706
|
+
return { state, cancelled: true, setupFailure: false };
|
|
207821
208707
|
if (outcome.iterationError) {
|
|
207822
208708
|
emitStatus(stepIndex, "fail");
|
|
207823
208709
|
recordSkippedSteps(
|
|
@@ -207856,7 +208742,8 @@ async function executeSteps(args) {
|
|
|
207856
208742
|
turnOrdinal
|
|
207857
208743
|
});
|
|
207858
208744
|
applyOutcome(state, outcome, turnOrdinal);
|
|
207859
|
-
if (outcome.cancelled)
|
|
208745
|
+
if (outcome.cancelled)
|
|
208746
|
+
return { state, cancelled: true, setupFailure: false };
|
|
207860
208747
|
if (outcome.iterationError) {
|
|
207861
208748
|
emitStatus(stepIndex, "fail");
|
|
207862
208749
|
recordSkippedSteps(
|
|
@@ -208511,6 +209398,7 @@ function buildLocalStepHandlers(ctx) {
|
|
|
208511
209398
|
const acc = ctx.acc;
|
|
208512
209399
|
async function drive(promptTurn, turnOrdinal) {
|
|
208513
209400
|
const messagesBefore = acc.conversationMessages.length;
|
|
209401
|
+
const spansBefore = acc.capturedSpans.length;
|
|
208514
209402
|
const inBefore = acc.accumulatedUsage.inputTokens ?? 0;
|
|
208515
209403
|
const outBefore = acc.accumulatedUsage.outputTokens ?? 0;
|
|
208516
209404
|
const totalBefore = acc.accumulatedUsage.totalTokens ?? 0;
|
|
@@ -208523,6 +209411,7 @@ function buildLocalStepHandlers(ctx) {
|
|
|
208523
209411
|
sinks: buildSinks?.(turnOrdinal, promptTurn.prompt)
|
|
208524
209412
|
});
|
|
208525
209413
|
const messages = acc.conversationMessages.slice(messagesBefore);
|
|
209414
|
+
const spans = acc.capturedSpans.slice(spansBefore);
|
|
208526
209415
|
const toolCalls = acc.toolsCalledByPrompt[turnOrdinal] ?? [];
|
|
208527
209416
|
const toolErrors = acc.toolErrorsByPrompt[turnOrdinal] ?? [];
|
|
208528
209417
|
const usage = {
|
|
@@ -208535,6 +209424,7 @@ function buildLocalStepHandlers(ctx) {
|
|
|
208535
209424
|
...messages.length ? { messages } : {},
|
|
208536
209425
|
...toolCalls.length ? { toolCalls } : {},
|
|
208537
209426
|
...toolErrors.length ? { toolErrors } : {},
|
|
209427
|
+
...spans.length ? { spans } : {},
|
|
208538
209428
|
usage,
|
|
208539
209429
|
// Optional-chained: `driveLocalEvalTurn` always returns an outcome, but
|
|
208540
209430
|
// this bridge has no business crashing over one it did not get.
|
|
@@ -208578,6 +209468,7 @@ function buildLocalStepHandlers(ctx) {
|
|
|
208578
209468
|
...messages.length ? { messages } : {},
|
|
208579
209469
|
...toolCalls.length ? { toolCalls } : {},
|
|
208580
209470
|
...toolErrors.length ? { toolErrors } : {},
|
|
209471
|
+
...spans.length ? { spans } : {},
|
|
208581
209472
|
usage,
|
|
208582
209473
|
// Optional-chained: `driveLocalEvalTurn` always returns an outcome, but
|
|
208583
209474
|
// this bridge has no business crashing over one it did not get.
|
|
@@ -208618,6 +209509,7 @@ function buildHostedStepHandlers(ctx) {
|
|
|
208618
209509
|
const acc = ctx.acc;
|
|
208619
209510
|
async function drivePrompt(prompt, turnOrdinal) {
|
|
208620
209511
|
const messagesBefore = acc.messageHistory.length;
|
|
209512
|
+
const spansBefore = acc.capturedSpans.length;
|
|
208621
209513
|
const inBefore = acc.accumulatedUsage.inputTokens ?? 0;
|
|
208622
209514
|
const outBefore = acc.accumulatedUsage.outputTokens ?? 0;
|
|
208623
209515
|
const totalBefore = acc.accumulatedUsage.totalTokens ?? 0;
|
|
@@ -208628,6 +209520,7 @@ function buildHostedStepHandlers(ctx) {
|
|
|
208628
209520
|
acc
|
|
208629
209521
|
});
|
|
208630
209522
|
const messages = acc.messageHistory.slice(messagesBefore);
|
|
209523
|
+
const spans = acc.capturedSpans.slice(spansBefore);
|
|
208631
209524
|
const toolCalls = acc.toolsCalledByPrompt[turnOrdinal] ?? [];
|
|
208632
209525
|
const usage = {
|
|
208633
209526
|
inputTokens: (acc.accumulatedUsage.inputTokens ?? 0) - inBefore,
|
|
@@ -208637,6 +209530,7 @@ function buildHostedStepHandlers(ctx) {
|
|
|
208637
209530
|
return {
|
|
208638
209531
|
...messages.length ? { messages } : {},
|
|
208639
209532
|
...toolCalls.length ? { toolCalls } : {},
|
|
209533
|
+
...spans.length ? { spans } : {},
|
|
208640
209534
|
usage,
|
|
208641
209535
|
...outcome.kind === "cancelled" ? { cancelled: true } : {},
|
|
208642
209536
|
...outcome.kind === "failed" ? {
|
|
@@ -208685,6 +209579,7 @@ function buildHostedStepHandlers(ctx) {
|
|
|
208685
209579
|
...messages.length ? { messages } : {},
|
|
208686
209580
|
...toolCalls.length ? { toolCalls } : {},
|
|
208687
209581
|
...toolErrors.length ? { toolErrors } : {},
|
|
209582
|
+
...spans.length ? { spans } : {},
|
|
208688
209583
|
usage,
|
|
208689
209584
|
...outcome.kind === "cancelled" ? { cancelled: true } : {},
|
|
208690
209585
|
...outcome.kind === "failed" ? {
|
|
@@ -210020,6 +210915,7 @@ async function commitEnvironmentQuickRun(convexClient6, args) {
|
|
|
210020
210915
|
startedAt: args.startedAt,
|
|
210021
210916
|
environment: {
|
|
210022
210917
|
environmentId: args.resolved.environmentRef.environmentId,
|
|
210918
|
+
...args.resolved.runtimeVenue === "local" ? { runtimeVenue: "local" } : {},
|
|
210023
210919
|
expectedRevision: args.resolved.environmentRef.revision,
|
|
210024
210920
|
...args.resolved.hostConfigId ? { expectedHostConfigId: args.resolved.hostConfigId } : {},
|
|
210025
210921
|
expectedServerIds: environmentEffectiveServerIds(args.resolved)
|
|
@@ -218323,6 +219219,7 @@ async function prepareEvalRun(clientManager, request3) {
|
|
|
218323
219219
|
}
|
|
218324
219220
|
environmentLaunch = resolvedEnvironment && resolvedEnvironment.environmentRef.environmentId === environmentId ? resolvedEnvironment : await resolveEnvironmentForLaunch(convexClient6, {
|
|
218325
219221
|
serverSource: EVAL_LAUNCH_SERVER_SOURCE,
|
|
219222
|
+
...request3.runtimeVenue ? { runtimeVenue: request3.runtimeVenue } : {},
|
|
218326
219223
|
projectId,
|
|
218327
219224
|
environmentId
|
|
218328
219225
|
}).catch((error) => {
|
|
@@ -218418,6 +219315,7 @@ async function prepareEvalRun(clientManager, request3) {
|
|
|
218418
219315
|
// environment resolves to at an unchanged revision. Echoing all three lets
|
|
218419
219316
|
// the mutation reject that drift instead of starting a run whose tool
|
|
218420
219317
|
// snapshot describes a different configuration than it executes.
|
|
219318
|
+
...environmentLaunch?.runtimeVenue === "local" ? { runtimeVenue: "local" } : {},
|
|
218421
219319
|
expectedEnvironmentRevision: environmentLaunch?.environmentRef.revision,
|
|
218422
219320
|
expectedEnvironmentHostConfigId: environmentLaunch?.hostConfigId,
|
|
218423
219321
|
expectedEnvironmentServerIds: environmentLaunch ? environmentEffectiveServerIds(environmentLaunch) : void 0,
|
|
@@ -218818,6 +219716,7 @@ async function prepareSingleCaseExecution(clientManager, request3) {
|
|
|
218818
219716
|
assertNoConflictingEnvironmentOverrides(request3);
|
|
218819
219717
|
environmentLaunch = request3.resolvedEnvironment && request3.resolvedEnvironment.environmentRef.environmentId === environmentId ? request3.resolvedEnvironment : await resolveEnvironmentForLaunch(convexClient6, {
|
|
218820
219718
|
serverSource: EVAL_LAUNCH_SERVER_SOURCE,
|
|
219719
|
+
...request3.runtimeVenue ? { runtimeVenue: request3.runtimeVenue } : {},
|
|
218821
219720
|
projectId,
|
|
218822
219721
|
environmentId
|
|
218823
219722
|
}).catch((error) => {
|
|
@@ -219166,6 +220065,7 @@ async function resolveGenerationServers(clientManager, request3) {
|
|
|
219166
220065
|
createConvexClients(request3.convexAuthToken).convexClient,
|
|
219167
220066
|
{
|
|
219168
220067
|
serverSource: EVAL_LAUNCH_SERVER_SOURCE,
|
|
220068
|
+
...request3.runtimeVenue ? { runtimeVenue: request3.runtimeVenue } : {},
|
|
219169
220069
|
projectId: request3.projectId,
|
|
219170
220070
|
environmentId: request3.environmentId
|
|
219171
220071
|
}
|
|
@@ -227421,10 +228321,10 @@ evals2.post("/run", async (c) => {
|
|
|
227421
228321
|
400
|
|
227422
228322
|
);
|
|
227423
228323
|
}
|
|
227424
|
-
const prepared = await prepareEvalRun(
|
|
227425
|
-
|
|
227426
|
-
|
|
227427
|
-
);
|
|
228324
|
+
const prepared = await prepareEvalRun(c.mcpClientManager, {
|
|
228325
|
+
...validationResult.data,
|
|
228326
|
+
runtimeVenue: "local"
|
|
228327
|
+
});
|
|
227428
228328
|
detachPreparedEvalRun({
|
|
227429
228329
|
prepared,
|
|
227430
228330
|
convexAuthToken: validationResult.data.convexAuthToken,
|
|
@@ -227590,10 +228490,10 @@ evals2.post("/run-test-case", async (c) => {
|
|
|
227590
228490
|
);
|
|
227591
228491
|
}
|
|
227592
228492
|
return c.json(
|
|
227593
|
-
await runEvalTestCaseWithManager(
|
|
227594
|
-
|
|
227595
|
-
|
|
227596
|
-
)
|
|
228493
|
+
await runEvalTestCaseWithManager(c.mcpClientManager, {
|
|
228494
|
+
...validationResult.data,
|
|
228495
|
+
runtimeVenue: "local"
|
|
228496
|
+
})
|
|
227597
228497
|
);
|
|
227598
228498
|
} catch (error) {
|
|
227599
228499
|
reportRouteFailure("[Error running test case]", error, {
|
|
@@ -227619,7 +228519,7 @@ evals2.post("/stream-test-case", async (c) => {
|
|
|
227619
228519
|
}
|
|
227620
228520
|
const stream = await streamEvalTestCaseWithManager(
|
|
227621
228521
|
c.mcpClientManager,
|
|
227622
|
-
validationResult.data,
|
|
228522
|
+
{ ...validationResult.data, runtimeVenue: "local" },
|
|
227623
228523
|
// Client disconnect aborts the run (including any awaited task).
|
|
227624
228524
|
{ requestSignal: c.req.raw.signal }
|
|
227625
228525
|
);
|
|
@@ -227686,10 +228586,10 @@ evals2.post("/generate-tests", async (c) => {
|
|
|
227686
228586
|
);
|
|
227687
228587
|
}
|
|
227688
228588
|
return c.json(
|
|
227689
|
-
await generateEvalTestsWithManager(
|
|
227690
|
-
|
|
227691
|
-
|
|
227692
|
-
)
|
|
228589
|
+
await generateEvalTestsWithManager(c.mcpClientManager, {
|
|
228590
|
+
...validationResult.data,
|
|
228591
|
+
runtimeVenue: "local"
|
|
228592
|
+
})
|
|
227693
228593
|
);
|
|
227694
228594
|
} catch (error) {
|
|
227695
228595
|
reportRouteFailure("Error in /evals/generate-tests", error, {
|
|
@@ -227713,10 +228613,10 @@ evals2.post("/generate-negative-tests", async (c) => {
|
|
|
227713
228613
|
);
|
|
227714
228614
|
}
|
|
227715
228615
|
return c.json(
|
|
227716
|
-
await generateNegativeEvalTestsWithManager(
|
|
227717
|
-
|
|
227718
|
-
|
|
227719
|
-
)
|
|
228616
|
+
await generateNegativeEvalTestsWithManager(c.mcpClientManager, {
|
|
228617
|
+
...validationResult.data,
|
|
228618
|
+
runtimeVenue: "local"
|
|
228619
|
+
})
|
|
227720
228620
|
);
|
|
227721
228621
|
} catch (error) {
|
|
227722
228622
|
reportRouteFailure("Error in /evals/generate-negative-tests", error, {
|
|
@@ -230171,8 +231071,8 @@ function formatDisplayPath2(fullPath) {
|
|
|
230171
231071
|
}
|
|
230172
231072
|
async function directoryExists3(dirPath) {
|
|
230173
231073
|
try {
|
|
230174
|
-
const
|
|
230175
|
-
return
|
|
231074
|
+
const stat8 = await fs3.stat(dirPath);
|
|
231075
|
+
return stat8.isDirectory();
|
|
230176
231076
|
} catch {
|
|
230177
231077
|
return false;
|
|
230178
231078
|
}
|
|
@@ -230569,8 +231469,8 @@ skills2.post("/read-file", async (c) => {
|
|
|
230569
231469
|
}
|
|
230570
231470
|
const fullPath = path7.join(skillDir, filePath);
|
|
230571
231471
|
try {
|
|
230572
|
-
const
|
|
230573
|
-
if (!
|
|
231472
|
+
const stat8 = await fs3.stat(fullPath);
|
|
231473
|
+
if (!stat8.isFile()) {
|
|
230574
231474
|
return c.json({ success: false, error: "Path is not a file" }, 400);
|
|
230575
231475
|
}
|
|
230576
231476
|
const mimeType = getMimeType(filePath);
|
|
@@ -230580,15 +231480,15 @@ skills2.post("/read-file", async (c) => {
|
|
|
230580
231480
|
path: filePath,
|
|
230581
231481
|
name: fileName,
|
|
230582
231482
|
mimeType,
|
|
230583
|
-
size:
|
|
231483
|
+
size: stat8.size,
|
|
230584
231484
|
isText
|
|
230585
231485
|
};
|
|
230586
231486
|
const maxSize = isText ? 1024 * 1024 : 5 * 1024 * 1024;
|
|
230587
|
-
if (
|
|
231487
|
+
if (stat8.size > maxSize) {
|
|
230588
231488
|
return c.json(
|
|
230589
231489
|
{
|
|
230590
231490
|
success: false,
|
|
230591
|
-
error: `File too large (${(
|
|
231491
|
+
error: `File too large (${(stat8.size / 1024 / 1024).toFixed(2)}MB). Maximum is ${maxSize / 1024 / 1024}MB`
|
|
230592
231492
|
},
|
|
230593
231493
|
400
|
|
230594
231494
|
);
|
|
@@ -256573,14 +257473,17 @@ async function requestArchiveDestination(c, projectId) {
|
|
|
256573
257473
|
typeof body?.error === "string" ? body.error : "Backend error"
|
|
256574
257474
|
);
|
|
256575
257475
|
}
|
|
256576
|
-
if (!isUsableStorageDestination(body?.uploadUrl)) {
|
|
257476
|
+
if (!isUsableStorageDestination(body?.uploadUrl) || body?.uploadGrantId !== void 0 && (typeof body.uploadGrantId !== "string" || !body.uploadGrantId)) {
|
|
256577
257477
|
throw new WebRouteError(
|
|
256578
257478
|
502,
|
|
256579
257479
|
ErrorCode.INTERNAL_ERROR,
|
|
256580
257480
|
UPLOAD_FAILED_MESSAGE
|
|
256581
257481
|
);
|
|
256582
257482
|
}
|
|
256583
|
-
return
|
|
257483
|
+
return {
|
|
257484
|
+
uploadUrl: body.uploadUrl,
|
|
257485
|
+
uploadGrantId: body.uploadGrantId
|
|
257486
|
+
};
|
|
256584
257487
|
} catch (error) {
|
|
256585
257488
|
if (error instanceof WebRouteError) throw error;
|
|
256586
257489
|
if (controller.signal.aborted) {
|
|
@@ -256659,7 +257562,7 @@ function handleBrowserProfileUpload(c) {
|
|
|
256659
257562
|
ARCHIVE_UPLOAD_TIMEOUT_MS
|
|
256660
257563
|
);
|
|
256661
257564
|
try {
|
|
256662
|
-
const response = await fetch(destination, {
|
|
257565
|
+
const response = await fetch(destination.uploadUrl, {
|
|
256663
257566
|
method: "POST",
|
|
256664
257567
|
headers: {
|
|
256665
257568
|
"Content-Type": "application/octet-stream",
|
|
@@ -256684,6 +257587,14 @@ function handleBrowserProfileUpload(c) {
|
|
|
256684
257587
|
UPLOAD_FAILED_MESSAGE
|
|
256685
257588
|
);
|
|
256686
257589
|
}
|
|
257590
|
+
await confirmUploadedObject({
|
|
257591
|
+
convexHttpUrl: convexHttpUrl2(),
|
|
257592
|
+
bearer: await getConvexBearerForRequest(c),
|
|
257593
|
+
serviceToken: getConfiguredInspectorServiceToken(),
|
|
257594
|
+
uploadGrantId: destination.uploadGrantId,
|
|
257595
|
+
storageId: stored.storageId,
|
|
257596
|
+
signal: AbortSignal.any([controller.signal, c.req.raw.signal])
|
|
257597
|
+
});
|
|
256687
257598
|
return { storageId: stored.storageId };
|
|
256688
257599
|
} catch (error) {
|
|
256689
257600
|
if (error instanceof WebRouteError) throw error;
|
|
@@ -274186,7 +275097,11 @@ var AGENT_API_BASE_PROMPT_LINES = [
|
|
|
274186
275097
|
"- Author cases as `steps` arrays; prefer a `prompt` step plus `toolCalledWith`-style assertions on the tools the conversation showed. Set `expectedOutput` when the user stated one.",
|
|
274187
275098
|
"- For new AI-authored cases, use generate_eval_cases and its spend approval flow. Use create/update case tools only for explicit user payloads or already reviewed drafts. Keep each full workflow as ordered steps.",
|
|
274188
275099
|
`- When creating a suite, set the suite \`model\` explicitly to \`${DEFAULT_SUITE_MODEL}\` unless the user asks for a different model.`,
|
|
274189
|
-
|
|
275100
|
+
// Names the marker `buildGatedProposalTools` puts on every gated tool's
|
|
275101
|
+
// description instead of enumerating the gated operations. The tier follows
|
|
275102
|
+
// each operation's `risk`, so a list here went stale with every new gated
|
|
275103
|
+
// op; the examples are illustrative, not the definition.
|
|
275104
|
+
"- Tools whose description says REQUIRES HUMAN APPROVAL do NOT perform the action when you call them (for example running a suite, a case, or a goal run, generating cases, cancelling a run, setting a schedule, or calling a tool on the user's own server). Calling one PROPOSES the action and returns an approval id, and a person must click to confirm. Say that you've proposed it and what it will do. NEVER say it has started, is running, or has been cancelled.",
|
|
274190
275105
|
"- If a proposal tool is not available to you, you cannot run anything at all. Say so plainly and report the ids the user needs \u2014 do not imply you started something.",
|
|
274191
275106
|
"- Always report the ids of anything you created.",
|
|
274192
275107
|
"- When a tool result carries a `permalinks` array, hand the user that `url` EXACTLY as written. NEVER invent, shorten, or rewrite an MCPJam app URL, and never build one from an id: a hand-made link opens whichever project the reader last selected, which is usually not the one you are talking about. If a result has no permalink, give the id and say where to find it.",
|
|
@@ -289420,7 +290335,10 @@ var BENCH_SERVICE_ROUTES = {
|
|
|
289420
290335
|
async function postServiceRoute5(path8, body, options) {
|
|
289421
290336
|
const env = benchServiceConfig();
|
|
289422
290337
|
const controller = new AbortController();
|
|
289423
|
-
const timeout = setTimeout(
|
|
290338
|
+
const timeout = setTimeout(
|
|
290339
|
+
() => controller.abort(),
|
|
290340
|
+
SERVICE_ROUTE_TIMEOUT_MS5
|
|
290341
|
+
);
|
|
289424
290342
|
try {
|
|
289425
290343
|
const response = await fetch(`${env.convexUrl}${path8}`, {
|
|
289426
290344
|
method: "POST",
|
|
@@ -289530,7 +290448,12 @@ function decodeClaimedJob(body) {
|
|
|
289530
290448
|
grant,
|
|
289531
290449
|
...readNumber3(credentials, "grantExpiresAt") !== void 0 ? { grantExpiresAt: readNumber3(credentials, "grantExpiresAt") } : {},
|
|
289532
290450
|
runnerBearer,
|
|
289533
|
-
...readNumber3(credentials, "runnerBearerExpiresAt") !== void 0 ? {
|
|
290451
|
+
...readNumber3(credentials, "runnerBearerExpiresAt") !== void 0 ? {
|
|
290452
|
+
runnerBearerExpiresAt: readNumber3(
|
|
290453
|
+
credentials,
|
|
290454
|
+
"runnerBearerExpiresAt"
|
|
290455
|
+
)
|
|
290456
|
+
} : {},
|
|
289534
290457
|
...Array.isArray(body.artifacts) ? { artifacts: body.artifacts } : {}
|
|
289535
290458
|
};
|
|
289536
290459
|
}
|
|
@@ -289653,7 +290576,9 @@ async function reportExecutionComplete(args) {
|
|
|
289653
290576
|
{ grant: args.job.grant }
|
|
289654
290577
|
);
|
|
289655
290578
|
if (isLeaseLostResponse(status, body)) {
|
|
289656
|
-
throw new LeaseLostError2(
|
|
290579
|
+
throw new LeaseLostError2(
|
|
290580
|
+
"execution-complete rejected the lease generation"
|
|
290581
|
+
);
|
|
289657
290582
|
}
|
|
289658
290583
|
if (status !== 200 || !body?.ok) {
|
|
289659
290584
|
throw new Error(`execution-complete rejected (${status})`);
|
|
@@ -289736,7 +290661,7 @@ function resolveEvalCellSpec(job, entry3) {
|
|
|
289736
290661
|
const missing = [
|
|
289737
290662
|
cellId ? null : "cellId (roster row)",
|
|
289738
290663
|
suiteId ? null : "suiteId (claim pins)",
|
|
289739
|
-
environmentId
|
|
290664
|
+
environmentId ? null : "environmentId (the cell's pinned execution configuration)"
|
|
289740
290665
|
].filter((field) => field !== null);
|
|
289741
290666
|
if (missing.length > 0) {
|
|
289742
290667
|
throw new JobUnexecutableError(
|
|
@@ -289818,7 +290743,9 @@ function assertClaimExecutable(job) {
|
|
|
289818
290743
|
throw new JobUnexecutableError("the claim carried no benchmark run id");
|
|
289819
290744
|
}
|
|
289820
290745
|
if (!job.pins?.definitionHash) {
|
|
289821
|
-
throw new JobUnexecutableError(
|
|
290746
|
+
throw new JobUnexecutableError(
|
|
290747
|
+
"the claim carried no pinned definition hash"
|
|
290748
|
+
);
|
|
289822
290749
|
}
|
|
289823
290750
|
for (const entry3 of job.roster ?? []) {
|
|
289824
290751
|
if (TERMINAL_EVIDENCE_STATUSES.has(entry3.status)) continue;
|
|
@@ -289882,6 +290809,11 @@ function buildWriteGuard(cell, benchmarkRunId, ledger) {
|
|
|
289882
290809
|
}
|
|
289883
290810
|
async function defaultRunEvalCell(args) {
|
|
289884
290811
|
const { job, cell } = args;
|
|
290812
|
+
if (!cell.environmentId) {
|
|
290813
|
+
throw new JobUnexecutableError(
|
|
290814
|
+
"Benchmark cell has no prepared environment."
|
|
290815
|
+
);
|
|
290816
|
+
}
|
|
289885
290817
|
const writeGuard = buildWriteGuard(cell, job.benchmarkRunId, args.ledger);
|
|
289886
290818
|
const authorized = await createAuthorizedManager(
|
|
289887
290819
|
{},
|
|
@@ -289916,11 +290848,9 @@ async function defaultRunEvalCell(args) {
|
|
|
289916
290848
|
// child ends up filed under the wrong benchmark.
|
|
289917
290849
|
source: "benchmark",
|
|
289918
290850
|
benchmarkRunId: job.benchmarkRunId,
|
|
289919
|
-
//
|
|
289920
|
-
|
|
289921
|
-
|
|
289922
|
-
// host pin is only the fallback for a backend that sends none.
|
|
289923
|
-
...cell.environmentId ? { environmentId: cell.environmentId, ephemeralEnvironment: true } : cell.namedHostId ? { namedHostId: cell.namedHostId } : {},
|
|
290851
|
+
// Benchmark cells share the canonical environment launch core.
|
|
290852
|
+
environmentId: cell.environmentId,
|
|
290853
|
+
ephemeralEnvironment: true,
|
|
289924
290854
|
// The CELL's pinned repetition count, not the suite's `runs` default.
|
|
289925
290855
|
// The scorer's `minimumRepetitionsPerRequiredCell` is a publication
|
|
289926
290856
|
// floor, so a cell declared at 3 that runs once is not merely thinner
|