agent-qa 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/capability-catalog/capability-catalog.json +51 -327
- package/dist/cli-main.cjs +5658 -11153
- package/dist/cli-main.cjs.map +1 -1
- package/dist/cli-main.js +4451 -10024
- package/dist/cli-main.js.map +1 -1
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +332 -3875
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -348
- package/dist/index.d.ts +12 -348
- package/dist/index.js +282 -3963
- package/dist/index.js.map +1 -1
- package/package.json +31 -23
- package/skills/agent-qa-authoring/SKILL.md +5 -11
- package/skills/agent-qa-authoring/references/agent-qa-contracts.json +2 -4
- package/skills/agent-qa-debug-fix/SKILL.md +1 -4
- package/skills/agent-qa-knowledge/SKILL.md +2 -4
- package/skills/agent-qa-knowledge/references/knowledge-reads.md +19 -22
- package/skills/agent-qa-result-triage/SKILL.md +0 -2
- package/skills/agent-qa-result-triage/references/triage-categories.md +1 -1
package/dist/index.js
CHANGED
|
@@ -9,29 +9,6 @@ var __export = (target, all) => {
|
|
|
9
9
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
10
|
};
|
|
11
11
|
|
|
12
|
-
// src/hard-cut.ts
|
|
13
|
-
function INVALID_CONFIG_FIX(configPath) {
|
|
14
|
-
return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
|
|
15
|
-
}
|
|
16
|
-
function isRetiredKeyIssue(issue) {
|
|
17
|
-
return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.retired === true;
|
|
18
|
-
}
|
|
19
|
-
var RE_INITIALIZE_URL, HARD_CUT_STEPS;
|
|
20
|
-
var init_hard_cut = __esm({
|
|
21
|
-
"src/hard-cut.ts"() {
|
|
22
|
-
"use strict";
|
|
23
|
-
RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
|
|
24
|
-
HARD_CUT_STEPS = [
|
|
25
|
-
"This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
|
|
26
|
-
" 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
|
|
27
|
-
" 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
|
|
28
|
-
" 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
|
|
29
|
-
" 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
|
|
30
|
-
`What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
|
|
31
|
-
].join("\n");
|
|
32
|
-
}
|
|
33
|
-
});
|
|
34
|
-
|
|
35
12
|
// src/provider-plugins.ts
|
|
36
13
|
var provider_plugins_exports = {};
|
|
37
14
|
__export(provider_plugins_exports, {
|
|
@@ -56,26 +33,11 @@ var init_provider_plugins = __esm({
|
|
|
56
33
|
}
|
|
57
34
|
});
|
|
58
35
|
|
|
36
|
+
// src/index.ts
|
|
37
|
+
import { getAgentQaVersion } from "@vostride/agent-qa-core";
|
|
38
|
+
export * from "@vostride/agent-qa-core";
|
|
39
|
+
|
|
59
40
|
// src/config.ts
|
|
60
|
-
var config_exports = {};
|
|
61
|
-
__export(config_exports, {
|
|
62
|
-
ConfigFileNotFoundError: () => ConfigFileNotFoundError,
|
|
63
|
-
ConfigFileNotYamlError: () => ConfigFileNotYamlError,
|
|
64
|
-
deepMerge: () => deepMerge,
|
|
65
|
-
formatConfigDebug: () => formatConfigDebug,
|
|
66
|
-
loadConfigFile: () => loadConfigFile,
|
|
67
|
-
loadConfigPlugins: () => loadConfigPlugins,
|
|
68
|
-
loadEnvOverrides: () => loadEnvOverrides,
|
|
69
|
-
loadRequiredConfigFile: () => loadRequiredConfigFile,
|
|
70
|
-
mergeConfigs: () => mergeConfigs,
|
|
71
|
-
mergeUseBlocks: () => mergeUseBlocks,
|
|
72
|
-
mergeWithTestConfig: () => mergeWithTestConfig,
|
|
73
|
-
normalizeConfigForValidation: () => normalizeConfigForValidation,
|
|
74
|
-
readAnalyticsPrivacyDeclaration: () => readAnalyticsPrivacyDeclaration,
|
|
75
|
-
resolveConfig: () => resolveConfig,
|
|
76
|
-
resolveConfiguredKnowledgeRoot: () => resolveConfiguredKnowledgeRoot,
|
|
77
|
-
resolveRawConfigKnowledgeRoot: () => resolveRawConfigKnowledgeRoot
|
|
78
|
-
});
|
|
79
41
|
import { readFile } from "fs/promises";
|
|
80
42
|
import { dirname as dirname2, resolve as resolvePath2 } from "path";
|
|
81
43
|
import { parse as parseYaml } from "yaml";
|
|
@@ -87,6 +49,32 @@ import {
|
|
|
87
49
|
resolveKnowledgeRoot,
|
|
88
50
|
yamlParseFailureMessage
|
|
89
51
|
} from "@vostride/agent-qa-core";
|
|
52
|
+
|
|
53
|
+
// src/hard-cut.ts
|
|
54
|
+
var RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
|
|
55
|
+
var HARD_CUT_STEPS = [
|
|
56
|
+
"This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
|
|
57
|
+
" 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
|
|
58
|
+
" 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
|
|
59
|
+
" 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
|
|
60
|
+
" 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
|
|
61
|
+
`What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
|
|
62
|
+
].join("\n");
|
|
63
|
+
function INVALID_CONFIG_FIX(configPath) {
|
|
64
|
+
return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
|
|
65
|
+
}
|
|
66
|
+
function isHardCutIssue(issue) {
|
|
67
|
+
return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.hardCut === true;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// src/config.ts
|
|
71
|
+
var ENV_MAPPING = {
|
|
72
|
+
AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
|
|
73
|
+
AGENT_QA_MCP_PORT: "services.mcp.port",
|
|
74
|
+
AGENT_QA_CACHE_DIR: "services.cache.dir",
|
|
75
|
+
AGENT_QA_LOG_LEVEL: "services.logging.level",
|
|
76
|
+
AGENT_QA_HEADLESS: "use.browser.headless"
|
|
77
|
+
};
|
|
90
78
|
function setNestedValue(obj, path, value) {
|
|
91
79
|
const parts = path.split(".");
|
|
92
80
|
let current = obj;
|
|
@@ -111,6 +99,15 @@ function parseEnvValue(key, value) {
|
|
|
111
99
|
}
|
|
112
100
|
return value;
|
|
113
101
|
}
|
|
102
|
+
var ConfigFileNotYamlError = class extends Error {
|
|
103
|
+
configPath;
|
|
104
|
+
constructor(configPath, cause) {
|
|
105
|
+
const positionOnly = `${configPath}: ${yamlParseFailureMessage(cause)}`;
|
|
106
|
+
super(positionOnly, { cause: new Error(positionOnly) });
|
|
107
|
+
this.name = "ConfigFileNotYamlError";
|
|
108
|
+
this.configPath = configPath;
|
|
109
|
+
}
|
|
110
|
+
};
|
|
114
111
|
async function loadConfigFile(configPath) {
|
|
115
112
|
let content;
|
|
116
113
|
try {
|
|
@@ -125,6 +122,14 @@ async function loadConfigFile(configPath) {
|
|
|
125
122
|
throw new ConfigFileNotYamlError(configPath, err);
|
|
126
123
|
}
|
|
127
124
|
}
|
|
125
|
+
var ConfigFileNotFoundError = class extends Error {
|
|
126
|
+
configPath;
|
|
127
|
+
constructor(configPath) {
|
|
128
|
+
super(`Config file not found: ${configPath}`);
|
|
129
|
+
this.name = "ConfigFileNotFoundError";
|
|
130
|
+
this.configPath = configPath;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
128
133
|
async function loadRequiredConfigFile(configPath) {
|
|
129
134
|
let content;
|
|
130
135
|
try {
|
|
@@ -190,6 +195,7 @@ function mergeUseBlocks(globalUse, suiteUse, testUse, cliFlags) {
|
|
|
190
195
|
cliFlags
|
|
191
196
|
);
|
|
192
197
|
}
|
|
198
|
+
var CIRCULAR_MARKER = "<circular>";
|
|
193
199
|
function flattenObject(obj, prefix = "", ancestors = []) {
|
|
194
200
|
const result = {};
|
|
195
201
|
if (typeof obj !== "object" || obj === null) {
|
|
@@ -287,7 +293,7 @@ Hint: services.dashboard.enabled was removed. Dashboard behavior is enabled by t
|
|
|
287
293
|
const baseUrlHint = baseUrlRefused ? `
|
|
288
294
|
|
|
289
295
|
Hint: services.dashboard.baseUrl must now carry a scheme \u2014 write an absolute URL such as http://127.0.0.1:3470 rather than a bare host like localhost:3470, or remove the key entirely. An empty string is not an absence: absence is a legal and deliberate posture here \u2014 a standalone \`agent-qa mcp\` with no dashboard has no base, and the carrier tool result then names that absence rather than fabricating a URL. The value is refused because it is assigned to an href inside the served App document and appended verbatim to every carrier tool result, so a schemeless or empty base composes a link that resolves against nothing.` : "";
|
|
290
|
-
const closing = result.error.issues.some(
|
|
296
|
+
const closing = result.error.issues.some(isHardCutIssue) ? HARD_CUT_STEPS : INVALID_CONFIG_FIX(configPath);
|
|
291
297
|
throw new Error(
|
|
292
298
|
`Config validation failed:
|
|
293
299
|
${messages.join("\n")}
|
|
@@ -319,39 +325,6 @@ function resolveRawConfigKnowledgeRoot(config, configPath) {
|
|
|
319
325
|
return null;
|
|
320
326
|
}
|
|
321
327
|
}
|
|
322
|
-
var ENV_MAPPING, ConfigFileNotYamlError, ConfigFileNotFoundError, CIRCULAR_MARKER;
|
|
323
|
-
var init_config = __esm({
|
|
324
|
-
"src/config.ts"() {
|
|
325
|
-
"use strict";
|
|
326
|
-
init_hard_cut();
|
|
327
|
-
ENV_MAPPING = {
|
|
328
|
-
AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
|
|
329
|
-
AGENT_QA_MCP_PORT: "services.mcp.port",
|
|
330
|
-
AGENT_QA_CACHE_DIR: "services.cache.dir",
|
|
331
|
-
AGENT_QA_CACHE_TTL: "services.cache.ttl",
|
|
332
|
-
AGENT_QA_LOG_LEVEL: "services.logging.level",
|
|
333
|
-
AGENT_QA_HEADLESS: "use.browser.headless"
|
|
334
|
-
};
|
|
335
|
-
ConfigFileNotYamlError = class extends Error {
|
|
336
|
-
configPath;
|
|
337
|
-
constructor(configPath, cause) {
|
|
338
|
-
const positionOnly = `${configPath}: ${yamlParseFailureMessage(cause)}`;
|
|
339
|
-
super(positionOnly, { cause: new Error(positionOnly) });
|
|
340
|
-
this.name = "ConfigFileNotYamlError";
|
|
341
|
-
this.configPath = configPath;
|
|
342
|
-
}
|
|
343
|
-
};
|
|
344
|
-
ConfigFileNotFoundError = class extends Error {
|
|
345
|
-
configPath;
|
|
346
|
-
constructor(configPath) {
|
|
347
|
-
super(`Config file not found: ${configPath}`);
|
|
348
|
-
this.name = "ConfigFileNotFoundError";
|
|
349
|
-
this.configPath = configPath;
|
|
350
|
-
}
|
|
351
|
-
};
|
|
352
|
-
CIRCULAR_MARKER = "<circular>";
|
|
353
|
-
}
|
|
354
|
-
});
|
|
355
328
|
|
|
356
329
|
// src/targets.ts
|
|
357
330
|
function resolveTarget(config, targetName) {
|
|
@@ -369,8 +342,7 @@ function resolveTarget(config, targetName) {
|
|
|
369
342
|
appPackage: target.appPackage,
|
|
370
343
|
appActivity: target.appActivity,
|
|
371
344
|
device: target.device,
|
|
372
|
-
// Carried
|
|
373
|
-
// target whose author did not declare one (D-06).
|
|
345
|
+
// Carried unchanged, `undefined` included: an undeclared target gets no filled-in class.
|
|
374
346
|
environment: target.environment
|
|
375
347
|
};
|
|
376
348
|
}
|
|
@@ -381,3261 +353,19 @@ function listTargets(config) {
|
|
|
381
353
|
const typedEntry = entry;
|
|
382
354
|
return {
|
|
383
355
|
name,
|
|
384
|
-
platform: typedEntry.platform
|
|
385
|
-
};
|
|
386
|
-
});
|
|
387
|
-
}
|
|
388
|
-
var init_targets = __esm({
|
|
389
|
-
"src/targets.ts"() {
|
|
390
|
-
"use strict";
|
|
391
|
-
}
|
|
392
|
-
});
|
|
393
|
-
|
|
394
|
-
// src/composition/declared-secrets.ts
|
|
395
|
-
import { existsSync as existsSync2, readFileSync as readFileSync4, statSync as statSync2 } from "fs";
|
|
396
|
-
import { parse as parseYaml2 } from "yaml";
|
|
397
|
-
import { resolveWorkspacePaths } from "@vostride/agent-qa-core";
|
|
398
|
-
import { SecretRedactor, SecretStore } from "@vostride/agent-qa-runner";
|
|
399
|
-
function readDeclaredSecrets(secretsPath) {
|
|
400
|
-
if (!existsSync2(secretsPath)) {
|
|
401
|
-
throw new DeclaredSecretsUnavailableError([
|
|
402
|
-
`Error: Secrets file not found: ${secretsPath}`,
|
|
403
|
-
"Create the file before running tests. It may be empty, but it must exist."
|
|
404
|
-
]);
|
|
405
|
-
}
|
|
406
|
-
let content;
|
|
407
|
-
try {
|
|
408
|
-
content = readFileSync4(secretsPath, "utf-8");
|
|
409
|
-
} catch {
|
|
410
|
-
throw new DeclaredSecretsUnavailableError([
|
|
411
|
-
`Error: Secrets file could not be read: ${secretsPath}`,
|
|
412
|
-
"Check the path and file permissions before running tests."
|
|
413
|
-
]);
|
|
414
|
-
}
|
|
415
|
-
try {
|
|
416
|
-
return SecretStore.fromEnvContent(content);
|
|
417
|
-
} catch (err) {
|
|
418
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
419
|
-
throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
function resolveSecretsPath(configPath) {
|
|
423
|
-
const file = parseYaml2(readFileSync4(configPath, "utf-8")) ?? {};
|
|
424
|
-
const config = mergeConfigs(file, loadEnvOverrides(), {});
|
|
425
|
-
const configured = config?.workspace?.secretsFile;
|
|
426
|
-
if (typeof configured !== "string" || configured.trim().length === 0) {
|
|
427
|
-
throw DeclaredSecretsUnavailableError.required();
|
|
428
|
-
}
|
|
429
|
-
const workspace = resolveWorkspacePaths({ config, configPath });
|
|
430
|
-
return workspace.secretsFile.absolutePath;
|
|
431
|
-
}
|
|
432
|
-
function statKey(path) {
|
|
433
|
-
const stat2 = statSync2(path);
|
|
434
|
-
return `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
435
|
-
}
|
|
436
|
-
function createDeclaredSecretMask(configPath) {
|
|
437
|
-
let resolved;
|
|
438
|
-
let cached;
|
|
439
|
-
const currentSecretsPath = () => {
|
|
440
|
-
const configKey = statKey(configPath);
|
|
441
|
-
if (resolved?.configKey !== configKey) {
|
|
442
|
-
resolved = void 0;
|
|
443
|
-
resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
|
|
444
|
-
}
|
|
445
|
-
return resolved.secretsPath;
|
|
446
|
-
};
|
|
447
|
-
const redactor = () => {
|
|
448
|
-
const secretsPath = currentSecretsPath();
|
|
449
|
-
let key;
|
|
450
|
-
try {
|
|
451
|
-
key = `${secretsPath}\0${statKey(secretsPath)}`;
|
|
452
|
-
} catch {
|
|
453
|
-
key = void 0;
|
|
454
|
-
}
|
|
455
|
-
if (key !== void 0 && cached?.key === key) return cached.redactor;
|
|
456
|
-
const current = new SecretRedactor(readDeclaredSecrets(secretsPath));
|
|
457
|
-
cached = key === void 0 ? void 0 : { key, redactor: current };
|
|
458
|
-
return current;
|
|
459
|
-
};
|
|
460
|
-
return { mask: (text) => redactor().redactString(text), redactor };
|
|
461
|
-
}
|
|
462
|
-
var DeclaredSecretsUnavailableError;
|
|
463
|
-
var init_declared_secrets = __esm({
|
|
464
|
-
"src/composition/declared-secrets.ts"() {
|
|
465
|
-
"use strict";
|
|
466
|
-
init_config();
|
|
467
|
-
DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
|
|
468
|
-
lines;
|
|
469
|
-
constructor(lines) {
|
|
470
|
-
super(lines.join(" "));
|
|
471
|
-
this.name = "DeclaredSecretsUnavailableError";
|
|
472
|
-
this.lines = lines;
|
|
473
|
-
}
|
|
474
|
-
static required() {
|
|
475
|
-
return new _DeclaredSecretsUnavailableError([
|
|
476
|
-
"Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
|
|
477
|
-
]);
|
|
478
|
-
}
|
|
479
|
-
};
|
|
480
|
-
}
|
|
481
|
-
});
|
|
482
|
-
|
|
483
|
-
// src/curation/promotion-posture.ts
|
|
484
|
-
import { existsSync as existsSync3, readdirSync as readdirSync3 } from "fs";
|
|
485
|
-
import { dirname as dirname7, join as join8, resolve as resolve3 } from "path";
|
|
486
|
-
function hasNoiseFloorArtefacts(committedRoot) {
|
|
487
|
-
if (existsSync3(join8(committedRoot, "reports", "noise-floor.json"))) return true;
|
|
488
|
-
try {
|
|
489
|
-
return readdirSync3(join8(committedRoot, "reports", "noise-floor")).some((name) => name.endsWith(".json"));
|
|
490
|
-
} catch (error) {
|
|
491
|
-
return error.code !== "ENOENT";
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
function renderPromotionLine(entry) {
|
|
495
|
-
return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
|
|
496
|
-
}
|
|
497
|
-
var init_promotion_posture = __esm({
|
|
498
|
-
"src/curation/promotion-posture.ts"() {
|
|
499
|
-
"use strict";
|
|
500
|
-
}
|
|
501
|
-
});
|
|
502
|
-
|
|
503
|
-
// src/eval-report-render.ts
|
|
504
|
-
function counterLines(title, counts) {
|
|
505
|
-
const lines = [`### ${title}`, ""];
|
|
506
|
-
const keys = Object.keys(counts).sort();
|
|
507
|
-
if (keys.length === 0) {
|
|
508
|
-
lines.push("- none");
|
|
509
|
-
} else {
|
|
510
|
-
for (const key of keys) lines.push(`- ${key}: ${counts[key]}`);
|
|
511
|
-
}
|
|
512
|
-
lines.push("");
|
|
513
|
-
return lines;
|
|
514
|
-
}
|
|
515
|
-
function rateLine(name, metric) {
|
|
516
|
-
const rate = metric.rate === void 0 ? "n/a (zero denominator)" : `${metric.rate}`;
|
|
517
|
-
return `- ${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
|
|
518
|
-
}
|
|
519
|
-
function metricLine(name, metric) {
|
|
520
|
-
if (metric === void 0) return `${name}: ${notSupplied}`;
|
|
521
|
-
const rate = metric.denominator === 0 ? "unavailable (zero denominator)" : metric.rate ?? notSupplied;
|
|
522
|
-
return `${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
|
|
523
|
-
}
|
|
524
|
-
function recoveryLines(name, recovery) {
|
|
525
|
-
const weightSum = recovery?.weightSum;
|
|
526
|
-
const weighted = weightSum === 0 ? "unavailable (zero weightSum)" : recovery?.weightedAccuracy ?? notSupplied;
|
|
527
|
-
return [
|
|
528
|
-
metricLine(`${name}.recoveryAccuracy`, recovery === void 0 ? void 0 : {
|
|
529
|
-
numerator: recovery.recoveredPositiveCount,
|
|
530
|
-
denominator: recovery.eligiblePositiveCount,
|
|
531
|
-
...recovery.recoveryAccuracy === void 0 ? {} : { rate: recovery.recoveryAccuracy }
|
|
532
|
-
}),
|
|
533
|
-
`${name}.weightedAccuracy: ${weighted} (weightSum: ${weightSum ?? notSupplied})`,
|
|
534
|
-
`${name}.pairsWithoutWeight: ${recovery?.pairsWithoutWeight ?? notSupplied}`
|
|
535
|
-
];
|
|
536
|
-
}
|
|
537
|
-
function splitLines(name, cell) {
|
|
538
|
-
const cost = cell?.perDecisionCost;
|
|
539
|
-
return [
|
|
540
|
-
metricLine(`${name}.firstAttemptDecisionAccuracy`, cell?.firstAttemptDecisionAccuracy),
|
|
541
|
-
metricLine(`${name}.verifierFalsePassRate`, cell?.verifierFalsePassRate),
|
|
542
|
-
...recoveryLines(`${name}.recovery`, cell?.recovery),
|
|
543
|
-
`${name}.perDecisionCost: ` + (cost === void 0 ? notSupplied : `decisionCount=${cost.decisionCount} promptTokens=${cost.promptTokens} completionTokens=${cost.completionTokens} totalTokens=${cost.totalTokens} durationMs=${cost.durationMs}`)
|
|
544
|
-
];
|
|
545
|
-
}
|
|
546
|
-
function renderReportLabelMetricLines(report) {
|
|
547
|
-
const calibration = report.calibration;
|
|
548
|
-
return [
|
|
549
|
-
metricLine("verifierFalsePassRate", report.headline.verifierFalsePassRate),
|
|
550
|
-
`calibration.status: ${calibration?.status ?? notSupplied}`,
|
|
551
|
-
`calibration.n_gold: ${calibration?.n_gold ?? notSupplied}`,
|
|
552
|
-
`calibration.precision: ${calibration?.precision ?? notSupplied}`,
|
|
553
|
-
metricLine("calibration.verifierFalsePassRate", calibration?.verifierFalsePassRate),
|
|
554
|
-
...recoveryLines("recovery", report.recovery),
|
|
555
|
-
...splitLines("perSplit.natural", report.perSplit?.natural),
|
|
556
|
-
...splitLines("perSplit.held-out.shared", report.perSplit?.["held-out"]?.shared),
|
|
557
|
-
...splitLines("perSplit.held-out.private", report.perSplit?.["held-out"]?.private),
|
|
558
|
-
...splitLines("perSplit.adversarial", report.perSplit?.adversarial),
|
|
559
|
-
`perSplit.unassignedCount: ${report.perSplit?.unassignedCount ?? notSupplied}`
|
|
560
|
-
];
|
|
561
|
-
}
|
|
562
|
-
function renderEvalReportMarkdown(report) {
|
|
563
|
-
const lines = ["# Eval Baseline Report", ""];
|
|
564
|
-
lines.push("## Provenance", "");
|
|
565
|
-
lines.push(`- batchKey: ${report.batchKey}`);
|
|
566
|
-
lines.push(`- matcherVersion: ${report.matcherVersion}`);
|
|
567
|
-
if (report.judgeIdentity === "mixed") {
|
|
568
|
-
lines.push(
|
|
569
|
-
`- judge: mixed identities across score records (${(report.mixedJudgeModelIds ?? []).join(", ")}) \u2014 no judge block, not a gate comparator`
|
|
570
|
-
);
|
|
571
|
-
} else if (report.judge === void 0) {
|
|
572
|
-
lines.push("- judge: no judge invoked");
|
|
573
|
-
} else {
|
|
574
|
-
lines.push(
|
|
575
|
-
`- judge: judgeModelId=${report.judge.judgeModelId} judgeFamily=${report.judge.judgeFamily} judgeVersion=${report.judge.judgeVersion} confidenceThreshold=${report.judge.confidenceThreshold}`
|
|
576
|
-
);
|
|
577
|
-
if (report.judge.familyOverride !== void 0) {
|
|
578
|
-
lines.push(
|
|
579
|
-
`- judge family override: ${report.judge.familyOverride} (self-preference bias condition disclosed; owner override of D-10, 2026-07-12)`
|
|
580
|
-
);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
const calibration = report.judge?.calibration;
|
|
584
|
-
if (calibration !== void 0) {
|
|
585
|
-
lines.push("- judge calibration: " + (calibration.status === "capability-gap" ? `capability-gap reason=${calibration.reason}` : calibration.status === "uncalibrated" ? `uncalibrated n=${calibration.n}` + (calibration.provenance === "unavailable" ? " (unavailable)" : " (copied)") : `calibrated n=${calibration.n} pooledAgreement=${calibration.pooledAgreement} wilson95=[${calibration.wilson95.lower},${calibration.wilson95.upper}] halfWidth=${calibration.wilson95.halfWidth} (copied)`));
|
|
586
|
-
}
|
|
587
|
-
lines.push("");
|
|
588
|
-
const headline = report.headline;
|
|
589
|
-
lines.push("## Headline", "");
|
|
590
|
-
lines.push(rateLine("firstAttemptDecisionAccuracy", headline.firstAttemptDecisionAccuracy));
|
|
591
|
-
lines.push("- perDecisionCost:");
|
|
592
|
-
lines.push(` - decisionCount: ${headline.perDecisionCost.decisionCount}`);
|
|
593
|
-
lines.push(` - promptTokens: ${headline.perDecisionCost.promptTokens}`);
|
|
594
|
-
lines.push(` - completionTokens: ${headline.perDecisionCost.completionTokens}`);
|
|
595
|
-
lines.push(` - totalTokens: ${headline.perDecisionCost.totalTokens}`);
|
|
596
|
-
lines.push(` - durationMs: ${headline.perDecisionCost.durationMs}`);
|
|
597
|
-
lines.push("");
|
|
598
|
-
lines.push("### Label metrics", "", ...renderReportLabelMetricLines(report).map((line) => `- ${line}`), "");
|
|
599
|
-
const honesty = report.honesty;
|
|
600
|
-
lines.push("## Honesty Counters", "");
|
|
601
|
-
lines.push(...counterLines("tieCounts", honesty.tieCounts));
|
|
602
|
-
lines.push(...counterLines("unscorableIdentityCounts", honesty.unscorableIdentityCounts));
|
|
603
|
-
lines.push(...counterLines("refusalCounts", honesty.refusalCounts));
|
|
604
|
-
lines.push(...counterLines("missByFailureCounts", honesty.missByFailureCounts));
|
|
605
|
-
lines.push(`- harnessErrorCount: ${honesty.harnessErrorCount}`, "");
|
|
606
|
-
lines.push(...counterLines("neverScoredCounts", honesty.neverScoredCounts));
|
|
607
|
-
lines.push(...counterLines("skippedCounts", honesty.skippedCounts));
|
|
608
|
-
lines.push(...counterLines("judge invoked", honesty.judge.invoked));
|
|
609
|
-
lines.push(...counterLines("judge settled", honesty.judge.settled));
|
|
610
|
-
lines.push(...counterLines("judge abstained", honesty.judge.abstained));
|
|
611
|
-
lines.push(...counterLines("judge errors", honesty.judge.errors));
|
|
612
|
-
lines.push(...counterLines("judge notInvoked", honesty.judge.notInvoked));
|
|
613
|
-
lines.push("### judge cost", "");
|
|
614
|
-
lines.push(`- calls: ${honesty.judge.cost.calls}`);
|
|
615
|
-
lines.push(`- inputTokens: ${honesty.judge.cost.inputTokens}`);
|
|
616
|
-
lines.push(`- outputTokens: ${honesty.judge.cost.outputTokens}`);
|
|
617
|
-
lines.push(`- outcomesMissingUsage: ${honesty.judge.cost.outcomesMissingUsage}`);
|
|
618
|
-
lines.push("");
|
|
619
|
-
lines.push("## Axis Distributions", "");
|
|
620
|
-
lines.push(...counterLines("elementIdentity", report.axisDistributions.elementIdentity));
|
|
621
|
-
lines.push(...counterLines("value", report.axisDistributions.value));
|
|
622
|
-
lines.push(...counterLines("opClass", report.axisDistributions.opClass));
|
|
623
|
-
lines.push(...counterLines("sideEffect", report.axisDistributions.sideEffect));
|
|
624
|
-
lines.push(...counterLines("loop", report.axisDistributions.loop));
|
|
625
|
-
const recovery = report.recovery;
|
|
626
|
-
lines.push("## Recovery", "");
|
|
627
|
-
lines.push(`- selfHealPairCount: ${recovery.selfHealPairCount}`);
|
|
628
|
-
lines.push(`- recoveredPositiveCount: ${recovery.recoveredPositiveCount}`);
|
|
629
|
-
lines.push(`- eligiblePositiveCount: ${recovery.eligiblePositiveCount}`);
|
|
630
|
-
lines.push(
|
|
631
|
-
`- recoveryAccuracy: ${recovery.recoveryAccuracy === void 0 ? "n/a (no eligible positives)" : recovery.recoveryAccuracy}`
|
|
632
|
-
);
|
|
633
|
-
lines.push(`- negativeDecisionCount: ${recovery.negativeDecisionCount}`);
|
|
634
|
-
lines.push(`- negativeFullMatchCount: ${recovery.negativeFullMatchCount}`);
|
|
635
|
-
lines.push("");
|
|
636
|
-
return lines.join("\n");
|
|
637
|
-
}
|
|
638
|
-
var notSupplied;
|
|
639
|
-
var init_eval_report_render = __esm({
|
|
640
|
-
"src/eval-report-render.ts"() {
|
|
641
|
-
"use strict";
|
|
642
|
-
notSupplied = "unavailable (not supplied)";
|
|
643
|
-
}
|
|
644
|
-
});
|
|
645
|
-
|
|
646
|
-
// src/model-defaults.ts
|
|
647
|
-
var DEFAULT_ANTHROPIC_MODEL;
|
|
648
|
-
var init_model_defaults = __esm({
|
|
649
|
-
"src/model-defaults.ts"() {
|
|
650
|
-
"use strict";
|
|
651
|
-
DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-5-5";
|
|
652
|
-
}
|
|
653
|
-
});
|
|
654
|
-
|
|
655
|
-
// src/llm-utils.ts
|
|
656
|
-
var llm_utils_exports = {};
|
|
657
|
-
__export(llm_utils_exports, {
|
|
658
|
-
applyResolvedAuthToModelConfig: () => applyResolvedAuthToModelConfig,
|
|
659
|
-
buildAnalysisTokenBudget: () => buildAnalysisTokenBudget,
|
|
660
|
-
buildJobModelCapability: () => buildJobModelCapability,
|
|
661
|
-
loadAuthPluginsForRawConfig: () => loadAuthPluginsForRawConfig,
|
|
662
|
-
loadProviderPluginsForRawConfig: () => loadProviderPluginsForRawConfig,
|
|
663
|
-
resolveCliAnalysisTokenBudget: () => resolveCliAnalysisTokenBudget,
|
|
664
|
-
resolveCliChatCapabilities: () => resolveCliChatCapabilities,
|
|
665
|
-
resolveCliJobModel: () => resolveCliJobModel,
|
|
666
|
-
resolveCredentials: () => resolveCredentials,
|
|
667
|
-
resolveLLMModels: () => resolveLLMModels,
|
|
668
|
-
resolveModelAuth: () => resolveModelAuth,
|
|
669
|
-
resolveNamedConfig: () => resolveNamedConfig,
|
|
670
|
-
toModelProfileAndAuth: () => toModelProfileAndAuth
|
|
671
|
-
});
|
|
672
|
-
import { dirname as dirname8, resolve as resolvePath3 } from "path";
|
|
673
|
-
import { createModel as createModel2, createTokenAccounting as createTokenAccounting2, getProviderOptions, resolveCapabilities as resolveCapabilities2, resolveVisionGate } from "@vostride/agent-qa-model";
|
|
674
|
-
import { generateText as generateText2 } from "ai";
|
|
675
|
-
function normalizeLLMConfig(raw) {
|
|
676
|
-
return {
|
|
677
|
-
name: raw.name,
|
|
678
|
-
provider: raw.provider,
|
|
679
|
-
model: raw.model,
|
|
680
|
-
...raw.baseURL !== void 0 ? { baseURL: raw.baseURL } : {},
|
|
681
|
-
...raw.providerHeaders !== void 0 ? { providerHeaders: raw.providerHeaders } : {},
|
|
682
|
-
...raw.vision !== void 0 ? { vision: raw.vision } : {},
|
|
683
|
-
...raw.contextWindow !== void 0 ? { contextWindow: raw.contextWindow } : {}
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
|
-
async function loadAuthPluginsForRawConfig(raw, configPath = "agent-qa.config.yaml") {
|
|
687
|
-
const declarations = raw?.plugins?.auth;
|
|
688
|
-
if (!Array.isArray(declarations) || declarations.length === 0) return;
|
|
689
|
-
const { loadLLMAuthPlugins: loadLLMAuthPlugins2 } = await import("@vostride/agent-qa-core");
|
|
690
|
-
await loadLLMAuthPlugins2(declarations, {
|
|
691
|
-
baseDir: dirname8(resolvePath3(configPath))
|
|
692
|
-
});
|
|
693
|
-
}
|
|
694
|
-
async function resolveNamedConfig(configName, configPath = "agent-qa.config.yaml", options = {}) {
|
|
695
|
-
const { loadConfigFile: loadConfigFile2 } = await Promise.resolve().then(() => (init_config(), config_exports));
|
|
696
|
-
const raw = await loadConfigFile2(configPath);
|
|
697
|
-
if (options.keyOnly !== true) {
|
|
698
|
-
await loadAuthPluginsForRawConfig(raw, configPath);
|
|
699
|
-
await loadProviderPluginsForRawConfig(raw, configPath);
|
|
700
|
-
}
|
|
701
|
-
const llms = raw?.registry?.llms?.map(normalizeLLMConfig);
|
|
702
|
-
const defaultLLM = raw?.use?.llm;
|
|
703
|
-
if (!llms?.length) {
|
|
704
|
-
throw new Error("No LLM configs found. Run `agent-qa init` to set up your config.");
|
|
705
|
-
}
|
|
706
|
-
const targetName = configName ?? defaultLLM;
|
|
707
|
-
if (!targetName) {
|
|
708
|
-
throw new Error("No config name specified and no use.llm set in config.");
|
|
709
|
-
}
|
|
710
|
-
const found = llms.find((c) => c.name === targetName);
|
|
711
|
-
if (!found) {
|
|
712
|
-
throw new Error(`Config "${targetName}" not found. Available: ${llms.map((c) => c.name).join(", ")}`);
|
|
713
|
-
}
|
|
714
|
-
return { config: found, allConfigs: llms, defaultName: defaultLLM ?? "" };
|
|
715
|
-
}
|
|
716
|
-
function resolveLLMModels(config) {
|
|
717
|
-
const defaults = { provider: "anthropic-subscription", model: DEFAULT_ANTHROPIC_MODEL };
|
|
718
|
-
if (!config.registry?.llms || !config.use?.llm) {
|
|
719
|
-
return { planner: defaults, verifier: defaults, configName: "" };
|
|
720
|
-
}
|
|
721
|
-
const found = config.registry.llms.find((c) => c.name === config.use.llm);
|
|
722
|
-
if (!found) {
|
|
723
|
-
return { planner: defaults, verifier: defaults, configName: "" };
|
|
724
|
-
}
|
|
725
|
-
const { name, ...modelFields } = normalizeLLMConfig(found);
|
|
726
|
-
return { planner: modelFields, verifier: modelFields, configName: name };
|
|
727
|
-
}
|
|
728
|
-
async function resolveModelAuth(configName, llmConfig) {
|
|
729
|
-
const { resolveLLMAuth } = await import("@vostride/agent-qa-core");
|
|
730
|
-
return resolveLLMAuth(configName, llmConfig);
|
|
731
|
-
}
|
|
732
|
-
function applyResolvedAuthToModelConfig(llmConfig, auth) {
|
|
733
|
-
switch (auth.kind) {
|
|
734
|
-
case "api-key":
|
|
735
|
-
return { ...llmConfig, apiKey: auth.apiKey };
|
|
736
|
-
case "bearer-token":
|
|
737
|
-
return { ...llmConfig, authToken: auth.token };
|
|
738
|
-
case "auth-fetch":
|
|
739
|
-
return { ...llmConfig, fetch: auth.fetch, modelAdapter: auth.modelAdapter };
|
|
740
|
-
case "unauthenticated":
|
|
741
|
-
return { ...llmConfig };
|
|
742
|
-
case "missing":
|
|
743
|
-
throw new Error(auth.message);
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
function toModelProfileAndAuth(llmConfig, resolved) {
|
|
747
|
-
const providerOptions = getProviderOptions({
|
|
748
|
-
provider: llmConfig.provider,
|
|
749
|
-
modelAdapter: resolved.kind === "auth-fetch" ? resolved.modelAdapter : void 0,
|
|
750
|
-
fetch: resolved.kind === "auth-fetch" ? resolved.fetch : void 0
|
|
751
|
-
});
|
|
752
|
-
const profile = {
|
|
753
|
-
provider: llmConfig.provider,
|
|
754
|
-
model: llmConfig.model,
|
|
755
|
-
...llmConfig.baseURL !== void 0 ? { baseURL: llmConfig.baseURL } : {},
|
|
756
|
-
...llmConfig.providerHeaders !== void 0 ? { headers: llmConfig.providerHeaders } : {},
|
|
757
|
-
...providerOptions ? { providerOptions } : {},
|
|
758
|
-
auth: toAuthDeclaration(resolved)
|
|
759
|
-
};
|
|
760
|
-
switch (resolved.kind) {
|
|
761
|
-
case "api-key":
|
|
762
|
-
return { profile, auth: { kind: "api-key", apiKey: resolved.apiKey } };
|
|
763
|
-
case "bearer-token":
|
|
764
|
-
return { profile, auth: { kind: "bearer-token", token: resolved.token } };
|
|
765
|
-
case "auth-fetch":
|
|
766
|
-
return { profile, auth: { kind: "auth-fetch", modelAdapter: resolved.modelAdapter, fetch: resolved.fetch } };
|
|
767
|
-
case "unauthenticated":
|
|
768
|
-
return { profile, auth: { kind: "none" } };
|
|
769
|
-
case "missing":
|
|
770
|
-
throw new Error(resolved.message);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
function toAuthDeclaration(resolved) {
|
|
774
|
-
switch (resolved.kind) {
|
|
775
|
-
case "api-key":
|
|
776
|
-
return { mode: "api-key" };
|
|
777
|
-
case "bearer-token":
|
|
778
|
-
return { mode: "bearer-token" };
|
|
779
|
-
case "auth-fetch":
|
|
780
|
-
return { mode: "subscription", plugin: resolved.provider };
|
|
781
|
-
case "unauthenticated":
|
|
782
|
-
case "missing":
|
|
783
|
-
return { mode: "none" };
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
function buildJobModelCapability(profile, auth, mask, deps = {}) {
|
|
787
|
-
const model = (deps.createModelImpl ?? createModel2)(profile, auth, mask);
|
|
788
|
-
const accounting = (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
|
|
789
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
|
|
790
|
-
});
|
|
791
|
-
const generate = deps.generateTextImpl ?? generateText2;
|
|
792
|
-
const vision = (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile).vision;
|
|
793
|
-
return {
|
|
794
|
-
modelId: `${profile.provider}:${profile.model}`,
|
|
795
|
-
vision,
|
|
796
|
-
async generate(request2) {
|
|
797
|
-
const base = {
|
|
798
|
-
model,
|
|
799
|
-
maxRetries: 0,
|
|
800
|
-
temperature: 0,
|
|
801
|
-
maxOutputTokens: accounting.getCeilings().maxOutput,
|
|
802
|
-
...request2.system === void 0 ? {} : { system: request2.system },
|
|
803
|
-
...request2.signal === void 0 ? {} : { abortSignal: request2.signal }
|
|
804
|
-
};
|
|
805
|
-
if (request2.images !== void 0 && request2.images.length > 0 && resolveVisionGate({ vision }).attachImage) {
|
|
806
|
-
const result2 = await generate({
|
|
807
|
-
...base,
|
|
808
|
-
messages: [
|
|
809
|
-
{
|
|
810
|
-
role: "user",
|
|
811
|
-
content: [
|
|
812
|
-
{ type: "text", text: request2.prompt },
|
|
813
|
-
...request2.images.map((image) => ({ type: "image", image }))
|
|
814
|
-
]
|
|
815
|
-
}
|
|
816
|
-
]
|
|
817
|
-
});
|
|
818
|
-
return { text: result2.text };
|
|
819
|
-
}
|
|
820
|
-
const result = await generate({
|
|
821
|
-
...base,
|
|
822
|
-
prompt: request2.prompt
|
|
823
|
-
});
|
|
824
|
-
return { text: result.text };
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
}
|
|
828
|
-
function buildAnalysisTokenBudget(profile, deps = {}) {
|
|
829
|
-
const accounting = (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
|
|
830
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
|
|
831
|
-
});
|
|
832
|
-
const ceilings = accounting.getCeilings();
|
|
833
|
-
return {
|
|
834
|
-
estimate: (text) => accounting.estimate([{ role: "user", content: text }]),
|
|
835
|
-
inputBudget: Math.max(0, ceilings.contextWindow - ceilings.maxOutput)
|
|
836
|
-
};
|
|
837
|
-
}
|
|
838
|
-
async function resolveCliAnalysisTokenBudget(configPath, deps = {}) {
|
|
839
|
-
try {
|
|
840
|
-
const { config } = await resolveNamedConfig(void 0, configPath);
|
|
841
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
842
|
-
const { profile } = toModelProfileAndAuth(config, resolved);
|
|
843
|
-
return buildAnalysisTokenBudget(profile, deps);
|
|
844
|
-
} catch {
|
|
845
|
-
return void 0;
|
|
846
|
-
}
|
|
847
|
-
}
|
|
848
|
-
async function resolveCliJobModel(configPath, deps = {}) {
|
|
849
|
-
const { config } = await resolveNamedConfig(void 0, configPath);
|
|
850
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
851
|
-
const { profile, auth } = toModelProfileAndAuth(config, resolved);
|
|
852
|
-
return buildJobModelCapability(profile, auth, createDeclaredSecretMask(configPath).mask, deps);
|
|
853
|
-
}
|
|
854
|
-
async function resolveCliChatCapabilities(configPath, deps = {}) {
|
|
855
|
-
const { mask } = createDeclaredSecretMask(configPath);
|
|
856
|
-
const build = async (configName) => {
|
|
857
|
-
const { config } = await resolveNamedConfig(configName, configPath);
|
|
858
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
859
|
-
const { profile, auth } = toModelProfileAndAuth(config, resolved);
|
|
860
|
-
return {
|
|
861
|
-
model: (deps.createModelImpl ?? createModel2)(profile, auth, mask),
|
|
862
|
-
accounting: (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
|
|
863
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
|
|
864
|
-
})
|
|
865
|
-
};
|
|
866
|
-
};
|
|
867
|
-
try {
|
|
868
|
-
const opened = await build();
|
|
869
|
-
return {
|
|
870
|
-
model: opened.model,
|
|
871
|
-
accounting: opened.accounting,
|
|
872
|
-
// A THROW rather than a degrade on this arm: `/model nonsense` is a person naming something,
|
|
873
|
-
// and the honest answer is the refusal that says which name failed. The turn engine's one
|
|
874
|
-
// refusal path carries it, leaving the same countable row any other refusal leaves.
|
|
875
|
-
modelFor: async (name) => (await build(name)).model
|
|
876
|
-
};
|
|
877
|
-
} catch {
|
|
878
|
-
return void 0;
|
|
879
|
-
}
|
|
880
|
-
}
|
|
881
|
-
async function resolveCredentials(configName, llmConfig) {
|
|
882
|
-
try {
|
|
883
|
-
const auth = await resolveModelAuth(configName, llmConfig);
|
|
884
|
-
if (auth.kind === "api-key") return { apiKey: auth.apiKey };
|
|
885
|
-
if (auth.kind === "bearer-token") return { authToken: auth.token };
|
|
886
|
-
if (auth.kind === "auth-fetch") return { authFetch: auth.fetch };
|
|
887
|
-
return {};
|
|
888
|
-
} catch {
|
|
889
|
-
return {};
|
|
890
|
-
}
|
|
891
|
-
}
|
|
892
|
-
var init_llm_utils = __esm({
|
|
893
|
-
"src/llm-utils.ts"() {
|
|
894
|
-
"use strict";
|
|
895
|
-
init_declared_secrets();
|
|
896
|
-
init_model_defaults();
|
|
897
|
-
init_provider_plugins();
|
|
898
|
-
}
|
|
899
|
-
});
|
|
900
|
-
|
|
901
|
-
// src/judge-panel-preflight.ts
|
|
902
|
-
async function preflightJudgePanel(session, options = {}, deps = {}) {
|
|
903
|
-
const minN = resolveJudgeCalibrationMinimum(options.minN);
|
|
904
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
905
|
-
if (names.length < 2) return {
|
|
906
|
-
kind: "capability-gap",
|
|
907
|
-
status: "capability-gap",
|
|
908
|
-
reason: "panel-not-configured",
|
|
909
|
-
minN,
|
|
910
|
-
entries: []
|
|
911
|
-
};
|
|
912
|
-
const named = deps.resolveNamedConfig ?? resolveNamedConfig;
|
|
913
|
-
const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
|
|
914
|
-
const rows = [];
|
|
915
|
-
for (const [index, name] of names.entries()) {
|
|
916
|
-
const diagnostic = {
|
|
917
|
-
index,
|
|
918
|
-
name,
|
|
919
|
-
families: [],
|
|
920
|
-
credential: "resolution-failed",
|
|
921
|
-
failures: []
|
|
922
|
-
};
|
|
923
|
-
const row = { diagnostic };
|
|
924
|
-
try {
|
|
925
|
-
const { config } = await named(name, session.configPath, { keyOnly: true });
|
|
926
|
-
row.config = config;
|
|
927
|
-
diagnostic.modelId = config.model;
|
|
928
|
-
diagnostic.families = [...resolveJudgeFamilies(config.model)].sort();
|
|
929
|
-
} catch {
|
|
930
|
-
diagnostic.failures = ["config-unresolved"];
|
|
931
|
-
}
|
|
932
|
-
if (row.config) {
|
|
933
|
-
try {
|
|
934
|
-
const auth = await resolveAuth(row.config.name, row.config);
|
|
935
|
-
row.auth = auth;
|
|
936
|
-
diagnostic.credential = auth.kind === "missing" || auth.kind === "unauthenticated" ? auth.kind : "present";
|
|
937
|
-
} catch {
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
if (diagnostic.credential !== "present") diagnostic.failures = [...diagnostic.failures, "no-credential"];
|
|
941
|
-
rows.push(row);
|
|
942
|
-
}
|
|
943
|
-
let replayProfile;
|
|
944
|
-
try {
|
|
945
|
-
replayProfile = await resolveReplayProfileDeclaration(session, named);
|
|
946
|
-
} catch {
|
|
947
|
-
throw new Error("Cannot resolve the replay profile for judge panel preflight. Check use.llm and its registry.llms entry.");
|
|
948
|
-
}
|
|
949
|
-
const replayFamilies = resolveJudgeFamilies(replayProfile.model);
|
|
950
|
-
const sets = rows.map((row) => resolveJudgeFamilies(row.config?.model ?? ""));
|
|
951
|
-
for (const [index, row] of rows.entries()) {
|
|
952
|
-
const failures = [...row.diagnostic.failures];
|
|
953
|
-
if (row.config && (sets[index].size === 0 || replayFamilies.size === 0)) failures.push("unknown-family");
|
|
954
|
-
if (judgeFamiliesIntersect(sets[index], replayFamilies)) failures.push("replay-family-collision");
|
|
955
|
-
if (sets.some((set, other) => other !== index && judgeFamiliesIntersect(sets[index], set))) failures.push("member-family-collision");
|
|
956
|
-
row.diagnostic.failures = failures;
|
|
957
|
-
}
|
|
958
|
-
const entries = rows.map(({ diagnostic }) => ({
|
|
959
|
-
...diagnostic,
|
|
960
|
-
name: session.mask(diagnostic.name),
|
|
961
|
-
...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
|
|
962
|
-
}));
|
|
963
|
-
const modelIds = rows.flatMap((row) => row.config ? [row.config.model] : []);
|
|
964
|
-
const complete = modelIds.length === names.length;
|
|
965
|
-
const identity = complete ? createPanelIdentity(modelIds) : void 0;
|
|
966
|
-
const safeIdentity = [replayProfile.model, ...modelIds].every((id) => session.mask(id) === id) ? identity : void 0;
|
|
967
|
-
const gap = (reason) => ({
|
|
968
|
-
kind: "capability-gap",
|
|
969
|
-
status: "capability-gap",
|
|
970
|
-
reason,
|
|
971
|
-
minN,
|
|
972
|
-
entries,
|
|
973
|
-
...safeIdentity === void 0 ? {} : { identity: safeIdentity }
|
|
974
|
-
});
|
|
975
|
-
if (rows.some((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
|
|
976
|
-
const eligibility = assessPanelEligibility({ memberModelIds: modelIds, replayedModelId: replayProfile.model });
|
|
977
|
-
if (!eligibility.eligible) return gap("single-family-host");
|
|
978
|
-
return {
|
|
979
|
-
kind: "eligible",
|
|
980
|
-
status: "uncalibrated",
|
|
981
|
-
n: 0,
|
|
982
|
-
minN,
|
|
983
|
-
entries,
|
|
984
|
-
identity,
|
|
985
|
-
replayProfile,
|
|
986
|
-
members: rows.map((row) => toModelProfileAndAuth(row.config, row.auth))
|
|
987
|
-
};
|
|
988
|
-
}
|
|
989
|
-
var init_judge_panel_preflight = __esm({
|
|
990
|
-
"src/judge-panel-preflight.ts"() {
|
|
991
|
-
"use strict";
|
|
992
|
-
init_eval_session_composition();
|
|
993
|
-
init_llm_utils();
|
|
994
|
-
}
|
|
995
|
-
});
|
|
996
|
-
|
|
997
|
-
// src/adversarial-probe-preflight.ts
|
|
998
|
-
var adversarial_probe_preflight_exports = {};
|
|
999
|
-
__export(adversarial_probe_preflight_exports, {
|
|
1000
|
-
resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability
|
|
1001
|
-
});
|
|
1002
|
-
function authoringFamily(modelId) {
|
|
1003
|
-
return [...resolveJudgeFamilies(modelId)].sort().join(",");
|
|
1004
|
-
}
|
|
1005
|
-
async function resolveAdversarialProbeCapability(session, deps = {}) {
|
|
1006
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
1007
|
-
if (names.length === 0) {
|
|
1008
|
-
return { kind: "no-probe-model", reason: "panel-not-configured", entries: [] };
|
|
1009
|
-
}
|
|
1010
|
-
const named = deps.resolveNamedConfig ?? resolveNamedConfig;
|
|
1011
|
-
const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
|
|
1012
|
-
const rows = [];
|
|
1013
|
-
for (const [index, name] of names.entries()) {
|
|
1014
|
-
const diagnostic = {
|
|
1015
|
-
index,
|
|
1016
|
-
name,
|
|
1017
|
-
families: [],
|
|
1018
|
-
credential: "resolution-failed",
|
|
1019
|
-
failures: []
|
|
1020
|
-
};
|
|
1021
|
-
const row = { diagnostic };
|
|
1022
|
-
try {
|
|
1023
|
-
const { config } = await named(name, session.configPath, { keyOnly: true });
|
|
1024
|
-
row.config = config;
|
|
1025
|
-
diagnostic.modelId = config.model;
|
|
1026
|
-
diagnostic.families = [...resolveJudgeFamilies(config.model)].sort();
|
|
1027
|
-
} catch {
|
|
1028
|
-
diagnostic.failures = ["config-unresolved"];
|
|
1029
|
-
}
|
|
1030
|
-
if (row.config) {
|
|
1031
|
-
try {
|
|
1032
|
-
const auth2 = await resolveAuth(row.config.name, row.config);
|
|
1033
|
-
row.auth = auth2;
|
|
1034
|
-
diagnostic.credential = auth2.kind === "missing" || auth2.kind === "unauthenticated" ? auth2.kind : "present";
|
|
1035
|
-
} catch {
|
|
1036
|
-
}
|
|
1037
|
-
}
|
|
1038
|
-
if (diagnostic.credential !== "present") {
|
|
1039
|
-
diagnostic.failures = [...diagnostic.failures, "no-credential"];
|
|
1040
|
-
}
|
|
1041
|
-
rows.push(row);
|
|
1042
|
-
}
|
|
1043
|
-
let replayProfile;
|
|
1044
|
-
try {
|
|
1045
|
-
replayProfile = await resolveReplayProfileDeclaration(session, named);
|
|
1046
|
-
} catch {
|
|
1047
|
-
throw new Error("Cannot resolve the replay profile for adversarial probe preflight. Check use.llm and its registry.llms entry.");
|
|
1048
|
-
}
|
|
1049
|
-
const replayFamilies = resolveJudgeFamilies(replayProfile.model);
|
|
1050
|
-
for (const row of rows) {
|
|
1051
|
-
if (!row.config) continue;
|
|
1052
|
-
const families = resolveJudgeFamilies(row.config.model);
|
|
1053
|
-
row.eligible = assessPanelEligibility({
|
|
1054
|
-
memberModelIds: [row.config.model],
|
|
1055
|
-
replayedModelId: replayProfile.model
|
|
1056
|
-
}).eligible;
|
|
1057
|
-
const failures = [...row.diagnostic.failures];
|
|
1058
|
-
if (families.size === 0 || replayFamilies.size === 0) failures.push("unknown-family");
|
|
1059
|
-
else if (!row.eligible) failures.push("replay-family-collision");
|
|
1060
|
-
row.diagnostic.failures = failures;
|
|
1061
|
-
}
|
|
1062
|
-
const entries = rows.map(({ diagnostic }) => ({
|
|
1063
|
-
...diagnostic,
|
|
1064
|
-
name: session.mask(diagnostic.name),
|
|
1065
|
-
...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
|
|
1066
|
-
}));
|
|
1067
|
-
const gap = (reason) => ({ kind: "no-probe-model", reason, entries });
|
|
1068
|
-
if (rows.every((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
|
|
1069
|
-
const selected = rows.find((row) => row.diagnostic.credential === "present" && row.config !== void 0 && row.eligible === true);
|
|
1070
|
-
if (!selected) return gap("single-family-host");
|
|
1071
|
-
const { profile, auth } = toModelProfileAndAuth(selected.config, selected.auth);
|
|
1072
|
-
return {
|
|
1073
|
-
kind: "probe-author",
|
|
1074
|
-
entries,
|
|
1075
|
-
author: {
|
|
1076
|
-
index: selected.diagnostic.index,
|
|
1077
|
-
name: selected.config.name,
|
|
1078
|
-
authoringModel: { id: profile.model, family: authoringFamily(profile.model) },
|
|
1079
|
-
profile,
|
|
1080
|
-
auth
|
|
1081
|
-
}
|
|
1082
|
-
};
|
|
1083
|
-
}
|
|
1084
|
-
var init_adversarial_probe_preflight = __esm({
|
|
1085
|
-
"src/adversarial-probe-preflight.ts"() {
|
|
1086
|
-
"use strict";
|
|
1087
|
-
init_eval_session_composition();
|
|
1088
|
-
init_llm_utils();
|
|
1089
|
-
}
|
|
1090
|
-
});
|
|
1091
|
-
|
|
1092
|
-
// src/adversarial-probe-author.ts
|
|
1093
|
-
var adversarial_probe_author_exports = {};
|
|
1094
|
-
__export(adversarial_probe_author_exports, {
|
|
1095
|
-
PROBE_AUTHOR_MAX_OUTPUT_TOKENS: () => PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
|
|
1096
|
-
PROBE_AUTHOR_TIMEOUT_MS: () => PROBE_AUTHOR_TIMEOUT_MS,
|
|
1097
|
-
createAdversarialProbeAuthor: () => createAdversarialProbeAuthor
|
|
1098
|
-
});
|
|
1099
|
-
import { generateText as generateText3 } from "ai";
|
|
1100
|
-
import { z as z7 } from "zod";
|
|
1101
|
-
function createAdversarialProbeAuthor(input) {
|
|
1102
|
-
const { author } = input;
|
|
1103
|
-
const create = input.createModelImpl ?? createModel3;
|
|
1104
|
-
const generate = input.generateTextImpl ?? generateText3;
|
|
1105
|
-
return {
|
|
1106
|
-
async author(request2) {
|
|
1107
|
-
if (request2.authoringModel.id !== author.authoringModel.id) {
|
|
1108
|
-
throw new Error("This probe author is bound to a different authoring model than the request names.");
|
|
1109
|
-
}
|
|
1110
|
-
await input.onCallAttempted?.({
|
|
1111
|
-
seedCassetteId: request2.seedCassetteId,
|
|
1112
|
-
kind: request2.kind,
|
|
1113
|
-
promptVersion: request2.promptVersion,
|
|
1114
|
-
authoringModelId: author.authoringModel.id
|
|
1115
|
-
});
|
|
1116
|
-
const profile = {
|
|
1117
|
-
...author.profile,
|
|
1118
|
-
...author.profile.capabilities === void 0 ? {} : { capabilities: { ...author.profile.capabilities } }
|
|
1119
|
-
};
|
|
1120
|
-
return await runSafetyScope({ executionContext: input.executionContext }, async () => {
|
|
1121
|
-
const model = create(profile, author.auth, input.mask);
|
|
1122
|
-
const prompt = renderAdversarialProbePrompt(request2);
|
|
1123
|
-
const result = await generate({
|
|
1124
|
-
model,
|
|
1125
|
-
maxRetries: 0,
|
|
1126
|
-
temperature: 0,
|
|
1127
|
-
maxOutputTokens: PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
|
|
1128
|
-
abortSignal: AbortSignal.timeout(PROBE_AUTHOR_TIMEOUT_MS),
|
|
1129
|
-
system: prompt.system,
|
|
1130
|
-
prompt: prompt.user
|
|
1131
|
-
});
|
|
1132
|
-
const parsed = AUTHORED_NAME.safeParse(parseJson2(result.text));
|
|
1133
|
-
if (!parsed.success) {
|
|
1134
|
-
throw new Error("The probe author returned a response that is not the single-field name object the frozen prompt requires.");
|
|
1135
|
-
}
|
|
1136
|
-
return {
|
|
1137
|
-
text: parsed.data.name,
|
|
1138
|
-
// Reported spend, carried as reported -- and ABSENT when the provider reported
|
|
1139
|
-
// nothing, never zero. A zero here would be a measured claim about a call whose
|
|
1140
|
-
// cost we do not know, and the disclosure that reads this cannot tell the two
|
|
1141
|
-
// apart once they are the same number. The probe budget is denominated in CALLS
|
|
1142
|
-
// for exactly this reason, so an unreported token still never buys an extra call.
|
|
1143
|
-
usage: {
|
|
1144
|
-
...reportedCount(result.usage?.inputTokens) === void 0 ? {} : { promptTokens: reportedCount(result.usage?.inputTokens) },
|
|
1145
|
-
...reportedCount(result.usage?.outputTokens) === void 0 ? {} : { completionTokens: reportedCount(result.usage?.outputTokens) }
|
|
1146
|
-
}
|
|
1147
|
-
};
|
|
1148
|
-
});
|
|
1149
|
-
}
|
|
1150
|
-
};
|
|
1151
|
-
}
|
|
1152
|
-
function parseJson2(text) {
|
|
1153
|
-
try {
|
|
1154
|
-
return JSON.parse(text);
|
|
1155
|
-
} catch {
|
|
1156
|
-
return void 0;
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
function reportedCount(value) {
|
|
1160
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : void 0;
|
|
1161
|
-
}
|
|
1162
|
-
var PROBE_AUTHOR_MAX_OUTPUT_TOKENS, PROBE_AUTHOR_TIMEOUT_MS, AUTHORED_NAME;
|
|
1163
|
-
var init_adversarial_probe_author = __esm({
|
|
1164
|
-
"src/adversarial-probe-author.ts"() {
|
|
1165
|
-
"use strict";
|
|
1166
|
-
init_eval_session_composition();
|
|
1167
|
-
PROBE_AUTHOR_MAX_OUTPUT_TOKENS = 200;
|
|
1168
|
-
PROBE_AUTHOR_TIMEOUT_MS = 2e4;
|
|
1169
|
-
AUTHORED_NAME = z7.object({ name: z7.string() }).strict();
|
|
1170
|
-
}
|
|
1171
|
-
});
|
|
1172
|
-
|
|
1173
|
-
// src/curation/trust-priors.ts
|
|
1174
|
-
import { dirname as dirname9, resolve as resolve4 } from "path";
|
|
1175
|
-
function buildTrustPriorsResolver(input) {
|
|
1176
|
-
const configDir = dirname9(resolve4(input.configPath));
|
|
1177
|
-
const committedRoot = resolve4(configDir, "agent-qa-eval");
|
|
1178
|
-
const read = readLabelCalibration(committedRoot);
|
|
1179
|
-
if (read.kind === "refused") return Object.freeze({ kind: "refused", reason: REFUSAL });
|
|
1180
|
-
const table = {};
|
|
1181
|
-
const nByOrigin = { "agent-verified": 0, "agent-observed": 0 };
|
|
1182
|
-
if (read.kind === "loaded") {
|
|
1183
|
-
for (const [origin, channel] of [["agent-verified", "recorded-verdict"], ["agent-observed", "self-heal-pair"]]) {
|
|
1184
|
-
nByOrigin[origin] = read.artifact.channels[channel].n;
|
|
1185
|
-
const prior = read.artifact.priors[origin];
|
|
1186
|
-
if (prior !== void 0) table[origin] = Object.freeze({ alpha: prior.alpha, beta: prior.beta });
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
const priors = Object.keys(table).length === 0 ? void 0 : Object.freeze(table);
|
|
1190
|
-
const disclosure = Object.freeze({
|
|
1191
|
-
status: priors === void 0 ? "provisional" : "calibrated",
|
|
1192
|
-
...read.kind === "loaded" ? { digest: read.artifact.generationId } : {},
|
|
1193
|
-
nByOrigin: Object.freeze(nByOrigin)
|
|
1194
|
-
});
|
|
1195
|
-
let calibration;
|
|
1196
|
-
if (input.inspectCalibration) {
|
|
1197
|
-
if (read.kind === "absent") calibration = "label calibration: absent";
|
|
1198
|
-
else {
|
|
1199
|
-
const identity = readCalibrationGoldIdentity({ committedRoot, agentQaRoot: resolve4(configDir, ".agent-qa"), calibrationRead: read });
|
|
1200
|
-
const freshness = identity.kind === "unavailable" ? "unavailable" : identity.goldSetDigest === read.artifact.goldSetDigest ? "current" : "stale";
|
|
1201
|
-
const channels = ["recorded-verdict", "self-heal-pair"].map((channel) => {
|
|
1202
|
-
const row = read.artifact.channels[channel];
|
|
1203
|
-
return `${channel} ${row.status} n=${row.n}`;
|
|
1204
|
-
}).join(", ");
|
|
1205
|
-
calibration = `label calibration: ${channels}; goldDigest ${freshness}`;
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
return Object.freeze({
|
|
1209
|
-
kind: "ready",
|
|
1210
|
-
resolve: () => priors,
|
|
1211
|
-
disclosure,
|
|
1212
|
-
...calibration === void 0 ? {} : { calibration }
|
|
1213
|
-
});
|
|
1214
|
-
}
|
|
1215
|
-
function trustPriorsForHost(input) {
|
|
1216
|
-
const result = buildTrustPriorsResolver(input);
|
|
1217
|
-
if (result.kind === "refused") throw new TrustPriorsRefusedError();
|
|
1218
|
-
return result;
|
|
1219
|
-
}
|
|
1220
|
-
var REFUSAL, TrustPriorsRefusedError;
|
|
1221
|
-
var init_trust_priors = __esm({
|
|
1222
|
-
"src/curation/trust-priors.ts"() {
|
|
1223
|
-
"use strict";
|
|
1224
|
-
init_eval_session_composition();
|
|
1225
|
-
REFUSAL = "Trust prior calibration is invalid or unavailable; inspect the selected config's committed calibration marker and referenced revisions, following https://vostride.com/docs/agent-qa/self-improvement/evals#if-calibration-is-refused before composing this host.";
|
|
1226
|
-
TrustPriorsRefusedError = class extends Error {
|
|
1227
|
-
kind = "trust-priors-refused";
|
|
1228
|
-
constructor() {
|
|
1229
|
-
super(REFUSAL);
|
|
1230
|
-
this.name = "TrustPriorsRefusedError";
|
|
1231
|
-
}
|
|
1232
|
-
};
|
|
1233
|
-
}
|
|
1234
|
-
});
|
|
1235
|
-
|
|
1236
|
-
// src/gates/replay-gate-host.ts
|
|
1237
|
-
var replay_gate_host_exports = {};
|
|
1238
|
-
__export(replay_gate_host_exports, {
|
|
1239
|
-
createReplayGateHost: () => createReplayGateHost,
|
|
1240
|
-
createReplayGatePortForHost: () => createReplayGatePortForHost
|
|
1241
|
-
});
|
|
1242
|
-
import { dirname as dirname10, resolve as resolve5 } from "path";
|
|
1243
|
-
import { Gate1ReceiptNotPersistedError } from "@vostride/agent-qa-agent";
|
|
1244
|
-
import {
|
|
1245
|
-
createKnowledgeReader,
|
|
1246
|
-
LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES2
|
|
1247
|
-
} from "@vostride/agent-qa-agent-local";
|
|
1248
|
-
import {
|
|
1249
|
-
GateReceiptSchema,
|
|
1250
|
-
receiptDigestOf
|
|
1251
|
-
} from "@vostride/agent-qa-schema";
|
|
1252
|
-
function createReplayGatePortForHost(input) {
|
|
1253
|
-
const resolveTrustPriors = input.resolveTrustPriors ?? trustPriorsForHost(input).resolve;
|
|
1254
|
-
return {
|
|
1255
|
-
async runReplayGate(candidateId) {
|
|
1256
|
-
const { resolveEvalSession: resolveEvalSession2, runReplayGateVerb: runReplayGateVerb2 } = await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports));
|
|
1257
|
-
const session = await resolveEvalSession2({ configPath: input.configPath, keyOnly: true });
|
|
1258
|
-
const borrowed = input.resolveClaimAuthority?.();
|
|
1259
|
-
const result = await runReplayGateVerb2(session, candidateId, {
|
|
1260
|
-
resolveTrustPriors,
|
|
1261
|
-
agentQaRoot: input.agentQaRoot,
|
|
1262
|
-
onProgress: () => void 0,
|
|
1263
|
-
...borrowed === void 0 ? {} : { claimAuthority: borrowed }
|
|
1264
|
-
});
|
|
1265
|
-
if (result.kind === "refused") throw new Error(result.refusal.reason);
|
|
1266
|
-
return checkedReceipt(acknowledged(result, candidateId), candidateId);
|
|
1267
|
-
}
|
|
1268
|
-
};
|
|
1269
|
-
}
|
|
1270
|
-
function acknowledged(result, candidateId) {
|
|
1271
|
-
if (!ACKNOWLEDGED_RECORDINGS.includes(result.recorded)) {
|
|
1272
|
-
throw new Gate1ReceiptNotPersistedError(candidateId, result.recorded);
|
|
1273
|
-
}
|
|
1274
|
-
return result.receipt;
|
|
1275
|
-
}
|
|
1276
|
-
function checkedReceipt(value, candidateId) {
|
|
1277
|
-
const receipt = GateReceiptSchema.parse(value);
|
|
1278
|
-
if (receipt.gate !== 1 || receipt.candidateId !== candidateId || receipt.receiptDigest !== receiptDigestOf(receipt))
|
|
1279
|
-
throw new Error("Gate 1 returned a foreign candidate, wrong gate or invalid receipt digest");
|
|
1280
|
-
return receipt;
|
|
1281
|
-
}
|
|
1282
|
-
function createReplayGateHost(deps) {
|
|
1283
|
-
const resolveTrustPriors = deps.resolveTrustPriors ?? trustPriorsForHost(deps).resolve;
|
|
1284
|
-
const configDir = dirname10(resolve5(deps.configPath));
|
|
1285
|
-
const knowledgeRoot = resolveConfiguredKnowledgeRoot(deps.session.config, configDir);
|
|
1286
|
-
const reader = deps.reader ?? createKnowledgeReader(() => knowledgeRoot, void 0, void 0, resolveTrustPriors);
|
|
1287
|
-
const curation = deps.curation ?? LOCAL_CURATION_CAPABILITIES2;
|
|
1288
|
-
async function run(candidateId) {
|
|
1289
|
-
const read = reader.candidateInjection;
|
|
1290
|
-
if (read === void 0) throw new Error("knowledge reader binds no candidateInjection");
|
|
1291
|
-
const found = await read.call(reader, { candidateId });
|
|
1292
|
-
if (found.kind === "not-found")
|
|
1293
|
-
return {
|
|
1294
|
-
kind: "refused",
|
|
1295
|
-
refusal: {
|
|
1296
|
-
path: knowledgeRoot,
|
|
1297
|
-
reason: `candidate ${candidateId} not found`
|
|
1298
|
-
},
|
|
1299
|
-
notices: [],
|
|
1300
|
-
exit: 2
|
|
1301
|
-
};
|
|
1302
|
-
if (found.injection.candidateId !== candidateId)
|
|
1303
|
-
throw new Error("Gate 1 reader returned a foreign candidate");
|
|
1304
|
-
const runGate = deps.runReplayGate ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).runReplayGate;
|
|
1305
|
-
const result = await runGate({
|
|
1306
|
-
...deps.composition,
|
|
1307
|
-
injection: found.injection,
|
|
1308
|
-
committedRoot: resolve5(configDir, "agent-qa-eval"),
|
|
1309
|
-
agentQaRoot: deps.agentQaRoot,
|
|
1310
|
-
mask: deps.session.mask,
|
|
1311
|
-
producedBy: { opId: "eval.replay-gate" }
|
|
1312
|
-
});
|
|
1313
|
-
if (result.kind === "refused") return { ...result, exit: 2 };
|
|
1314
|
-
const receipt = checkedReceipt(result.receipt, candidateId);
|
|
1315
|
-
const answer = {
|
|
1316
|
-
kind: "receipt",
|
|
1317
|
-
receipt,
|
|
1318
|
-
lines: result.lines,
|
|
1319
|
-
notices: [...result.notices],
|
|
1320
|
-
recorded: "dry-run",
|
|
1321
|
-
exit: receipt.verdict === "pass" ? 0 : receipt.verdict === "fail" ? 1 : 2
|
|
1322
|
-
};
|
|
1323
|
-
if (!deps.record) return answer;
|
|
1324
|
-
const borrowed = deps.resolveClaimAuthority?.();
|
|
1325
|
-
if (borrowed !== void 0) {
|
|
1326
|
-
const outcome = await curation.recordGateReceipt({
|
|
1327
|
-
resolveTrustPriors,
|
|
1328
|
-
receipt,
|
|
1329
|
-
lease: { leaseId: borrowed.leaseId, epoch: borrowed.epoch },
|
|
1330
|
-
knowledgeRoot,
|
|
1331
|
-
executionContext: deps.session.executionContext,
|
|
1332
|
-
sourceBuildOrdinal: receipt.producedAtOrdinal,
|
|
1333
|
-
...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
|
|
1334
|
-
});
|
|
1335
|
-
answer.recorded = outcome.kind;
|
|
1336
|
-
return answer;
|
|
1337
|
-
}
|
|
1338
|
-
const lease = curation.maintenance.acquireWriterLease(knowledgeRoot);
|
|
1339
|
-
if (lease.kind !== "acquired") {
|
|
1340
|
-
answer.recorded = lease.kind === "held" ? "lease-held" : "lease-refused";
|
|
1341
|
-
answer.exit = 2;
|
|
1342
|
-
answer.notices.push(
|
|
1343
|
-
lease.kind === "held" ? "writer lease held by another process; receipt printed, not recorded \u2014 re-run when the daemon releases it" : `writer lease refused: ${lease.reason}; receipt printed, not recorded`
|
|
1344
|
-
);
|
|
1345
|
-
return answer;
|
|
1346
|
-
}
|
|
1347
|
-
try {
|
|
1348
|
-
const outcome = await curation.recordGateReceipt({
|
|
1349
|
-
resolveTrustPriors,
|
|
1350
|
-
receipt,
|
|
1351
|
-
lease: { leaseId: lease.leaseId, epoch: lease.epoch },
|
|
1352
|
-
knowledgeRoot,
|
|
1353
|
-
executionContext: deps.session.executionContext,
|
|
1354
|
-
sourceBuildOrdinal: receipt.producedAtOrdinal,
|
|
1355
|
-
...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
|
|
1356
|
-
});
|
|
1357
|
-
answer.recorded = outcome.kind;
|
|
1358
|
-
return answer;
|
|
1359
|
-
} finally {
|
|
1360
|
-
curation.maintenance.releaseLease({ knowledgeRoot, leaseId: lease.leaseId });
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
return {
|
|
1364
|
-
run,
|
|
1365
|
-
port: {
|
|
1366
|
-
async runReplayGate(id) {
|
|
1367
|
-
const result = await run(id);
|
|
1368
|
-
if (result.kind === "refused") throw new Error(result.refusal.reason);
|
|
1369
|
-
return acknowledged(result, id);
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
};
|
|
1373
|
-
}
|
|
1374
|
-
var ACKNOWLEDGED_RECORDINGS;
|
|
1375
|
-
var init_replay_gate_host = __esm({
|
|
1376
|
-
"src/gates/replay-gate-host.ts"() {
|
|
1377
|
-
"use strict";
|
|
1378
|
-
init_trust_priors();
|
|
1379
|
-
init_config();
|
|
1380
|
-
ACKNOWLEDGED_RECORDINGS = ["appended", "duplicate"];
|
|
1381
|
-
}
|
|
1382
|
-
});
|
|
1383
|
-
|
|
1384
|
-
// src/eval-session-composition.ts
|
|
1385
|
-
var eval_session_composition_exports = {};
|
|
1386
|
-
__export(eval_session_composition_exports, {
|
|
1387
|
-
ADVERSARIAL_STAGING_SUBDIR: () => ADVERSARIAL_STAGING_SUBDIR,
|
|
1388
|
-
ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS: () => ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS,
|
|
1389
|
-
EvalJudgeDeclarationError: () => EvalJudgeDeclarationError,
|
|
1390
|
-
EvalNamedInputError: () => EvalNamedInputError,
|
|
1391
|
-
EvalPartitionFlagsRequiredError: () => EvalPartitionFlagsRequiredError,
|
|
1392
|
-
EvalPartitionManifestConflictError: () => EvalPartitionManifestConflictError,
|
|
1393
|
-
FLOOR_DEFAULT_BOOTSTRAP_SEED: () => FLOOR_DEFAULT_BOOTSTRAP_SEED,
|
|
1394
|
-
FLOOR_DEFAULT_REPEATS: () => FLOOR_DEFAULT_REPEATS,
|
|
1395
|
-
FLOOR_DEFAULT_SEEDS: () => FLOOR_DEFAULT_SEEDS,
|
|
1396
|
-
FsHandoffSource: () => FsHandoffSource,
|
|
1397
|
-
FsMarkLedger: () => FsMarkLedger,
|
|
1398
|
-
GATE1_DEFAULT_K: () => GATE1_DEFAULT_K,
|
|
1399
|
-
GATE1_DEFAULT_MAX_SCREENS: () => GATE1_DEFAULT_MAX_SCREENS,
|
|
1400
|
-
GATE2_DEFAULT_CANARY_HORIZON_BUILDS: () => GATE2_DEFAULT_CANARY_HORIZON_BUILDS,
|
|
1401
|
-
GATE2_DEFAULT_MAX_ABLATIONS: () => GATE2_DEFAULT_MAX_ABLATIONS,
|
|
1402
|
-
HANDOFF_FILE: () => HANDOFF_FILE2,
|
|
1403
|
-
LABEL_CALIBRATION_MIN_GOLD: () => LABEL_CALIBRATION_MIN_GOLD,
|
|
1404
|
-
MARK_INTENT_FILE: () => MARK_INTENT_FILE,
|
|
1405
|
-
MarkLedgerCorruptError: () => MarkLedgerCorruptError,
|
|
1406
|
-
NO_PROBE_MODEL_REASONS: () => NO_PROBE_MODEL_REASONS,
|
|
1407
|
-
RUNS_SUBDIR: () => RUNS_SUBDIR2,
|
|
1408
|
-
SEED_POLICIES: () => SEED_POLICIES2,
|
|
1409
|
-
adversarialHostOptions: () => adversarialHostOptions,
|
|
1410
|
-
assessPanelEligibility: () => assessPanelEligibility,
|
|
1411
|
-
canonicalJson: () => canonicalJson6,
|
|
1412
|
-
computeCorpusDigest: () => computeCorpusDigest2,
|
|
1413
|
-
computeReplayProfileFingerprintFor: () => computeReplayProfileFingerprintFor,
|
|
1414
|
-
createAdversarialProbeAuthor: () => createAdversarialProbeAuthor,
|
|
1415
|
-
createModel: () => createModel3,
|
|
1416
|
-
createPanelIdentity: () => createPanelIdentity,
|
|
1417
|
-
feedbackSeedHostOptions: () => feedbackSeedHostOptions,
|
|
1418
|
-
freezeGoldenTrajectory: () => freezeGoldenTrajectory,
|
|
1419
|
-
isFeedbackStateNovel: () => isFeedbackStateNovel,
|
|
1420
|
-
judgeCapabilityOf: () => judgeCapabilityOf,
|
|
1421
|
-
judgeFamiliesIntersect: () => judgeFamiliesIntersect,
|
|
1422
|
-
latestMarkReceiptFor: () => latestMarkReceiptFor,
|
|
1423
|
-
listAdversarialBatches: () => listAdversarialBatches,
|
|
1424
|
-
listStagedAdversarial: () => listStagedAdversarial2,
|
|
1425
|
-
locateMisdirectionDecision: () => locateMisdirectionDecision,
|
|
1426
|
-
markedDecisionCount: () => markedDecisionCount,
|
|
1427
|
-
preflightJudgePanel: () => preflightJudgePanel,
|
|
1428
|
-
previewMarkLabels: () => previewMarkLabels,
|
|
1429
|
-
projectMatcherConfigContract: () => projectMatcherConfigContract,
|
|
1430
|
-
readActiveCommittedCorpus: () => readActiveCommittedCorpus2,
|
|
1431
|
-
readAdversarialRefreshRecords: () => readAdversarialRefreshRecords,
|
|
1432
|
-
readCalibrationGoldIdentity: () => readCalibrationGoldIdentity,
|
|
1433
|
-
readCanaryCorpusDigest: () => readCanaryCorpusDigest,
|
|
1434
|
-
readCommittedCorpus: () => readCommittedCorpus2,
|
|
1435
|
-
readGatePostureFor: () => readGatePostureFor,
|
|
1436
|
-
readJudgeCalibration: () => readJudgeCalibration,
|
|
1437
|
-
readLabelCalibration: () => readLabelCalibration,
|
|
1438
|
-
readMisdirectionRootPosture: () => readMisdirectionRootPosture,
|
|
1439
|
-
renderAdversarialProbePrompt: () => renderAdversarialProbePrompt,
|
|
1440
|
-
renderBaselineComparatorRefusal: () => renderBaselineComparatorRefusal2,
|
|
1441
|
-
renderCensusMarkdown: () => renderCensusMarkdown,
|
|
1442
|
-
renderJudgeTargetedCalibration: () => renderJudgeTargetedCalibration,
|
|
1443
|
-
resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability,
|
|
1444
|
-
resolveEvalJudge: () => resolveEvalJudge,
|
|
1445
|
-
resolveEvalSession: () => resolveEvalSession,
|
|
1446
|
-
resolveGate2Caps: () => resolveGate2Caps,
|
|
1447
|
-
resolveGateProfileIdentity: () => resolveGateProfileIdentity,
|
|
1448
|
-
resolveJudgeCalibrationMinimum: () => resolveJudgeCalibrationMinimum,
|
|
1449
|
-
resolveJudgeFamilies: () => resolveJudgeFamilies,
|
|
1450
|
-
resolveMarkLabelTargets: () => resolveMarkLabelTargets,
|
|
1451
|
-
resolveMarkedDecisions: () => resolveMarkedDecisions,
|
|
1452
|
-
resolveMinimumGateSet: () => resolveMinimumGateSet,
|
|
1453
|
-
resolveMisdirectionCaps: () => resolveMisdirectionCaps,
|
|
1454
|
-
resolveProductPartition: () => resolveProductPartition2,
|
|
1455
|
-
resolveReplayPlanner: () => resolveReplayPlanner,
|
|
1456
|
-
resolveReplayProfileDeclaration: () => resolveReplayProfileDeclaration,
|
|
1457
|
-
resolveReplayProfileFingerprint: () => resolveReplayProfileFingerprint,
|
|
1458
|
-
resolveTargetEnvironmentForProduct: () => resolveTargetEnvironmentForProduct,
|
|
1459
|
-
runAdversarialVerb: () => runAdversarialVerb,
|
|
1460
|
-
runCalibrateVerb: () => runCalibrateVerb,
|
|
1461
|
-
runCanaryAblation: () => runCanaryAblation,
|
|
1462
|
-
runCensusVerb: () => runCensusVerb,
|
|
1463
|
-
runExtractVerb: () => runExtractVerb,
|
|
1464
|
-
runFeedbackStaging: () => runFeedbackStaging,
|
|
1465
|
-
runFloorVerb: () => runFloorVerb,
|
|
1466
|
-
runGateVerb: () => runGateVerb,
|
|
1467
|
-
runJudgeCalibrateVerb: () => runJudgeCalibrateVerb,
|
|
1468
|
-
runMisdirectionCheck: () => runMisdirectionCheck,
|
|
1469
|
-
runReplayGate: () => runReplayGate,
|
|
1470
|
-
runReplayGateVerb: () => runReplayGateVerb,
|
|
1471
|
-
runReplayVerb: () => runReplayVerb,
|
|
1472
|
-
runReportVerb: () => runReportVerb,
|
|
1473
|
-
runRetireAdversarialVerb: () => runRetireAdversarialVerb,
|
|
1474
|
-
runSafetyScope: () => runSafetyScope,
|
|
1475
|
-
runSeedFromFeedbackVerb: () => runSeedFromFeedbackVerb,
|
|
1476
|
-
runSeedVerb: () => runSeedVerb,
|
|
1477
|
-
selectGateSet: () => selectGateSet,
|
|
1478
|
-
selectedEvalJudgeNames: () => selectedEvalJudgeNames,
|
|
1479
|
-
serializeReplayGateReceipt: () => serializeReplayGateReceipt,
|
|
1480
|
-
slugBranch: () => slugBranch2
|
|
1481
|
-
});
|
|
1482
|
-
import { GATE2_DEFAULT_MAX_LOWER_TRUST_PER_SCOPE } from "@vostride/agent-qa-schema";
|
|
1483
|
-
import { resolveKnowledgeRoot as resolveKnowledgeRoot2 } from "@vostride/agent-qa-core";
|
|
1484
|
-
import { createHash as createHash2, randomBytes } from "crypto";
|
|
1485
|
-
import { existsSync as existsSync4, lstatSync as lstatSync2, readFileSync as readFileSync5, readdirSync as readdirSync4, realpathSync as realpathSync2 } from "fs";
|
|
1486
|
-
import { readFile as readFile2 } from "fs/promises";
|
|
1487
|
-
import { dirname as dirname11, isAbsolute as isAbsolute2, join as join9, relative, resolve as resolve6 } from "path";
|
|
1488
|
-
import {
|
|
1489
|
-
HANDOFF_FILE,
|
|
1490
|
-
RUNS_SUBDIR,
|
|
1491
|
-
loadCompletedRunHandoff,
|
|
1492
|
-
slugBranch
|
|
1493
|
-
} from "@vostride/agent-qa-runner-local";
|
|
1494
|
-
import {
|
|
1495
|
-
assertBaselineComparator,
|
|
1496
|
-
buildEvalReport,
|
|
1497
|
-
buildSeedingPlan,
|
|
1498
|
-
admitSeedMember,
|
|
1499
|
-
canonicalJson as canonicalJson5,
|
|
1500
|
-
composeBaselineProfileKey,
|
|
1501
|
-
computeBatchKey,
|
|
1502
|
-
computeCorpusDigest,
|
|
1503
|
-
createPanelIdentity,
|
|
1504
|
-
decisionCorrectnessVectors,
|
|
1505
|
-
deriveCensus,
|
|
1506
|
-
renderCensusMarkdown,
|
|
1507
|
-
EXCLUSION_CLASSES,
|
|
1508
|
-
extractRunCassettes,
|
|
1509
|
-
FLOOR_PROTOCOL_MINIMUM,
|
|
1510
|
-
guardClassifierRepeatability,
|
|
1511
|
-
guardCampaignManifest,
|
|
1512
|
-
JUDGE_CONFIG,
|
|
1513
|
-
judgeIdentitySchema,
|
|
1514
|
-
maskCorpusText,
|
|
1515
|
-
partitionParamsOf,
|
|
1516
|
-
projectBaselinePointer,
|
|
1517
|
-
projectDecisionVectorsDocument,
|
|
1518
|
-
projectPartitionManifest,
|
|
1519
|
-
projectRunExclusion,
|
|
1520
|
-
renderBaselineComparatorRefusal,
|
|
1521
|
-
renderFloorRefusal,
|
|
1522
|
-
resolveJudgeFamily,
|
|
1523
|
-
resolveProductPartition,
|
|
1524
|
-
SEED_POLICIES
|
|
1525
|
-
} from "@vostride/agent-qa-eval";
|
|
1526
|
-
import {
|
|
1527
|
-
COMMITTED_CASSETTES_DIR,
|
|
1528
|
-
DEFAULT_EVAL_CASSETTES_SUBDIR,
|
|
1529
|
-
DEFAULT_EVAL_REPLAYS_SUBDIR,
|
|
1530
|
-
DEFAULT_EVAL_REPORTS_SUBDIR,
|
|
1531
|
-
FloorProtocolError,
|
|
1532
|
-
FsCassetteStore,
|
|
1533
|
-
FsReplayRecordStore,
|
|
1534
|
-
FsReportStore,
|
|
1535
|
-
readReportProfileProof,
|
|
1536
|
-
FsRunArtifactSource,
|
|
1537
|
-
FsScoreStore,
|
|
1538
|
-
assembleScoredBatch,
|
|
1539
|
-
copyJudgeCalibrationToReport,
|
|
1540
|
-
readJudgeCalibration,
|
|
1541
|
-
ablateCanaryStep,
|
|
1542
|
-
locateTargetCassettes,
|
|
1543
|
-
GATE2_AUDIT_SUBDIR,
|
|
1544
|
-
isNovelState,
|
|
1545
|
-
stageFeedbackEntry,
|
|
1546
|
-
admitStagedFeedback,
|
|
1547
|
-
ADVERSARIAL_STAGING_SUBDIR,
|
|
1548
|
-
listStagedAdversarial,
|
|
1549
|
-
refreshAdversarialSplit,
|
|
1550
|
-
retireStagedAdversarialBatch,
|
|
1551
|
-
resolveAdversarialProbeCallCap,
|
|
1552
|
-
resolveAdversarialSeedCap,
|
|
1553
|
-
stageAdversarialProposal,
|
|
1554
|
-
FEEDBACK_STAGING_SUBDIR,
|
|
1555
|
-
readCommittedCorpusDigest,
|
|
1556
|
-
readProbeAttempt,
|
|
1557
|
-
cassetteProductPartition,
|
|
1558
|
-
commitCensus,
|
|
1559
|
-
commitEvalReport,
|
|
1560
|
-
committedExclusionPath,
|
|
1561
|
-
committedNoiseFloorPath,
|
|
1562
|
-
committedPartitionManifestPath,
|
|
1563
|
-
computeReplayProfileFingerprintFor,
|
|
1564
|
-
listBaselinePointers,
|
|
1565
|
-
listCommittedReports,
|
|
1566
|
-
listNoiseFloorArtefacts,
|
|
1567
|
-
loadCassetteBundles,
|
|
1568
|
-
loadLabelEvidence,
|
|
1569
|
-
loadReplayRecord,
|
|
1570
|
-
loadScoreRecord,
|
|
1571
|
-
readCommittedCorpus,
|
|
1572
|
-
readActiveCommittedCorpus,
|
|
1573
|
-
readPartitionManifests,
|
|
1574
|
-
readRunExclusions,
|
|
1575
|
-
replayJudgeRequest,
|
|
1576
|
-
runNoiseFloorMeasurement,
|
|
1577
|
-
runReleaseGate,
|
|
1578
|
-
runLabelCalibration,
|
|
1579
|
-
runJudgeCalibration,
|
|
1580
|
-
maskJudgeCalibrationText,
|
|
1581
|
-
seedCommittedCorpus,
|
|
1582
|
-
sweepCassetteReplays,
|
|
1583
|
-
writeBaselinePointer,
|
|
1584
|
-
writePartitionManifest,
|
|
1585
|
-
writeRunExclusion
|
|
1586
|
-
} from "@vostride/agent-qa-eval-local";
|
|
1587
|
-
import { runSafetyScope as runSafetyScope2 } from "@vostride/agent-qa-safety-local";
|
|
1588
|
-
import {
|
|
1589
|
-
ANALYSIS_FILE,
|
|
1590
|
-
admitAnalysisArtifactForRun,
|
|
1591
|
-
guardRunAnalysisArtifact,
|
|
1592
|
-
resolveRunVerdicts,
|
|
1593
|
-
sha256hex as sha256hex3
|
|
1594
|
-
} from "@vostride/agent-qa-schema";
|
|
1595
|
-
import {
|
|
1596
|
-
LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES3,
|
|
1597
|
-
resolveRunExecutionContext
|
|
1598
|
-
} from "@vostride/agent-qa-agent-local";
|
|
1599
|
-
import { renderBaselineComparatorRefusal as renderBaselineComparatorRefusal2, SEED_POLICIES as SEED_POLICIES2 } from "@vostride/agent-qa-eval";
|
|
1600
|
-
import { computeCorpusDigest as computeCorpusDigest2, selectGateSet } from "@vostride/agent-qa-eval";
|
|
1601
|
-
import { judgeCapabilityOf } from "@vostride/agent-qa-eval";
|
|
1602
|
-
import { readActiveCommittedCorpus as readActiveCommittedCorpus2, readCommittedCorpus as readCommittedCorpus2, readGatePostureFor, resolveMinimumGateSet } from "@vostride/agent-qa-eval-local";
|
|
1603
|
-
import {
|
|
1604
|
-
assessPanelEligibility,
|
|
1605
|
-
judgeFamiliesIntersect,
|
|
1606
|
-
resolveJudgeFamilies,
|
|
1607
|
-
resolveJudgeCalibrationMinimum
|
|
1608
|
-
} from "@vostride/agent-qa-eval";
|
|
1609
|
-
import { projectMatcherConfigContract } from "@vostride/agent-qa-eval";
|
|
1610
|
-
import { runReplayGate, serializeReplayGateReceipt, readLabelCalibration, readCalibrationGoldIdentity } from "@vostride/agent-qa-eval-local";
|
|
1611
|
-
import { listStagedAdversarial as listStagedAdversarial2, readAdversarialRefreshRecords } from "@vostride/agent-qa-eval-local";
|
|
1612
|
-
import { canonicalJson as canonicalJson6, markedDecisionCount, previewMarkLabels, resolveMarkLabelTargets, resolveMarkedDecisions, resolveProductPartition as resolveProductPartition2 } from "@vostride/agent-qa-eval";
|
|
1613
|
-
import { FsHandoffSource, FsMarkLedger, MARK_INTENT_FILE, MarkLedgerCorruptError, freezeGoldenTrajectory, latestMarkReceiptFor } from "@vostride/agent-qa-eval-local";
|
|
1614
|
-
import { HANDOFF_FILE as HANDOFF_FILE2, RUNS_SUBDIR as RUNS_SUBDIR2, slugBranch as slugBranch2 } from "@vostride/agent-qa-runner-local";
|
|
1615
|
-
import { runSafetyScope } from "@vostride/agent-qa-safety-local";
|
|
1616
|
-
import { createModel as createModel3 } from "@vostride/agent-qa-model";
|
|
1617
|
-
import { NO_PROBE_MODEL_REASONS, renderAdversarialProbePrompt } from "@vostride/agent-qa-eval";
|
|
1618
|
-
function selectedEvalJudgeNames(config) {
|
|
1619
|
-
if (config.judge !== void 0 && config.judges !== void 0) {
|
|
1620
|
-
throw new EvalJudgeDeclarationError("Set either eval.judge or eval.judges, never both; remove one declaration.");
|
|
1621
|
-
}
|
|
1622
|
-
if (config.judge !== void 0 && typeof config.judge !== "string") {
|
|
1623
|
-
throw new EvalJudgeDeclarationError("eval.judge must be one nonempty registry name; use eval.judges for a list.");
|
|
1624
|
-
}
|
|
1625
|
-
const names = config.judge !== void 0 ? [config.judge] : config.judges === void 0 ? [] : config.judges;
|
|
1626
|
-
if (!Array.isArray(names) || Array.from(names).some((name) => typeof name !== "string" || name.trim().length === 0)) {
|
|
1627
|
-
throw new EvalJudgeDeclarationError("eval.judges must be an array of nonempty registry names.");
|
|
1628
|
-
}
|
|
1629
|
-
if (names.length > JUDGE_CONFIG.maxPanelMembers) {
|
|
1630
|
-
throw new EvalJudgeDeclarationError(`Choose at most ${JUDGE_CONFIG.maxPanelMembers} explicit judge names.`);
|
|
1631
|
-
}
|
|
1632
|
-
return [...names];
|
|
1633
|
-
}
|
|
1634
|
-
function withJudgeOverrides(judge, modelOverrides) {
|
|
1635
|
-
if (judge === void 0 || modelOverrides === void 0) return judge;
|
|
1636
|
-
return "members" in judge ? { ...judge, members: judge.members.map((member) => ({ ...member, modelOverrides })) } : { ...judge, modelOverrides };
|
|
1637
|
-
}
|
|
1638
|
-
async function resolveEvalSession(input = {}) {
|
|
1639
|
-
const configPath = input.configPath ?? "agent-qa.config.yaml";
|
|
1640
|
-
const configDir = dirname11(resolve6(configPath));
|
|
1641
|
-
const raw = await loadConfigFile(configPath);
|
|
1642
|
-
if (raw?.eval && typeof raw.eval === "object") selectedEvalJudgeNames(raw.eval);
|
|
1643
|
-
const config = await resolveConfig({ configPath, loadAuthPlugins: input.keyOnly !== true });
|
|
1644
|
-
const evalConfig = mergeEvalConfig(config.eval, input.flags);
|
|
1645
|
-
if (input.keyOnly === true) {
|
|
1646
|
-
return {
|
|
1647
|
-
config,
|
|
1648
|
-
configPath,
|
|
1649
|
-
configDir,
|
|
1650
|
-
evalConfig,
|
|
1651
|
-
mask: () => {
|
|
1652
|
-
throw new Error("A key-only eval session has no declared secrets to mask with");
|
|
1653
|
-
},
|
|
1654
|
-
executionContext: resolveRunExecutionContext()
|
|
1655
|
-
};
|
|
1656
|
-
}
|
|
1657
|
-
const secretStore = await loadEvalSecretStore(config, configDir);
|
|
1658
|
-
const { SecretRedactor: SecretRedactor2 } = await import("@vostride/agent-qa-runner");
|
|
1659
|
-
const redactor = new SecretRedactor2(secretStore);
|
|
1660
|
-
return {
|
|
1661
|
-
config,
|
|
1662
|
-
configPath,
|
|
1663
|
-
configDir,
|
|
1664
|
-
evalConfig,
|
|
1665
|
-
mask: (text) => redactor.redactString(text),
|
|
1666
|
-
executionContext: resolveRunExecutionContext()
|
|
1667
|
-
};
|
|
1668
|
-
}
|
|
1669
|
-
function committedCorpusMask(session) {
|
|
1670
|
-
const declared = createDeclaredSecretMask(session.configPath);
|
|
1671
|
-
declared.redactor();
|
|
1672
|
-
return declared.mask;
|
|
1673
|
-
}
|
|
1674
|
-
async function runExtractVerb(session, filters = {}) {
|
|
1675
|
-
return await runSafetyScope2(
|
|
1676
|
-
{ executionContext: session.executionContext },
|
|
1677
|
-
async () => sweepAndExtract(resolve6(session.configDir, ".agent-qa"), filters)
|
|
1678
|
-
);
|
|
1679
|
-
}
|
|
1680
|
-
function sweepAndExtract(agentQaRoot, filters) {
|
|
1681
|
-
const runsRoot = resolve6(agentQaRoot, RUNS_SUBDIR);
|
|
1682
|
-
let runSlugs = listRunSlugsWithHandoff(runsRoot);
|
|
1683
|
-
if (filters.run !== void 0) {
|
|
1684
|
-
const named = filters.run;
|
|
1685
|
-
runSlugs = runSlugs.filter((slug) => slug === named || slug === slugBranch(named));
|
|
1686
|
-
if (runSlugs.length === 0) {
|
|
1687
|
-
throw new EvalNamedInputError("run", named, runsRoot);
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
const store = new FsCassetteStore({ root: agentQaRoot });
|
|
1691
|
-
const runsSkipped = [];
|
|
1692
|
-
const completeness = {};
|
|
1693
|
-
let runsFiltered = 0;
|
|
1694
|
-
let cassettesWritten = 0;
|
|
1695
|
-
for (const runSlug of runSlugs) {
|
|
1696
|
-
let extraction;
|
|
1697
|
-
try {
|
|
1698
|
-
const handoff = loadCompletedRunHandoff(resolve6(runsRoot, runSlug, HANDOFF_FILE));
|
|
1699
|
-
if (filters.product !== void 0 && resolveProductPartition(handoff.scope) !== filters.product) {
|
|
1700
|
-
runsFiltered++;
|
|
1701
|
-
continue;
|
|
1702
|
-
}
|
|
1703
|
-
extraction = extractRunCassettes({
|
|
1704
|
-
handoff,
|
|
1705
|
-
artifacts: new FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
|
|
1706
|
-
});
|
|
1707
|
-
} catch (err) {
|
|
1708
|
-
runsSkipped.push({ runSlug, reason: err instanceof Error ? err.message : String(err) });
|
|
1709
|
-
continue;
|
|
1710
|
-
}
|
|
1711
|
-
for (const extracted of extraction.cassettes) {
|
|
1712
|
-
store.write(extracted);
|
|
1713
|
-
cassettesWritten++;
|
|
1714
|
-
}
|
|
1715
|
-
for (const emitted of extraction.report.emitted) {
|
|
1716
|
-
bumpCounter(completeness, emitted.completenessStatus);
|
|
1717
|
-
}
|
|
1718
|
-
for (const skip of extraction.report.skips) {
|
|
1719
|
-
bumpCounter(completeness, skip.reason);
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
1722
|
-
return {
|
|
1723
|
-
runsSwept: runSlugs.length,
|
|
1724
|
-
runsSkipped,
|
|
1725
|
-
runsFiltered,
|
|
1726
|
-
cassettesWritten,
|
|
1727
|
-
completeness,
|
|
1728
|
-
cassettesRoot: resolve6(agentQaRoot, DEFAULT_EVAL_CASSETTES_SUBDIR)
|
|
1729
|
-
};
|
|
1730
|
-
}
|
|
1731
|
-
function listRunSlugsWithHandoff(runsRoot) {
|
|
1732
|
-
let entries;
|
|
1733
|
-
try {
|
|
1734
|
-
entries = readdirSync4(runsRoot, { withFileTypes: true });
|
|
1735
|
-
} catch (err) {
|
|
1736
|
-
if (err.code === "ENOENT") return [];
|
|
1737
|
-
throw err;
|
|
1738
|
-
}
|
|
1739
|
-
return entries.filter((entry) => entry.isDirectory() && existsSync4(resolve6(runsRoot, entry.name, HANDOFF_FILE))).map((entry) => entry.name).sort();
|
|
1740
|
-
}
|
|
1741
|
-
function bumpCounter(counters, key) {
|
|
1742
|
-
counters[key] = (counters[key] ?? 0) + 1;
|
|
1743
|
-
}
|
|
1744
|
-
function resolveCassettesRoot(session, source) {
|
|
1745
|
-
if (source === "committed") {
|
|
1746
|
-
const cassettesRoot = join9(
|
|
1747
|
-
resolve6(session.configDir, "agent-qa-eval"),
|
|
1748
|
-
COMMITTED_CASSETTES_DIR
|
|
1749
|
-
);
|
|
1750
|
-
return { cassettesRoot, notice: `cassette source: committed root ${cassettesRoot}` };
|
|
1751
|
-
}
|
|
1752
|
-
return {
|
|
1753
|
-
cassettesRoot: join9(resolve6(session.configDir, ".agent-qa"), DEFAULT_EVAL_CASSETTES_SUBDIR)
|
|
1754
|
-
};
|
|
1755
|
-
}
|
|
1756
|
-
function replayInvocationSpend(entries, seed, modelCalls, started, calibration) {
|
|
1757
|
-
const report = buildEvalReport(entries, void 0, void 0, { declaredSeed: seed, calibration });
|
|
1758
|
-
const cost = report.totalCost;
|
|
1759
|
-
const judgeCost = report.honesty.judge.cost;
|
|
1760
|
-
return {
|
|
1761
|
-
seed: seed ?? null,
|
|
1762
|
-
seedHonored: cost.sampleCount === 0 ? null : report.seedHonored ?? null,
|
|
1763
|
-
decisions: report.headline.perDecisionCost.decisionCount,
|
|
1764
|
-
modelCalls,
|
|
1765
|
-
promptTokens: cost.promptTokens + judgeCost.inputTokens,
|
|
1766
|
-
completionTokens: cost.completionTokens + judgeCost.outputTokens,
|
|
1767
|
-
wallClockMs: Math.max(0, Math.round(performance.now() - started)),
|
|
1768
|
-
sampleCount: cost.sampleCount,
|
|
1769
|
-
samplesMissingTokenUsage: cost.samplesMissingTokenUsage,
|
|
1770
|
-
judgeCost
|
|
1771
|
-
};
|
|
1772
|
-
}
|
|
1773
|
-
async function runReplayVerb(session, filters = {}, opts = {}) {
|
|
1774
|
-
const started = performance.now();
|
|
1775
|
-
const agentQaRoot = resolve6(session.configDir, ".agent-qa");
|
|
1776
|
-
const source = opts.source ?? "staged";
|
|
1777
|
-
const sourceRoot = resolveCassettesRoot(session, source);
|
|
1778
|
-
const cassettesRoot = sourceRoot.cassettesRoot;
|
|
1779
|
-
const replaysRoot = join9(agentQaRoot, DEFAULT_EVAL_REPLAYS_SUBDIR);
|
|
1780
|
-
const { swept, loaded, loadRefusals } = loadCassetteBundles(cassettesRoot);
|
|
1781
|
-
const activeCorpus = readActiveCommittedCorpus(resolve6(session.configDir, "agent-qa-eval"), agentQaRoot);
|
|
1782
|
-
const activeIds = source !== "committed" ? void 0 : new Set(activeCorpus.snapshot.entries.map((entry) => entry.cassetteId));
|
|
1783
|
-
let selected = activeIds === void 0 ? loaded : loaded.filter((entry) => activeIds.has(entry.cassette.id));
|
|
1784
|
-
if (filters.cassette !== void 0) {
|
|
1785
|
-
const named = filters.cassette;
|
|
1786
|
-
selected = selected.filter((entry) => entry.cassette.id === named);
|
|
1787
|
-
if (selected.length === 0) {
|
|
1788
|
-
throw new EvalNamedInputError("cassette", named, cassettesRoot, "replay");
|
|
1789
|
-
}
|
|
1790
|
-
}
|
|
1791
|
-
if (filters.run !== void 0) {
|
|
1792
|
-
const named = filters.run;
|
|
1793
|
-
selected = selected.filter((entry) => {
|
|
1794
|
-
const head = entry.cassette.id.split("/")[0];
|
|
1795
|
-
return head === named || head === slugBranch(named);
|
|
1796
|
-
});
|
|
1797
|
-
if (selected.length === 0) {
|
|
1798
|
-
throw new EvalNamedInputError("run", named, cassettesRoot, "replay");
|
|
1799
|
-
}
|
|
1800
|
-
}
|
|
1801
|
-
if (filters.product !== void 0) {
|
|
1802
|
-
const named = filters.product;
|
|
1803
|
-
selected = selected.filter((entry) => cassetteProductPartition(entry.cassette) === named);
|
|
1804
|
-
}
|
|
1805
|
-
selected.sort((a, b) => a.cassette.id < b.cassette.id ? -1 : a.cassette.id > b.cassette.id ? 1 : 0);
|
|
1806
|
-
const notices = [];
|
|
1807
|
-
if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
|
|
1808
|
-
notices.push(...activeCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
|
|
1809
|
-
const zeroJudge = (configured) => ({
|
|
1810
|
-
configured,
|
|
1811
|
-
invoked: 0,
|
|
1812
|
-
settled: 0,
|
|
1813
|
-
abstained: 0,
|
|
1814
|
-
errors: 0,
|
|
1815
|
-
notInvoked: 0
|
|
1816
|
-
});
|
|
1817
|
-
if (selected.length === 0) {
|
|
1818
|
-
notices.push(`empty sweep \u2014 no ${source} cassette bundles matched`);
|
|
1819
|
-
return {
|
|
1820
|
-
batchKey: "empty",
|
|
1821
|
-
swept,
|
|
1822
|
-
replayed: 0,
|
|
1823
|
-
loadRefusals,
|
|
1824
|
-
replayRefusals: {},
|
|
1825
|
-
scored: 0,
|
|
1826
|
-
judge: zeroJudge(selectedEvalJudgeNames(session.evalConfig).length > 0),
|
|
1827
|
-
replaysRoot,
|
|
1828
|
-
source,
|
|
1829
|
-
notices,
|
|
1830
|
-
spend: replayInvocationSpend([], session.evalConfig.seed, 0, started, activeCorpus?.calibration)
|
|
1831
|
-
};
|
|
1832
|
-
}
|
|
1833
|
-
const batchKey = computeBatchKey(selected.map((entry) => entry.cassette.id));
|
|
1834
|
-
const planner = await resolveReplayPlanner(session);
|
|
1835
|
-
const judgeResolution = await resolveEvalJudge(session);
|
|
1836
|
-
if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
|
|
1837
|
-
const recordStore = new FsReplayRecordStore({ root: agentQaRoot });
|
|
1838
|
-
const scoreStore = new FsScoreStore({ root: agentQaRoot });
|
|
1839
|
-
let modelCalls = 0;
|
|
1840
|
-
const { replayed, replayRefusals, replayedDirs, notices: replayNotices } = await sweepCassetteReplays({
|
|
1841
|
-
selected,
|
|
1842
|
-
labelRecords: activeCorpus?.snapshot.labelsById,
|
|
1843
|
-
profile: planner.profile,
|
|
1844
|
-
auth: planner.auth,
|
|
1845
|
-
modelCallObservation: { onProviderCall: () => {
|
|
1846
|
-
modelCalls++;
|
|
1847
|
-
} },
|
|
1848
|
-
...opts.plannerOverrides !== void 0 ? { modelOverrides: opts.plannerOverrides } : {},
|
|
1849
|
-
...session.evalConfig.k !== void 0 ? { k: session.evalConfig.k } : {},
|
|
1850
|
-
...session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed } : {},
|
|
1851
|
-
...judgeResolution.judge !== void 0 ? {
|
|
1852
|
-
judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides)
|
|
1853
|
-
} : {},
|
|
1854
|
-
mask: session.mask,
|
|
1855
|
-
recordStore,
|
|
1856
|
-
scoreStore,
|
|
1857
|
-
replaysRoot,
|
|
1858
|
-
// Progress ALWAYS rides stderr — stdout stays reserved for the rendered DTO.
|
|
1859
|
-
onProgress: (id) => process.stderr.write(`replay: ${id}
|
|
1860
|
-
`)
|
|
1861
|
-
});
|
|
1862
|
-
notices.push(...replayNotices);
|
|
1863
|
-
const judge = zeroJudge(judgeResolution.judge !== void 0);
|
|
1864
|
-
let scored = 0;
|
|
1865
|
-
for (const recordDir of replayedDirs) {
|
|
1866
|
-
const replay = loadReplayRecord(recordDir);
|
|
1867
|
-
const load = loadScoreRecord(recordDir);
|
|
1868
|
-
if (replay.kind !== "loaded") {
|
|
1869
|
-
notices.push(`spend replay read refused under ${recordDir}: ${replay.refusal.detail}`);
|
|
1870
|
-
}
|
|
1871
|
-
if (load.kind === "absent") continue;
|
|
1872
|
-
if (load.kind === "refused") {
|
|
1873
|
-
notices.push(`score read refused under ${recordDir}: ${load.refusal.detail}`);
|
|
1874
|
-
continue;
|
|
1875
|
-
}
|
|
1876
|
-
scored++;
|
|
1877
|
-
foldJudgeDispositions(load.record.score, judge);
|
|
1878
|
-
}
|
|
1879
|
-
const spendBatch = assembleScoredBatch({
|
|
1880
|
-
replaysRoot,
|
|
1881
|
-
cassettesRoot,
|
|
1882
|
-
source,
|
|
1883
|
-
recordDirs: replayedDirs,
|
|
1884
|
-
labelRecords: activeCorpus?.snapshot.labelsById,
|
|
1885
|
-
corpusEntriesById: source === "committed" ? activeCorpus.corpusEntriesById : void 0,
|
|
1886
|
-
excludedLabelIds: activeCorpus?.snapshot.excludedLabelIds,
|
|
1887
|
-
calibration: activeCorpus?.calibration
|
|
1888
|
-
});
|
|
1889
|
-
notices.push(...spendBatch.notices);
|
|
1890
|
-
return {
|
|
1891
|
-
batchKey,
|
|
1892
|
-
swept,
|
|
1893
|
-
replayed,
|
|
1894
|
-
loadRefusals,
|
|
1895
|
-
replayRefusals,
|
|
1896
|
-
scored,
|
|
1897
|
-
judge,
|
|
1898
|
-
replaysRoot,
|
|
1899
|
-
source,
|
|
1900
|
-
notices,
|
|
1901
|
-
spend: replayInvocationSpend(spendBatch.entries, session.evalConfig.seed, modelCalls, started, spendBatch.calibration)
|
|
1902
|
-
};
|
|
1903
|
-
}
|
|
1904
|
-
function foldJudgeDispositions(score, judge) {
|
|
1905
|
-
const entries = score.kind === "leaf" ? [score.entry] : score.kind === "procedure" ? score.entries : [];
|
|
1906
|
-
for (const entry of entries) {
|
|
1907
|
-
if (entry.kind !== "decision") continue;
|
|
1908
|
-
const decision = entry.score;
|
|
1909
|
-
if (decision.kind === "unresolved-tie") {
|
|
1910
|
-
const outcome = decision.judgeOutcome;
|
|
1911
|
-
if (outcome === void 0) continue;
|
|
1912
|
-
if (outcome.kind === "abstained") {
|
|
1913
|
-
judge.abstained++;
|
|
1914
|
-
judge.invoked++;
|
|
1915
|
-
} else if (outcome.kind === "judge-error") {
|
|
1916
|
-
judge.errors++;
|
|
1917
|
-
judge.invoked++;
|
|
1918
|
-
} else {
|
|
1919
|
-
judge.notInvoked++;
|
|
1920
|
-
}
|
|
1921
|
-
} else if (decision.kind === "scored" && decision.judgeSettled !== void 0) {
|
|
1922
|
-
judge.settled++;
|
|
1923
|
-
judge.invoked++;
|
|
1924
|
-
}
|
|
1925
|
-
}
|
|
1926
|
-
}
|
|
1927
|
-
async function resolveEvalJudge(session) {
|
|
1928
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
1929
|
-
if (names.length === 0) {
|
|
1930
|
-
return {
|
|
1931
|
-
notice: "no judge configured \u2014 unresolved ties will record not-invoked{no-judge-configured}"
|
|
1932
|
-
};
|
|
1933
|
-
}
|
|
1934
|
-
if (names.length > 1) {
|
|
1935
|
-
const members = [];
|
|
1936
|
-
for (const name of names) members.push(await resolveNamedProfile(name, session.configPath));
|
|
1937
|
-
return { judge: { members }, ...session.evalConfig.judgeAllowSameFamily === true ? { notice: "eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint." } : {} };
|
|
1938
|
-
}
|
|
1939
|
-
const resolved = await resolveNamedProfile(names[0], session.configPath);
|
|
1940
|
-
const planner = await resolveReplayPlanner(session);
|
|
1941
|
-
const judgeFamily = resolveJudgeFamily(resolved.profile.model);
|
|
1942
|
-
const plannerFamily = resolveJudgeFamily(planner.profile.model);
|
|
1943
|
-
const overrideActive = session.evalConfig.judgeAllowSameFamily === true;
|
|
1944
|
-
const judge = overrideActive ? { ...resolved, allowSameFamily: true } : resolved;
|
|
1945
|
-
if (judgeFamily !== void 0 && plannerFamily !== void 0 && judgeFamily === plannerFamily && overrideActive) {
|
|
1946
|
-
return {
|
|
1947
|
-
judge,
|
|
1948
|
-
notice: `D-10 override active (eval.judgeAllowSameFamily, owner decision 2026-07-12): judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" share the judge family '${judgeFamily}' \u2014 same-family judging runs and every scored entry discloses familyOverride: same-family-allowed`
|
|
1949
|
-
};
|
|
1950
|
-
}
|
|
1951
|
-
if (judgeFamily === void 0 || plannerFamily === void 0 || judgeFamily === plannerFamily) {
|
|
1952
|
-
return {
|
|
1953
|
-
judge,
|
|
1954
|
-
notice: `judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" resolve to the same or unknown family \u2014 per-cassette judge eligibility will refuse self-judging and ties may record not-invoked`
|
|
1955
|
-
};
|
|
1956
|
-
}
|
|
1957
|
-
return { judge };
|
|
1958
|
-
}
|
|
1959
|
-
async function resolveReplayPlanner(session) {
|
|
1960
|
-
const plannerName = session.config.use?.llm;
|
|
1961
|
-
if (!plannerName) {
|
|
1962
|
-
throw new Error(
|
|
1963
|
-
"use.llm is not set in the config \u2014 the replay planner is the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
|
|
1964
|
-
);
|
|
1965
|
-
}
|
|
1966
|
-
return resolveNamedProfile(plannerName, session.configPath);
|
|
1967
|
-
}
|
|
1968
|
-
async function runJudgeCalibrateVerb(session, opts = {}) {
|
|
1969
|
-
if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
|
|
1970
|
-
const preflight = await preflightJudgePanel(session, { minN: opts.minN });
|
|
1971
|
-
if (preflight.kind === "capability-gap" && !preflight.identity) return {
|
|
1972
|
-
...await runJudgeCalibration({
|
|
1973
|
-
kind: "capability-gap",
|
|
1974
|
-
reason: preflight.reason,
|
|
1975
|
-
minN: preflight.minN,
|
|
1976
|
-
commit: opts.commit,
|
|
1977
|
-
replace: opts.replace
|
|
1978
|
-
}),
|
|
1979
|
-
entries: preflight.entries
|
|
1980
|
-
};
|
|
1981
|
-
const replayProfile = preflight.kind === "eligible" ? preflight.replayProfile : await resolveReplayProfileDeclaration(session);
|
|
1982
|
-
const fingerprint = computeReplayProfileFingerprintFor({
|
|
1983
|
-
profile: replayProfile,
|
|
1984
|
-
seedPolicy: "declared"
|
|
1985
|
-
});
|
|
1986
|
-
const context = {
|
|
1987
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
1988
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
1989
|
-
// Saved evidence must prove this actual planner/profile; judging does not rerun a planner.
|
|
1990
|
-
profile: {
|
|
1991
|
-
plannerModelId: replayProfile.model,
|
|
1992
|
-
fingerprint,
|
|
1993
|
-
matcherVersion: (await import("@vostride/agent-qa-eval")).projectMatcherConfigContract().version,
|
|
1994
|
-
judgeModelId: preflight.identity.judgeModelId,
|
|
1995
|
-
judgeVersion: preflight.identity.judgeVersion
|
|
1996
|
-
},
|
|
1997
|
-
identity: preflight.identity,
|
|
1998
|
-
minN: preflight.minN
|
|
1999
|
-
};
|
|
2000
|
-
const result = await runJudgeCalibration(preflight.kind === "eligible" ? { kind: "probe", commit: opts.commit, replace: opts.replace, probe: {
|
|
2001
|
-
...context,
|
|
2002
|
-
replayProfile,
|
|
2003
|
-
members: preflight.members.map((member) => ({ ...member, modelOverrides: opts.judgeOverrides })),
|
|
2004
|
-
mask: session.mask
|
|
2005
|
-
} } : {
|
|
2006
|
-
kind: "capability-gap",
|
|
2007
|
-
reason: preflight.reason,
|
|
2008
|
-
minN: preflight.minN,
|
|
2009
|
-
publication: context,
|
|
2010
|
-
commit: opts.commit,
|
|
2011
|
-
replace: opts.replace
|
|
2012
|
-
});
|
|
2013
|
-
return maskJudgeCalibrationText({ ...result, entries: preflight.entries }, session.mask);
|
|
2014
|
-
}
|
|
2015
|
-
function renderJudgeTargetedCalibration(disclosure) {
|
|
2016
|
-
if (disclosure === void 0) {
|
|
2017
|
-
return ["judge-targeted adversarial bait: not recorded (this artifact predates the disclosure)"];
|
|
2018
|
-
}
|
|
2019
|
-
if (disclosure.source === "unavailable") {
|
|
2020
|
-
return [`judge-targeted adversarial bait: not determined (${disclosure.reason})`];
|
|
2021
|
-
}
|
|
2022
|
-
const lines = [
|
|
2023
|
-
`judge-targeted adversarial bait: judge-targeted=${disclosure.judgeTargeted} self-authored-by-this-panel=${disclosure.selfAuthored} observed-in-measured-population=${disclosure.observedInCalibration}`,
|
|
2024
|
-
`adversarial provenance: model-authored=${disclosure.byProvenance.probe} table-derived=${disclosure.byProvenance.table} legacy-unattributed=${disclosure.byProvenance.absent} malformed=${disclosure.byProvenance.malformed} unresolved=${disclosure.metadataUnavailable} of ${disclosure.total} committed members (arms are counted apart, never summed)`
|
|
2025
|
-
];
|
|
2026
|
-
for (const row of disclosure.byAuthoringModel) {
|
|
2027
|
-
lines.push(`author ${row.authoringModel} (${row.authoringFamily})${row.panelMember ? " [panel member \u2014 self-authored]" : ""}: judge-targeted=${row.judgeTargeted} authored=${row.probeAuthored}`);
|
|
2028
|
-
}
|
|
2029
|
-
lines.push("Bait counts above are RAW: the agreement, interval, per-member rows and minN are not adjusted for them.");
|
|
2030
|
-
return lines;
|
|
2031
|
-
}
|
|
2032
|
-
async function runCalibrateVerb(session, opts = {}) {
|
|
2033
|
-
const minGold = opts.minGold ?? LABEL_CALIBRATION_MIN_GOLD;
|
|
2034
|
-
if (!Number.isSafeInteger(minGold) || minGold < 2) {
|
|
2035
|
-
return { kind: "refused", reason: "--min-gold must be a safe integer of at least 2." };
|
|
2036
|
-
}
|
|
2037
|
-
if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
|
|
2038
|
-
const result = await runLabelCalibration({
|
|
2039
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
2040
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
2041
|
-
commit: opts.commit,
|
|
2042
|
-
replace: opts.replace,
|
|
2043
|
-
minGold: { value: minGold, source: opts.minGold === void 0 ? "default" : "cli" }
|
|
2044
|
-
});
|
|
2045
|
-
if ("artifact" in result) return result;
|
|
2046
|
-
const reasons = {
|
|
2047
|
-
refused: "Calibration inputs unavailable or invalid; inspect committed labels and run evidence.",
|
|
2048
|
-
busy: "Another calibration publisher holds the lock; retry after it finishes.",
|
|
2049
|
-
"input-drift": "Calibration inputs changed; retry from the current snapshot.",
|
|
2050
|
-
"publication-failed": "Calibration publication interrupted; retry the identical command.",
|
|
2051
|
-
"write-conflict": "Calibration bytes conflict; review the dry-run, then use --commit --replace to replace the marker. Immutable revision conflicts require inspection."
|
|
2052
|
-
};
|
|
2053
|
-
return { kind: result.kind, reason: reasons[result.kind] };
|
|
2054
|
-
}
|
|
2055
|
-
function resolveGate2Caps(session, opts = {}) {
|
|
2056
|
-
const maxAblations = opts.maxAblations ?? session.evalConfig.gate2?.maxAblations ?? GATE2_DEFAULT_MAX_ABLATIONS;
|
|
2057
|
-
if (!Number.isSafeInteger(maxAblations) || maxAblations < 1) {
|
|
2058
|
-
throw new TypeError("--max-ablations must be a positive integer");
|
|
2059
|
-
}
|
|
2060
|
-
const maxAblationsSource = opts.maxAblations !== void 0 ? "flag" : session.evalConfig.gate2?.maxAblations !== void 0 ? "config" : "default";
|
|
2061
|
-
const canaryHorizonBuilds = session.evalConfig.gate2?.canaryHorizonBuilds ?? GATE2_DEFAULT_CANARY_HORIZON_BUILDS;
|
|
2062
|
-
const maxLowerTrustPerScope = session.evalConfig.gate2?.maxLowerTrustPerScope ?? GATE2_DEFAULT_MAX_LOWER_TRUST_PER_SCOPE;
|
|
2063
|
-
for (const [name, value] of Object.entries({ canaryHorizonBuilds, maxLowerTrustPerScope })) {
|
|
2064
|
-
if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`eval.gate2.${name} must be a positive integer`);
|
|
2065
|
-
}
|
|
2066
|
-
return {
|
|
2067
|
-
maxAblations,
|
|
2068
|
-
maxAblationsSource,
|
|
2069
|
-
canaryHorizonBuilds,
|
|
2070
|
-
maxLowerTrustPerScope,
|
|
2071
|
-
canaryHorizonBuildsSource: session.evalConfig.gate2?.canaryHorizonBuilds === void 0 ? "default" : "config",
|
|
2072
|
-
maxLowerTrustPerScopeSource: session.evalConfig.gate2?.maxLowerTrustPerScope === void 0 ? "default" : "config"
|
|
2073
|
-
};
|
|
2074
|
-
}
|
|
2075
|
-
function readCanaryCorpusDigest(session) {
|
|
2076
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2077
|
-
try {
|
|
2078
|
-
lstatSync2(committedRoot);
|
|
2079
|
-
} catch (error) {
|
|
2080
|
-
return error.code === "ENOENT" ? { kind: "no-eval-root" } : { kind: "unavailable", reason: error instanceof Error ? error.message : String(error) };
|
|
2081
|
-
}
|
|
2082
|
-
return readCommittedCorpusDigest({ committedRoot });
|
|
2083
|
-
}
|
|
2084
|
-
async function runCanaryAblation(session, input) {
|
|
2085
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2086
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2087
|
-
if (digest2.kind !== "digest") return digest2;
|
|
2088
|
-
const k = input.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
2089
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
2090
|
-
throw new TypeError("--k must be an odd integer of at least 3");
|
|
2091
|
-
}
|
|
2092
|
-
const identity = await resolveGateProfileIdentity(session, input.plannerOverrides);
|
|
2093
|
-
const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
|
|
2094
|
-
(entry) => entry.artefact.profileKey === identity.profileKey
|
|
2095
|
-
)?.artefact;
|
|
2096
|
-
const baseSeed = input.seed ?? session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
|
|
2097
|
-
const execution = await resolveEvalSession({ configPath: session.configPath });
|
|
2098
|
-
const planner = await resolveReplayPlanner(execution);
|
|
2099
|
-
return ablateCanaryStep({
|
|
2100
|
-
agentQaRoot: input.agentQaRoot,
|
|
2101
|
-
committedRoot,
|
|
2102
|
-
recordId: input.recordId,
|
|
2103
|
-
target: input.target,
|
|
2104
|
-
attributedObservationId: input.attributedObservationId,
|
|
2105
|
-
k,
|
|
2106
|
-
baseSeed,
|
|
2107
|
-
replay: {
|
|
2108
|
-
profile: planner.profile,
|
|
2109
|
-
auth: planner.auth,
|
|
2110
|
-
modelOverrides: input.plannerOverrides,
|
|
2111
|
-
mask: execution.mask
|
|
2112
|
-
}
|
|
2113
|
-
});
|
|
2114
|
-
}
|
|
2115
|
-
async function readMisdirectionRootPosture(session) {
|
|
2116
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2117
|
-
const floorless = !hasNoiseFloorArtefacts(resolve6(session.configDir, "agent-qa-eval"));
|
|
2118
|
-
if (digest2.kind === "no-eval-root" || floorless) return { kind: "no-eval-root" };
|
|
2119
|
-
if (digest2.kind !== "digest") return { kind: "broken-eval-root", reason: "eval-root-unreadable" };
|
|
2120
|
-
const identity = await resolveGateProfileIdentity(session);
|
|
2121
|
-
return {
|
|
2122
|
-
kind: "eval-root",
|
|
2123
|
-
corpusDigest: digest2.corpusDigest,
|
|
2124
|
-
profileDigest: sha256hex3(identity.profileKey)
|
|
2125
|
-
};
|
|
2126
|
-
}
|
|
2127
|
-
function resolveMisdirectionCaps(_session, opts = {}) {
|
|
2128
|
-
const maxChecks = opts.maxChecks ?? ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS;
|
|
2129
|
-
if (!Number.isSafeInteger(maxChecks) || maxChecks < 0) {
|
|
2130
|
-
throw new TypeError("--max-misdirection-checks must be a non-negative integer");
|
|
2131
|
-
}
|
|
2132
|
-
return { maxChecks, source: opts.maxChecks === void 0 ? "default" : "flag" };
|
|
2133
|
-
}
|
|
2134
|
-
async function locateMisdirectionDecision(session, target) {
|
|
2135
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2136
|
-
const corpus = readCommittedCorpus(committedRoot);
|
|
2137
|
-
const located = await locateTargetCassettes({
|
|
2138
|
-
committedRoot,
|
|
2139
|
-
agentQaRoot: target.agentQaRoot,
|
|
2140
|
-
gate1Dir: misdirectionAuditDir(target),
|
|
2141
|
-
corpusEntries: corpus.snapshot.entries,
|
|
2142
|
-
target: { runId: target.runId, testId: target.testId, stepIndex: target.stepIndex },
|
|
2143
|
-
source: "handoff"
|
|
2144
|
-
});
|
|
2145
|
-
if (located.kind === "unavailable") {
|
|
2146
|
-
return { kind: "unavailable", reason: located.detail.reason ?? "target-unavailable" };
|
|
2147
|
-
}
|
|
2148
|
-
if (located.cassettes.length !== 1) {
|
|
2149
|
-
return { kind: "unavailable", reason: "ambiguous live step: multiple recorded decisions" };
|
|
2150
|
-
}
|
|
2151
|
-
return { kind: "located", decisionOrdinal: located.cassettes[0].cassette.decisionOrdinal };
|
|
2152
|
-
}
|
|
2153
|
-
async function runMisdirectionCheck(session, input) {
|
|
2154
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2155
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2156
|
-
if (digest2.kind !== "digest") {
|
|
2157
|
-
return { kind: "unavailable", reason: digest2.kind === "no-eval-root" ? "no-eval-root" : digest2.reason };
|
|
2158
|
-
}
|
|
2159
|
-
const k = session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
2160
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
2161
|
-
throw new TypeError("eval.k must be an odd integer of at least 3");
|
|
2162
|
-
}
|
|
2163
|
-
const identity = await resolveGateProfileIdentity(session);
|
|
2164
|
-
const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
|
|
2165
|
-
(entry) => entry.artefact.profileKey === identity.profileKey
|
|
2166
|
-
)?.artefact;
|
|
2167
|
-
const baseSeed = session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
|
|
2168
|
-
const execution = await resolveEvalSession({ configPath: session.configPath });
|
|
2169
|
-
const planner = await resolveReplayPlanner(execution);
|
|
2170
|
-
const result = await ablateCanaryStep({
|
|
2171
|
-
agentQaRoot: input.agentQaRoot,
|
|
2172
|
-
committedRoot,
|
|
2173
|
-
recordId: input.recordId,
|
|
2174
|
-
target: { runId: input.runId, testId: input.testId, stepIndex: input.stepIndex },
|
|
2175
|
-
attributedObservationId: misdirectionAuditKey(input),
|
|
2176
|
-
k,
|
|
2177
|
-
baseSeed,
|
|
2178
|
-
admission: { failingPathCandidate: input.authorization },
|
|
2179
|
-
replay: {
|
|
2180
|
-
profile: planner.profile,
|
|
2181
|
-
auth: planner.auth,
|
|
2182
|
-
mask: execution.mask
|
|
2183
|
-
}
|
|
2184
|
-
});
|
|
2185
|
-
if (result.kind !== "ablated") return { kind: "unavailable", reason: result.reason };
|
|
2186
|
-
return {
|
|
2187
|
-
kind: "checked",
|
|
2188
|
-
crossCheck: result.crossCheck,
|
|
2189
|
-
seedHonored: result.seedHonored,
|
|
2190
|
-
samples: { k: result.k, seeds: [...result.seeds], modelCalls: result.spend.modelCalls }
|
|
2191
|
-
};
|
|
2192
|
-
}
|
|
2193
|
-
function misdirectionAuditKey(target) {
|
|
2194
|
-
return `misdirection/${slugBranch(target.runId)}/${slugBranch(target.testId)}/${String(target.stepIndex)}${target.recordId === void 0 ? "" : `/${slugBranch(target.recordId)}`}`;
|
|
2195
|
-
}
|
|
2196
|
-
function misdirectionAuditDir(target) {
|
|
2197
|
-
return join9(target.agentQaRoot, GATE2_AUDIT_SUBDIR, slugBranch(misdirectionAuditKey(target)));
|
|
2198
|
-
}
|
|
2199
|
-
function isFeedbackStateNovel(session, input) {
|
|
2200
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2201
|
-
if (digest2.kind === "no-eval-root") return false;
|
|
2202
|
-
if (digest2.kind !== "digest") throw new TypeError("feedback corpus unavailable");
|
|
2203
|
-
return isNovelState({ ...input, committedRoot: resolve6(session.configDir, "agent-qa-eval") });
|
|
2204
|
-
}
|
|
2205
|
-
async function runFeedbackStaging(session, input) {
|
|
2206
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2207
|
-
if (digest2.kind !== "digest") return digest2;
|
|
2208
|
-
return stageFeedbackEntry({
|
|
2209
|
-
...input,
|
|
2210
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
2211
|
-
mask: committedCorpusMask(session)
|
|
2212
|
-
});
|
|
2213
|
-
}
|
|
2214
|
-
async function runSeedFromFeedbackVerb(session, opts) {
|
|
2215
|
-
const stagingDir = join9(opts.agentQaRoot, FEEDBACK_STAGING_SUBDIR);
|
|
2216
|
-
const configArg = `'${resolve6(session.configPath).replaceAll("'", `'"'"'`)}'`;
|
|
2217
|
-
const floorCommand = `agent-qa --config ${configArg} eval floor`;
|
|
2218
|
-
const base = { entries: [], refusals: [], staleReceipts: [], writesStarted: false, failures: [], stagingDir, floorCommand };
|
|
2219
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2220
|
-
if (digest2.kind !== "digest") {
|
|
2221
|
-
const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
|
|
2222
|
-
return { ...base, kind: "refused", reason, failures: [{ phase: "before-write", reason }] };
|
|
2223
|
-
}
|
|
2224
|
-
try {
|
|
2225
|
-
const receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
|
|
2226
|
-
const result = await admitStagedFeedback({
|
|
2227
|
-
...opts,
|
|
2228
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
2229
|
-
mask: committedCorpusMask(session)
|
|
2230
|
-
});
|
|
2231
|
-
const selected = opts.dryRun ? result.entries : result.admitted;
|
|
2232
|
-
const entries = selected.map((entry) => ({
|
|
2233
|
-
cassetteId: entry.extracted.cassette.id,
|
|
2234
|
-
product: entry.product,
|
|
2235
|
-
split: "natural",
|
|
2236
|
-
source: "canary-feedback",
|
|
2237
|
-
benefit: entry.benefit,
|
|
2238
|
-
attributedObservationId: entry.attributedObservationId,
|
|
2239
|
-
runId: entry.runId,
|
|
2240
|
-
stepIndex: entry.stepIndex,
|
|
2241
|
-
captured_at_ordinal: entry.extracted.cassette.receipt.buildOrdinal
|
|
2242
|
-
}));
|
|
2243
|
-
const moves = opts.dryRun ? entries.length > 0 : result.nextCorpusDigest === void 0 ? void 0 : result.nextCorpusDigest !== result.previousCorpusDigest;
|
|
2244
|
-
const refused = result.refusals.length > 0 || result.failures.length > 0;
|
|
2245
|
-
return {
|
|
2246
|
-
kind: refused ? "refused" : result.entries.length === 0 ? "nothing-staged" : opts.dryRun ? "dry-run" : "admitted",
|
|
2247
|
-
...refused ? { reason: "admission-refused" } : {},
|
|
2248
|
-
entries,
|
|
2249
|
-
refusals: result.refusals,
|
|
2250
|
-
writesStarted: result.writesStarted,
|
|
2251
|
-
failures: result.failures,
|
|
2252
|
-
staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === result.previousCorpusDigest) : [],
|
|
2253
|
-
previousCorpusDigest: result.previousCorpusDigest,
|
|
2254
|
-
...result.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: result.nextCorpusDigest },
|
|
2255
|
-
stagingDir,
|
|
2256
|
-
floorCommand
|
|
2257
|
-
};
|
|
2258
|
-
} catch {
|
|
2259
|
-
return {
|
|
2260
|
-
...base,
|
|
2261
|
-
kind: "refused",
|
|
2262
|
-
reason: "feedback-unavailable",
|
|
2263
|
-
failures: [{ phase: "before-write", reason: "feedback-unavailable" }],
|
|
2264
|
-
previousCorpusDigest: digest2.corpusDigest
|
|
2265
|
-
};
|
|
2266
|
-
}
|
|
2267
|
-
}
|
|
2268
|
-
function authoringModelDisclosureOf(model) {
|
|
2269
|
-
return {
|
|
2270
|
-
idDigest: createHash2("sha256").update(model.id, "utf8").digest("hex"),
|
|
2271
|
-
family: model.family
|
|
2272
|
-
};
|
|
2273
|
-
}
|
|
2274
|
-
function adversarialHostOptions(session) {
|
|
2275
|
-
return {
|
|
2276
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
2277
|
-
knowledgeRoot: resolveKnowledgeRoot2(session.config.knowledge, session.configDir),
|
|
2278
|
-
readStandingGateReceipts: LOCAL_CURATION_CAPABILITIES3.readStandingGateReceipts
|
|
2279
|
-
};
|
|
2280
|
-
}
|
|
2281
|
-
function runRetireAdversarialVerb(session, opts) {
|
|
2282
|
-
const result = retireStagedAdversarialBatch({
|
|
2283
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
2284
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
2285
|
-
product: opts.product,
|
|
2286
|
-
batchDigest: opts.batch
|
|
2287
|
-
});
|
|
2288
|
-
return {
|
|
2289
|
-
...result,
|
|
2290
|
-
modelCalls: 0,
|
|
2291
|
-
admitsNothing: true,
|
|
2292
|
-
settledWorkPreserved: true,
|
|
2293
|
-
newMeasurementStaleness: false
|
|
2294
|
-
};
|
|
2295
|
-
}
|
|
2296
|
-
async function runAdversarialVerb(session, opts) {
|
|
2297
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2298
|
-
const product = opts.product ?? "default";
|
|
2299
|
-
const stagingDir = join9(opts.agentQaRoot, ADVERSARIAL_STAGING_SUBDIR, slugBranch(product));
|
|
2300
|
-
const configArg = `'${resolve6(session.configPath).replaceAll("'", `'"'"'`)}'`;
|
|
2301
|
-
const floorCommand = `agent-qa --config ${configArg} eval floor`;
|
|
2302
|
-
const cap = resolveAdversarialSeedCap(
|
|
2303
|
-
opts.maxSeeds !== void 0 ? { value: opts.maxSeeds, source: "flag" } : session.evalConfig.adversarial?.maxSeedsPerRefresh !== void 0 ? { value: session.evalConfig.adversarial.maxSeedsPerRefresh, source: "config" } : {}
|
|
2304
|
-
);
|
|
2305
|
-
const probeCap = resolveAdversarialProbeCallCap(
|
|
2306
|
-
opts.maxProbeCalls !== void 0 ? { value: opts.maxProbeCalls, source: "flag" } : session.evalConfig.adversarial?.maxProbeCalls !== void 0 ? { value: session.evalConfig.adversarial.maxProbeCalls, source: "config" } : {}
|
|
2307
|
-
);
|
|
2308
|
-
const base = {
|
|
2309
|
-
kind: "refused",
|
|
2310
|
-
product,
|
|
2311
|
-
stagingDir,
|
|
2312
|
-
staged: [],
|
|
2313
|
-
admitted: [],
|
|
2314
|
-
unchanged: [],
|
|
2315
|
-
remaining: [],
|
|
2316
|
-
refusals: [],
|
|
2317
|
-
settledRefusals: [],
|
|
2318
|
-
stagingSkips: [],
|
|
2319
|
-
skips: [],
|
|
2320
|
-
seedsConsidered: 0,
|
|
2321
|
-
seedsDeferred: 0,
|
|
2322
|
-
eligibleSeeds: 0,
|
|
2323
|
-
cap,
|
|
2324
|
-
writesStarted: false,
|
|
2325
|
-
staleReceipts: [],
|
|
2326
|
-
floorCommand,
|
|
2327
|
-
probeCap,
|
|
2328
|
-
spend: { modelCalls: 0, promptTokens: 0, completionTokens: 0, usageKnown: true },
|
|
2329
|
-
probeSkips: [],
|
|
2330
|
-
probeRefusals: []
|
|
2331
|
-
};
|
|
2332
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2333
|
-
if (digest2.kind !== "digest") {
|
|
2334
|
-
const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
|
|
2335
|
-
return { ...base, kind: "refused", reason };
|
|
2336
|
-
}
|
|
2337
|
-
if (opts.admit === true) {
|
|
2338
|
-
if (opts.batch === void 0) {
|
|
2339
|
-
return { ...base, kind: "refused", reason: "admit-requires-batch" };
|
|
2340
|
-
}
|
|
2341
|
-
let receipts = [];
|
|
2342
|
-
const refreshed = refreshAdversarialSplit({
|
|
2343
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2344
|
-
committedRoot,
|
|
2345
|
-
product,
|
|
2346
|
-
batchDigest: opts.batch,
|
|
2347
|
-
beforeWrites: () => {
|
|
2348
|
-
receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
|
|
2349
|
-
},
|
|
2350
|
-
...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun },
|
|
2351
|
-
mask: committedCorpusMask(session)
|
|
2352
|
-
});
|
|
2353
|
-
const moves = refreshed.nextCorpusDigest === void 0 ? refreshed.writesStarted ? void 0 : false : refreshed.nextCorpusDigest !== refreshed.previousCorpusDigest;
|
|
2354
|
-
return {
|
|
2355
|
-
...base,
|
|
2356
|
-
kind: refreshed.kind,
|
|
2357
|
-
...refreshed.reason === void 0 ? {} : { reason: refreshed.reason },
|
|
2358
|
-
batchDigest: refreshed.batchDigest,
|
|
2359
|
-
admitted: refreshed.admitted,
|
|
2360
|
-
unchanged: refreshed.unchanged,
|
|
2361
|
-
remaining: refreshed.remaining,
|
|
2362
|
-
refusals: refreshed.refusals,
|
|
2363
|
-
stagingSkips: refreshed.stagingSkips,
|
|
2364
|
-
writesStarted: refreshed.writesStarted,
|
|
2365
|
-
...refreshed.retirement === void 0 ? {} : { retirement: refreshed.retirement },
|
|
2366
|
-
...refreshed.lifecycleBusy === void 0 ? {} : { lifecycleBusy: refreshed.lifecycleBusy },
|
|
2367
|
-
staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === refreshed.previousCorpusDigest) : [],
|
|
2368
|
-
...refreshed.previousCorpusDigest === void 0 ? {} : { previousCorpusDigest: refreshed.previousCorpusDigest },
|
|
2369
|
-
...refreshed.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: refreshed.nextCorpusDigest }
|
|
2370
|
-
};
|
|
2371
|
-
}
|
|
2372
|
-
const probe = await resolveProbeBinding(session, opts, product);
|
|
2373
|
-
const result = await stageAdversarialProposal({
|
|
2374
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2375
|
-
committedRoot,
|
|
2376
|
-
product,
|
|
2377
|
-
...opts.mutators === void 0 ? {} : { mutators: opts.mutators },
|
|
2378
|
-
// Only a NON-default cap is pushed down: the composition owns the recorded default, and
|
|
2379
|
-
// handing it back an unattributed value would have it record an operator choice nobody made.
|
|
2380
|
-
...cap.source === "default" ? {} : { maxSeeds: cap.value, maxSeedsSource: cap.source },
|
|
2381
|
-
...probeCap.source === "default" ? {} : { maxProbeCalls: probeCap.value, maxProbeCallsSource: probeCap.source },
|
|
2382
|
-
...probe.noProbeModel === void 0 ? {} : { noProbeModel: probe.noProbeModel },
|
|
2383
|
-
// A DRY RUN authors nothing. A preview that spends money is not a preview, and there is no
|
|
2384
|
-
// way to un-spend it once the batch it previewed is discarded.
|
|
2385
|
-
...probe.author === void 0 || opts.dryRun === true ? {} : { probeAuthor: probe.author, authoringModel: probe.authoringModel },
|
|
2386
|
-
...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun }
|
|
2387
|
-
});
|
|
2388
|
-
return {
|
|
2389
|
-
...base,
|
|
2390
|
-
kind: result.kind,
|
|
2391
|
-
...result.batchDigest === void 0 ? {} : { batchDigest: result.batchDigest },
|
|
2392
|
-
staged: result.staged,
|
|
2393
|
-
refusals: result.refusals,
|
|
2394
|
-
settledRefusals: result.settledRefusals,
|
|
2395
|
-
stagingSkips: result.stagingSkips,
|
|
2396
|
-
skips: result.skips,
|
|
2397
|
-
seedsConsidered: result.seedsConsidered,
|
|
2398
|
-
seedsDeferred: result.seedsDeferred,
|
|
2399
|
-
eligibleSeeds: result.eligibleSeeds,
|
|
2400
|
-
cap: result.cap,
|
|
2401
|
-
tableVersion: result.tableVersion,
|
|
2402
|
-
tableHash: result.tableHash,
|
|
2403
|
-
previousCorpusDigest: digest2.corpusDigest,
|
|
2404
|
-
// Staging commits nothing, so nothing it does can stale a standing receipt.
|
|
2405
|
-
staleReceipts: [],
|
|
2406
|
-
...result.noProbeModel === void 0 ? {} : { noProbeModel: result.noProbeModel },
|
|
2407
|
-
// The raw id stays on the binding, where derivation and the committed stamp need it. What
|
|
2408
|
-
// leaves on the result is the family and a digest -- see AdversarialVerbResult.
|
|
2409
|
-
...result.authoringModel === void 0 ? {} : { authoringModel: authoringModelDisclosureOf(result.authoringModel) },
|
|
2410
|
-
probeCap: result.probeCap,
|
|
2411
|
-
spend: result.spend,
|
|
2412
|
-
probeSkips: result.probeSkips,
|
|
2413
|
-
probeRefusals: result.probeRefusals
|
|
2414
|
-
};
|
|
2415
|
-
}
|
|
2416
|
-
async function resolveProbeBinding(session, opts, product) {
|
|
2417
|
-
const capability = opts.probeCapability ?? await (await Promise.resolve().then(() => (init_adversarial_probe_preflight(), adversarial_probe_preflight_exports))).resolveAdversarialProbeCapability(session);
|
|
2418
|
-
if (capability.kind !== "probe-author") {
|
|
2419
|
-
return { noProbeModel: { kind: "no-probe-model", reason: capability.reason } };
|
|
2420
|
-
}
|
|
2421
|
-
const bind = opts.bindProbeAuthor ?? await defaultProbeAuthorBinder(session);
|
|
2422
|
-
const binding = {
|
|
2423
|
-
onCallAttempted: (attempt) => {
|
|
2424
|
-
const state = readProbeAttempt({
|
|
2425
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2426
|
-
key: {
|
|
2427
|
-
product,
|
|
2428
|
-
seedCassetteId: attempt.seedCassetteId,
|
|
2429
|
-
kind: attempt.kind,
|
|
2430
|
-
promptVersion: attempt.promptVersion
|
|
2431
|
-
}
|
|
2432
|
-
});
|
|
2433
|
-
if (state.kind === "unattempted") {
|
|
2434
|
-
throw new Error("This probe call holds no durable reservation. Nothing reaches a provider before the spend is recorded.");
|
|
2435
|
-
}
|
|
2436
|
-
}
|
|
2437
|
-
};
|
|
2438
|
-
return { author: bind(capability.author, binding), authoringModel: capability.author.authoringModel };
|
|
2439
|
-
}
|
|
2440
|
-
async function defaultProbeAuthorBinder(session) {
|
|
2441
|
-
const { createAdversarialProbeAuthor: createAdversarialProbeAuthor2 } = await Promise.resolve().then(() => (init_adversarial_probe_author(), adversarial_probe_author_exports));
|
|
2442
|
-
return (author, binding) => createAdversarialProbeAuthor2({
|
|
2443
|
-
author,
|
|
2444
|
-
executionContext: session.executionContext,
|
|
2445
|
-
mask: session.mask,
|
|
2446
|
-
onCallAttempted: binding.onCallAttempted
|
|
2447
|
-
});
|
|
2448
|
-
}
|
|
2449
|
-
function listAdversarialBatches(session, opts) {
|
|
2450
|
-
void session;
|
|
2451
|
-
return listStagedAdversarial(opts).map((batch) => ({
|
|
2452
|
-
dir: batch.dir,
|
|
2453
|
-
batchDigest: batch.manifest.batchDigest,
|
|
2454
|
-
product: batch.manifest.product,
|
|
2455
|
-
tableVersion: batch.manifest.tableVersion,
|
|
2456
|
-
tableHash: batch.manifest.tableHash,
|
|
2457
|
-
stagedAt: batch.manifest.stagedAt,
|
|
2458
|
-
cases: batch.manifest.cases.map((row) => ({
|
|
2459
|
-
cassetteId: row.cassetteId,
|
|
2460
|
-
seedCassetteId: row.seedCassetteId,
|
|
2461
|
-
mutator: row.mutator,
|
|
2462
|
-
kind: row.kind,
|
|
2463
|
-
tableEntryId: row.tableEntryId
|
|
2464
|
-
}))
|
|
2465
|
-
}));
|
|
2466
|
-
}
|
|
2467
|
-
function feedbackSeedHostOptions(session) {
|
|
2468
|
-
return {
|
|
2469
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
2470
|
-
knowledgeRoot: resolveKnowledgeRoot2(session.config.knowledge, session.configDir),
|
|
2471
|
-
readStandingGateReceipts: LOCAL_CURATION_CAPABILITIES3.readStandingGateReceipts
|
|
2472
|
-
};
|
|
2473
|
-
}
|
|
2474
|
-
async function runReplayGateVerb(session, candidateId, opts = {}) {
|
|
2475
|
-
const k = opts.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
2476
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
2477
|
-
throw new TypeError("--k must be an odd integer of at least 3");
|
|
2478
|
-
}
|
|
2479
|
-
const kSource = opts.k !== void 0 ? "flag" : session.evalConfig.k !== void 0 ? "config" : "default";
|
|
2480
|
-
const maxScreens = opts.maxScreens ?? session.evalConfig.gate1?.maxScreens ?? GATE1_DEFAULT_MAX_SCREENS;
|
|
2481
|
-
const maxScreensSource = opts.maxScreens !== void 0 ? "flag" : session.evalConfig.gate1?.maxScreens !== void 0 ? "config" : "default";
|
|
2482
|
-
if (!Number.isSafeInteger(maxScreens) || maxScreens < 1) {
|
|
2483
|
-
throw new TypeError("--max-screens must be a positive integer");
|
|
2484
|
-
}
|
|
2485
|
-
const identity = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2486
|
-
const profile = await resolveReplayProfileDeclaration(session);
|
|
2487
|
-
let execution;
|
|
2488
|
-
let mask = session.mask;
|
|
2489
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
2490
|
-
const prepare = () => execution ??= resolveEvalSession({
|
|
2491
|
-
configPath: session.configPath,
|
|
2492
|
-
flags: { judge: names }
|
|
2493
|
-
}).then(async (full) => {
|
|
2494
|
-
if ((await resolveGateProfileIdentity(full, opts.plannerOverrides)).profileKey !== identity.profileKey) {
|
|
2495
|
-
throw new EvalJudgeDeclarationError("Gate profile identity changed before execution; re-run eval replay-gate.");
|
|
2496
|
-
}
|
|
2497
|
-
mask = full.mask;
|
|
2498
|
-
return full;
|
|
2499
|
-
});
|
|
2500
|
-
const declarations = await resolveEvalJudgeDeclarations(session);
|
|
2501
|
-
const lazyJudges = declarations.map(({ name, profile: profile2 }) => ({
|
|
2502
|
-
profile: profile2,
|
|
2503
|
-
modelOverrides: opts.judgeOverrides,
|
|
2504
|
-
resolveAuth: async () => {
|
|
2505
|
-
const full = await prepare();
|
|
2506
|
-
return (await resolveNamedProfile(name, full.configPath)).auth;
|
|
2507
|
-
}
|
|
2508
|
-
}));
|
|
2509
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2510
|
-
const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
|
|
2511
|
-
(entry) => entry.path === join9(committedRoot, committedNoiseFloorPath(identity.profileKey)) && entry.artefact.profileKey === identity.profileKey
|
|
2512
|
-
)?.artefact;
|
|
2513
|
-
const { createReplayGateHost: createReplayGateHost2 } = await Promise.resolve().then(() => (init_replay_gate_host(), replay_gate_host_exports));
|
|
2514
|
-
const result = await createReplayGateHost2({
|
|
2515
|
-
...opts.resolveTrustPriors === void 0 ? {} : { resolveTrustPriors: opts.resolveTrustPriors },
|
|
2516
|
-
// 422 D-14: the lease the caller lent this verb, re-presented to the bridge as a resolver
|
|
2517
|
-
// over the value this call already holds. The bridge calls it once, at its own claim.
|
|
2518
|
-
...opts.claimAuthority === void 0 ? {} : { resolveClaimAuthority: () => opts.claimAuthority },
|
|
2519
|
-
configPath: session.configPath,
|
|
2520
|
-
agentQaRoot: opts.agentQaRoot ?? resolve6(session.configDir, ".agent-qa"),
|
|
2521
|
-
record: opts.dryRun !== true,
|
|
2522
|
-
session: {
|
|
2523
|
-
...session,
|
|
2524
|
-
mask: (text) => mask(text)
|
|
2525
|
-
},
|
|
2526
|
-
composition: {
|
|
2527
|
-
identity,
|
|
2528
|
-
baselineWarmup: {
|
|
2529
|
-
configPath: session.configPath,
|
|
2530
|
-
...names.length > 1 ? { judges: names } : { judge: names[0] }
|
|
2531
|
-
},
|
|
2532
|
-
k,
|
|
2533
|
-
kSource,
|
|
2534
|
-
maxScreens,
|
|
2535
|
-
planner: {
|
|
2536
|
-
profile,
|
|
2537
|
-
modelOverrides: opts.plannerOverrides,
|
|
2538
|
-
resolveAuth: async () => (await resolveReplayPlanner(await prepare())).auth
|
|
2539
|
-
},
|
|
2540
|
-
...lazyJudges.length > 1 ? { judgeMembers: lazyJudges } : { judge: lazyJudges[0] },
|
|
2541
|
-
judgeOverrideConfigured: names.length <= 1 && session.evalConfig.judgeAllowSameFamily === true,
|
|
2542
|
-
baselineSeed: session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed, source: "config" } : { floorSeeds: floor?.seeds ?? [] },
|
|
2543
|
-
clock: opts.clock,
|
|
2544
|
-
onProgress: opts.onProgress
|
|
2545
|
-
}
|
|
2546
|
-
}).run(candidateId);
|
|
2547
|
-
const capNotice = `cap: maxScreens=${maxScreens} (${maxScreensSource})`;
|
|
2548
|
-
return {
|
|
2549
|
-
...result,
|
|
2550
|
-
maxScreens,
|
|
2551
|
-
maxScreensSource,
|
|
2552
|
-
k,
|
|
2553
|
-
kSource,
|
|
2554
|
-
notices: [...result.notices, ...names.length > 1 && session.evalConfig.judgeAllowSameFamily === true ? ["eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint."] : [], capNotice],
|
|
2555
|
-
...result.kind === "receipt" ? { lines: [...result.lines, capNotice] } : {}
|
|
2556
|
-
};
|
|
2557
|
-
}
|
|
2558
|
-
async function runFloorVerb(session, opts = {}) {
|
|
2559
|
-
const repeats = opts.repeats ?? FLOOR_DEFAULT_REPEATS;
|
|
2560
|
-
const seeds = [...new Set(opts.seeds ?? FLOOR_DEFAULT_SEEDS)];
|
|
2561
|
-
const bootstrapSeed = opts.bootstrapSeed ?? FLOOR_DEFAULT_BOOTSTRAP_SEED;
|
|
2562
|
-
if (!Number.isSafeInteger(repeats) || repeats < FLOOR_PROTOCOL_MINIMUM.repeats || seeds.length < FLOOR_PROTOCOL_MINIMUM.seeds || !seeds.every(Number.isSafeInteger)) {
|
|
2563
|
-
throw new FloorProtocolError();
|
|
2564
|
-
}
|
|
2565
|
-
if (!Number.isSafeInteger(bootstrapSeed)) throw new TypeError("--bootstrap-seed must be a safe integer");
|
|
2566
|
-
let classifierRepeatability;
|
|
2567
|
-
if (opts.classifierRepeatability !== void 0) {
|
|
2568
|
-
const path = resolve6(session.configDir, opts.classifierRepeatability);
|
|
2569
|
-
let value;
|
|
2570
|
-
try {
|
|
2571
|
-
value = JSON.parse(await readFile2(path, "utf8"));
|
|
2572
|
-
} catch (error) {
|
|
2573
|
-
throw new TypeError(`classifier repeatability unreadable at ${path}: ${error.message}`);
|
|
2574
|
-
}
|
|
2575
|
-
const guarded = guardClassifierRepeatability(value);
|
|
2576
|
-
if (guarded.kind === "rejected") {
|
|
2577
|
-
throw new TypeError(`classifier repeatability rejected at ${path}: ${guarded.reason}`);
|
|
2578
|
-
}
|
|
2579
|
-
classifierRepeatability = guarded.repeatability;
|
|
2580
|
-
}
|
|
2581
|
-
const planner = opts.fromMeasured === void 0 ? await resolveReplayPlanner(session) : { profile: await resolveReplayProfileDeclaration(session) };
|
|
2582
|
-
const judgeResolution = opts.fromMeasured === void 0 ? await resolveEvalJudge(session) : {};
|
|
2583
|
-
const judge = withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides);
|
|
2584
|
-
const { plannerModelId, matcherVersion, judgeModelId, judgeVersion } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2585
|
-
if (opts.fromMeasured === void 0 && replayJudgeRequest(judge)?.judgeModelId !== judgeModelId) {
|
|
2586
|
-
throw new EvalJudgeDeclarationError("Floor judge identity changed before execution; re-run eval floor.");
|
|
2587
|
-
}
|
|
2588
|
-
const profileKeyInputs = {
|
|
2589
|
-
plannerModelId,
|
|
2590
|
-
matcherVersion,
|
|
2591
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
|
|
2592
|
-
};
|
|
2593
|
-
const costLines = [];
|
|
2594
|
-
const result = await runNoiseFloorMeasurement({
|
|
2595
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
2596
|
-
cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
|
|
2597
|
-
agentQaRoot: resolve6(session.configDir, ".agent-qa"),
|
|
2598
|
-
...planner,
|
|
2599
|
-
...opts.fromMeasured === void 0 ? {} : { fromMeasured: resolve6(session.configDir, opts.fromMeasured) },
|
|
2600
|
-
modelOverrides: opts.plannerOverrides,
|
|
2601
|
-
judge,
|
|
2602
|
-
mask: session.mask,
|
|
2603
|
-
profileKeyInputs,
|
|
2604
|
-
seeds,
|
|
2605
|
-
repeats,
|
|
2606
|
-
bootstrapSeed,
|
|
2607
|
-
classifierRepeatability,
|
|
2608
|
-
explanation: opts.explanation,
|
|
2609
|
-
commit: opts.commit === true,
|
|
2610
|
-
onProgress: (line) => {
|
|
2611
|
-
if (/^floor r\d+ seed=/.test(line)) costLines.push(line);
|
|
2612
|
-
process.stderr.write(line + "\n");
|
|
2613
|
-
}
|
|
2614
|
-
});
|
|
2615
|
-
const notices = [...result.notices];
|
|
2616
|
-
if (judgeResolution.notice !== void 0) notices.unshift(judgeResolution.notice);
|
|
2617
|
-
if (result.committed?.kind === "refused") notices.push(renderFloorRefusal(result.committed.refusal));
|
|
2618
|
-
return {
|
|
2619
|
-
...result,
|
|
2620
|
-
notices,
|
|
2621
|
-
repeats,
|
|
2622
|
-
seeds,
|
|
2623
|
-
bootstrapSeed,
|
|
2624
|
-
costLines,
|
|
2625
|
-
recordedDefaults: opts.repeats === void 0 && opts.seeds === void 0 && opts.bootstrapSeed === void 0,
|
|
2626
|
-
corpusDigest: result.artefact.corpusDigest
|
|
2627
|
-
};
|
|
2628
|
-
}
|
|
2629
|
-
async function runGateVerb(session, opts = {}) {
|
|
2630
|
-
if (opts.stagedDir !== void 0 && opts.report === void 0) {
|
|
2631
|
-
throw new TypeError("--staged-dir requires --report <batchKey>");
|
|
2632
|
-
}
|
|
2633
|
-
const declaredSeed = opts.seed ?? session.evalConfig.seed;
|
|
2634
|
-
if (declaredSeed !== void 0 && !Number.isSafeInteger(declaredSeed)) {
|
|
2635
|
-
throw new TypeError("--seed must be a safe integer");
|
|
2636
|
-
}
|
|
2637
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2638
|
-
const agentQaRoot = resolve6(session.configDir, ".agent-qa");
|
|
2639
|
-
const { plannerModelId, judgeModelId, judgeVersion, fingerprint, matcherVersion, profileKey } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2640
|
-
const profileKeyInputs = {
|
|
2641
|
-
plannerModelId,
|
|
2642
|
-
matcherVersion,
|
|
2643
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
|
|
2644
|
-
};
|
|
2645
|
-
const floorPath = join9(committedRoot, committedNoiseFloorPath(profileKey));
|
|
2646
|
-
const floorSeed = declaredSeed === void 0 ? listNoiseFloorArtefacts(committedRoot).artefacts.find((entry) => entry.path === floorPath && entry.profileKey === profileKey)?.artefact.seeds[0] : void 0;
|
|
2647
|
-
const seed = declaredSeed ?? floorSeed ?? FLOOR_DEFAULT_SEEDS[0];
|
|
2648
|
-
const seedSource = opts.seed !== void 0 ? "flag" : session.evalConfig.seed !== void 0 ? "config" : floorSeed !== void 0 ? "floor" : "default";
|
|
2649
|
-
const probedDirs = [];
|
|
2650
|
-
const notices = [];
|
|
2651
|
-
let mode;
|
|
2652
|
-
if (opts.report !== void 0) {
|
|
2653
|
-
const choices = opts.stagedDir !== void 0 ? [{ dir: resolve6(session.configDir, opts.stagedDir), label: "--staged-dir" }] : [
|
|
2654
|
-
{ dir: join9(agentQaRoot, "eval", "gate", slugBranch(profileKey), "reports"), label: "gate root" },
|
|
2655
|
-
{ dir: join9(agentQaRoot, DEFAULT_EVAL_REPORTS_SUBDIR), label: "default staging root" }
|
|
2656
|
-
];
|
|
2657
|
-
const files = [];
|
|
2658
|
-
let stagedOutputDir;
|
|
2659
|
-
for (const choice of choices) {
|
|
2660
|
-
probedDirs.push(choice.dir);
|
|
2661
|
-
const file = join9(choice.dir, `${slugBranch(opts.report)}.json`);
|
|
2662
|
-
files.push(file);
|
|
2663
|
-
try {
|
|
2664
|
-
lstatSync2(file);
|
|
2665
|
-
} catch (error) {
|
|
2666
|
-
if (error.code === "ENOENT") continue;
|
|
2667
|
-
}
|
|
2668
|
-
stagedOutputDir = choice.dir;
|
|
2669
|
-
notices.push(`gate --report: reading ${file} (${choice.label})`);
|
|
2670
|
-
break;
|
|
2671
|
-
}
|
|
2672
|
-
if (stagedOutputDir === void 0) {
|
|
2673
|
-
return {
|
|
2674
|
-
kind: "refused",
|
|
2675
|
-
refusal: {
|
|
2676
|
-
path: files[files.length - 1],
|
|
2677
|
-
reason: `staged report absent; probed ${files.join(", ")}`
|
|
2678
|
-
},
|
|
2679
|
-
notices,
|
|
2680
|
-
seed,
|
|
2681
|
-
seedSource,
|
|
2682
|
-
probedDirs
|
|
2683
|
-
};
|
|
2684
|
-
}
|
|
2685
|
-
mode = {
|
|
2686
|
-
kind: "report",
|
|
2687
|
-
batchKey: opts.report,
|
|
2688
|
-
stagedOutputDir,
|
|
2689
|
-
profile: await resolveReplayProfileDeclaration(session),
|
|
2690
|
-
modelOverrides: opts.plannerOverrides
|
|
2691
|
-
};
|
|
2692
|
-
} else {
|
|
2693
|
-
const planner = await resolveReplayPlanner(session);
|
|
2694
|
-
const judgeResolution = await resolveEvalJudge(session);
|
|
2695
|
-
if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
|
|
2696
|
-
mode = {
|
|
2697
|
-
kind: "fresh",
|
|
2698
|
-
...planner,
|
|
2699
|
-
modelOverrides: opts.plannerOverrides,
|
|
2700
|
-
judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides),
|
|
2701
|
-
mask: session.mask,
|
|
2702
|
-
seed,
|
|
2703
|
-
cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
|
|
2704
|
-
onProgress: (line) => process.stderr.write(line + "\n")
|
|
2705
|
-
};
|
|
2706
|
-
}
|
|
2707
|
-
const result = await runReleaseGate({ committedRoot, agentQaRoot, profileKeyInputs, mode });
|
|
2708
|
-
return { ...result, notices: [...notices, ...result.notices], seed, seedSource, probedDirs };
|
|
2709
|
-
}
|
|
2710
|
-
function isExclusionVerdict(verdict) {
|
|
2711
|
-
return EXCLUDED_VERDICT_CLASSES.has(verdict.class);
|
|
2712
|
-
}
|
|
2713
|
-
async function runSeedVerb(session, filters = {}, params = {}) {
|
|
2714
|
-
const agentQaRoot = resolve6(session.configDir, ".agent-qa");
|
|
2715
|
-
const runsRoot = resolve6(agentQaRoot, RUNS_SUBDIR);
|
|
2716
|
-
let runSlugs = listRunSlugsWithHandoff(runsRoot);
|
|
2717
|
-
if (filters.run !== void 0) {
|
|
2718
|
-
const named = filters.run;
|
|
2719
|
-
runSlugs = runSlugs.filter((slug) => slug === named || slug === slugBranch(named));
|
|
2720
|
-
if (runSlugs.length === 0) {
|
|
2721
|
-
throw new EvalNamedInputError("run", named, runsRoot, "seed");
|
|
2722
|
-
}
|
|
2723
|
-
}
|
|
2724
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
2725
|
-
const silverPerTestCap = params.silverPerTestCap ?? session.evalConfig.seeding?.silverPerTestCap;
|
|
2726
|
-
const result = {
|
|
2727
|
-
membersExcluded: [],
|
|
2728
|
-
admissionConflicts: [],
|
|
2729
|
-
silverExclusionRows: [],
|
|
2730
|
-
supersededOriginalCount: 0,
|
|
2731
|
-
retirementCounts: { missing: 0, conflict: 0 },
|
|
2732
|
-
runsSwept: runSlugs.length,
|
|
2733
|
-
runsSkipped: [],
|
|
2734
|
-
runsFiltered: 0,
|
|
2735
|
-
runsExcluded: [],
|
|
2736
|
-
runsAdmitted: [],
|
|
2737
|
-
notices: [],
|
|
2738
|
-
handoffsSeeded: 0,
|
|
2739
|
-
seededCount: 0,
|
|
2740
|
-
unchangedCount: 0,
|
|
2741
|
-
refusals: [],
|
|
2742
|
-
pairSkips: 0,
|
|
2743
|
-
silverExclusions: 0,
|
|
2744
|
-
pairCounts: { total: 0, sameState: 0, differingState: 0 },
|
|
2745
|
-
contracts: {},
|
|
2746
|
-
scaffold: {},
|
|
2747
|
-
partitions: [],
|
|
2748
|
-
exclusionsWritten: [],
|
|
2749
|
-
committedRoot
|
|
2750
|
-
};
|
|
2751
|
-
const admitted = [];
|
|
2752
|
-
const initialCommitted = readCommittedCorpus(committedRoot);
|
|
2753
|
-
for (const runSlug of runSlugs) {
|
|
2754
|
-
try {
|
|
2755
|
-
const handoffPath = resolve6(runsRoot, runSlug, HANDOFF_FILE);
|
|
2756
|
-
const located = relative(realpathSync2(agentQaRoot), realpathSync2(handoffPath));
|
|
2757
|
-
if (located === ".." || located.startsWith("../") || located.startsWith("..\\") || isAbsolute2(located)) {
|
|
2758
|
-
throw new Error("seed handoff path is outside the operator run root");
|
|
2759
|
-
}
|
|
2760
|
-
const handoff = loadCompletedRunHandoff(handoffPath);
|
|
2761
|
-
if (slugBranch(handoff.runId) !== runSlug) throw new Error("seed handoff run identity does not match its directory");
|
|
2762
|
-
const product = resolveProductPartition(handoff.scope);
|
|
2763
|
-
if (filters.product !== void 0 && product !== filters.product) {
|
|
2764
|
-
result.runsFiltered++;
|
|
2765
|
-
continue;
|
|
2766
|
-
}
|
|
2767
|
-
const verified = loadLabelEvidence({ snapshot: initialCommitted.snapshot, agentQaRoot, incoming: handoff });
|
|
2768
|
-
if (!verified.handoffs.some((value) => value.runId === handoff.runId)) {
|
|
2769
|
-
throw new Error("seed incoming handoff evidence could not be positively verified before admission");
|
|
2770
|
-
}
|
|
2771
|
-
const analysis = readRunAnalysisArtifact(runsRoot, runSlug, handoff.runId, result.notices);
|
|
2772
|
-
const row = {
|
|
2773
|
-
runSlug,
|
|
2774
|
-
product,
|
|
2775
|
-
verdicts: resolveSeedVerdictRows(analysis.artifact, handoff)
|
|
2776
|
-
};
|
|
2777
|
-
const excluding = row.verdicts.filter(isExclusionVerdict);
|
|
2778
|
-
const memberVerdicts = resolveSeedMemberVerdicts(analysis.artifact, handoff);
|
|
2779
|
-
const eligible = handoff.tests.filter((member) => admitSeedMember(member, memberVerdicts.get(member.testIndex)));
|
|
2780
|
-
const hasEligibleRerun = eligible.some((member) => member.isRerun);
|
|
2781
|
-
if (excluding.length > 0 && !hasEligibleRerun) {
|
|
2782
|
-
result.runsExcluded.push(row);
|
|
2783
|
-
const record = projectRunExclusion({
|
|
2784
|
-
runId: handoff.runId,
|
|
2785
|
-
runSlug,
|
|
2786
|
-
product,
|
|
2787
|
-
verdicts: excluding,
|
|
2788
|
-
analysisDigest: analysis.text === void 0 ? sha256hex3(canonicalJson5(row.verdicts)) : sha256hex3(analysis.text)
|
|
2789
|
-
});
|
|
2790
|
-
const outcome = writeRunExclusion({
|
|
2791
|
-
record,
|
|
2792
|
-
committedRoot
|
|
2793
|
-
});
|
|
2794
|
-
result.exclusionsWritten.push({
|
|
2795
|
-
runSlug,
|
|
2796
|
-
product,
|
|
2797
|
-
path: committedExclusionPath(product, runSlug),
|
|
2798
|
-
outcome: outcome.kind
|
|
2799
|
-
});
|
|
2800
|
-
continue;
|
|
2801
|
-
}
|
|
2802
|
-
const exclusionPath = committedExclusionPath(product, runSlug);
|
|
2803
|
-
if (existsSync4(join9(committedRoot, exclusionPath))) {
|
|
2804
|
-
result.admissionConflicts.push({ runSlug, product, path: exclusionPath, reason: "existing-run-exclusion" });
|
|
2805
|
-
result.notices.push(`seed ${runSlug}: existing run exclusion requires the reviewed exclusion path before admission`);
|
|
2806
|
-
continue;
|
|
2807
|
-
}
|
|
2808
|
-
const testIndexes = new Set(eligible.map((member) => member.testIndex));
|
|
2809
|
-
for (const member of handoff.tests) {
|
|
2810
|
-
const verdict = memberVerdicts.get(member.testIndex);
|
|
2811
|
-
if (!testIndexes.has(member.testIndex) && verdict) {
|
|
2812
|
-
result.membersExcluded.push({ runSlug, product, testIndex: member.testIndex, verdict });
|
|
2813
|
-
}
|
|
2814
|
-
}
|
|
2815
|
-
result.runsAdmitted.push(row);
|
|
2816
|
-
admitted.push({ runSlug, handoff, product, testIndexes });
|
|
2817
|
-
} catch (err) {
|
|
2818
|
-
result.runsSkipped.push({
|
|
2819
|
-
runSlug,
|
|
2820
|
-
reason: err instanceof Error ? err.message : String(err)
|
|
2821
|
-
});
|
|
2822
|
-
continue;
|
|
2823
|
-
}
|
|
2824
|
-
}
|
|
2825
|
-
const manifests = ensurePartitionManifests(session, admitted, committedRoot, result, params);
|
|
2826
|
-
for (const { runSlug, handoff, product, testIndexes } of admitted) {
|
|
2827
|
-
let report;
|
|
2828
|
-
try {
|
|
2829
|
-
const committed = readCommittedCorpus(committedRoot);
|
|
2830
|
-
const evidence = loadLabelEvidence({ snapshot: committed.snapshot, agentQaRoot, incoming: handoff });
|
|
2831
|
-
for (const refusal of evidence.refusals) result.notices.push(`SOFT evidence ${refusal.runId}: ${refusal.reason}`);
|
|
2832
|
-
if (!evidence.handoffs.some((value) => value.runId === handoff.runId)) {
|
|
2833
|
-
throw new Error("seed incoming handoff evidence could not be positively verified");
|
|
2834
|
-
}
|
|
2835
|
-
const mask = committedCorpusMask(session);
|
|
2836
|
-
const admittedHandoff = maskCorpusText(
|
|
2837
|
-
{ ...handoff, tests: handoff.tests.filter((member) => testIndexes.has(member.testIndex)) },
|
|
2838
|
-
mask
|
|
2839
|
-
);
|
|
2840
|
-
const populationHandoffs = evidence.handoffs.map((source) => {
|
|
2841
|
-
if (source.runId === handoff.runId) return admittedHandoff;
|
|
2842
|
-
const sourceAnalysis = readRunAnalysisArtifact(runsRoot, slugBranch(source.runId), source.runId, result.notices);
|
|
2843
|
-
const verdicts = resolveSeedMemberVerdicts(sourceAnalysis.artifact, source);
|
|
2844
|
-
return maskCorpusText(
|
|
2845
|
-
{ ...source, tests: source.tests.filter((member) => admitSeedMember(member, verdicts.get(member.testIndex))) },
|
|
2846
|
-
mask
|
|
2847
|
-
);
|
|
2848
|
-
});
|
|
2849
|
-
const manifest = manifests.get(product);
|
|
2850
|
-
if (manifest === void 0) {
|
|
2851
|
-
throw new Error(`no committed partition manifest for product ${product}`);
|
|
2852
|
-
}
|
|
2853
|
-
const extraction = extractRunCassettes({
|
|
2854
|
-
handoff: admittedHandoff,
|
|
2855
|
-
artifacts: new FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
|
|
2856
|
-
});
|
|
2857
|
-
const targetEnvironment = resolveTargetEnvironmentForProduct(session.config, product);
|
|
2858
|
-
const plan = buildSeedingPlan({
|
|
2859
|
-
handoff: admittedHandoff,
|
|
2860
|
-
populationContext: { snapshot: committed.snapshot, handoffs: populationHandoffs, marks: evidence.marks },
|
|
2861
|
-
cassettes: extraction.cassettes,
|
|
2862
|
-
// 414 D-07: the split comes from the product's COMMITTED manifest params — the cli adds
|
|
2863
|
-
// no selection or relabel logic of its own.
|
|
2864
|
-
partition: partitionParamsOf(manifest),
|
|
2865
|
-
capturedAt: new Date(handoff.endedAt).toISOString(),
|
|
2866
|
-
...targetEnvironment === void 0 ? {} : { targetEnvironment },
|
|
2867
|
-
params: {
|
|
2868
|
-
...silverPerTestCap !== void 0 ? { silverPerTestCap } : {}
|
|
2869
|
-
},
|
|
2870
|
-
mask
|
|
2871
|
-
});
|
|
2872
|
-
report = seedCommittedCorpus({
|
|
2873
|
-
plan,
|
|
2874
|
-
cassettes: extraction.cassettes,
|
|
2875
|
-
committedRoot,
|
|
2876
|
-
mask
|
|
2877
|
-
});
|
|
2878
|
-
for (const skip of report.softEvidence?.skips ?? []) {
|
|
2879
|
-
result.notices.push(`SOFT evidence ${skip.runId}/${skip.testIndex}/${skip.stepIndex}/${skip.attempt}: ${skip.reason}`);
|
|
2880
|
-
}
|
|
2881
|
-
} catch (err) {
|
|
2882
|
-
result.runsSkipped.push({
|
|
2883
|
-
runSlug,
|
|
2884
|
-
reason: err instanceof Error ? err.message : String(err)
|
|
2885
|
-
});
|
|
2886
|
-
continue;
|
|
2887
|
-
}
|
|
2888
|
-
result.handoffsSeeded++;
|
|
2889
|
-
result.seededCount += report.seededCount;
|
|
2890
|
-
result.unchangedCount += report.unchangedCount;
|
|
2891
|
-
result.supersededOriginalCount += report.supersededOriginalCount;
|
|
2892
|
-
result.retirementCounts.missing += report.retirementCounts.missing;
|
|
2893
|
-
result.retirementCounts.conflict += report.retirementCounts.conflict;
|
|
2894
|
-
result.silverExclusionRows.push(...report.silverExclusions.map((row) => ({ ...row, runSlug, product })));
|
|
2895
|
-
result.refusals.push(...report.refusals);
|
|
2896
|
-
result.pairSkips += report.pairSkips.length;
|
|
2897
|
-
result.silverExclusions += report.silverExclusions.length;
|
|
2898
|
-
result.pairCounts.total += report.pairCounts.total;
|
|
2899
|
-
result.pairCounts.sameState += report.pairCounts.sameState;
|
|
2900
|
-
result.pairCounts.differingState += report.pairCounts.differingState;
|
|
2901
|
-
for (const contract of report.contracts) {
|
|
2902
|
-
bumpCounter(result.contracts, contract.outcome);
|
|
2903
|
-
}
|
|
2904
|
-
bumpCounter(result.scaffold, report.scaffold.gitignore);
|
|
2905
|
-
bumpCounter(result.scaffold, report.scaffold.marker);
|
|
2906
|
-
}
|
|
2907
|
-
return result;
|
|
2908
|
-
}
|
|
2909
|
-
function ensurePartitionManifests(session, admitted, committedRoot, result, params) {
|
|
2910
|
-
const byProduct = /* @__PURE__ */ new Map();
|
|
2911
|
-
const read = readPartitionManifests(committedRoot);
|
|
2912
|
-
for (const manifest of read.manifests) byProduct.set(manifest.product, manifest);
|
|
2913
|
-
result.notices.push(...read.notices);
|
|
2914
|
-
for (const refusal of read.refusals) {
|
|
2915
|
-
result.notices.push(`partition manifest refused at ${refusal.path}: ${refusal.reason}`);
|
|
2916
|
-
}
|
|
2917
|
-
const products = [...new Set(admitted.map((entry) => entry.product))].sort();
|
|
2918
|
-
const { heldOutFraction, privateFraction } = params;
|
|
2919
|
-
const unpartitioned = products.filter((product) => !byProduct.has(product));
|
|
2920
|
-
if (unpartitioned.length > 0 && (heldOutFraction === void 0 || privateFraction === void 0)) {
|
|
2921
|
-
throw new EvalPartitionFlagsRequiredError(unpartitioned);
|
|
2922
|
-
}
|
|
2923
|
-
for (const product of products) {
|
|
2924
|
-
const committed = byProduct.get(product);
|
|
2925
|
-
const relPath = committedPartitionManifestPath(product);
|
|
2926
|
-
if (committed === void 0) {
|
|
2927
|
-
const saltHex = randomBytes(32).toString("hex");
|
|
2928
|
-
const manifest = projectPartitionManifest({
|
|
2929
|
-
product,
|
|
2930
|
-
saltHex,
|
|
2931
|
-
heldOutFraction,
|
|
2932
|
-
privateFraction
|
|
2933
|
-
});
|
|
2934
|
-
const outcome2 = writePartitionManifest({
|
|
2935
|
-
manifest,
|
|
2936
|
-
committedRoot
|
|
2937
|
-
});
|
|
2938
|
-
byProduct.set(product, manifest);
|
|
2939
|
-
result.partitions.push(partitionRow(manifest, relPath, requireManifestWritten(outcome2, product, relPath)));
|
|
2940
|
-
continue;
|
|
2941
|
-
}
|
|
2942
|
-
if (heldOutFraction === void 0 || privateFraction === void 0) {
|
|
2943
|
-
result.partitions.push(partitionRow(committed, relPath, "unchanged"));
|
|
2944
|
-
continue;
|
|
2945
|
-
}
|
|
2946
|
-
const reprojected = projectPartitionManifest({
|
|
2947
|
-
product,
|
|
2948
|
-
saltHex: committed.salt,
|
|
2949
|
-
heldOutFraction,
|
|
2950
|
-
privateFraction
|
|
2951
|
-
});
|
|
2952
|
-
const outcome = writePartitionManifest({
|
|
2953
|
-
manifest: reprojected,
|
|
2954
|
-
committedRoot
|
|
2955
|
-
});
|
|
2956
|
-
result.partitions.push(
|
|
2957
|
-
partitionRow(committed, relPath, requireManifestWritten(outcome, product, relPath))
|
|
2958
|
-
);
|
|
2959
|
-
}
|
|
2960
|
-
return byProduct;
|
|
2961
|
-
}
|
|
2962
|
-
function partitionRow(manifest, path, outcome) {
|
|
2963
|
-
return {
|
|
2964
|
-
product: manifest.product,
|
|
2965
|
-
path,
|
|
2966
|
-
outcome,
|
|
2967
|
-
saltDigest: sha256hex3(manifest.salt),
|
|
2968
|
-
heldOutFraction: manifest.heldOutFraction,
|
|
2969
|
-
privateFraction: manifest.privateFraction,
|
|
2970
|
-
partitionVersion: manifest.partitionVersion
|
|
2971
|
-
};
|
|
2972
|
-
}
|
|
2973
|
-
function resolveTargetEnvironmentForProduct(config, product) {
|
|
2974
|
-
const declared = /* @__PURE__ */ new Set();
|
|
2975
|
-
for (const summary of listTargets(config)) {
|
|
2976
|
-
const target = resolveTarget(config, summary.name);
|
|
2977
|
-
if (target.product !== product) continue;
|
|
2978
|
-
if (target.environment === void 0) continue;
|
|
2979
|
-
declared.add(target.environment);
|
|
2980
|
-
}
|
|
2981
|
-
return declared.size === 1 ? [...declared][0] : void 0;
|
|
2982
|
-
}
|
|
2983
|
-
function requireManifestWritten(outcome, product, path) {
|
|
2984
|
-
if (outcome.kind === "write-conflict") {
|
|
2985
|
-
throw new EvalPartitionManifestConflictError(product, path);
|
|
2986
|
-
}
|
|
2987
|
-
return outcome.kind;
|
|
2988
|
-
}
|
|
2989
|
-
function readRunAnalysisArtifact(runsRoot, runSlug, runId, notices) {
|
|
2990
|
-
const path = join9(runsRoot, runSlug, ANALYSIS_FILE);
|
|
2991
|
-
if (!existsSync4(path)) return {};
|
|
2992
|
-
try {
|
|
2993
|
-
const text = readFileSync5(path, "utf8");
|
|
2994
|
-
const admission = admitAnalysisArtifactForRun(
|
|
2995
|
-
guardRunAnalysisArtifact(JSON.parse(text)),
|
|
2996
|
-
runId
|
|
2997
|
-
);
|
|
2998
|
-
if (admission.kind === "admitted") return { artifact: admission.artifact, text };
|
|
2999
|
-
notices.push(
|
|
3000
|
-
`${ANALYSIS_FILE} for ${runSlug} records run ${admission.recordedRunId}, not ${admission.resolvingRunId} \u2014 deterministic floor used`
|
|
3001
|
-
);
|
|
3002
|
-
return {};
|
|
3003
|
-
} catch (err) {
|
|
3004
|
-
notices.push(
|
|
3005
|
-
`${ANALYSIS_FILE} unreadable for ${runSlug}: ${err instanceof Error ? err.message : String(err)} \u2014 deterministic floor used`
|
|
3006
|
-
);
|
|
3007
|
-
return {};
|
|
3008
|
-
}
|
|
3009
|
-
}
|
|
3010
|
-
function resolveSeedVerdictRows(artifact, handoff) {
|
|
3011
|
-
return [...resolveSeedMemberVerdicts(artifact, handoff).entries()].sort(([first], [second]) => first - second).map(([, verdict]) => verdict);
|
|
3012
|
-
}
|
|
3013
|
-
function resolveSeedMemberVerdicts(artifact, handoff) {
|
|
3014
|
-
const byPerTest = /* @__PURE__ */ new Map();
|
|
3015
|
-
for (const member of handoff.tests) {
|
|
3016
|
-
if (member.rootCause === void 0) continue;
|
|
3017
|
-
byPerTest.set(member.testIndex, {
|
|
3018
|
-
class: member.rootCause.class,
|
|
3019
|
-
producer: member.rootCause.producer
|
|
3020
|
-
});
|
|
3021
|
-
}
|
|
3022
|
-
for (const verdict of resolveRunVerdicts(artifact, handoff)) {
|
|
3023
|
-
byPerTest.set(verdict.perTest, { class: verdict.class, producer: verdict.producer });
|
|
3024
|
-
}
|
|
3025
|
-
return byPerTest;
|
|
3026
|
-
}
|
|
3027
|
-
async function runReportVerb(session, filters = {}, opts = {}) {
|
|
3028
|
-
if (opts.fromCommitted !== void 0) {
|
|
3029
|
-
if (opts.baseline !== true) throw new TypeError("--from-committed requires --baseline");
|
|
3030
|
-
if (opts.seedPolicy === void 0) {
|
|
3031
|
-
throw new TypeError("--from-committed requires --seed-policy <none|declared>");
|
|
3032
|
-
}
|
|
3033
|
-
if (opts.commit === true) throw new TypeError("--from-committed cannot be combined with --commit");
|
|
3034
|
-
if (!/^[0-9a-f]{64}$/.test(opts.fromCommitted)) {
|
|
3035
|
-
throw new TypeError("--from-committed requires a batchKey of 64 lowercase hexadecimal characters");
|
|
3036
|
-
}
|
|
3037
|
-
if (!SEED_POLICIES.includes(opts.seedPolicy)) {
|
|
3038
|
-
throw new TypeError('--seed-policy must be "none" or "declared"');
|
|
3039
|
-
}
|
|
3040
|
-
const committedRoot2 = resolve6(session.configDir, "agent-qa-eval");
|
|
3041
|
-
const reportList = listCommittedReports(committedRoot2);
|
|
3042
|
-
const requestedFile = join9(committedRoot2, "reports", opts.fromCommitted, "report.json");
|
|
3043
|
-
const entry = reportList.reports.find((candidate) => candidate.batchKey === opts.fromCommitted);
|
|
3044
|
-
if (entry === void 0) {
|
|
3045
|
-
const refusal = reportList.refusals.find((candidate) => candidate.path === requestedFile);
|
|
3046
|
-
throw new EvalNamedInputError(
|
|
3047
|
-
"batchKey",
|
|
3048
|
-
opts.fromCommitted,
|
|
3049
|
-
refusal?.path ?? join9(committedRoot2, "reports"),
|
|
3050
|
-
"designate",
|
|
3051
|
-
refusal?.reason
|
|
3052
|
-
);
|
|
3053
|
-
}
|
|
3054
|
-
if (entry.path !== requestedFile || entry.report.batchKey !== opts.fromCommitted) {
|
|
3055
|
-
throw new EvalNamedInputError(
|
|
3056
|
-
"batchKey",
|
|
3057
|
-
opts.fromCommitted,
|
|
3058
|
-
entry.path,
|
|
3059
|
-
"designate",
|
|
3060
|
-
`report batchKey "${entry.report.batchKey}" does not match the requested directory`
|
|
3061
|
-
);
|
|
3062
|
-
}
|
|
3063
|
-
const notices2 = reportList.refusals.map(
|
|
3064
|
-
(refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`
|
|
3065
|
-
);
|
|
3066
|
-
const designation = await designateReportBaseline(
|
|
3067
|
-
session,
|
|
3068
|
-
entry.report,
|
|
3069
|
-
opts.seedPolicy,
|
|
3070
|
-
opts.plannerOverrides,
|
|
3071
|
-
notices2
|
|
3072
|
-
);
|
|
3073
|
-
return {
|
|
3074
|
-
batchKey: entry.report.batchKey,
|
|
3075
|
-
entries: 0,
|
|
3076
|
-
report: entry.report,
|
|
3077
|
-
committedFile: entry.path,
|
|
3078
|
-
judgeIdentity: entry.report.judgeIdentity === "mixed" ? "mixed" : entry.report.judge === void 0 ? "absent" : "present",
|
|
3079
|
-
source: "committed",
|
|
3080
|
-
...designation,
|
|
3081
|
-
notices: notices2
|
|
3082
|
-
};
|
|
3083
|
-
}
|
|
3084
|
-
if (opts.baseline === true && opts.commit !== true) {
|
|
3085
|
-
throw new Error(
|
|
3086
|
-
"--baseline requires --commit: the pointer designates a COMMITTED report (413 D-19)"
|
|
3087
|
-
);
|
|
3088
|
-
}
|
|
3089
|
-
const agentQaRoot = resolve6(session.configDir, ".agent-qa");
|
|
3090
|
-
const replaysRoot = join9(agentQaRoot, DEFAULT_EVAL_REPLAYS_SUBDIR);
|
|
3091
|
-
const source = opts.source ?? "staged";
|
|
3092
|
-
const sourceRoot = resolveCassettesRoot(session, source);
|
|
3093
|
-
const cassettesRoot = sourceRoot.cassettesRoot;
|
|
3094
|
-
const notices = [];
|
|
3095
|
-
if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
|
|
3096
|
-
const labelCorpus = readActiveCommittedCorpus(resolve6(session.configDir, "agent-qa-eval"), agentQaRoot);
|
|
3097
|
-
const committedCorpus = source === "committed" ? labelCorpus : void 0;
|
|
3098
|
-
notices.push(...labelCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
|
|
3099
|
-
const {
|
|
3100
|
-
entries: assembled,
|
|
3101
|
-
judgeBlocks,
|
|
3102
|
-
partitionsById,
|
|
3103
|
-
missingBundleCount,
|
|
3104
|
-
notices: assemblyNotices
|
|
3105
|
-
} = assembleScoredBatch({
|
|
3106
|
-
replaysRoot,
|
|
3107
|
-
cassettesRoot,
|
|
3108
|
-
source,
|
|
3109
|
-
labelRecords: labelCorpus.snapshot.labelsById,
|
|
3110
|
-
excludedLabelIds: labelCorpus.snapshot.excludedLabelIds,
|
|
3111
|
-
...committedCorpus === void 0 ? {} : {
|
|
3112
|
-
productByCassetteId: new Map(committedCorpus.snapshot.entries.map((entry) => [entry.cassetteId, entry.product])),
|
|
3113
|
-
eligibleCassetteIds: new Set(committedCorpus.snapshot.entries.map((entry) => entry.cassetteId)),
|
|
3114
|
-
corpusEntriesById: committedCorpus.corpusEntriesById
|
|
3115
|
-
}
|
|
3116
|
-
});
|
|
3117
|
-
notices.push(...assemblyNotices);
|
|
3118
|
-
if (assembled.some((entry) => entry.split === void 0)) {
|
|
3119
|
-
notices.push("corpus split ownership unknown; entries without corpus metadata are unassigned");
|
|
3120
|
-
}
|
|
3121
|
-
if (missingBundleCount > 0) {
|
|
3122
|
-
notices.push(
|
|
3123
|
-
`${missingBundleCount} replay record(s) have no ${source} cassette bundle \u2014 label legs absent`
|
|
3124
|
-
);
|
|
3125
|
-
}
|
|
3126
|
-
let entries = assembled;
|
|
3127
|
-
if (filters.cassette !== void 0) {
|
|
3128
|
-
const named = filters.cassette;
|
|
3129
|
-
entries = entries.filter((entry) => entry.cassetteId === named);
|
|
3130
|
-
if (entries.length === 0) {
|
|
3131
|
-
throw new EvalNamedInputError("cassette", named, replaysRoot, "report");
|
|
3132
|
-
}
|
|
3133
|
-
}
|
|
3134
|
-
if (filters.run !== void 0) {
|
|
3135
|
-
const named = filters.run;
|
|
3136
|
-
entries = entries.filter((entry) => {
|
|
3137
|
-
const head = entry.cassetteId.split("/")[0];
|
|
3138
|
-
return head === named || head === slugBranch(named);
|
|
3139
|
-
});
|
|
3140
|
-
if (entries.length === 0) {
|
|
3141
|
-
throw new EvalNamedInputError("run", named, replaysRoot, "report");
|
|
3142
|
-
}
|
|
3143
|
-
}
|
|
3144
|
-
if (filters.product !== void 0) {
|
|
3145
|
-
const named = filters.product;
|
|
3146
|
-
entries = entries.filter((entry) => partitionsById.get(entry.cassetteId) === named);
|
|
3147
|
-
}
|
|
3148
|
-
entries.sort((a, b) => a.cassetteId < b.cassetteId ? -1 : a.cassetteId > b.cassetteId ? 1 : 0);
|
|
3149
|
-
let judgeIdentity;
|
|
3150
|
-
let judgeForReport;
|
|
3151
|
-
let mixedIdentity;
|
|
3152
|
-
if (judgeBlocks.length === 0) {
|
|
3153
|
-
judgeIdentity = "absent";
|
|
3154
|
-
notices.push("no judge identity on any persisted score record \u2014 the report carries no judge block");
|
|
3155
|
-
} else {
|
|
3156
|
-
const identities = judgeBlocks.map((block) => judgeIdentitySchema.strip().parse(block));
|
|
3157
|
-
const first = canonicalJson5(identities[0]);
|
|
3158
|
-
if (identities.every((block) => canonicalJson5(block) === first)) {
|
|
3159
|
-
judgeIdentity = "present";
|
|
3160
|
-
judgeForReport = identities[0];
|
|
3161
|
-
} else {
|
|
3162
|
-
judgeIdentity = "mixed";
|
|
3163
|
-
mixedIdentity = {
|
|
3164
|
-
judgeIdentity: "mixed",
|
|
3165
|
-
judgeModelIds: [...new Set(judgeBlocks.map((block) => block.judgeModelId))].sort()
|
|
3166
|
-
};
|
|
3167
|
-
notices.push(
|
|
3168
|
-
"mixed judge identities across persisted score records \u2014 the report carries no judge block (never fabricated)"
|
|
3169
|
-
);
|
|
3170
|
-
}
|
|
3171
|
-
}
|
|
3172
|
-
const report = buildEvalReport(entries, judgeForReport, mixedIdentity, { calibration: labelCorpus.calibration });
|
|
3173
|
-
const reportSeedPolicy = session.evalConfig.seed !== void 0 ? "declared" : "none";
|
|
3174
|
-
let reportProfileProof;
|
|
3175
|
-
if (report.judge?.panelSize !== void 0) {
|
|
3176
|
-
let calibrationRead;
|
|
3177
|
-
try {
|
|
3178
|
-
const replayProfile = await resolveReplayProfileDeclaration(session);
|
|
3179
|
-
const actualModels = entries.flatMap((entry) => {
|
|
3180
|
-
if ("kind" in entry.outcome) return [];
|
|
3181
|
-
const results = entry.outcome.granularity === "leaf" ? [entry.outcome] : entry.outcome.steps.flatMap((step) => step.kind === "decision" ? [step.result] : []);
|
|
3182
|
-
return results.map((result) => result.replayedModel.modelId);
|
|
3183
|
-
});
|
|
3184
|
-
if (actualModels.length === 0 || actualModels.some((model) => model !== replayProfile.model)) {
|
|
3185
|
-
throw new Error("report replay profile unavailable");
|
|
3186
|
-
}
|
|
3187
|
-
const fingerprint = computeReplayProfileFingerprintFor({
|
|
3188
|
-
profile: replayProfile,
|
|
3189
|
-
seedPolicy: reportSeedPolicy,
|
|
3190
|
-
modelOverrides: opts.plannerOverrides
|
|
3191
|
-
});
|
|
3192
|
-
reportProfileProof = readReportProfileProof({
|
|
3193
|
-
agentQaRoot,
|
|
3194
|
-
replaysRoot,
|
|
3195
|
-
report,
|
|
3196
|
-
entries,
|
|
3197
|
-
fingerprint,
|
|
3198
|
-
corpusDigest: computeCorpusDigest(labelCorpus.snapshot.entries),
|
|
3199
|
-
seed: session.evalConfig.seed
|
|
3200
|
-
});
|
|
3201
|
-
if (!reportProfileProof) throw new Error("report full profile proof unavailable");
|
|
3202
|
-
calibrationRead = readJudgeCalibration({
|
|
3203
|
-
committedRoot: resolve6(session.configDir, "agent-qa-eval"),
|
|
3204
|
-
agentQaRoot,
|
|
3205
|
-
identity: report.judge,
|
|
3206
|
-
profile: {
|
|
3207
|
-
plannerModelId: replayProfile.model,
|
|
3208
|
-
fingerprint,
|
|
3209
|
-
matcherVersion: report.matcherVersion,
|
|
3210
|
-
judgeModelId: report.judge.judgeModelId,
|
|
3211
|
-
judgeVersion: report.judge.judgeVersion
|
|
3212
|
-
}
|
|
3213
|
-
});
|
|
3214
|
-
} catch {
|
|
3215
|
-
}
|
|
3216
|
-
copyJudgeCalibrationToReport(report, calibrationRead);
|
|
3217
|
-
if (calibrationRead?.kind !== "loaded") notices.push("judge calibration unavailable: no matching guarded artifact");
|
|
3218
|
-
}
|
|
3219
|
-
const vectors = decisionCorrectnessVectors(entries, partitionsById);
|
|
3220
|
-
const decisions = projectDecisionVectorsDocument({
|
|
3221
|
-
batchKey: report.batchKey,
|
|
3222
|
-
matcherVersion: report.matcherVersion,
|
|
3223
|
-
vectors
|
|
3224
|
-
});
|
|
3225
|
-
if (reportProfileProof) report.runIdentity = reportProfileProof.runIdentity;
|
|
3226
|
-
const staged = new FsReportStore({ root: agentQaRoot }).save(report, decisions, reportProfileProof?.replayGeneration);
|
|
3227
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
3228
|
-
let committed;
|
|
3229
|
-
if (opts.commit === true) {
|
|
3230
|
-
committed = commitEvalReport({
|
|
3231
|
-
report,
|
|
3232
|
-
decisions,
|
|
3233
|
-
markdown: renderEvalReportMarkdown(report),
|
|
3234
|
-
committedRoot
|
|
3235
|
-
});
|
|
3236
|
-
}
|
|
3237
|
-
let baseline;
|
|
3238
|
-
let profileKey;
|
|
3239
|
-
if (opts.baseline === true) {
|
|
3240
|
-
if (committed?.kind !== "committed") {
|
|
3241
|
-
notices.push(
|
|
3242
|
-
`baseline pointer skipped: report commit outcome was ${committed?.kind ?? "absent"}`
|
|
3243
|
-
);
|
|
3244
|
-
} else {
|
|
3245
|
-
const designation = await designateReportBaseline(
|
|
3246
|
-
session,
|
|
3247
|
-
report,
|
|
3248
|
-
reportSeedPolicy,
|
|
3249
|
-
opts.plannerOverrides,
|
|
3250
|
-
notices
|
|
3251
|
-
);
|
|
3252
|
-
profileKey = designation.profileKey;
|
|
3253
|
-
baseline = designation.baseline;
|
|
3254
|
-
}
|
|
3255
|
-
}
|
|
3256
|
-
return {
|
|
3257
|
-
batchKey: report.batchKey,
|
|
3258
|
-
entries: entries.length,
|
|
3259
|
-
report,
|
|
3260
|
-
stagedFile: staged.file,
|
|
3261
|
-
decisionVectorCount: decisions.vectors.length,
|
|
3262
|
-
...committed !== void 0 ? { committed } : {},
|
|
3263
|
-
judgeIdentity,
|
|
3264
|
-
source,
|
|
3265
|
-
...baseline !== void 0 ? { baseline } : {},
|
|
3266
|
-
...profileKey !== void 0 ? { profileKey } : {},
|
|
3267
|
-
notices
|
|
3268
|
-
};
|
|
3269
|
-
}
|
|
3270
|
-
async function designateReportBaseline(session, report, seedPolicy, plannerOverrides, notices) {
|
|
3271
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
3272
|
-
const read = readActiveCommittedCorpus(committedRoot, resolve6(session.configDir, ".agent-qa"));
|
|
3273
|
-
notices.push(...read.notices);
|
|
3274
|
-
for (const refusal of read.refusals) {
|
|
3275
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3276
|
-
}
|
|
3277
|
-
const pointer = projectBaselinePointer({
|
|
3278
|
-
plannerModelId: await resolvePlannerModelId(session),
|
|
3279
|
-
fingerprint: await resolveReplayProfileFingerprint(session, seedPolicy, plannerOverrides),
|
|
3280
|
-
corpusDigest: computeCorpusDigest(read.snapshot.entries),
|
|
3281
|
-
matcherVersion: report.matcherVersion,
|
|
3282
|
-
...report.judge === void 0 ? {} : {
|
|
3283
|
-
judgeModelId: report.judge.judgeModelId,
|
|
3284
|
-
...report.judge.judgeVersion === void 0 ? {} : { judgeVersion: report.judge.judgeVersion }
|
|
3285
|
-
},
|
|
3286
|
-
batchKey: report.batchKey
|
|
3287
|
-
});
|
|
3288
|
-
const baseline = writeBaselinePointer({ report, pointer, committedRoot });
|
|
3289
|
-
if (baseline.kind === "refused") notices.push(renderBaselineComparatorRefusal(baseline.refusal));
|
|
3290
|
-
if (seedPolicy === "none" && (baseline.kind === "written" || baseline.kind === "unchanged")) {
|
|
3291
|
-
notices.push(
|
|
3292
|
-
"baseline designated under seed policy none \u2014 eval gate compares against a declared-policy pointer; re-designate with eval report --baseline --from-committed <batchKey> --seed-policy declared"
|
|
3293
|
-
);
|
|
3294
|
-
}
|
|
3295
|
-
return { profileKey: pointer.profileKey, baseline };
|
|
3296
|
-
}
|
|
3297
|
-
async function resolvePlannerModelId(session) {
|
|
3298
|
-
return (await resolveBaselinePlannerConfig(session)).model;
|
|
3299
|
-
}
|
|
3300
|
-
async function resolveGateProfileIdentity(session, modelOverrides) {
|
|
3301
|
-
const plannerModelId = await resolvePlannerModelId(session);
|
|
3302
|
-
const fingerprint = await resolveReplayProfileFingerprint(session, "declared", modelOverrides);
|
|
3303
|
-
const judgeProfiles = await resolveEvalJudgeDeclarations(session);
|
|
3304
|
-
const judgeMemberModelIds = judgeProfiles.map((member) => member.profile.model);
|
|
3305
|
-
const judgeIdentity = judgeMemberModelIds.length === 0 ? void 0 : createPanelIdentity(judgeMemberModelIds);
|
|
3306
|
-
const judgeModelId = judgeIdentity?.judgeModelId;
|
|
3307
|
-
const identity = {
|
|
3308
|
-
plannerModelId,
|
|
3309
|
-
fingerprint,
|
|
3310
|
-
matcherVersion: buildEvalReport([]).matcherVersion,
|
|
3311
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion: JUDGE_CONFIG.judgeVersion }
|
|
3312
|
-
};
|
|
3313
|
-
return {
|
|
3314
|
-
...identity,
|
|
3315
|
-
profileKey: composeBaselineProfileKey(identity),
|
|
3316
|
-
...judgeMemberModelIds.length > 1 ? { judgeMemberModelIds } : {}
|
|
3317
|
-
};
|
|
3318
|
-
}
|
|
3319
|
-
async function resolveEvalJudgeDeclarations(session) {
|
|
3320
|
-
const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3321
|
-
const members = [];
|
|
3322
|
-
for (const name of selectedEvalJudgeNames(session.evalConfig)) {
|
|
3323
|
-
const { config } = await resolveNamedConfig2(name, session.configPath, { keyOnly: true });
|
|
3324
|
-
members.push({ name, profile: {
|
|
3325
|
-
provider: config.provider,
|
|
3326
|
-
model: config.model,
|
|
3327
|
-
auth: { mode: "none" },
|
|
3328
|
-
...config.baseURL === void 0 ? {} : { baseURL: config.baseURL },
|
|
3329
|
-
...config.providerHeaders === void 0 ? {} : { headers: config.providerHeaders }
|
|
3330
|
-
} });
|
|
3331
|
-
}
|
|
3332
|
-
return members;
|
|
3333
|
-
}
|
|
3334
|
-
async function resolveReplayProfileFingerprint(session, seedPolicy, modelOverrides) {
|
|
3335
|
-
const profile = await resolveReplayProfileDeclaration(session);
|
|
3336
|
-
return computeReplayProfileFingerprintFor({ profile, seedPolicy, modelOverrides });
|
|
3337
|
-
}
|
|
3338
|
-
async function resolveReplayProfileDeclaration(session, namedResolver) {
|
|
3339
|
-
const config = await resolveBaselinePlannerConfig(session, namedResolver);
|
|
3340
|
-
return {
|
|
3341
|
-
provider: config.provider,
|
|
3342
|
-
model: config.model,
|
|
3343
|
-
...config.baseURL !== void 0 ? { baseURL: config.baseURL } : {},
|
|
3344
|
-
...config.providerHeaders !== void 0 ? { headers: config.providerHeaders } : {},
|
|
3345
|
-
auth: { mode: "none" }
|
|
3346
|
-
};
|
|
3347
|
-
}
|
|
3348
|
-
async function resolveBaselinePlannerConfig(session, namedResolver) {
|
|
3349
|
-
const plannerName = session.config.use?.llm;
|
|
3350
|
-
if (!plannerName) {
|
|
3351
|
-
throw new Error(
|
|
3352
|
-
"use.llm is not set in the config \u2014 the baseline profile key is keyed on the replay planner, the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
|
|
3353
|
-
);
|
|
3354
|
-
}
|
|
3355
|
-
const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3356
|
-
const { config } = await (namedResolver ?? resolveNamedConfig2)(plannerName, session.configPath, { keyOnly: true });
|
|
3357
|
-
return config;
|
|
3358
|
-
}
|
|
3359
|
-
async function runCensusVerb(session, opts = {}) {
|
|
3360
|
-
const committedRoot = resolve6(session.configDir, "agent-qa-eval");
|
|
3361
|
-
const notices = [];
|
|
3362
|
-
const read = readActiveCommittedCorpus(committedRoot, resolve6(session.configDir, ".agent-qa"));
|
|
3363
|
-
const manifestRead = readPartitionManifests(committedRoot);
|
|
3364
|
-
const exclusionRead = readRunExclusions(committedRoot);
|
|
3365
|
-
notices.push(...read.notices, ...manifestRead.notices);
|
|
3366
|
-
for (const refusal of [...read.refusals, ...manifestRead.refusals, ...exclusionRead.refusals]) {
|
|
3367
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3368
|
-
}
|
|
3369
|
-
let campaign;
|
|
3370
|
-
let campaignManifest;
|
|
3371
|
-
if (opts.campaign !== void 0) {
|
|
3372
|
-
campaignManifest = resolve6(process.cwd(), opts.campaign);
|
|
3373
|
-
const manifest = guardCampaignManifest(JSON.parse(readFileSync5(campaignManifest, "utf8")));
|
|
3374
|
-
campaign = { legs: manifest.legs, manifestDigest: sha256hex3(canonicalJson5(manifest)) };
|
|
3375
|
-
}
|
|
3376
|
-
const reportList = listCommittedReports(committedRoot);
|
|
3377
|
-
const pointerList = listBaselinePointers(committedRoot);
|
|
3378
|
-
for (const refusal of [...reportList.refusals, ...pointerList.refusals]) {
|
|
3379
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3380
|
-
}
|
|
3381
|
-
const reports = [];
|
|
3382
|
-
for (const entry of reportList.reports) {
|
|
3383
|
-
const comparator = assertBaselineComparator(entry.report);
|
|
3384
|
-
const designatedBy = pointerList.pointers.filter((pointer) => pointer.batchKey === entry.batchKey).map((pointer) => pointer.profileKey).sort();
|
|
3385
|
-
notices.push(
|
|
3386
|
-
`cross-check ${entry.batchKey}: report decisions ${entry.report.headline.perDecisionCost.decisionCount} vs corpus cassettes ${read.snapshot.entries.length} \u2014 batch membership differs (batchKey only)`
|
|
3387
|
-
);
|
|
3388
|
-
if (!comparator.ok) notices.push(renderBaselineComparatorRefusal(comparator.refusal));
|
|
3389
|
-
reports.push({ report: entry.report, comparator, designatedBy });
|
|
3390
|
-
}
|
|
3391
|
-
const floorList = listNoiseFloorArtefacts(committedRoot);
|
|
3392
|
-
for (const refusal of floorList.refusals) {
|
|
3393
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3394
|
-
}
|
|
3395
|
-
const noiseFloor = floorList.artefacts.map(({ profileKey, path, artefact }) => {
|
|
3396
|
-
const repeatability = artefact.classifierRepeatability;
|
|
3397
|
-
return {
|
|
3398
|
-
profileKey,
|
|
3399
|
-
path,
|
|
3400
|
-
...artefact.minimumGateSet.value === void 0 ? {} : { minimumGateSet: artefact.minimumGateSet.value },
|
|
3401
|
-
minimumGateSetDerivation: artefact.minimumGateSet.derivation,
|
|
3402
|
-
...repeatability === void 0 ? {} : {
|
|
3403
|
-
classifierRepeatability: {
|
|
3404
|
-
n: repeatability.n,
|
|
3405
|
-
k: repeatability.k,
|
|
3406
|
-
...repeatability.classFlipRate === void 0 ? {} : { classFlipRate: repeatability.classFlipRate },
|
|
3407
|
-
...repeatability.admitExcludeFlipRate === void 0 ? {} : { admitExcludeFlipRate: repeatability.admitExcludeFlipRate }
|
|
3408
|
-
}
|
|
3409
|
-
}
|
|
3410
|
-
};
|
|
3411
|
-
});
|
|
3412
|
-
const census = deriveCensus({
|
|
3413
|
-
snapshot: read.snapshot,
|
|
3414
|
-
...read.corpusEntriesById === void 0 ? {} : { corpusEntriesById: read.corpusEntriesById },
|
|
3415
|
-
...read.retiredMarkedCassetteIds === void 0 ? {} : { retiredMarkedCassetteIds: read.retiredMarkedCassetteIds },
|
|
3416
|
-
...campaign === void 0 ? {} : { campaign },
|
|
3417
|
-
reports,
|
|
3418
|
-
manifests: manifestRead.manifests,
|
|
3419
|
-
exclusions: exclusionRead.exclusions,
|
|
3420
|
-
schemaVersionsSeen: read.schemaVersionsSeen
|
|
3421
|
-
});
|
|
3422
|
-
let committed;
|
|
3423
|
-
if (opts.commit === true) {
|
|
3424
|
-
committed = commitCensus({
|
|
3425
|
-
census,
|
|
3426
|
-
markdown: renderCensusMarkdown(census),
|
|
3427
|
-
committedRoot
|
|
3428
|
-
});
|
|
3429
|
-
}
|
|
3430
|
-
return {
|
|
3431
|
-
corpusDigest: census.corpusDigest,
|
|
3432
|
-
entryCount: census.entryCount,
|
|
3433
|
-
productsSeen: read.productsSeen,
|
|
3434
|
-
census,
|
|
3435
|
-
...campaignManifest !== void 0 ? { campaignManifest } : {},
|
|
3436
|
-
...committed !== void 0 ? { committed } : {},
|
|
3437
|
-
...noiseFloor.length === 0 ? {} : { noiseFloor },
|
|
3438
|
-
notices
|
|
3439
|
-
};
|
|
3440
|
-
}
|
|
3441
|
-
async function resolveNamedProfile(name, configPath) {
|
|
3442
|
-
const { resolveNamedConfig: resolveNamedConfig2, resolveModelAuth: resolveModelAuth2, toModelProfileAndAuth: toModelProfileAndAuth2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3443
|
-
const { config } = await resolveNamedConfig2(name, configPath);
|
|
3444
|
-
const auth = await resolveModelAuth2(config.name, config);
|
|
3445
|
-
if (auth.kind === "missing") {
|
|
3446
|
-
throw new Error(
|
|
3447
|
-
`No credentials for LLM config "${config.name}": ${auth.message} Run \`agent-qa auth login\` to authenticate.`
|
|
3448
|
-
);
|
|
3449
|
-
}
|
|
3450
|
-
return toModelProfileAndAuth2(config, auth);
|
|
3451
|
-
}
|
|
3452
|
-
async function loadEvalSecretStore(config, configDir) {
|
|
3453
|
-
const secretsPath = resolve6(configDir, config.workspace.secretsFile);
|
|
3454
|
-
let content;
|
|
3455
|
-
try {
|
|
3456
|
-
content = await readFile2(secretsPath, "utf-8");
|
|
3457
|
-
} catch {
|
|
3458
|
-
throw new Error(
|
|
3459
|
-
`Secrets file not found or unreadable: ${secretsPath}. Create the file before running eval verbs (it may be empty, but it must exist).`
|
|
3460
|
-
);
|
|
3461
|
-
}
|
|
3462
|
-
try {
|
|
3463
|
-
const { SecretStore: SecretStore2 } = await import("@vostride/agent-qa-runner");
|
|
3464
|
-
return SecretStore2.fromEnvContent(content);
|
|
3465
|
-
} catch (err) {
|
|
3466
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3467
|
-
throw new Error(`Secrets file is invalid: ${secretsPath}: ${message}`);
|
|
3468
|
-
}
|
|
3469
|
-
}
|
|
3470
|
-
function mergeEvalConfig(fromConfig, flags) {
|
|
3471
|
-
const configured = selectedEvalJudgeNames(fromConfig ?? {});
|
|
3472
|
-
const names = flags?.judge === void 0 ? configured : selectedEvalJudgeNames(typeof flags.judge === "string" ? { judge: flags.judge } : { judges: flags.judge });
|
|
3473
|
-
const judgeAllowSameFamily = fromConfig?.judgeAllowSameFamily;
|
|
3474
|
-
const k = flags?.k ?? fromConfig?.k;
|
|
3475
|
-
const seed = flags?.seed ?? fromConfig?.seed;
|
|
3476
|
-
const silverPerTestCap = flags?.silverCap ?? fromConfig?.seeding?.silverPerTestCap;
|
|
3477
|
-
return {
|
|
3478
|
-
...names.length === 1 ? { judge: names[0] } : names.length > 1 ? { judges: names } : {},
|
|
3479
|
-
...judgeAllowSameFamily !== void 0 ? { judgeAllowSameFamily } : {},
|
|
3480
|
-
...k !== void 0 ? { k } : {},
|
|
3481
|
-
...seed !== void 0 ? { seed } : {},
|
|
3482
|
-
...fromConfig?.gate2 === void 0 ? {} : { gate2: fromConfig.gate2 },
|
|
3483
|
-
...fromConfig?.adversarial === void 0 ? {} : { adversarial: fromConfig.adversarial },
|
|
3484
|
-
...silverPerTestCap !== void 0 ? { seeding: { silverPerTestCap } } : {},
|
|
3485
|
-
// 422-09: forwarded WHOLE, exactly as `gate2` and `adversarial` above are. The field-by-field
|
|
3486
|
-
// rebuild this replaces silently dropped every member the schema gained after it was written.
|
|
3487
|
-
...fromConfig?.gate1 === void 0 ? {} : { gate1: fromConfig.gate1 }
|
|
3488
|
-
};
|
|
3489
|
-
}
|
|
3490
|
-
var EvalJudgeDeclarationError, EvalNamedInputError, FLOOR_DEFAULT_REPEATS, FLOOR_DEFAULT_SEEDS, FLOOR_DEFAULT_BOOTSTRAP_SEED, LABEL_CALIBRATION_MIN_GOLD, GATE1_DEFAULT_MAX_SCREENS, GATE1_DEFAULT_K, GATE2_DEFAULT_MAX_ABLATIONS, GATE2_DEFAULT_CANARY_HORIZON_BUILDS, ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS, EXCLUDED_VERDICT_CLASSES, EvalPartitionFlagsRequiredError, EvalPartitionManifestConflictError;
|
|
3491
|
-
var init_eval_session_composition = __esm({
|
|
3492
|
-
"src/eval-session-composition.ts"() {
|
|
3493
|
-
"use strict";
|
|
3494
|
-
init_config();
|
|
3495
|
-
init_declared_secrets();
|
|
3496
|
-
init_targets();
|
|
3497
|
-
init_promotion_posture();
|
|
3498
|
-
init_eval_report_render();
|
|
3499
|
-
init_judge_panel_preflight();
|
|
3500
|
-
init_judge_panel_preflight();
|
|
3501
|
-
init_adversarial_probe_preflight();
|
|
3502
|
-
init_adversarial_probe_author();
|
|
3503
|
-
EvalJudgeDeclarationError = class extends TypeError {
|
|
3504
|
-
constructor(message) {
|
|
3505
|
-
super(message);
|
|
3506
|
-
this.name = "EvalJudgeDeclarationError";
|
|
3507
|
-
}
|
|
3508
|
-
};
|
|
3509
|
-
EvalNamedInputError = class extends Error {
|
|
3510
|
-
inputKind;
|
|
3511
|
-
inputName;
|
|
3512
|
-
searchRoot;
|
|
3513
|
-
constructor(inputKind, inputName, searchRoot, verb = "extract", detail) {
|
|
3514
|
-
super(
|
|
3515
|
-
detail === void 0 ? `No recorded ${inputKind} named "${inputName}" under ${searchRoot} \u2014 nothing to ${verb}.` : `Cannot ${verb} ${inputKind} "${inputName}" at ${searchRoot}: ${detail}`
|
|
3516
|
-
);
|
|
3517
|
-
this.name = "EvalNamedInputError";
|
|
3518
|
-
this.inputKind = inputKind;
|
|
3519
|
-
this.inputName = inputName;
|
|
3520
|
-
this.searchRoot = searchRoot;
|
|
3521
|
-
}
|
|
3522
|
-
};
|
|
3523
|
-
FLOOR_DEFAULT_REPEATS = 3;
|
|
3524
|
-
FLOOR_DEFAULT_SEEDS = [11, 29];
|
|
3525
|
-
FLOOR_DEFAULT_BOOTSTRAP_SEED = 7;
|
|
3526
|
-
LABEL_CALIBRATION_MIN_GOLD = 20;
|
|
3527
|
-
GATE1_DEFAULT_MAX_SCREENS = 64;
|
|
3528
|
-
GATE1_DEFAULT_K = 3;
|
|
3529
|
-
GATE2_DEFAULT_MAX_ABLATIONS = 4;
|
|
3530
|
-
GATE2_DEFAULT_CANARY_HORIZON_BUILDS = 5;
|
|
3531
|
-
ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS = 4;
|
|
3532
|
-
EXCLUDED_VERDICT_CLASSES = new Set(
|
|
3533
|
-
// 414 D-02: the local literal triplet is gone. `EXCLUSION_CLASSES` is the SAME closed
|
|
3534
|
-
// vocabulary, and it now also decides what the committed exclusion record admits — a second
|
|
3535
|
-
// spelling here could let the seeder drop a run the record schema would refuse to describe.
|
|
3536
|
-
EXCLUSION_CLASSES
|
|
3537
|
-
);
|
|
3538
|
-
EvalPartitionFlagsRequiredError = class extends Error {
|
|
3539
|
-
products;
|
|
3540
|
-
constructor(products) {
|
|
3541
|
-
super(
|
|
3542
|
-
`partition of ${products.join(", ")} requires --held-out-fraction and --private-fraction on its first seed (414 D-07); no default exists`
|
|
3543
|
-
);
|
|
3544
|
-
this.name = "EvalPartitionFlagsRequiredError";
|
|
3545
|
-
this.products = products;
|
|
3546
|
-
}
|
|
3547
|
-
};
|
|
3548
|
-
EvalPartitionManifestConflictError = class extends Error {
|
|
3549
|
-
product;
|
|
3550
|
-
path;
|
|
3551
|
-
constructor(product, path) {
|
|
3552
|
-
super(
|
|
3553
|
-
`partition manifest for ${product} at ${path}: write-conflict \u2014 the committed manifest disagrees with the supplied fractions; changing a manifest is a reviewed git edit and a named migration (414 D-05)`
|
|
3554
|
-
);
|
|
3555
|
-
this.name = "EvalPartitionManifestConflictError";
|
|
3556
|
-
this.product = product;
|
|
3557
|
-
this.path = path;
|
|
3558
|
-
}
|
|
356
|
+
platform: typedEntry.platform
|
|
3559
357
|
};
|
|
3560
|
-
}
|
|
3561
|
-
}
|
|
3562
|
-
|
|
3563
|
-
// src/index.ts
|
|
3564
|
-
init_config();
|
|
3565
|
-
init_targets();
|
|
3566
|
-
import { getAgentQaVersion } from "@vostride/agent-qa-core";
|
|
3567
|
-
export * from "@vostride/agent-qa-core";
|
|
358
|
+
});
|
|
359
|
+
}
|
|
3568
360
|
|
|
3569
361
|
// src/commands/cache.ts
|
|
3570
|
-
init_config();
|
|
3571
362
|
import { Command } from "commander";
|
|
3572
363
|
import pc2 from "picocolors";
|
|
3573
|
-
import {
|
|
3574
|
-
import {
|
|
364
|
+
import { dirname as dirname3, resolve } from "path";
|
|
365
|
+
import { purgeActionCache } from "@vostride/agent-qa-agent-local";
|
|
3575
366
|
import { DEFAULT_AGENT_QA_CACHE_DIR } from "@vostride/agent-qa-core";
|
|
3576
|
-
async function removeCacheEntries(cacheRoot) {
|
|
3577
|
-
let branchDirs;
|
|
3578
|
-
try {
|
|
3579
|
-
branchDirs = await readdir(cacheRoot);
|
|
3580
|
-
} catch (err) {
|
|
3581
|
-
if (err.code === "ENOENT") return 0;
|
|
3582
|
-
throw err;
|
|
3583
|
-
}
|
|
3584
|
-
let purged = 0;
|
|
3585
|
-
for (const branch of branchDirs) {
|
|
3586
|
-
const branchPath = join(cacheRoot, branch);
|
|
3587
|
-
let entryStat;
|
|
3588
|
-
try {
|
|
3589
|
-
entryStat = await stat(branchPath);
|
|
3590
|
-
} catch {
|
|
3591
|
-
continue;
|
|
3592
|
-
}
|
|
3593
|
-
if (!entryStat.isDirectory()) {
|
|
3594
|
-
await rm(branchPath, { force: true });
|
|
3595
|
-
continue;
|
|
3596
|
-
}
|
|
3597
|
-
let files;
|
|
3598
|
-
try {
|
|
3599
|
-
files = await readdir(branchPath);
|
|
3600
|
-
} catch {
|
|
3601
|
-
continue;
|
|
3602
|
-
}
|
|
3603
|
-
for (const file of files) {
|
|
3604
|
-
if (!file.endsWith(".json")) continue;
|
|
3605
|
-
await rm(join(branchPath, file), { force: true });
|
|
3606
|
-
purged++;
|
|
3607
|
-
}
|
|
3608
|
-
await rm(branchPath, { recursive: true, force: true }).catch(() => {
|
|
3609
|
-
});
|
|
3610
|
-
}
|
|
3611
|
-
return purged;
|
|
3612
|
-
}
|
|
3613
|
-
async function countCacheEntries(cacheRoot) {
|
|
3614
|
-
let branchDirs;
|
|
3615
|
-
try {
|
|
3616
|
-
branchDirs = await readdir(cacheRoot);
|
|
3617
|
-
} catch (err) {
|
|
3618
|
-
if (err.code === "ENOENT") return 0;
|
|
3619
|
-
throw err;
|
|
3620
|
-
}
|
|
3621
|
-
let count2 = 0;
|
|
3622
|
-
for (const branch of branchDirs) {
|
|
3623
|
-
try {
|
|
3624
|
-
const files = await readdir(join(cacheRoot, branch));
|
|
3625
|
-
count2 += files.filter((f) => f.endsWith(".json")).length;
|
|
3626
|
-
} catch {
|
|
3627
|
-
}
|
|
3628
|
-
}
|
|
3629
|
-
return count2;
|
|
3630
|
-
}
|
|
3631
|
-
async function purgeTest(_testPath, cacheDir) {
|
|
3632
|
-
return removeCacheEntries(resolve(cacheDir));
|
|
3633
|
-
}
|
|
3634
367
|
async function purgeAll(cacheDir) {
|
|
3635
|
-
|
|
3636
|
-
const total = await countCacheEntries(resolvedDir);
|
|
3637
|
-
if (total === 0) return 0;
|
|
3638
|
-
return removeCacheEntries(resolvedDir);
|
|
368
|
+
return purgeActionCache(resolve(cacheDir));
|
|
3639
369
|
}
|
|
3640
370
|
|
|
3641
371
|
// src/curation/curator-author.ts
|
|
@@ -3654,7 +384,7 @@ import {
|
|
|
3654
384
|
} from "@vostride/agent-qa-schema";
|
|
3655
385
|
|
|
3656
386
|
// src/curation/emit-candidates.ts
|
|
3657
|
-
import { isAbsolute, join
|
|
387
|
+
import { isAbsolute, join } from "path";
|
|
3658
388
|
import { z } from "zod";
|
|
3659
389
|
import {
|
|
3660
390
|
LOCAL_CURATION_CAPABILITIES
|
|
@@ -3965,9 +695,7 @@ var trajectoryCompositeValueSchema = z.union([
|
|
|
3965
695
|
z.record(z.string(), procedureJsonValueSchema)
|
|
3966
696
|
]);
|
|
3967
697
|
var trajectoryValueSlotSchema = z.object({
|
|
3968
|
-
//
|
|
3969
|
-
// fill.submit, scroll.maxSwipes/retryOpposite, durations, or scale here would author a
|
|
3970
|
-
// procedure whose behavior differs from the verified trajectory.
|
|
698
|
+
// Boolean and numeric action fields are replay semantics; dropping one changes the replay.
|
|
3971
699
|
observed: z.union([
|
|
3972
700
|
z.string(),
|
|
3973
701
|
z.number(),
|
|
@@ -3975,11 +703,7 @@ var trajectoryValueSlotSchema = z.object({
|
|
|
3975
703
|
trajectoryCompositeValueSchema
|
|
3976
704
|
]),
|
|
3977
705
|
source: z.enum(["literal", "test-variable", "masked-secret"]),
|
|
3978
|
-
//
|
|
3979
|
-
// a test variable matched against Plan 381-41's interpolation provenance, or a secret
|
|
3980
|
-
// recovered from its recorded masked form. `curator-author` forces the covering
|
|
3981
|
-
// parameter to exactly this name so consult-time fillability (D-08, which matches on
|
|
3982
|
-
// binding NAMES) succeeds. Absent when the source is an anonymous literal.
|
|
706
|
+
// The binding name behind this slot; `curator-author` names the covering parameter after it.
|
|
3983
707
|
sourceName: z.string().min(1).optional()
|
|
3984
708
|
}).strict();
|
|
3985
709
|
var trajectoryStepSchema = z.object({
|
|
@@ -4249,7 +973,7 @@ function projectCurationCandidates(handoff, mask) {
|
|
|
4249
973
|
function bindArtifactSources(projection, runDir) {
|
|
4250
974
|
return projection.artifactDescriptors.map((descriptor) => ({
|
|
4251
975
|
artifactId: descriptor.ref.id,
|
|
4252
|
-
sourcePath:
|
|
976
|
+
sourcePath: join(runDir, RUN_EVIDENCE_BLOBS_SUBDIR, descriptor.ref.sha)
|
|
4253
977
|
}));
|
|
4254
978
|
}
|
|
4255
979
|
function incrementReport(report, outcome) {
|
|
@@ -4449,9 +1173,9 @@ function buildPrompt(job, neighbors, accounting) {
|
|
|
4449
1173
|
const { contextWindow, maxOutput } = accounting.getCeilings();
|
|
4450
1174
|
const inputBudget = contextWindow - maxOutput;
|
|
4451
1175
|
if (inputBudget <= 0) return void 0;
|
|
4452
|
-
for (let
|
|
4453
|
-
const presented = limited.slice(0,
|
|
4454
|
-
const messages = renderPrompt(job, presented, neighbors.length -
|
|
1176
|
+
for (let count = limited.length; count >= 0; count -= 1) {
|
|
1177
|
+
const presented = limited.slice(0, count);
|
|
1178
|
+
const messages = renderPrompt(job, presented, neighbors.length - count);
|
|
4455
1179
|
if (accounting.estimate(messages) <= inputBudget) {
|
|
4456
1180
|
return {
|
|
4457
1181
|
messages,
|
|
@@ -5258,38 +1982,38 @@ function buildCuratorAuthor(deps) {
|
|
|
5258
1982
|
}
|
|
5259
1983
|
|
|
5260
1984
|
// src/curation/curation-tail.ts
|
|
5261
|
-
import { randomUUID as
|
|
1985
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
5262
1986
|
import {
|
|
5263
|
-
closeSync as
|
|
5264
|
-
existsSync as
|
|
5265
|
-
fsyncSync as
|
|
5266
|
-
openSync as
|
|
5267
|
-
readFileSync as
|
|
5268
|
-
readdirSync
|
|
5269
|
-
renameSync as
|
|
5270
|
-
rmSync as
|
|
5271
|
-
writeFileSync as
|
|
1987
|
+
closeSync as closeSync3,
|
|
1988
|
+
existsSync as existsSync3,
|
|
1989
|
+
fsyncSync as fsyncSync3,
|
|
1990
|
+
openSync as openSync3,
|
|
1991
|
+
readFileSync as readFileSync5,
|
|
1992
|
+
readdirSync,
|
|
1993
|
+
renameSync as renameSync2,
|
|
1994
|
+
rmSync as rmSync4,
|
|
1995
|
+
writeFileSync as writeFileSync3
|
|
5272
1996
|
} from "fs";
|
|
5273
|
-
import { join as
|
|
5274
|
-
import { resolveKnowledgeRoot as
|
|
1997
|
+
import { join as join7 } from "path";
|
|
1998
|
+
import { resolveKnowledgeRoot as resolveKnowledgeRoot2 } from "@vostride/agent-qa-core";
|
|
5275
1999
|
import {
|
|
5276
|
-
LOCAL_CURATION_CAPABILITIES as
|
|
2000
|
+
LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES3,
|
|
5277
2001
|
QUARANTINE_CLASS_DISPOSITIONS,
|
|
5278
2002
|
QUARANTINE_FLOOR_ELIGIBILITY
|
|
5279
2003
|
} from "@vostride/agent-qa-agent-local";
|
|
5280
2004
|
import {
|
|
5281
2005
|
EVENTS_FILE,
|
|
5282
|
-
HANDOFF_FILE
|
|
5283
|
-
RUNS_SUBDIR
|
|
5284
|
-
loadCompletedRunHandoff
|
|
5285
|
-
slugBranch
|
|
2006
|
+
HANDOFF_FILE,
|
|
2007
|
+
RUNS_SUBDIR,
|
|
2008
|
+
loadCompletedRunHandoff,
|
|
2009
|
+
slugBranch
|
|
5286
2010
|
} from "@vostride/agent-qa-runner-local";
|
|
5287
|
-
import { runSafetyScope
|
|
5288
|
-
import { canonicalJson as
|
|
2011
|
+
import { runSafetyScope } from "@vostride/agent-qa-safety-local";
|
|
2012
|
+
import { canonicalJson as canonicalJson3, sha256hex, CurationSnapshotSchema, CURATION_BREAKDOWN_BUCKETS } from "@vostride/agent-qa-schema";
|
|
5289
2013
|
|
|
5290
2014
|
// src/daemon/client.ts
|
|
5291
2015
|
import { request } from "http";
|
|
5292
|
-
import { dirname as
|
|
2016
|
+
import { dirname as dirname6, join as join4 } from "path";
|
|
5293
2017
|
import {
|
|
5294
2018
|
FunctionInputError,
|
|
5295
2019
|
FunctionNotBoundError,
|
|
@@ -5312,7 +2036,7 @@ import {
|
|
|
5312
2036
|
unlinkSync as unlinkSync2,
|
|
5313
2037
|
writeFileSync
|
|
5314
2038
|
} from "fs";
|
|
5315
|
-
import { dirname as
|
|
2039
|
+
import { dirname as dirname5, join as join3 } from "path";
|
|
5316
2040
|
import { LOCAL_DAEMON_CAPABILITIES } from "@vostride/agent-qa-agent-local";
|
|
5317
2041
|
import { z as z4 } from "zod";
|
|
5318
2042
|
|
|
@@ -5326,7 +2050,7 @@ import {
|
|
|
5326
2050
|
} from "http";
|
|
5327
2051
|
import { connect } from "net";
|
|
5328
2052
|
import { tmpdir } from "os";
|
|
5329
|
-
import { basename
|
|
2053
|
+
import { basename, dirname as dirname4, join as join2, resolve as resolve2, sep } from "path";
|
|
5330
2054
|
import { invokeFunction } from "@vostride/agent-qa-agent";
|
|
5331
2055
|
import { ANALYTICS_AGENT_PRODUCTS } from "@vostride/agent-qa-analytics";
|
|
5332
2056
|
import { z as z3 } from "zod";
|
|
@@ -5450,7 +2174,7 @@ var daemonRegistrationSchema = z4.object({
|
|
|
5450
2174
|
startedAtMs: z4.number().int().nonnegative()
|
|
5451
2175
|
}).strict();
|
|
5452
2176
|
function daemonRegistrationPath(agentQaRoot) {
|
|
5453
|
-
return
|
|
2177
|
+
return join3(agentQaRoot, DAEMON_REGISTRATION_FILE);
|
|
5454
2178
|
}
|
|
5455
2179
|
function readDaemonRegistrationFile(file) {
|
|
5456
2180
|
let raw;
|
|
@@ -5551,15 +2275,7 @@ async function classifyDaemonRegistration(input) {
|
|
|
5551
2275
|
);
|
|
5552
2276
|
switch (verdict.kind) {
|
|
5553
2277
|
case "fresh":
|
|
5554
|
-
|
|
5555
|
-
* CR-01: `unknown` answers held, with the same holder payload as `fresh`.
|
|
5556
|
-
*
|
|
5557
|
-
* This verdict selects registration deletion and socket unlink, and the two preconditions
|
|
5558
|
-
* are correlated rather than independent — fd exhaustion produces both the failed identity
|
|
5559
|
-
* handshake and the unreadable epoch at once. Treating "I could not read the authority
|
|
5560
|
-
* state" as "nobody holds it" therefore de-registers a live, healthy daemon precisely when
|
|
5561
|
-
* the host is under stress. The refusal is the only recoverable answer.
|
|
5562
|
-
*/
|
|
2278
|
+
// `unknown` answers held: fd exhaustion fails both checks at once, so unknown is not free.
|
|
5563
2279
|
case "unknown":
|
|
5564
2280
|
return {
|
|
5565
2281
|
kind: "unreachable-lease-fresh",
|
|
@@ -5578,7 +2294,7 @@ async function classifyDaemonRegistration(input) {
|
|
|
5578
2294
|
// src/daemon/client.ts
|
|
5579
2295
|
var DAEMON_NUDGE_TIMEOUT_MS = 5e3;
|
|
5580
2296
|
function defaultKnowledgeRoot(agentQaRoot) {
|
|
5581
|
-
return
|
|
2297
|
+
return join4(dirname6(agentQaRoot), "agent-qa-knowledge");
|
|
5582
2298
|
}
|
|
5583
2299
|
function parseJson(text, context) {
|
|
5584
2300
|
try {
|
|
@@ -5673,401 +2389,36 @@ async function nudgeDaemon(agentQaRoot, options = {}) {
|
|
|
5673
2389
|
}
|
|
5674
2390
|
|
|
5675
2391
|
// src/curation/emit-canary-outcomes.ts
|
|
5676
|
-
import { readFileSync as
|
|
5677
|
-
import { isAbsolute as
|
|
5678
|
-
import { LOCAL_CURATION_CAPABILITIES as
|
|
5679
|
-
import {
|
|
5680
|
-
admitAnalysisArtifactForRun as admitAnalysisArtifactForRun2,
|
|
5681
|
-
ANALYSIS_FILE as ANALYSIS_FILE2,
|
|
5682
|
-
canonicalJson as canonicalJson7,
|
|
5683
|
-
deriveMemoryOnPathSites,
|
|
5684
|
-
guardRunAnalysisArtifact as guardRunAnalysisArtifact2,
|
|
5685
|
-
misdirectionSourceDigest,
|
|
5686
|
-
normalizeAnchor as normalizeAnchor2,
|
|
5687
|
-
resolveRunVerdicts as resolveRunVerdicts2
|
|
5688
|
-
} from "@vostride/agent-qa-schema";
|
|
5689
|
-
|
|
5690
|
-
// src/curation/canary-run-state.ts
|
|
5691
|
-
import { closeSync as closeSync2, fsyncSync as fsyncSync2, linkSync, mkdirSync as mkdirSync3, openSync as openSync2, readdirSync, readFileSync as readFileSync2, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
5692
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
5693
|
-
import { join as join6 } from "path";
|
|
5694
|
-
import { z as z5 } from "zod";
|
|
5695
|
-
import { CROSS_CHECKS, canonicalJson as canonicalJson2, sha256hex } from "@vostride/agent-qa-schema";
|
|
5696
|
-
var POLICY_FILE = "canary-policy.json";
|
|
5697
|
-
function readRunCanaryPolicy(runDir, runId) {
|
|
5698
|
-
let text;
|
|
5699
|
-
try {
|
|
5700
|
-
text = readFileSync2(join6(runDir, POLICY_FILE), "utf8");
|
|
5701
|
-
} catch (error) {
|
|
5702
|
-
if (error.code === "ENOENT") return void 0;
|
|
5703
|
-
throw new Error("canary policy unavailable");
|
|
5704
|
-
}
|
|
5705
|
-
const policy = JSON.parse(text);
|
|
5706
|
-
if (typeof policy !== "object" || policy === null || Array.isArray(policy) || Object.keys(policy).sort().join(",") !== "maxAblations,runId,schemaVersion,source" || !("schemaVersion" in policy) || policy.schemaVersion !== 2 || !("runId" in policy) || policy.runId !== runId || !("maxAblations" in policy) || typeof policy.maxAblations !== "number" || !Number.isSafeInteger(policy.maxAblations) || policy.maxAblations < 1 || !("source" in policy) || typeof policy.source !== "string" || !["flag", "config", "default"].includes(policy.source)) {
|
|
5707
|
-
throw new Error("canary policy invalid");
|
|
5708
|
-
}
|
|
5709
|
-
return policy;
|
|
5710
|
-
}
|
|
5711
|
-
var count = z5.number().int().nonnegative();
|
|
5712
|
-
var digest = z5.string().regex(/^[a-f0-9]{64}$/);
|
|
5713
|
-
var completionSchema = z5.object({
|
|
5714
|
-
schemaVersion: z5.literal(2),
|
|
5715
|
-
runId: z5.string().min(1),
|
|
5716
|
-
candidateId: z5.string().min(1),
|
|
5717
|
-
attributedObservationId: z5.string().min(1),
|
|
5718
|
-
observedStateDigest: digest,
|
|
5719
|
-
result: z5.union([
|
|
5720
|
-
z5.object({
|
|
5721
|
-
kind: z5.literal("ablated"),
|
|
5722
|
-
crossCheck: z5.enum(CROSS_CHECKS),
|
|
5723
|
-
corpusDigest: digest,
|
|
5724
|
-
k: z5.number().int().min(3).refine((k) => k % 2 === 1),
|
|
5725
|
-
baseSeed: z5.number().int(),
|
|
5726
|
-
seeds: z5.array(z5.number().int()),
|
|
5727
|
-
seedHonored: z5.boolean(),
|
|
5728
|
-
spend: z5.object({ modelCalls: count, screens: count, promptTokens: count, completionTokens: count }).strict(),
|
|
5729
|
-
gate2Dir: z5.string().min(1)
|
|
5730
|
-
}).strict().refine((result) => result.seeds.length === result.k && result.seeds[0] === result.baseSeed),
|
|
5731
|
-
z5.object({ kind: z5.literal("error"), message: z5.literal("ablation-error") }).strict(),
|
|
5732
|
-
z5.object({ kind: z5.literal("unavailable"), reason: z5.literal("ablation-unavailable") }).strict(),
|
|
5733
|
-
z5.object({ kind: z5.literal("no-eval-root") }).strict()
|
|
5734
|
-
])
|
|
5735
|
-
}).strict();
|
|
5736
|
-
var COMPLETIONS_DIR = "canary-completions";
|
|
5737
|
-
function readCompletedCanaryAblations(runDir, runId) {
|
|
5738
|
-
const dir = join6(runDir, COMPLETIONS_DIR);
|
|
5739
|
-
let names;
|
|
5740
|
-
try {
|
|
5741
|
-
names = readdirSync(dir).sort();
|
|
5742
|
-
} catch (error) {
|
|
5743
|
-
if (error.code === "ENOENT") return [];
|
|
5744
|
-
throw new Error("completed canary evidence unavailable");
|
|
5745
|
-
}
|
|
5746
|
-
return names.filter((name) => !name.startsWith(".")).map((name) => {
|
|
5747
|
-
let value;
|
|
5748
|
-
try {
|
|
5749
|
-
value = completionSchema.parse(JSON.parse(readFileSync2(join6(dir, name), "utf8")));
|
|
5750
|
-
} catch {
|
|
5751
|
-
throw new Error("completed canary evidence invalid");
|
|
5752
|
-
}
|
|
5753
|
-
if (value.runId !== runId || name !== `${sha256hex(value.candidateId)}.json`) {
|
|
5754
|
-
throw new Error("completed canary evidence identity mismatch");
|
|
5755
|
-
}
|
|
5756
|
-
return value;
|
|
5757
|
-
});
|
|
5758
|
-
}
|
|
5759
|
-
function saveCompletedCanaryAblation(runDir, input) {
|
|
5760
|
-
const value = completionSchema.parse(input);
|
|
5761
|
-
const dir = join6(runDir, COMPLETIONS_DIR);
|
|
5762
|
-
mkdirSync3(dir, { recursive: true });
|
|
5763
|
-
const file = join6(dir, `${sha256hex(value.candidateId)}.json`);
|
|
5764
|
-
const temporary = join6(dir, `.${randomUUID2()}.tmp`);
|
|
5765
|
-
const text = canonicalJson2(value) + "\n";
|
|
5766
|
-
try {
|
|
5767
|
-
writeFileSync2(temporary, text, { flag: "wx", mode: 384 });
|
|
5768
|
-
const fd = openSync2(temporary, "r");
|
|
5769
|
-
try {
|
|
5770
|
-
fsyncSync2(fd);
|
|
5771
|
-
} finally {
|
|
5772
|
-
closeSync2(fd);
|
|
5773
|
-
}
|
|
5774
|
-
try {
|
|
5775
|
-
linkSync(temporary, file);
|
|
5776
|
-
} catch (error) {
|
|
5777
|
-
if (error.code !== "EEXIST" || readFileSync2(file, "utf8") !== text) {
|
|
5778
|
-
throw new Error("completed canary evidence conflict");
|
|
5779
|
-
}
|
|
5780
|
-
}
|
|
5781
|
-
const directory = openSync2(dir, "r");
|
|
5782
|
-
try {
|
|
5783
|
-
fsyncSync2(directory);
|
|
5784
|
-
} finally {
|
|
5785
|
-
closeSync2(directory);
|
|
5786
|
-
}
|
|
5787
|
-
} finally {
|
|
5788
|
-
rmSync3(temporary, { force: true });
|
|
5789
|
-
}
|
|
5790
|
-
}
|
|
5791
|
-
|
|
5792
|
-
// src/curation/misdirection-run-state.ts
|
|
5793
|
-
import {
|
|
5794
|
-
chmodSync as chmodSync2,
|
|
5795
|
-
closeSync as closeSync3,
|
|
5796
|
-
fsyncSync as fsyncSync3,
|
|
5797
|
-
linkSync as linkSync2,
|
|
5798
|
-
mkdirSync as mkdirSync4,
|
|
5799
|
-
openSync as openSync3,
|
|
5800
|
-
readFileSync as readFileSync3,
|
|
5801
|
-
readdirSync as readdirSync2,
|
|
5802
|
-
renameSync as renameSync2,
|
|
5803
|
-
rmSync as rmSync4,
|
|
5804
|
-
statSync,
|
|
5805
|
-
writeFileSync as writeFileSync3
|
|
5806
|
-
} from "fs";
|
|
5807
|
-
import { randomUUID as randomUUID3 } from "crypto";
|
|
5808
|
-
import { dirname as dirname6, join as join7 } from "path";
|
|
5809
|
-
import { z as z6 } from "zod";
|
|
2392
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
2393
|
+
import { isAbsolute as isAbsolute2, join as join5 } from "path";
|
|
2394
|
+
import { LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES2 } from "@vostride/agent-qa-agent-local";
|
|
5810
2395
|
import {
|
|
5811
|
-
|
|
5812
|
-
|
|
5813
|
-
|
|
2396
|
+
admitAnalysisArtifactForRun,
|
|
2397
|
+
ANALYSIS_FILE,
|
|
2398
|
+
canonicalJson as canonicalJson2,
|
|
2399
|
+
guardRunAnalysisArtifact,
|
|
2400
|
+
normalizeAnchor,
|
|
2401
|
+
resolveRunVerdicts
|
|
5814
2402
|
} from "@vostride/agent-qa-schema";
|
|
5815
|
-
var MISDIRECTION_COMPLETIONS_DIR = "misdirection-completions";
|
|
5816
|
-
function misdirectionCheckKeyDigest(key) {
|
|
5817
|
-
return sha256hex2(
|
|
5818
|
-
canonicalJson3({
|
|
5819
|
-
runId: key.runId,
|
|
5820
|
-
perTest: key.perTest,
|
|
5821
|
-
stepIndex: key.stepIndex,
|
|
5822
|
-
decisionOrdinal: key.decisionOrdinal,
|
|
5823
|
-
recordId: key.recordId
|
|
5824
|
-
})
|
|
5825
|
-
);
|
|
5826
|
-
}
|
|
5827
|
-
var policySchema = z6.object({
|
|
5828
|
-
schemaVersion: z6.literal(2),
|
|
5829
|
-
runId: z6.string().min(1),
|
|
5830
|
-
maxChecks: z6.number().int().nonnegative(),
|
|
5831
|
-
source: z6.enum(["flag", "default"])
|
|
5832
|
-
}).strict();
|
|
5833
|
-
var reservationSchema = z6.object({
|
|
5834
|
-
schemaVersion: z6.literal(2),
|
|
5835
|
-
runId: z6.string().min(1),
|
|
5836
|
-
perTest: z6.number().int().nonnegative(),
|
|
5837
|
-
stepIndex: z6.number().int().nonnegative(),
|
|
5838
|
-
decisionOrdinal: z6.number().int().nonnegative(),
|
|
5839
|
-
recordId: z6.string().min(1),
|
|
5840
|
-
handoffBaseDigest: z6.string().min(1),
|
|
5841
|
-
briefDigest: z6.string().min(1),
|
|
5842
|
-
corpusDigest: z6.string().min(1),
|
|
5843
|
-
profileDigest: z6.string().min(1)
|
|
5844
|
-
}).strict();
|
|
5845
|
-
function readDocuments(dir, unavailable) {
|
|
5846
|
-
let names;
|
|
5847
|
-
try {
|
|
5848
|
-
names = readdirSync2(dir).sort();
|
|
5849
|
-
} catch (error) {
|
|
5850
|
-
if (error.code === "ENOENT") return [];
|
|
5851
|
-
throw new Error(unavailable);
|
|
5852
|
-
}
|
|
5853
|
-
return names.filter((name) => !name.startsWith(".")).map((name) => ({ name, text: readFileSync3(join7(dir, name), "utf8") }));
|
|
5854
|
-
}
|
|
5855
|
-
function readCompletedMisdirectionChecks(runDir, runId) {
|
|
5856
|
-
return readDocuments(
|
|
5857
|
-
join7(runDir, MISDIRECTION_COMPLETIONS_DIR),
|
|
5858
|
-
"completed misdirection evidence unavailable"
|
|
5859
|
-
).map(({ name, text }) => {
|
|
5860
|
-
let value;
|
|
5861
|
-
try {
|
|
5862
|
-
value = MisdirectionRecordResultSchema.parse(JSON.parse(text));
|
|
5863
|
-
} catch {
|
|
5864
|
-
throw new Error("completed misdirection evidence invalid");
|
|
5865
|
-
}
|
|
5866
|
-
if (value.runId !== runId || name !== `${misdirectionCheckKeyDigest(value)}.json`) {
|
|
5867
|
-
throw new Error("completed misdirection evidence identity mismatch");
|
|
5868
|
-
}
|
|
5869
|
-
return value;
|
|
5870
|
-
});
|
|
5871
|
-
}
|
|
5872
|
-
var postureSchema = z6.object({
|
|
5873
|
-
schemaVersion: z6.literal(2),
|
|
5874
|
-
runId: z6.string().min(1),
|
|
5875
|
-
kind: z6.enum(["eval-root", "no-eval-root", "broken-eval-root"]),
|
|
5876
|
-
corpusDigest: z6.string().min(1).optional(),
|
|
5877
|
-
profileDigest: z6.string().min(1).optional(),
|
|
5878
|
-
reason: z6.string().min(1).optional()
|
|
5879
|
-
}).strict();
|
|
5880
2403
|
|
|
5881
2404
|
// src/curation/canary-credit.ts
|
|
5882
|
-
import {
|
|
5883
|
-
actedPair as actedPair3,
|
|
5884
|
-
canonicalJson as canonicalJson4,
|
|
5885
|
-
distilledScreenStateDigest,
|
|
5886
|
-
exercisedAnchors as exercisedAnchors2,
|
|
5887
|
-
normalizeAnchor
|
|
5888
|
-
} from "@vostride/agent-qa-schema";
|
|
5889
2405
|
function emptyCanaryCreditCounters() {
|
|
5890
|
-
return {
|
|
5891
|
-
credited: 0,
|
|
5892
|
-
rideAlongUncredited: 0,
|
|
5893
|
-
ablationCapExceeded: 0,
|
|
5894
|
-
ablationHeld: 0,
|
|
5895
|
-
ablationInconclusive: 0,
|
|
5896
|
-
ablationError: 0,
|
|
5897
|
-
rerunExcluded: 0,
|
|
5898
|
-
nonSignal: { environment: 0, timeout: 0, flake: 0 },
|
|
5899
|
-
staged: 0,
|
|
5900
|
-
stagingRefused: 0
|
|
5901
|
-
};
|
|
2406
|
+
return { rideAlongUncredited: 0, rerunExcluded: 0, nonSignal: { environment: 0, timeout: 0, flake: 0 } };
|
|
5902
2407
|
}
|
|
5903
|
-
|
|
5904
|
-
if (!Number.isSafeInteger(input.maxAblations) || input.maxAblations < 1) {
|
|
5905
|
-
throw new TypeError("--max-ablations must be a positive integer");
|
|
5906
|
-
}
|
|
2408
|
+
function creditCanaryAttributions(input) {
|
|
5907
2409
|
const counters = emptyCanaryCreditCounters();
|
|
5908
2410
|
const attributions = [];
|
|
5909
|
-
let digestRead = false;
|
|
5910
|
-
let corpusDigest;
|
|
5911
|
-
const ablatedCandidates = /* @__PURE__ */ new Set();
|
|
5912
|
-
const completedByCandidate = new Map(input.completedAblations.map((entry) => [entry.candidateId, entry]));
|
|
5913
|
-
let ablationsRun = completedByCandidate.size;
|
|
5914
2411
|
for (const row of input.attributions) {
|
|
5915
2412
|
if (row.kind !== "outcome") {
|
|
5916
2413
|
attributions.push(row);
|
|
5917
2414
|
continue;
|
|
5918
2415
|
}
|
|
5919
|
-
const served = row.source === "served-brief";
|
|
5920
|
-
if (served && input.ablation !== void 0 && !digestRead) {
|
|
5921
|
-
digestRead = true;
|
|
5922
|
-
try {
|
|
5923
|
-
const digest2 = await input.ablation.corpusDigest();
|
|
5924
|
-
if (digest2.kind === "digest") corpusDigest = digest2.corpusDigest;
|
|
5925
|
-
} catch {
|
|
5926
|
-
}
|
|
5927
|
-
}
|
|
5928
2416
|
const outcome = {
|
|
5929
2417
|
...row.outcome,
|
|
5930
|
-
|
|
5931
|
-
benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along",
|
|
5932
|
-
...served && corpusDigest !== void 0 ? { corpusDigest } : {}
|
|
2418
|
+
benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along"
|
|
5933
2419
|
};
|
|
5934
|
-
if (
|
|
5935
|
-
|
|
5936
|
-
delete outcome.ablation;
|
|
5937
|
-
if (served) {
|
|
5938
|
-
const step = input.handoff.tests.find((test) => test.testIndex === outcome.testIndex && test.testId === outcome.testId)?.steps.find((candidate) => candidate.stepIndex === outcome.stepIndex);
|
|
5939
|
-
const live = step?.subActions?.filter((action2) => action2.exchanges !== void 0) ?? [];
|
|
5940
|
-
const action = live.length === 1 ? live[0] : void 0;
|
|
5941
|
-
const acted = action === void 0 ? void 0 : actedSubAction(action);
|
|
5942
|
-
const observed = step !== void 0 && acted !== void 0 && exercisedAnchors2({ ...step, subActions: [acted] }).some((anchor) => canonicalJson4(normalizeAnchor(anchor)) === canonicalJson4(normalizeAnchor(outcome.exercisedAnchor))) ? actedPair3(acted).stateBefore : void 0;
|
|
5943
|
-
if (observed !== void 0) outcome.observedStateDigest = distilledScreenStateDigest(observed);
|
|
5944
|
-
const bundle = input.readBundleStateDigests({
|
|
5945
|
-
knowledgeRoot: input.knowledgeRoot,
|
|
5946
|
-
candidateId: outcome.candidateId
|
|
5947
|
-
});
|
|
5948
|
-
if (outcome.observedStateDigest !== void 0 && bundle.kind === "found" && bundle.digests.length > 0) {
|
|
5949
|
-
outcome.stateValidity = bundle.digests.includes(outcome.observedStateDigest) ? "valid" : "stale";
|
|
5950
|
-
}
|
|
5951
|
-
}
|
|
5952
|
-
let crossCheck = "not-run";
|
|
5953
|
-
let corpusDigestMoved;
|
|
5954
|
-
const completed = completedByCandidate.get(outcome.candidateId);
|
|
5955
|
-
if (outcome.runOutcome === "positive") {
|
|
5956
|
-
if (!served || input.ablation === void 0) {
|
|
5957
|
-
counters.rideAlongUncredited += 1;
|
|
5958
|
-
} else if (outcome.observedStateDigest === void 0) {
|
|
5959
|
-
counters.ablationInconclusive += 1;
|
|
5960
|
-
} else if (ablatedCandidates.has(outcome.candidateId)) {
|
|
5961
|
-
counters.rideAlongUncredited += 1;
|
|
5962
|
-
} else if (completed !== void 0 && (completed.attributedObservationId !== input.observationIdOf(outcome) || completed.observedStateDigest !== outcome.observedStateDigest)) {
|
|
5963
|
-
counters.rideAlongUncredited += 1;
|
|
5964
|
-
} else if (completed === void 0 && ablationsRun >= input.maxAblations) {
|
|
5965
|
-
counters.ablationCapExceeded += 1;
|
|
5966
|
-
} else {
|
|
5967
|
-
ablatedCandidates.add(outcome.candidateId);
|
|
5968
|
-
let result;
|
|
5969
|
-
if (completed !== void 0) result = completed.result;
|
|
5970
|
-
else {
|
|
5971
|
-
ablationsRun += 1;
|
|
5972
|
-
try {
|
|
5973
|
-
result = await input.ablation.ablate({
|
|
5974
|
-
handoff: input.handoff,
|
|
5975
|
-
runDir: input.runDir,
|
|
5976
|
-
candidateId: outcome.candidateId,
|
|
5977
|
-
runId: outcome.runId,
|
|
5978
|
-
testId: outcome.testId,
|
|
5979
|
-
testIndex: outcome.testIndex,
|
|
5980
|
-
stepIndex: outcome.stepIndex,
|
|
5981
|
-
attributedObservationId: input.observationIdOf(outcome)
|
|
5982
|
-
});
|
|
5983
|
-
} catch (error) {
|
|
5984
|
-
result = { kind: "error", message: error instanceof Error ? error.message : String(error) };
|
|
5985
|
-
}
|
|
5986
|
-
input.saveCompletedAblation({
|
|
5987
|
-
schemaVersion: 2,
|
|
5988
|
-
runId: outcome.runId,
|
|
5989
|
-
candidateId: outcome.candidateId,
|
|
5990
|
-
attributedObservationId: input.observationIdOf(outcome),
|
|
5991
|
-
observedStateDigest: outcome.observedStateDigest,
|
|
5992
|
-
result: result.kind === "ablated" ? {
|
|
5993
|
-
kind: "ablated",
|
|
5994
|
-
crossCheck: result.crossCheck,
|
|
5995
|
-
corpusDigest: result.corpusDigest,
|
|
5996
|
-
k: result.k,
|
|
5997
|
-
baseSeed: result.baseSeed,
|
|
5998
|
-
seeds: result.seeds,
|
|
5999
|
-
seedHonored: result.seedHonored,
|
|
6000
|
-
spend: result.spend,
|
|
6001
|
-
gate2Dir: result.gate2Dir
|
|
6002
|
-
} : result.kind === "error" ? { kind: "error", message: "ablation-error" } : result.kind === "unavailable" ? { kind: "unavailable", reason: "ablation-unavailable" } : { kind: "no-eval-root" }
|
|
6003
|
-
});
|
|
6004
|
-
}
|
|
6005
|
-
switch (result.kind) {
|
|
6006
|
-
case "ablated":
|
|
6007
|
-
crossCheck = result.crossCheck;
|
|
6008
|
-
if (corpusDigest !== void 0 && corpusDigest !== result.corpusDigest) corpusDigestMoved = true;
|
|
6009
|
-
outcome.corpusDigest = result.corpusDigest;
|
|
6010
|
-
outcome.ablation = {
|
|
6011
|
-
crossCheck,
|
|
6012
|
-
k: result.k,
|
|
6013
|
-
baseSeed: result.baseSeed,
|
|
6014
|
-
seeds: result.seeds,
|
|
6015
|
-
seedHonored: result.seedHonored,
|
|
6016
|
-
spend: result.spend
|
|
6017
|
-
};
|
|
6018
|
-
if (crossCheck === "flipped") {
|
|
6019
|
-
outcome.benefit = "exercised-benefited";
|
|
6020
|
-
counters.credited += 1;
|
|
6021
|
-
} else if (crossCheck === "held") counters.ablationHeld += 1;
|
|
6022
|
-
else counters.ablationInconclusive += 1;
|
|
6023
|
-
break;
|
|
6024
|
-
case "no-eval-root":
|
|
6025
|
-
counters.rideAlongUncredited += 1;
|
|
6026
|
-
break;
|
|
6027
|
-
case "unavailable":
|
|
6028
|
-
counters.ablationInconclusive += 1;
|
|
6029
|
-
break;
|
|
6030
|
-
case "error":
|
|
6031
|
-
counters.ablationError += 1;
|
|
6032
|
-
break;
|
|
6033
|
-
}
|
|
6034
|
-
}
|
|
6035
|
-
}
|
|
6036
|
-
let staged;
|
|
6037
|
-
if (served && input.ablation !== void 0 && outcome.observedStateDigest !== void 0) {
|
|
6038
|
-
try {
|
|
6039
|
-
const product = input.handoff.scope.tier === "product" && input.handoff.scope.key !== "" && input.handoff.scope.key !== "unknown" ? input.handoff.scope.key : "default";
|
|
6040
|
-
const stateDigest = outcome.observedStateDigest;
|
|
6041
|
-
if (await input.ablation.isNovel({ product, stateDigest })) {
|
|
6042
|
-
const stage = await input.ablation.stage({
|
|
6043
|
-
handoff: input.handoff,
|
|
6044
|
-
runDir: input.runDir,
|
|
6045
|
-
candidateId: outcome.candidateId,
|
|
6046
|
-
runId: outcome.runId,
|
|
6047
|
-
testId: outcome.testId,
|
|
6048
|
-
testIndex: outcome.testIndex,
|
|
6049
|
-
stepIndex: outcome.stepIndex,
|
|
6050
|
-
attributedObservationId: input.observationIdOf(outcome),
|
|
6051
|
-
product,
|
|
6052
|
-
stateDigest,
|
|
6053
|
-
benefit: outcome.benefit
|
|
6054
|
-
});
|
|
6055
|
-
if (stage.kind === "staged") {
|
|
6056
|
-
counters.staged += 1;
|
|
6057
|
-
staged = true;
|
|
6058
|
-
} else if (!["already-staged", "not-novel", "no-eval-root"].includes(stage.kind)) counters.stagingRefused += 1;
|
|
6059
|
-
}
|
|
6060
|
-
} catch {
|
|
6061
|
-
counters.stagingRefused += 1;
|
|
6062
|
-
}
|
|
6063
|
-
}
|
|
6064
|
-
attributions.push({
|
|
6065
|
-
...row,
|
|
6066
|
-
outcome,
|
|
6067
|
-
crossCheck,
|
|
6068
|
-
...staged === void 0 ? {} : { staged },
|
|
6069
|
-
...corpusDigestMoved === void 0 ? {} : { corpusDigestMoved }
|
|
6070
|
-
});
|
|
2420
|
+
if (outcome.runOutcome === "positive") counters.rideAlongUncredited += 1;
|
|
2421
|
+
attributions.push({ ...row, outcome });
|
|
6071
2422
|
}
|
|
6072
2423
|
return { attributions, counters };
|
|
6073
2424
|
}
|
|
@@ -6083,7 +2434,7 @@ function resolveAttributionClasses(artifact, handoff) {
|
|
|
6083
2434
|
for (const member of handoff.tests) {
|
|
6084
2435
|
if (member.rootCause !== void 0) classes.set(member.testIndex, member.rootCause.class);
|
|
6085
2436
|
}
|
|
6086
|
-
for (const verdict of
|
|
2437
|
+
for (const verdict of resolveRunVerdicts(artifact, handoff)) {
|
|
6087
2438
|
classes.set(verdict.perTest, verdict.class);
|
|
6088
2439
|
}
|
|
6089
2440
|
return classes;
|
|
@@ -6091,13 +2442,13 @@ function resolveAttributionClasses(artifact, handoff) {
|
|
|
6091
2442
|
function loadRunAnalysisArtifact(runDir, runId) {
|
|
6092
2443
|
let bytes;
|
|
6093
2444
|
try {
|
|
6094
|
-
bytes =
|
|
2445
|
+
bytes = readFileSync2(join5(runDir, ANALYSIS_FILE), "utf8");
|
|
6095
2446
|
} catch (error) {
|
|
6096
2447
|
if (error.code === "ENOENT") return void 0;
|
|
6097
2448
|
throw error;
|
|
6098
2449
|
}
|
|
6099
|
-
const admission =
|
|
6100
|
-
|
|
2450
|
+
const admission = admitAnalysisArtifactForRun(
|
|
2451
|
+
guardRunAnalysisArtifact(JSON.parse(bytes)),
|
|
6101
2452
|
runId
|
|
6102
2453
|
);
|
|
6103
2454
|
return admission.kind === "admitted" ? admission.artifact : void 0;
|
|
@@ -6108,7 +2459,7 @@ var STALENESS_MISS_REASONS = /* @__PURE__ */ new Set([
|
|
|
6108
2459
|
"page-anchor-mismatch"
|
|
6109
2460
|
]);
|
|
6110
2461
|
function normalizedKey(anchor) {
|
|
6111
|
-
return
|
|
2462
|
+
return canonicalJson2(normalizeAnchor(anchor));
|
|
6112
2463
|
}
|
|
6113
2464
|
function nonAttributableFailure(test, resolved) {
|
|
6114
2465
|
const resolvedClass = resolved.get(test.testIndex);
|
|
@@ -6119,45 +2470,6 @@ function attributedOutcome(test, step, resolved) {
|
|
|
6119
2470
|
if (nonAttributableFailure(test, resolved)) return void 0;
|
|
6120
2471
|
return "negative";
|
|
6121
2472
|
}
|
|
6122
|
-
var EXPOSURE_SEPARATOR = "";
|
|
6123
|
-
function exposureKey(perTest, stepIndex, recordId) {
|
|
6124
|
-
return [perTest, stepIndex, recordId].join(EXPOSURE_SEPARATOR);
|
|
6125
|
-
}
|
|
6126
|
-
function flippedExposures(handoff, evidence) {
|
|
6127
|
-
if (evidence === void 0 || evidence.length === 0) return /* @__PURE__ */ new Set();
|
|
6128
|
-
const runId = String(handoff.runId);
|
|
6129
|
-
const baseDigest = misdirectionSourceDigest(handoff);
|
|
6130
|
-
const exposed = /* @__PURE__ */ new Set();
|
|
6131
|
-
for (const test of handoff.tests) {
|
|
6132
|
-
for (const site of deriveMemoryOnPathSites(test.steps)) {
|
|
6133
|
-
exposed.add(exposureKey(test.testIndex, site.stepIndex, site.recordId));
|
|
6134
|
-
}
|
|
6135
|
-
}
|
|
6136
|
-
const flipped = /* @__PURE__ */ new Set();
|
|
6137
|
-
const answered = /* @__PURE__ */ new Set();
|
|
6138
|
-
for (const row of evidence) {
|
|
6139
|
-
if (row.runId !== runId) continue;
|
|
6140
|
-
if (row.handoffBaseDigest !== baseDigest) continue;
|
|
6141
|
-
const key = exposureKey(row.perTest, row.stepIndex, row.recordId);
|
|
6142
|
-
if (!exposed.has(key)) continue;
|
|
6143
|
-
if (answered.has(key)) {
|
|
6144
|
-
flipped.delete(key);
|
|
6145
|
-
continue;
|
|
6146
|
-
}
|
|
6147
|
-
answered.add(key);
|
|
6148
|
-
if (row.crossCheck === "flipped") flipped.add(key);
|
|
6149
|
-
}
|
|
6150
|
-
return flipped;
|
|
6151
|
-
}
|
|
6152
|
-
function causalRecordsOn(test, step, flipped) {
|
|
6153
|
-
if (flipped.size === 0 || step.verifierVerdict.pass) return [];
|
|
6154
|
-
const ids = [];
|
|
6155
|
-
for (const site of deriveMemoryOnPathSites([step])) {
|
|
6156
|
-
if (!flipped.has(exposureKey(test.testIndex, site.stepIndex, site.recordId))) continue;
|
|
6157
|
-
if (!ids.includes(site.recordId)) ids.push(site.recordId);
|
|
6158
|
-
}
|
|
6159
|
-
return ids;
|
|
6160
|
-
}
|
|
6161
2473
|
function procedureReplayOutcome(test, step, resolved) {
|
|
6162
2474
|
const source = step.planSource;
|
|
6163
2475
|
if (source === void 0 || source.kind !== "procedure-replay") return void 0;
|
|
@@ -6193,7 +2505,7 @@ function servedCanaryIds(step) {
|
|
|
6193
2505
|
}
|
|
6194
2506
|
function firstMatchingAnchor(candidateAnchors, stepAnchors) {
|
|
6195
2507
|
const exercisedByKey = new Map(
|
|
6196
|
-
stepAnchors.map((anchor) => [normalizedKey(anchor),
|
|
2508
|
+
stepAnchors.map((anchor) => [normalizedKey(anchor), normalizeAnchor(anchor)])
|
|
6197
2509
|
);
|
|
6198
2510
|
for (const candidateAnchor of candidateAnchors) {
|
|
6199
2511
|
const match = exercisedByKey.get(normalizedKey(candidateAnchor));
|
|
@@ -6218,10 +2530,6 @@ function projectCanaryAttributions(input) {
|
|
|
6218
2530
|
return resolutionByCandidate.get(id);
|
|
6219
2531
|
};
|
|
6220
2532
|
const resolved = resolveAttributionClasses(input.artifact, input.handoff);
|
|
6221
|
-
const flipped = flippedExposures(
|
|
6222
|
-
input.handoff,
|
|
6223
|
-
input.misdirectionEvidence ?? input.handoff.misdirectionEvidence
|
|
6224
|
-
);
|
|
6225
2533
|
const projected = [];
|
|
6226
2534
|
for (const test of input.handoff.tests) {
|
|
6227
2535
|
if (test.isRerun === true) {
|
|
@@ -6229,45 +2537,33 @@ function projectCanaryAttributions(input) {
|
|
|
6229
2537
|
continue;
|
|
6230
2538
|
}
|
|
6231
2539
|
for (const step of test.steps) {
|
|
6232
|
-
projectBriefAttributions(input, test, step, resolveAnchors, resolved,
|
|
2540
|
+
projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected);
|
|
6233
2541
|
projectProcedureAttribution(input, test, step, resolveAnchors, resolved, projected);
|
|
6234
2542
|
}
|
|
6235
2543
|
}
|
|
6236
2544
|
return projected;
|
|
6237
2545
|
}
|
|
6238
|
-
function projectBriefAttributions(input, test, step, resolveAnchors, resolved,
|
|
2546
|
+
function projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected) {
|
|
6239
2547
|
const runOutcome = attributedOutcome(test, step, resolved);
|
|
6240
2548
|
const resolvedClass = resolved.get(test.testIndex);
|
|
6241
|
-
|
|
6242
|
-
const causalSet = new Set(causal);
|
|
6243
|
-
const nonSignal = (candidateId) => {
|
|
2549
|
+
if (runOutcome === void 0) {
|
|
6244
2550
|
if (resolvedClass === void 0 || !isNonSignalClass(resolvedClass)) return;
|
|
6245
|
-
|
|
6246
|
-
|
|
6247
|
-
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
for (const candidateId of servedCanaryIds(step)) nonSignal(candidateId);
|
|
2551
|
+
for (const candidateId of servedCanaryIds(step)) {
|
|
2552
|
+
projected.push({
|
|
2553
|
+
kind: "non-signal",
|
|
2554
|
+
source: "served-brief",
|
|
2555
|
+
class: resolvedClass,
|
|
2556
|
+
candidateId,
|
|
2557
|
+
testId: test.testId,
|
|
2558
|
+
testIndex: test.testIndex,
|
|
2559
|
+
stepIndex: step.stepIndex
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
6257
2562
|
return;
|
|
6258
2563
|
}
|
|
6259
2564
|
const stepAnchors = exercisedAnchors(step);
|
|
6260
|
-
if (stepAnchors.length === 0
|
|
6261
|
-
const
|
|
6262
|
-
const candidateIds = [...served, ...causal.filter((id) => !served.includes(id))];
|
|
6263
|
-
for (const candidateId of candidateIds) {
|
|
6264
|
-
const causalRow = causalSet.has(candidateId);
|
|
6265
|
-
const rowOutcome = causalRow ? "negative" : runOutcome;
|
|
6266
|
-
if (rowOutcome === void 0) {
|
|
6267
|
-
nonSignal(candidateId);
|
|
6268
|
-
continue;
|
|
6269
|
-
}
|
|
6270
|
-
if (!causalRow && stepAnchors.length === 0) continue;
|
|
2565
|
+
if (stepAnchors.length === 0) return;
|
|
2566
|
+
for (const candidateId of servedCanaryIds(step)) {
|
|
6271
2567
|
const resolution = resolveAnchors(candidateId);
|
|
6272
2568
|
if (resolution.kind === "unresolved") {
|
|
6273
2569
|
projected.push({
|
|
@@ -6282,20 +2578,16 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
|
|
|
6282
2578
|
});
|
|
6283
2579
|
continue;
|
|
6284
2580
|
}
|
|
6285
|
-
const
|
|
6286
|
-
const fallback = resolution.anchors[0];
|
|
6287
|
-
const exercisedAnchor = contacted ?? (causalRow && fallback !== void 0 ? normalizeAnchor2(fallback) : void 0);
|
|
2581
|
+
const exercisedAnchor = firstMatchingAnchor(resolution.anchors, stepAnchors);
|
|
6288
2582
|
if (exercisedAnchor === void 0) continue;
|
|
6289
2583
|
projected.push({
|
|
6290
2584
|
kind: "outcome",
|
|
6291
2585
|
source: "served-brief",
|
|
6292
|
-
...causalRow ? { misdirectionFlipped: true } : {},
|
|
6293
2586
|
outcome: {
|
|
6294
2587
|
candidateId,
|
|
6295
|
-
runOutcome
|
|
6296
|
-
//
|
|
2588
|
+
runOutcome,
|
|
2589
|
+
// The credit pass sets the benefit from the run outcome.
|
|
6297
2590
|
benefit: "rode-along",
|
|
6298
|
-
stateValidity: "unknown",
|
|
6299
2591
|
runId: String(input.handoff.runId),
|
|
6300
2592
|
testId: test.testId,
|
|
6301
2593
|
testIndex: test.testIndex,
|
|
@@ -6304,10 +2596,9 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
|
|
|
6304
2596
|
briefDigest: step.servedBrief.briefDigest,
|
|
6305
2597
|
attribution: [
|
|
6306
2598
|
`Candidate ${candidateId}`,
|
|
6307
|
-
`matched ${
|
|
2599
|
+
`matched ${canonicalJson2(exercisedAnchor)}`,
|
|
6308
2600
|
`on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
|
|
6309
|
-
`with ${
|
|
6310
|
-
...causalRow ? ["after the misdirection cross-check flipped"] : []
|
|
2601
|
+
`with ${runOutcome} verdict`
|
|
6311
2602
|
].join(" "),
|
|
6312
2603
|
exercisedAnchor
|
|
6313
2604
|
}
|
|
@@ -6360,9 +2651,8 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
|
|
|
6360
2651
|
outcome: {
|
|
6361
2652
|
candidateId: source.procedureId,
|
|
6362
2653
|
runOutcome,
|
|
6363
|
-
//
|
|
2654
|
+
// The credit pass sets the benefit from the run outcome.
|
|
6364
2655
|
benefit: "rode-along",
|
|
6365
|
-
stateValidity: "unknown",
|
|
6366
2656
|
runId: String(input.handoff.runId),
|
|
6367
2657
|
testId: test.testId,
|
|
6368
2658
|
testIndex: test.testIndex,
|
|
@@ -6374,7 +2664,7 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
|
|
|
6374
2664
|
`on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
|
|
6375
2665
|
attributionDetail
|
|
6376
2666
|
].join(" "),
|
|
6377
|
-
exercisedAnchor:
|
|
2667
|
+
exercisedAnchor: normalizeAnchor(anchor)
|
|
6378
2668
|
}
|
|
6379
2669
|
});
|
|
6380
2670
|
}
|
|
@@ -6384,45 +2674,27 @@ function projectCanaryOutcomes(input) {
|
|
|
6384
2674
|
);
|
|
6385
2675
|
}
|
|
6386
2676
|
async function emitCanaryOutcomes(opts) {
|
|
6387
|
-
if (!
|
|
6388
|
-
if (!
|
|
2677
|
+
if (!isAbsolute2(opts.knowledgeRoot)) throw new TypeError("knowledgeRoot must be absolute");
|
|
2678
|
+
if (!isAbsolute2(opts.runDir)) throw new TypeError("runDir must be absolute");
|
|
6389
2679
|
const artifact = loadRunAnalysisArtifact(opts.runDir, String(opts.handoff.runId));
|
|
6390
|
-
const recovered = opts.handoff.misdirectionEvidence === void 0 ? readCompletedMisdirectionChecks(opts.runDir, String(opts.handoff.runId)) : void 0;
|
|
6391
2680
|
const projected = projectCanaryAttributions({
|
|
6392
2681
|
handoff: opts.handoff,
|
|
6393
2682
|
...artifact === void 0 ? {} : { artifact },
|
|
6394
|
-
|
|
6395
|
-
resolveCandidateAnchors: (id) => LOCAL_CURATION_CAPABILITIES4.readCanonicalBundleAnchors({
|
|
2683
|
+
resolveCandidateAnchors: (id) => LOCAL_CURATION_CAPABILITIES2.readCanonicalBundleAnchors({
|
|
6396
2684
|
knowledgeRoot: opts.knowledgeRoot,
|
|
6397
2685
|
bundleId: id
|
|
6398
2686
|
})
|
|
6399
2687
|
});
|
|
6400
|
-
const credit =
|
|
6401
|
-
completedAblations: readCompletedCanaryAblations(opts.runDir, String(opts.handoff.runId)),
|
|
6402
|
-
saveCompletedAblation: (entry) => saveCompletedCanaryAblation(opts.runDir, entry),
|
|
6403
|
-
attributions: projected,
|
|
6404
|
-
handoff: opts.handoff,
|
|
6405
|
-
runDir: opts.runDir,
|
|
6406
|
-
...opts.credit,
|
|
6407
|
-
knowledgeRoot: opts.knowledgeRoot,
|
|
6408
|
-
observationIdOf: LOCAL_CURATION_CAPABILITIES4.canaryOutcomeObservationId,
|
|
6409
|
-
maxAblations: opts.credit.maxAblations ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
|
|
6410
|
-
readBundleStateDigests: opts.credit.readBundleStateDigests ?? LOCAL_CURATION_CAPABILITIES4.readCanonicalBundleStateDigests
|
|
6411
|
-
});
|
|
6412
|
-
const canary = credit?.counters ?? emptyCanaryCreditCounters();
|
|
6413
|
-
if (credit === void 0) {
|
|
6414
|
-
canary.rideAlongUncredited = projected.filter((row) => row.kind === "outcome" && row.outcome.runOutcome === "positive").length;
|
|
6415
|
-
}
|
|
2688
|
+
const credit = creditCanaryAttributions({ attributions: projected });
|
|
6416
2689
|
const report = {
|
|
6417
|
-
canary,
|
|
2690
|
+
canary: credit.counters,
|
|
6418
2691
|
emitted: 0,
|
|
6419
2692
|
refused: 0,
|
|
6420
2693
|
unresolved: 0,
|
|
6421
2694
|
procedure: { emitted: 0, refused: 0, unresolved: 0 },
|
|
6422
2695
|
details: []
|
|
6423
2696
|
};
|
|
6424
|
-
const
|
|
6425
|
-
for (const attribution of attributions) {
|
|
2697
|
+
for (const attribution of credit.attributions) {
|
|
6426
2698
|
if (attribution.kind === "rerun-excluded") {
|
|
6427
2699
|
report.canary.rerunExcluded += 1;
|
|
6428
2700
|
report.details.push({
|
|
@@ -6462,7 +2734,6 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6462
2734
|
});
|
|
6463
2735
|
continue;
|
|
6464
2736
|
}
|
|
6465
|
-
const causal = attribution.misdirectionFlipped === true;
|
|
6466
2737
|
let outcome;
|
|
6467
2738
|
try {
|
|
6468
2739
|
opts.emit(attribution.outcome);
|
|
@@ -6477,10 +2748,6 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6477
2748
|
}
|
|
6478
2749
|
report.details.push({
|
|
6479
2750
|
benefit: attribution.outcome.benefit,
|
|
6480
|
-
crossCheck: attribution.crossCheck ?? "not-run",
|
|
6481
|
-
...attribution.staged === void 0 ? {} : { staged: attribution.staged },
|
|
6482
|
-
...attribution.corpusDigestMoved === void 0 ? {} : { corpusDigestMoved: attribution.corpusDigestMoved },
|
|
6483
|
-
...causal ? { misdirectionFlipped: true } : {},
|
|
6484
2751
|
candidateId: attribution.outcome.candidateId,
|
|
6485
2752
|
testId: attribution.outcome.testId,
|
|
6486
2753
|
testIndex: attribution.outcome.testIndex,
|
|
@@ -6491,42 +2758,126 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6491
2758
|
return report;
|
|
6492
2759
|
}
|
|
6493
2760
|
|
|
6494
|
-
// src/curation/
|
|
6495
|
-
|
|
2761
|
+
// src/curation/promotion-line.ts
|
|
2762
|
+
function renderPromotionLine(entry) {
|
|
2763
|
+
return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
|
|
2764
|
+
}
|
|
6496
2765
|
|
|
6497
2766
|
// src/curation/run-curation-context.ts
|
|
6498
|
-
|
|
6499
|
-
init_config();
|
|
6500
|
-
import { randomUUID as randomUUID4 } from "crypto";
|
|
2767
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
6501
2768
|
import {
|
|
6502
|
-
closeSync as
|
|
6503
|
-
fsyncSync as
|
|
6504
|
-
linkSync
|
|
6505
|
-
lstatSync as
|
|
6506
|
-
openSync as
|
|
6507
|
-
readFileSync as
|
|
6508
|
-
rmSync as
|
|
6509
|
-
writeFileSync as
|
|
2769
|
+
closeSync as closeSync2,
|
|
2770
|
+
fsyncSync as fsyncSync2,
|
|
2771
|
+
linkSync,
|
|
2772
|
+
lstatSync as lstatSync2,
|
|
2773
|
+
openSync as openSync2,
|
|
2774
|
+
readFileSync as readFileSync4,
|
|
2775
|
+
rmSync as rmSync3,
|
|
2776
|
+
writeFileSync as writeFileSync2
|
|
6510
2777
|
} from "fs";
|
|
6511
|
-
import { dirname as
|
|
6512
|
-
import { z as
|
|
2778
|
+
import { dirname as dirname7, isAbsolute as isAbsolute3, join as join6, resolve as resolve3 } from "path";
|
|
2779
|
+
import { z as z5 } from "zod";
|
|
2780
|
+
|
|
2781
|
+
// src/composition/declared-secrets.ts
|
|
2782
|
+
import { existsSync as existsSync2, readFileSync as readFileSync3, statSync } from "fs";
|
|
2783
|
+
import { parse as parseYaml2 } from "yaml";
|
|
2784
|
+
import { resolveWorkspacePaths } from "@vostride/agent-qa-core";
|
|
2785
|
+
import { SecretRedactor, SecretStore } from "@vostride/agent-qa-runner";
|
|
2786
|
+
var DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
|
|
2787
|
+
lines;
|
|
2788
|
+
constructor(lines) {
|
|
2789
|
+
super(lines.join(" "));
|
|
2790
|
+
this.name = "DeclaredSecretsUnavailableError";
|
|
2791
|
+
this.lines = lines;
|
|
2792
|
+
}
|
|
2793
|
+
static required() {
|
|
2794
|
+
return new _DeclaredSecretsUnavailableError([
|
|
2795
|
+
"Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
|
|
2796
|
+
]);
|
|
2797
|
+
}
|
|
2798
|
+
};
|
|
2799
|
+
function readDeclaredSecrets(secretsPath) {
|
|
2800
|
+
if (!existsSync2(secretsPath)) {
|
|
2801
|
+
throw new DeclaredSecretsUnavailableError([
|
|
2802
|
+
`Error: Secrets file not found: ${secretsPath}`,
|
|
2803
|
+
"Create the file before running tests. It may be empty, but it must exist."
|
|
2804
|
+
]);
|
|
2805
|
+
}
|
|
2806
|
+
let content;
|
|
2807
|
+
try {
|
|
2808
|
+
content = readFileSync3(secretsPath, "utf-8");
|
|
2809
|
+
} catch {
|
|
2810
|
+
throw new DeclaredSecretsUnavailableError([
|
|
2811
|
+
`Error: Secrets file could not be read: ${secretsPath}`,
|
|
2812
|
+
"Check the path and file permissions before running tests."
|
|
2813
|
+
]);
|
|
2814
|
+
}
|
|
2815
|
+
try {
|
|
2816
|
+
return SecretStore.fromEnvContent(content);
|
|
2817
|
+
} catch (err) {
|
|
2818
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2819
|
+
throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
function resolveSecretsPath(configPath) {
|
|
2823
|
+
const file = parseYaml2(readFileSync3(configPath, "utf-8")) ?? {};
|
|
2824
|
+
const config = mergeConfigs(file, loadEnvOverrides(), {});
|
|
2825
|
+
const configured = config?.workspace?.secretsFile;
|
|
2826
|
+
if (typeof configured !== "string" || configured.trim().length === 0) {
|
|
2827
|
+
throw DeclaredSecretsUnavailableError.required();
|
|
2828
|
+
}
|
|
2829
|
+
const workspace = resolveWorkspacePaths({ config, configPath });
|
|
2830
|
+
return workspace.secretsFile.absolutePath;
|
|
2831
|
+
}
|
|
2832
|
+
function statKey(path) {
|
|
2833
|
+
const stat = statSync(path);
|
|
2834
|
+
return `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
|
|
2835
|
+
}
|
|
2836
|
+
function createDeclaredSecretMask(configPath) {
|
|
2837
|
+
let resolved;
|
|
2838
|
+
let cached;
|
|
2839
|
+
const currentSecretsPath = () => {
|
|
2840
|
+
const configKey = statKey(configPath);
|
|
2841
|
+
if (resolved?.configKey !== configKey) {
|
|
2842
|
+
resolved = void 0;
|
|
2843
|
+
resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
|
|
2844
|
+
}
|
|
2845
|
+
return resolved.secretsPath;
|
|
2846
|
+
};
|
|
2847
|
+
const redactor = () => {
|
|
2848
|
+
const secretsPath = currentSecretsPath();
|
|
2849
|
+
let key;
|
|
2850
|
+
try {
|
|
2851
|
+
key = `${secretsPath}\0${statKey(secretsPath)}`;
|
|
2852
|
+
} catch {
|
|
2853
|
+
key = void 0;
|
|
2854
|
+
}
|
|
2855
|
+
if (key !== void 0 && cached?.key === key) return cached.redactor;
|
|
2856
|
+
const current = new SecretRedactor(readDeclaredSecrets(secretsPath));
|
|
2857
|
+
cached = key === void 0 ? void 0 : { key, redactor: current };
|
|
2858
|
+
return current;
|
|
2859
|
+
};
|
|
2860
|
+
return { mask: (text) => redactor().redactString(text), redactor };
|
|
2861
|
+
}
|
|
2862
|
+
|
|
2863
|
+
// src/curation/run-curation-context.ts
|
|
6513
2864
|
var RUN_CONFIG_RECORD_FILE = "run-config.json";
|
|
6514
2865
|
var RUN_CONFIG_RECORD_SCHEMA_VERSION = 2;
|
|
6515
|
-
var runConfigRecordSchema =
|
|
6516
|
-
schemaVersion:
|
|
6517
|
-
runId:
|
|
6518
|
-
configPath:
|
|
2866
|
+
var runConfigRecordSchema = z5.object({
|
|
2867
|
+
schemaVersion: z5.literal(RUN_CONFIG_RECORD_SCHEMA_VERSION),
|
|
2868
|
+
runId: z5.string().min(1),
|
|
2869
|
+
configPath: z5.string().min(1)
|
|
6519
2870
|
}).strict();
|
|
6520
2871
|
function runConfigRecordPath(agentQaRoot, runId) {
|
|
6521
|
-
return
|
|
2872
|
+
return join6(completedRunDir(agentQaRoot, runId), RUN_CONFIG_RECORD_FILE);
|
|
6522
2873
|
}
|
|
6523
2874
|
function readRunConfigRecord(agentQaRoot, runId) {
|
|
6524
2875
|
const path = runConfigRecordPath(agentQaRoot, runId);
|
|
6525
2876
|
try {
|
|
6526
|
-
if (!
|
|
6527
|
-
const parsed = runConfigRecordSchema.safeParse(JSON.parse(
|
|
2877
|
+
if (!lstatSync2(path).isFile()) return void 0;
|
|
2878
|
+
const parsed = runConfigRecordSchema.safeParse(JSON.parse(readFileSync4(path, "utf8")));
|
|
6528
2879
|
if (!parsed.success) return void 0;
|
|
6529
|
-
if (parsed.data.runId !== runId || !
|
|
2880
|
+
if (parsed.data.runId !== runId || !isAbsolute3(parsed.data.configPath)) return void 0;
|
|
6530
2881
|
return parsed.data;
|
|
6531
2882
|
} catch {
|
|
6532
2883
|
return void 0;
|
|
@@ -6554,13 +2905,13 @@ async function resolveRunCurationContext(input) {
|
|
|
6554
2905
|
let knowledgeRoot;
|
|
6555
2906
|
try {
|
|
6556
2907
|
const config = await resolveConfig({ configPath, loadAuthPlugins: false });
|
|
6557
|
-
knowledgeRoot = resolveConfiguredKnowledgeRoot(config,
|
|
2908
|
+
knowledgeRoot = resolveConfiguredKnowledgeRoot(config, dirname7(configPath));
|
|
6558
2909
|
} catch {
|
|
6559
2910
|
return {
|
|
6560
2911
|
handBack: `Run ${runId} was recorded under ${configPath}, which cannot be read as an agent-qa config, so it is left for a run that can read it.`
|
|
6561
2912
|
};
|
|
6562
2913
|
}
|
|
6563
|
-
if (
|
|
2914
|
+
if (resolve3(knowledgeRoot) !== resolve3(input.drainerKnowledgeRoot)) {
|
|
6564
2915
|
return {
|
|
6565
2916
|
handBack: `Run ${runId} keeps its knowledge in ${knowledgeRoot}; it is left for a run of its own project.`
|
|
6566
2917
|
};
|
|
@@ -6588,18 +2939,18 @@ var CURATION_TAIL_RECEIPT_FILE = "curation-tail-receipt.json";
|
|
|
6588
2939
|
var CURATION_TAIL_RECEIPT_SCHEMA_VERSION = 2;
|
|
6589
2940
|
function defaultOperations() {
|
|
6590
2941
|
return {
|
|
6591
|
-
loadCompletedRunHandoff
|
|
6592
|
-
resolveKnowledgeRoot:
|
|
6593
|
-
runSafetyScope
|
|
2942
|
+
loadCompletedRunHandoff,
|
|
2943
|
+
resolveKnowledgeRoot: resolveKnowledgeRoot2,
|
|
2944
|
+
runSafetyScope,
|
|
6594
2945
|
emitCurationCandidates,
|
|
6595
|
-
openKnowledgeReader:
|
|
2946
|
+
openKnowledgeReader: LOCAL_CURATION_CAPABILITIES3.openKnowledgeReader,
|
|
6596
2947
|
emitCanaryOutcomes,
|
|
6597
|
-
foldRunOutcomes:
|
|
6598
|
-
reconcileRunOutcomesProjection:
|
|
6599
|
-
acquireWriterLease:
|
|
6600
|
-
releaseLease:
|
|
2948
|
+
foldRunOutcomes: LOCAL_CURATION_CAPABILITIES3.foldRunOutcomes,
|
|
2949
|
+
reconcileRunOutcomesProjection: LOCAL_CURATION_CAPABILITIES3.reconcileRunOutcomesProjection,
|
|
2950
|
+
acquireWriterLease: LOCAL_CURATION_CAPABILITIES3.maintenance.acquireWriterLease,
|
|
2951
|
+
releaseLease: LOCAL_CURATION_CAPABILITIES3.maintenance.releaseLease,
|
|
6601
2952
|
buildCuratorAuthor,
|
|
6602
|
-
drainQueue:
|
|
2953
|
+
drainQueue: LOCAL_CURATION_CAPABILITIES3.drainQueue,
|
|
6603
2954
|
listCompletedRunDirs,
|
|
6604
2955
|
readCurationTailReceipt,
|
|
6605
2956
|
writeCurationTailReceipt,
|
|
@@ -6608,15 +2959,15 @@ function defaultOperations() {
|
|
|
6608
2959
|
};
|
|
6609
2960
|
}
|
|
6610
2961
|
function completedRunDir(agentQaRoot, runId) {
|
|
6611
|
-
return
|
|
2962
|
+
return join7(agentQaRoot, RUNS_SUBDIR, slugBranch(runId));
|
|
6612
2963
|
}
|
|
6613
2964
|
function completedRunsRoot(agentQaRoot) {
|
|
6614
|
-
return
|
|
2965
|
+
return join7(agentQaRoot, RUNS_SUBDIR);
|
|
6615
2966
|
}
|
|
6616
2967
|
function listCompletedRunDirs(agentQaRoot) {
|
|
6617
2968
|
const runsRoot = completedRunsRoot(agentQaRoot);
|
|
6618
|
-
if (!
|
|
6619
|
-
return
|
|
2969
|
+
if (!existsSync3(runsRoot)) return [];
|
|
2970
|
+
return readdirSync(runsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join7(runsRoot, entry.name)).filter((runDir) => existsSync3(join7(runDir, HANDOFF_FILE))).sort();
|
|
6620
2971
|
}
|
|
6621
2972
|
function isSortedCandidateIds(value) {
|
|
6622
2973
|
if (!Array.isArray(value)) return false;
|
|
@@ -6628,12 +2979,12 @@ function isSortedCandidateIds(value) {
|
|
|
6628
2979
|
function isCanaryCreditCounters(value) {
|
|
6629
2980
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
6630
2981
|
const record = value;
|
|
6631
|
-
if (Object.keys(record).sort().join(",") !== "
|
|
6632
|
-
const
|
|
6633
|
-
if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" ||
|
|
2982
|
+
if (Object.keys(record).sort().join(",") !== "nonSignal,rerunExcluded,rideAlongUncredited") return false;
|
|
2983
|
+
const count = (entry) => typeof entry === "number" && Number.isInteger(entry) && entry >= 0;
|
|
2984
|
+
if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" || count(entry))) return false;
|
|
6634
2985
|
const nonSignal = record.nonSignal;
|
|
6635
2986
|
if (typeof nonSignal !== "object" || nonSignal === null || Array.isArray(nonSignal)) return false;
|
|
6636
|
-
return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(
|
|
2987
|
+
return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(count);
|
|
6637
2988
|
}
|
|
6638
2989
|
function isCurationTailReceipt(value) {
|
|
6639
2990
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
@@ -6644,7 +2995,7 @@ function isCurationTailReceipt(value) {
|
|
|
6644
2995
|
function readCurationTailReceipt(runDir) {
|
|
6645
2996
|
try {
|
|
6646
2997
|
const parsed = JSON.parse(
|
|
6647
|
-
|
|
2998
|
+
readFileSync5(join7(runDir, CURATION_TAIL_RECEIPT_FILE), "utf8")
|
|
6648
2999
|
);
|
|
6649
3000
|
return isCurationTailReceipt(parsed) ? parsed : void 0;
|
|
6650
3001
|
} catch {
|
|
@@ -6652,36 +3003,36 @@ function readCurationTailReceipt(runDir) {
|
|
|
6652
3003
|
}
|
|
6653
3004
|
}
|
|
6654
3005
|
function fsyncFile(file) {
|
|
6655
|
-
const fd =
|
|
3006
|
+
const fd = openSync3(file, "r");
|
|
6656
3007
|
try {
|
|
6657
|
-
|
|
3008
|
+
fsyncSync3(fd);
|
|
6658
3009
|
} finally {
|
|
6659
|
-
|
|
3010
|
+
closeSync3(fd);
|
|
6660
3011
|
}
|
|
6661
3012
|
}
|
|
6662
3013
|
function writeCurationTailReceipt(runDir, receipt) {
|
|
6663
3014
|
if (!isCurationTailReceipt(receipt)) throw new TypeError("invalid curation tail receipt");
|
|
6664
|
-
const file =
|
|
6665
|
-
const staged =
|
|
3015
|
+
const file = join7(runDir, CURATION_TAIL_RECEIPT_FILE);
|
|
3016
|
+
const staged = join7(runDir, `.${CURATION_TAIL_RECEIPT_FILE}.${randomUUID3()}.tmp`);
|
|
6666
3017
|
try {
|
|
6667
|
-
|
|
3018
|
+
writeFileSync3(staged, `${canonicalJson3(receipt)}
|
|
6668
3019
|
`, { encoding: "utf8", mode: 384 });
|
|
6669
3020
|
fsyncFile(staged);
|
|
6670
|
-
|
|
3021
|
+
renameSync2(staged, file);
|
|
6671
3022
|
fsyncFile(runDir);
|
|
6672
3023
|
} finally {
|
|
6673
|
-
|
|
3024
|
+
rmSync4(staged, { force: true });
|
|
6674
3025
|
}
|
|
6675
3026
|
}
|
|
6676
3027
|
function curationTailHandoffDigest(handoff) {
|
|
6677
|
-
return
|
|
3028
|
+
return sha256hex(canonicalJson3(handoff));
|
|
6678
3029
|
}
|
|
6679
3030
|
function receiptIdentityForHandoff(handoff) {
|
|
6680
3031
|
return {
|
|
6681
3032
|
schemaVersion: CURATION_TAIL_RECEIPT_SCHEMA_VERSION,
|
|
6682
3033
|
runId: String(handoff.runId),
|
|
6683
3034
|
handoffDigest: curationTailHandoffDigest(handoff),
|
|
6684
|
-
projectionVersion:
|
|
3035
|
+
projectionVersion: LOCAL_CURATION_CAPABILITIES3.runOutcomesProjectionVersion
|
|
6685
3036
|
};
|
|
6686
3037
|
}
|
|
6687
3038
|
function receiptMatches(receipt, expected) {
|
|
@@ -6706,12 +3057,6 @@ function candidateSummaryLine(runId, report) {
|
|
|
6706
3057
|
});
|
|
6707
3058
|
return `Knowledge curation candidates for ${runId}: ${segments.join("; ")}.`;
|
|
6708
3059
|
}
|
|
6709
|
-
function misdirectionHarmLine(runId, rows) {
|
|
6710
|
-
const landed = rows.filter((row) => row.outcome.kind === "emitted");
|
|
6711
|
-
const refused = rows.length - landed.length;
|
|
6712
|
-
const named = rows.map((row) => row.candidateId).join(", ");
|
|
6713
|
-
return `Misdirection harm for ${runId}: ${landed.length} harmed outcome(s) emitted from flipped cross-check evidence regardless of verdict class (${named})${refused === 0 ? "" : `; ${refused} refused by the queue`}.`;
|
|
6714
|
-
}
|
|
6715
3060
|
function curationOutcomeLine(input) {
|
|
6716
3061
|
if (input.committed <= 0) return void 0;
|
|
6717
3062
|
const proposed = Math.min(Math.max(input.downgraded, 0), input.committed);
|
|
@@ -6779,7 +3124,6 @@ async function curateCompletedRun(input) {
|
|
|
6779
3124
|
deprecated: previous.deprecated ?? 0,
|
|
6780
3125
|
complete: true,
|
|
6781
3126
|
drainKind: previous.drainKind,
|
|
6782
|
-
candidateIds: previous.candidateIds,
|
|
6783
3127
|
snapshot: previous.snapshot
|
|
6784
3128
|
};
|
|
6785
3129
|
}
|
|
@@ -6794,7 +3138,7 @@ async function curateCompletedRun(input) {
|
|
|
6794
3138
|
const budgetMs = input.deadlineMs === void 0 ? void 0 : Math.max(0, input.deadlineMs - tailStartedAtMs);
|
|
6795
3139
|
await operations.runSafetyScope(
|
|
6796
3140
|
{
|
|
6797
|
-
//
|
|
3141
|
+
// Posture is persisted run truth, never an ambient re-detection.
|
|
6798
3142
|
executionContext: input.handoff.executionContext
|
|
6799
3143
|
},
|
|
6800
3144
|
async () => {
|
|
@@ -6805,29 +3149,13 @@ async function curateCompletedRun(input) {
|
|
|
6805
3149
|
...input.mask === void 0 ? {} : { mask: input.mask }
|
|
6806
3150
|
});
|
|
6807
3151
|
input.log.info?.(candidateSummaryLine(receiptIdentity.runId, candidates));
|
|
6808
|
-
const reader = operations.openKnowledgeReader({
|
|
6809
|
-
knowledgeRoot: input.knowledgeRoot,
|
|
6810
|
-
priors: input.resolveTrustPriors?.()
|
|
6811
|
-
});
|
|
6812
|
-
const ablation = operations.canaryAblation;
|
|
6813
|
-
const recordedPolicy = readRunCanaryPolicy(input.runDir, String(input.handoff.runId));
|
|
6814
|
-
const caps = ablation === void 0 ? void 0 : recordedPolicy ?? (input.maxAblations !== void 0 ? { maxAblations: input.maxAblations, source: "flag" } : await ablation.resolveCaps?.() ?? {
|
|
6815
|
-
maxAblations: (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
|
|
6816
|
-
source: "default"
|
|
6817
|
-
});
|
|
6818
|
-
if (ablation !== void 0 && recordedPolicy === void 0) input.log.info?.("canary cap: legacy run without recorded policy; using explicit run override or current config/default");
|
|
6819
|
-
if (caps !== void 0) input.log.info?.(`canary cap: maxAblations=${caps.maxAblations} (${caps.source})`);
|
|
3152
|
+
const reader = operations.openKnowledgeReader({ knowledgeRoot: input.knowledgeRoot });
|
|
6820
3153
|
const canaries = await operations.emitCanaryOutcomes({
|
|
6821
3154
|
handoff: input.handoff,
|
|
6822
3155
|
knowledgeRoot: input.knowledgeRoot,
|
|
6823
3156
|
runDir: input.runDir,
|
|
6824
|
-
emit: reader.emitCanaryOutcome.bind(reader)
|
|
6825
|
-
...operations.canaryAblation === void 0 ? {} : { credit: { ablation: operations.canaryAblation, maxAblations: caps.maxAblations, readBundleStateDigests: LOCAL_CURATION_CAPABILITIES5.readCanonicalBundleStateDigests } }
|
|
3157
|
+
emit: reader.emitCanaryOutcome.bind(reader)
|
|
6826
3158
|
});
|
|
6827
|
-
const causalHarm = (canaries.details ?? []).filter((row) => row.misdirectionFlipped === true);
|
|
6828
|
-
if (causalHarm.length > 0) {
|
|
6829
|
-
input.log.info?.(misdirectionHarmLine(receiptIdentity.runId, causalHarm));
|
|
6830
|
-
}
|
|
6831
3159
|
try {
|
|
6832
3160
|
const fold = operations.foldRunOutcomes({
|
|
6833
3161
|
knowledgeRoot: input.knowledgeRoot,
|
|
@@ -6852,15 +3180,13 @@ async function curateCompletedRun(input) {
|
|
|
6852
3180
|
}));
|
|
6853
3181
|
const drain = await operations.drainQueue({
|
|
6854
3182
|
sourceRunId: String(input.handoff.runId),
|
|
6855
|
-
observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [
|
|
3183
|
+
observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [LOCAL_CURATION_CAPABILITIES3.canaryOutcomeObservationId({
|
|
6856
3184
|
candidateId: row.candidateId,
|
|
6857
3185
|
runId: input.handoff.runId,
|
|
6858
3186
|
testId: row.testId,
|
|
6859
3187
|
testIndex: row.testIndex,
|
|
6860
3188
|
stepIndex: row.stepIndex
|
|
6861
3189
|
})]),
|
|
6862
|
-
priors: input.resolveTrustPriors?.(),
|
|
6863
|
-
...input.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: input.resolvePromotionPosture },
|
|
6864
3190
|
knowledgeRoot: input.knowledgeRoot,
|
|
6865
3191
|
author,
|
|
6866
3192
|
executionContext: input.handoff.executionContext,
|
|
@@ -6902,8 +3228,7 @@ async function curateCompletedRun(input) {
|
|
|
6902
3228
|
drainKind: drain.kind,
|
|
6903
3229
|
handoffHolderPid: drain.kind === "handed-off" ? drain.holder.pid : null,
|
|
6904
3230
|
candidateIds: sortedCandidateIds(candidates),
|
|
6905
|
-
//
|
|
6906
|
-
// answers from this receipt; a handed-off receipt has no counts of its own to record.
|
|
3231
|
+
// Counts travel with the snapshot, when the rerun short-circuit answers from the receipt.
|
|
6907
3232
|
...validated?.success === true ? { snapshot: validated.data, curated, deprecated } : {}
|
|
6908
3233
|
};
|
|
6909
3234
|
}
|
|
@@ -6948,7 +3273,6 @@ async function curateCompletedRun(input) {
|
|
|
6948
3273
|
deprecated,
|
|
6949
3274
|
complete: true,
|
|
6950
3275
|
drainKind,
|
|
6951
|
-
candidateIds: completedReceipt.candidateIds,
|
|
6952
3276
|
...completedReceipt.snapshot === void 0 ? {} : { snapshot: completedReceipt.snapshot }
|
|
6953
3277
|
};
|
|
6954
3278
|
}
|
|
@@ -6961,7 +3285,7 @@ async function runCurationTail(opts) {
|
|
|
6961
3285
|
let currentHandoff;
|
|
6962
3286
|
try {
|
|
6963
3287
|
currentHandoff = operations.loadCompletedRunHandoff(
|
|
6964
|
-
|
|
3288
|
+
join7(currentRunDir, HANDOFF_FILE)
|
|
6965
3289
|
);
|
|
6966
3290
|
} catch {
|
|
6967
3291
|
opts.log.warn(
|
|
@@ -7003,7 +3327,7 @@ async function runCurationTail(opts) {
|
|
|
7003
3327
|
opts.log.warn("curation: run-outcome projection not reconciled (no writer authority)");
|
|
7004
3328
|
} else if (reconcile.kind === "rebuilt") {
|
|
7005
3329
|
opts.log.info?.(
|
|
7006
|
-
`curation: run-outcome projection rebuilt at version ${
|
|
3330
|
+
`curation: run-outcome projection rebuilt at version ${LOCAL_CURATION_CAPABILITIES3.runOutcomesProjectionVersion} (folded ${reconcile.report.folded}, skipped ${reconcile.report.skippedHandoffs} handoff, ${reconcile.report.skippedArtifacts} artifact)`
|
|
7007
3331
|
);
|
|
7008
3332
|
}
|
|
7009
3333
|
} catch (error) {
|
|
@@ -7025,7 +3349,7 @@ async function runCurationTail(opts) {
|
|
|
7025
3349
|
for (const runDir of runDirs) {
|
|
7026
3350
|
let handoff;
|
|
7027
3351
|
try {
|
|
7028
|
-
handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff(
|
|
3352
|
+
handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff(join7(runDir, HANDOFF_FILE));
|
|
7029
3353
|
} catch {
|
|
7030
3354
|
opts.log.warn(`Skipping post-verdict knowledge curation for ${runDir}: handoff is malformed.`);
|
|
7031
3355
|
continue;
|
|
@@ -7047,11 +3371,7 @@ async function runCurationTail(opts) {
|
|
|
7047
3371
|
}
|
|
7048
3372
|
runMask = context.mask;
|
|
7049
3373
|
}
|
|
7050
|
-
const resolvePromotionPosture = opts.resolvePromotionPostureForRun === void 0 ? opts.resolvePromotionPosture : await opts.resolvePromotionPostureForRun(handoff.buildOrdinal, handoff.scope);
|
|
7051
3374
|
const folded = await curateCompletedRun({
|
|
7052
|
-
resolveTrustPriors: opts.resolveTrustPriors,
|
|
7053
|
-
...runDir !== currentRunDir || opts.maxAblations === void 0 ? {} : { maxAblations: opts.maxAblations },
|
|
7054
|
-
...resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture },
|
|
7055
3375
|
runDir,
|
|
7056
3376
|
handoff,
|
|
7057
3377
|
knowledgeRoot,
|
|
@@ -7133,7 +3453,6 @@ export {
|
|
|
7133
3453
|
normalizeConfigForValidation,
|
|
7134
3454
|
projectCanaryOutcomes,
|
|
7135
3455
|
purgeAll,
|
|
7136
|
-
purgeTest,
|
|
7137
3456
|
readAnalyticsPrivacyDeclaration,
|
|
7138
3457
|
resolveConfig,
|
|
7139
3458
|
resolveConfiguredKnowledgeRoot,
|