agent-qa 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -3
- package/capability-catalog/capability-catalog.json +51 -327
- package/dist/cli-main.cjs +5658 -11153
- package/dist/cli-main.cjs.map +1 -1
- package/dist/cli-main.js +4451 -10024
- package/dist/cli-main.js.map +1 -1
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +332 -3875
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -348
- package/dist/index.d.ts +12 -348
- package/dist/index.js +282 -3963
- package/dist/index.js.map +1 -1
- package/package.json +31 -23
- package/skills/agent-qa-authoring/SKILL.md +5 -11
- package/skills/agent-qa-authoring/references/agent-qa-contracts.json +2 -4
- package/skills/agent-qa-debug-fix/SKILL.md +1 -4
- package/skills/agent-qa-knowledge/SKILL.md +2 -4
- package/skills/agent-qa-knowledge/references/knowledge-reads.md +19 -22
- package/skills/agent-qa-result-triage/SKILL.md +0 -2
- package/skills/agent-qa-result-triage/references/triage-categories.md +1 -1
package/dist/index.cjs
CHANGED
|
@@ -32,29 +32,6 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
32
32
|
));
|
|
33
33
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
34
34
|
|
|
35
|
-
// src/hard-cut.ts
|
|
36
|
-
function INVALID_CONFIG_FIX(configPath) {
|
|
37
|
-
return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
|
|
38
|
-
}
|
|
39
|
-
function isRetiredKeyIssue(issue) {
|
|
40
|
-
return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.retired === true;
|
|
41
|
-
}
|
|
42
|
-
var RE_INITIALIZE_URL, HARD_CUT_STEPS;
|
|
43
|
-
var init_hard_cut = __esm({
|
|
44
|
-
"src/hard-cut.ts"() {
|
|
45
|
-
"use strict";
|
|
46
|
-
RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
|
|
47
|
-
HARD_CUT_STEPS = [
|
|
48
|
-
"This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
|
|
49
|
-
" 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
|
|
50
|
-
" 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
|
|
51
|
-
" 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
|
|
52
|
-
" 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
|
|
53
|
-
`What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
|
|
54
|
-
].join("\n");
|
|
55
|
-
}
|
|
56
|
-
});
|
|
57
|
-
|
|
58
35
|
// src/provider-plugins.ts
|
|
59
36
|
var provider_plugins_exports = {};
|
|
60
37
|
__export(provider_plugins_exports, {
|
|
@@ -80,13 +57,22 @@ var init_provider_plugins = __esm({
|
|
|
80
57
|
}
|
|
81
58
|
});
|
|
82
59
|
|
|
83
|
-
// src/
|
|
84
|
-
var
|
|
85
|
-
__export(
|
|
60
|
+
// src/index.ts
|
|
61
|
+
var index_exports = {};
|
|
62
|
+
__export(index_exports, {
|
|
63
|
+
CURATION_TAIL_RECEIPT_FILE: () => CURATION_TAIL_RECEIPT_FILE,
|
|
64
|
+
CURATION_TAIL_RECEIPT_SCHEMA_VERSION: () => CURATION_TAIL_RECEIPT_SCHEMA_VERSION,
|
|
65
|
+
CURATOR_MAX_NEIGHBORS: () => CURATOR_MAX_NEIGHBORS,
|
|
66
|
+
CURATOR_RETRY_CAP: () => CURATOR_RETRY_CAP,
|
|
86
67
|
ConfigFileNotFoundError: () => ConfigFileNotFoundError,
|
|
87
68
|
ConfigFileNotYamlError: () => ConfigFileNotYamlError,
|
|
69
|
+
VERSION: () => VERSION,
|
|
70
|
+
buildCuratorAuthor: () => buildCuratorAuthor,
|
|
71
|
+
curationTailHandoffDigest: () => curationTailHandoffDigest,
|
|
88
72
|
deepMerge: () => deepMerge,
|
|
73
|
+
emitCanaryOutcomes: () => emitCanaryOutcomes,
|
|
89
74
|
formatConfigDebug: () => formatConfigDebug,
|
|
75
|
+
listTargets: () => listTargets,
|
|
90
76
|
loadConfigFile: () => loadConfigFile,
|
|
91
77
|
loadConfigPlugins: () => loadConfigPlugins,
|
|
92
78
|
loadEnvOverrides: () => loadEnvOverrides,
|
|
@@ -95,11 +81,52 @@ __export(config_exports, {
|
|
|
95
81
|
mergeUseBlocks: () => mergeUseBlocks,
|
|
96
82
|
mergeWithTestConfig: () => mergeWithTestConfig,
|
|
97
83
|
normalizeConfigForValidation: () => normalizeConfigForValidation,
|
|
84
|
+
projectCanaryOutcomes: () => projectCanaryOutcomes,
|
|
85
|
+
purgeAll: () => purgeAll,
|
|
98
86
|
readAnalyticsPrivacyDeclaration: () => readAnalyticsPrivacyDeclaration,
|
|
99
87
|
resolveConfig: () => resolveConfig,
|
|
100
88
|
resolveConfiguredKnowledgeRoot: () => resolveConfiguredKnowledgeRoot,
|
|
101
|
-
resolveRawConfigKnowledgeRoot: () => resolveRawConfigKnowledgeRoot
|
|
89
|
+
resolveRawConfigKnowledgeRoot: () => resolveRawConfigKnowledgeRoot,
|
|
90
|
+
resolveTarget: () => resolveTarget,
|
|
91
|
+
runCurationTail: () => runCurationTail,
|
|
92
|
+
runVerdictFirstCurationTail: () => runVerdictFirstCurationTail
|
|
102
93
|
});
|
|
94
|
+
module.exports = __toCommonJS(index_exports);
|
|
95
|
+
var import_agent_qa_core5 = require("@vostride/agent-qa-core");
|
|
96
|
+
__reExport(index_exports, require("@vostride/agent-qa-core"), module.exports);
|
|
97
|
+
|
|
98
|
+
// src/config.ts
|
|
99
|
+
var import_promises = require("fs/promises");
|
|
100
|
+
var import_node_path2 = require("path");
|
|
101
|
+
var import_yaml = require("yaml");
|
|
102
|
+
var import_picocolors = __toESM(require("picocolors"), 1);
|
|
103
|
+
var import_agent_qa_core = require("@vostride/agent-qa-core");
|
|
104
|
+
|
|
105
|
+
// src/hard-cut.ts
|
|
106
|
+
var RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
|
|
107
|
+
var HARD_CUT_STEPS = [
|
|
108
|
+
"This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
|
|
109
|
+
" 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
|
|
110
|
+
" 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
|
|
111
|
+
" 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
|
|
112
|
+
" 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
|
|
113
|
+
`What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
|
|
114
|
+
].join("\n");
|
|
115
|
+
function INVALID_CONFIG_FIX(configPath) {
|
|
116
|
+
return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
|
|
117
|
+
}
|
|
118
|
+
function isHardCutIssue(issue) {
|
|
119
|
+
return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.hardCut === true;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// src/config.ts
|
|
123
|
+
var ENV_MAPPING = {
|
|
124
|
+
AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
|
|
125
|
+
AGENT_QA_MCP_PORT: "services.mcp.port",
|
|
126
|
+
AGENT_QA_CACHE_DIR: "services.cache.dir",
|
|
127
|
+
AGENT_QA_LOG_LEVEL: "services.logging.level",
|
|
128
|
+
AGENT_QA_HEADLESS: "use.browser.headless"
|
|
129
|
+
};
|
|
103
130
|
function setNestedValue(obj, path, value) {
|
|
104
131
|
const parts = path.split(".");
|
|
105
132
|
let current = obj;
|
|
@@ -124,6 +151,15 @@ function parseEnvValue(key, value) {
|
|
|
124
151
|
}
|
|
125
152
|
return value;
|
|
126
153
|
}
|
|
154
|
+
var ConfigFileNotYamlError = class extends Error {
|
|
155
|
+
configPath;
|
|
156
|
+
constructor(configPath, cause) {
|
|
157
|
+
const positionOnly = `${configPath}: ${(0, import_agent_qa_core.yamlParseFailureMessage)(cause)}`;
|
|
158
|
+
super(positionOnly, { cause: new Error(positionOnly) });
|
|
159
|
+
this.name = "ConfigFileNotYamlError";
|
|
160
|
+
this.configPath = configPath;
|
|
161
|
+
}
|
|
162
|
+
};
|
|
127
163
|
async function loadConfigFile(configPath) {
|
|
128
164
|
let content;
|
|
129
165
|
try {
|
|
@@ -138,6 +174,14 @@ async function loadConfigFile(configPath) {
|
|
|
138
174
|
throw new ConfigFileNotYamlError(configPath, err);
|
|
139
175
|
}
|
|
140
176
|
}
|
|
177
|
+
var ConfigFileNotFoundError = class extends Error {
|
|
178
|
+
configPath;
|
|
179
|
+
constructor(configPath) {
|
|
180
|
+
super(`Config file not found: ${configPath}`);
|
|
181
|
+
this.name = "ConfigFileNotFoundError";
|
|
182
|
+
this.configPath = configPath;
|
|
183
|
+
}
|
|
184
|
+
};
|
|
141
185
|
async function loadRequiredConfigFile(configPath) {
|
|
142
186
|
let content;
|
|
143
187
|
try {
|
|
@@ -203,6 +247,7 @@ function mergeUseBlocks(globalUse, suiteUse, testUse, cliFlags) {
|
|
|
203
247
|
cliFlags
|
|
204
248
|
);
|
|
205
249
|
}
|
|
250
|
+
var CIRCULAR_MARKER = "<circular>";
|
|
206
251
|
function flattenObject(obj, prefix = "", ancestors = []) {
|
|
207
252
|
const result = {};
|
|
208
253
|
if (typeof obj !== "object" || obj === null) {
|
|
@@ -300,7 +345,7 @@ Hint: services.dashboard.enabled was removed. Dashboard behavior is enabled by t
|
|
|
300
345
|
const baseUrlHint = baseUrlRefused ? `
|
|
301
346
|
|
|
302
347
|
Hint: services.dashboard.baseUrl must now carry a scheme \u2014 write an absolute URL such as http://127.0.0.1:3470 rather than a bare host like localhost:3470, or remove the key entirely. An empty string is not an absence: absence is a legal and deliberate posture here \u2014 a standalone \`agent-qa mcp\` with no dashboard has no base, and the carrier tool result then names that absence rather than fabricating a URL. The value is refused because it is assigned to an href inside the served App document and appended verbatim to every carrier tool result, so a schemeless or empty base composes a link that resolves against nothing.` : "";
|
|
303
|
-
const closing = result.error.issues.some(
|
|
348
|
+
const closing = result.error.issues.some(isHardCutIssue) ? HARD_CUT_STEPS : INVALID_CONFIG_FIX(configPath);
|
|
304
349
|
throw new Error(
|
|
305
350
|
`Config validation failed:
|
|
306
351
|
${messages.join("\n")}
|
|
@@ -311,3270 +356,68 @@ ${messages.join("\n")}
|
|
|
311
356
|
if (options.loadAuthPlugins !== false) await loadConfigPlugins(result.data, configPath);
|
|
312
357
|
return result.data;
|
|
313
358
|
}
|
|
314
|
-
async function loadConfigPlugins(config, configPath) {
|
|
315
|
-
await (0, import_agent_qa_core.loadLLMAuthPlugins)(config.plugins?.auth, {
|
|
316
|
-
baseDir: (0, import_node_path2.dirname)((0, import_node_path2.resolve)(configPath))
|
|
317
|
-
});
|
|
318
|
-
const { loadProviderPluginsForRawConfig: loadProviderPluginsForRawConfig2 } = await Promise.resolve().then(() => (init_provider_plugins(), provider_plugins_exports));
|
|
319
|
-
await loadProviderPluginsForRawConfig2(config, configPath);
|
|
320
|
-
}
|
|
321
|
-
function resolveConfiguredKnowledgeRoot(config, configDir) {
|
|
322
|
-
return (0, import_agent_qa_core.resolveKnowledgeRoot)(config.knowledge, configDir);
|
|
323
|
-
}
|
|
324
|
-
function resolveRawConfigKnowledgeRoot(config, configPath) {
|
|
325
|
-
if (config === null) return null;
|
|
326
|
-
try {
|
|
327
|
-
return (0, import_agent_qa_core.resolveKnowledgeRoot)(
|
|
328
|
-
import_agent_qa_core.KnowledgeConfigSchema.parse(config.knowledge ?? {}),
|
|
329
|
-
(0, import_node_path2.dirname)((0, import_node_path2.resolve)(configPath))
|
|
330
|
-
);
|
|
331
|
-
} catch {
|
|
332
|
-
return null;
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
var import_promises, import_node_path2, import_yaml, import_picocolors, import_agent_qa_core, ENV_MAPPING, ConfigFileNotYamlError, ConfigFileNotFoundError, CIRCULAR_MARKER;
|
|
336
|
-
var init_config = __esm({
|
|
337
|
-
"src/config.ts"() {
|
|
338
|
-
"use strict";
|
|
339
|
-
import_promises = require("fs/promises");
|
|
340
|
-
import_node_path2 = require("path");
|
|
341
|
-
import_yaml = require("yaml");
|
|
342
|
-
import_picocolors = __toESM(require("picocolors"), 1);
|
|
343
|
-
import_agent_qa_core = require("@vostride/agent-qa-core");
|
|
344
|
-
init_hard_cut();
|
|
345
|
-
ENV_MAPPING = {
|
|
346
|
-
AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
|
|
347
|
-
AGENT_QA_MCP_PORT: "services.mcp.port",
|
|
348
|
-
AGENT_QA_CACHE_DIR: "services.cache.dir",
|
|
349
|
-
AGENT_QA_CACHE_TTL: "services.cache.ttl",
|
|
350
|
-
AGENT_QA_LOG_LEVEL: "services.logging.level",
|
|
351
|
-
AGENT_QA_HEADLESS: "use.browser.headless"
|
|
352
|
-
};
|
|
353
|
-
ConfigFileNotYamlError = class extends Error {
|
|
354
|
-
configPath;
|
|
355
|
-
constructor(configPath, cause) {
|
|
356
|
-
const positionOnly = `${configPath}: ${(0, import_agent_qa_core.yamlParseFailureMessage)(cause)}`;
|
|
357
|
-
super(positionOnly, { cause: new Error(positionOnly) });
|
|
358
|
-
this.name = "ConfigFileNotYamlError";
|
|
359
|
-
this.configPath = configPath;
|
|
360
|
-
}
|
|
361
|
-
};
|
|
362
|
-
ConfigFileNotFoundError = class extends Error {
|
|
363
|
-
configPath;
|
|
364
|
-
constructor(configPath) {
|
|
365
|
-
super(`Config file not found: ${configPath}`);
|
|
366
|
-
this.name = "ConfigFileNotFoundError";
|
|
367
|
-
this.configPath = configPath;
|
|
368
|
-
}
|
|
369
|
-
};
|
|
370
|
-
CIRCULAR_MARKER = "<circular>";
|
|
371
|
-
}
|
|
372
|
-
});
|
|
373
|
-
|
|
374
|
-
// src/targets.ts
|
|
375
|
-
function resolveTarget(config, targetName) {
|
|
376
|
-
const targets = config.registry?.targets;
|
|
377
|
-
if (!targets || !(targetName in targets)) {
|
|
378
|
-
throw new Error(`Target "${targetName}" not found in config. Available targets: ${targets ? Object.keys(targets).join(", ") : "none"}`);
|
|
379
|
-
}
|
|
380
|
-
const target = targets[targetName];
|
|
381
|
-
return {
|
|
382
|
-
name: targetName,
|
|
383
|
-
product: target.product ?? targetName,
|
|
384
|
-
platform: target.platform,
|
|
385
|
-
url: target.url,
|
|
386
|
-
bundleId: target.bundleId,
|
|
387
|
-
appPackage: target.appPackage,
|
|
388
|
-
appActivity: target.appActivity,
|
|
389
|
-
device: target.device,
|
|
390
|
-
// Carried through UNCHANGED, undefined included: the resolver does not fill in a class for a
|
|
391
|
-
// target whose author did not declare one (D-06).
|
|
392
|
-
environment: target.environment
|
|
393
|
-
};
|
|
394
|
-
}
|
|
395
|
-
function listTargets(config) {
|
|
396
|
-
const targets = config.registry?.targets;
|
|
397
|
-
if (!targets) return [];
|
|
398
|
-
return Object.entries(targets).map(([name, entry]) => {
|
|
399
|
-
const typedEntry = entry;
|
|
400
|
-
return {
|
|
401
|
-
name,
|
|
402
|
-
platform: typedEntry.platform
|
|
403
|
-
};
|
|
404
|
-
});
|
|
405
|
-
}
|
|
406
|
-
var init_targets = __esm({
|
|
407
|
-
"src/targets.ts"() {
|
|
408
|
-
"use strict";
|
|
409
|
-
}
|
|
410
|
-
});
|
|
411
|
-
|
|
412
|
-
// src/composition/declared-secrets.ts
|
|
413
|
-
function readDeclaredSecrets(secretsPath) {
|
|
414
|
-
if (!(0, import_node_fs5.existsSync)(secretsPath)) {
|
|
415
|
-
throw new DeclaredSecretsUnavailableError([
|
|
416
|
-
`Error: Secrets file not found: ${secretsPath}`,
|
|
417
|
-
"Create the file before running tests. It may be empty, but it must exist."
|
|
418
|
-
]);
|
|
419
|
-
}
|
|
420
|
-
let content;
|
|
421
|
-
try {
|
|
422
|
-
content = (0, import_node_fs5.readFileSync)(secretsPath, "utf-8");
|
|
423
|
-
} catch {
|
|
424
|
-
throw new DeclaredSecretsUnavailableError([
|
|
425
|
-
`Error: Secrets file could not be read: ${secretsPath}`,
|
|
426
|
-
"Check the path and file permissions before running tests."
|
|
427
|
-
]);
|
|
428
|
-
}
|
|
429
|
-
try {
|
|
430
|
-
return import_agent_qa_runner.SecretStore.fromEnvContent(content);
|
|
431
|
-
} catch (err) {
|
|
432
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
433
|
-
throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
function resolveSecretsPath(configPath) {
|
|
437
|
-
const file = (0, import_yaml2.parse)((0, import_node_fs5.readFileSync)(configPath, "utf-8")) ?? {};
|
|
438
|
-
const config = mergeConfigs(file, loadEnvOverrides(), {});
|
|
439
|
-
const configured = config?.workspace?.secretsFile;
|
|
440
|
-
if (typeof configured !== "string" || configured.trim().length === 0) {
|
|
441
|
-
throw DeclaredSecretsUnavailableError.required();
|
|
442
|
-
}
|
|
443
|
-
const workspace = (0, import_agent_qa_core3.resolveWorkspacePaths)({ config, configPath });
|
|
444
|
-
return workspace.secretsFile.absolutePath;
|
|
445
|
-
}
|
|
446
|
-
function statKey(path) {
|
|
447
|
-
const stat2 = (0, import_node_fs5.statSync)(path);
|
|
448
|
-
return `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
|
|
449
|
-
}
|
|
450
|
-
function createDeclaredSecretMask(configPath) {
|
|
451
|
-
let resolved;
|
|
452
|
-
let cached;
|
|
453
|
-
const currentSecretsPath = () => {
|
|
454
|
-
const configKey = statKey(configPath);
|
|
455
|
-
if (resolved?.configKey !== configKey) {
|
|
456
|
-
resolved = void 0;
|
|
457
|
-
resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
|
|
458
|
-
}
|
|
459
|
-
return resolved.secretsPath;
|
|
460
|
-
};
|
|
461
|
-
const redactor = () => {
|
|
462
|
-
const secretsPath = currentSecretsPath();
|
|
463
|
-
let key;
|
|
464
|
-
try {
|
|
465
|
-
key = `${secretsPath}\0${statKey(secretsPath)}`;
|
|
466
|
-
} catch {
|
|
467
|
-
key = void 0;
|
|
468
|
-
}
|
|
469
|
-
if (key !== void 0 && cached?.key === key) return cached.redactor;
|
|
470
|
-
const current = new import_agent_qa_runner.SecretRedactor(readDeclaredSecrets(secretsPath));
|
|
471
|
-
cached = key === void 0 ? void 0 : { key, redactor: current };
|
|
472
|
-
return current;
|
|
473
|
-
};
|
|
474
|
-
return { mask: (text) => redactor().redactString(text), redactor };
|
|
475
|
-
}
|
|
476
|
-
var import_node_fs5, import_yaml2, import_agent_qa_core3, import_agent_qa_runner, DeclaredSecretsUnavailableError;
|
|
477
|
-
var init_declared_secrets = __esm({
|
|
478
|
-
"src/composition/declared-secrets.ts"() {
|
|
479
|
-
"use strict";
|
|
480
|
-
import_node_fs5 = require("fs");
|
|
481
|
-
import_yaml2 = require("yaml");
|
|
482
|
-
import_agent_qa_core3 = require("@vostride/agent-qa-core");
|
|
483
|
-
import_agent_qa_runner = require("@vostride/agent-qa-runner");
|
|
484
|
-
init_config();
|
|
485
|
-
DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
|
|
486
|
-
lines;
|
|
487
|
-
constructor(lines) {
|
|
488
|
-
super(lines.join(" "));
|
|
489
|
-
this.name = "DeclaredSecretsUnavailableError";
|
|
490
|
-
this.lines = lines;
|
|
491
|
-
}
|
|
492
|
-
static required() {
|
|
493
|
-
return new _DeclaredSecretsUnavailableError([
|
|
494
|
-
"Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
|
|
495
|
-
]);
|
|
496
|
-
}
|
|
497
|
-
};
|
|
498
|
-
}
|
|
499
|
-
});
|
|
500
|
-
|
|
501
|
-
// src/curation/promotion-posture.ts
|
|
502
|
-
function hasNoiseFloorArtefacts(committedRoot) {
|
|
503
|
-
if ((0, import_node_fs6.existsSync)((0, import_node_path10.join)(committedRoot, "reports", "noise-floor.json"))) return true;
|
|
504
|
-
try {
|
|
505
|
-
return (0, import_node_fs6.readdirSync)((0, import_node_path10.join)(committedRoot, "reports", "noise-floor")).some((name) => name.endsWith(".json"));
|
|
506
|
-
} catch (error) {
|
|
507
|
-
return error.code !== "ENOENT";
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
function renderPromotionLine(entry) {
|
|
511
|
-
return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
|
|
512
|
-
}
|
|
513
|
-
var import_node_fs6, import_node_path10;
|
|
514
|
-
var init_promotion_posture = __esm({
|
|
515
|
-
"src/curation/promotion-posture.ts"() {
|
|
516
|
-
"use strict";
|
|
517
|
-
import_node_fs6 = require("fs");
|
|
518
|
-
import_node_path10 = require("path");
|
|
519
|
-
}
|
|
520
|
-
});
|
|
521
|
-
|
|
522
|
-
// src/eval-report-render.ts
|
|
523
|
-
function counterLines(title, counts) {
|
|
524
|
-
const lines = [`### ${title}`, ""];
|
|
525
|
-
const keys = Object.keys(counts).sort();
|
|
526
|
-
if (keys.length === 0) {
|
|
527
|
-
lines.push("- none");
|
|
528
|
-
} else {
|
|
529
|
-
for (const key of keys) lines.push(`- ${key}: ${counts[key]}`);
|
|
530
|
-
}
|
|
531
|
-
lines.push("");
|
|
532
|
-
return lines;
|
|
533
|
-
}
|
|
534
|
-
function rateLine(name, metric) {
|
|
535
|
-
const rate = metric.rate === void 0 ? "n/a (zero denominator)" : `${metric.rate}`;
|
|
536
|
-
return `- ${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
|
|
537
|
-
}
|
|
538
|
-
function metricLine(name, metric) {
|
|
539
|
-
if (metric === void 0) return `${name}: ${notSupplied}`;
|
|
540
|
-
const rate = metric.denominator === 0 ? "unavailable (zero denominator)" : metric.rate ?? notSupplied;
|
|
541
|
-
return `${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
|
|
542
|
-
}
|
|
543
|
-
function recoveryLines(name, recovery) {
|
|
544
|
-
const weightSum = recovery?.weightSum;
|
|
545
|
-
const weighted = weightSum === 0 ? "unavailable (zero weightSum)" : recovery?.weightedAccuracy ?? notSupplied;
|
|
546
|
-
return [
|
|
547
|
-
metricLine(`${name}.recoveryAccuracy`, recovery === void 0 ? void 0 : {
|
|
548
|
-
numerator: recovery.recoveredPositiveCount,
|
|
549
|
-
denominator: recovery.eligiblePositiveCount,
|
|
550
|
-
...recovery.recoveryAccuracy === void 0 ? {} : { rate: recovery.recoveryAccuracy }
|
|
551
|
-
}),
|
|
552
|
-
`${name}.weightedAccuracy: ${weighted} (weightSum: ${weightSum ?? notSupplied})`,
|
|
553
|
-
`${name}.pairsWithoutWeight: ${recovery?.pairsWithoutWeight ?? notSupplied}`
|
|
554
|
-
];
|
|
555
|
-
}
|
|
556
|
-
function splitLines(name, cell) {
|
|
557
|
-
const cost = cell?.perDecisionCost;
|
|
558
|
-
return [
|
|
559
|
-
metricLine(`${name}.firstAttemptDecisionAccuracy`, cell?.firstAttemptDecisionAccuracy),
|
|
560
|
-
metricLine(`${name}.verifierFalsePassRate`, cell?.verifierFalsePassRate),
|
|
561
|
-
...recoveryLines(`${name}.recovery`, cell?.recovery),
|
|
562
|
-
`${name}.perDecisionCost: ` + (cost === void 0 ? notSupplied : `decisionCount=${cost.decisionCount} promptTokens=${cost.promptTokens} completionTokens=${cost.completionTokens} totalTokens=${cost.totalTokens} durationMs=${cost.durationMs}`)
|
|
563
|
-
];
|
|
564
|
-
}
|
|
565
|
-
function renderReportLabelMetricLines(report) {
|
|
566
|
-
const calibration = report.calibration;
|
|
567
|
-
return [
|
|
568
|
-
metricLine("verifierFalsePassRate", report.headline.verifierFalsePassRate),
|
|
569
|
-
`calibration.status: ${calibration?.status ?? notSupplied}`,
|
|
570
|
-
`calibration.n_gold: ${calibration?.n_gold ?? notSupplied}`,
|
|
571
|
-
`calibration.precision: ${calibration?.precision ?? notSupplied}`,
|
|
572
|
-
metricLine("calibration.verifierFalsePassRate", calibration?.verifierFalsePassRate),
|
|
573
|
-
...recoveryLines("recovery", report.recovery),
|
|
574
|
-
...splitLines("perSplit.natural", report.perSplit?.natural),
|
|
575
|
-
...splitLines("perSplit.held-out.shared", report.perSplit?.["held-out"]?.shared),
|
|
576
|
-
...splitLines("perSplit.held-out.private", report.perSplit?.["held-out"]?.private),
|
|
577
|
-
...splitLines("perSplit.adversarial", report.perSplit?.adversarial),
|
|
578
|
-
`perSplit.unassignedCount: ${report.perSplit?.unassignedCount ?? notSupplied}`
|
|
579
|
-
];
|
|
580
|
-
}
|
|
581
|
-
function renderEvalReportMarkdown(report) {
|
|
582
|
-
const lines = ["# Eval Baseline Report", ""];
|
|
583
|
-
lines.push("## Provenance", "");
|
|
584
|
-
lines.push(`- batchKey: ${report.batchKey}`);
|
|
585
|
-
lines.push(`- matcherVersion: ${report.matcherVersion}`);
|
|
586
|
-
if (report.judgeIdentity === "mixed") {
|
|
587
|
-
lines.push(
|
|
588
|
-
`- judge: mixed identities across score records (${(report.mixedJudgeModelIds ?? []).join(", ")}) \u2014 no judge block, not a gate comparator`
|
|
589
|
-
);
|
|
590
|
-
} else if (report.judge === void 0) {
|
|
591
|
-
lines.push("- judge: no judge invoked");
|
|
592
|
-
} else {
|
|
593
|
-
lines.push(
|
|
594
|
-
`- judge: judgeModelId=${report.judge.judgeModelId} judgeFamily=${report.judge.judgeFamily} judgeVersion=${report.judge.judgeVersion} confidenceThreshold=${report.judge.confidenceThreshold}`
|
|
595
|
-
);
|
|
596
|
-
if (report.judge.familyOverride !== void 0) {
|
|
597
|
-
lines.push(
|
|
598
|
-
`- judge family override: ${report.judge.familyOverride} (self-preference bias condition disclosed; owner override of D-10, 2026-07-12)`
|
|
599
|
-
);
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
const calibration = report.judge?.calibration;
|
|
603
|
-
if (calibration !== void 0) {
|
|
604
|
-
lines.push("- judge calibration: " + (calibration.status === "capability-gap" ? `capability-gap reason=${calibration.reason}` : calibration.status === "uncalibrated" ? `uncalibrated n=${calibration.n}` + (calibration.provenance === "unavailable" ? " (unavailable)" : " (copied)") : `calibrated n=${calibration.n} pooledAgreement=${calibration.pooledAgreement} wilson95=[${calibration.wilson95.lower},${calibration.wilson95.upper}] halfWidth=${calibration.wilson95.halfWidth} (copied)`));
|
|
605
|
-
}
|
|
606
|
-
lines.push("");
|
|
607
|
-
const headline = report.headline;
|
|
608
|
-
lines.push("## Headline", "");
|
|
609
|
-
lines.push(rateLine("firstAttemptDecisionAccuracy", headline.firstAttemptDecisionAccuracy));
|
|
610
|
-
lines.push("- perDecisionCost:");
|
|
611
|
-
lines.push(` - decisionCount: ${headline.perDecisionCost.decisionCount}`);
|
|
612
|
-
lines.push(` - promptTokens: ${headline.perDecisionCost.promptTokens}`);
|
|
613
|
-
lines.push(` - completionTokens: ${headline.perDecisionCost.completionTokens}`);
|
|
614
|
-
lines.push(` - totalTokens: ${headline.perDecisionCost.totalTokens}`);
|
|
615
|
-
lines.push(` - durationMs: ${headline.perDecisionCost.durationMs}`);
|
|
616
|
-
lines.push("");
|
|
617
|
-
lines.push("### Label metrics", "", ...renderReportLabelMetricLines(report).map((line) => `- ${line}`), "");
|
|
618
|
-
const honesty = report.honesty;
|
|
619
|
-
lines.push("## Honesty Counters", "");
|
|
620
|
-
lines.push(...counterLines("tieCounts", honesty.tieCounts));
|
|
621
|
-
lines.push(...counterLines("unscorableIdentityCounts", honesty.unscorableIdentityCounts));
|
|
622
|
-
lines.push(...counterLines("refusalCounts", honesty.refusalCounts));
|
|
623
|
-
lines.push(...counterLines("missByFailureCounts", honesty.missByFailureCounts));
|
|
624
|
-
lines.push(`- harnessErrorCount: ${honesty.harnessErrorCount}`, "");
|
|
625
|
-
lines.push(...counterLines("neverScoredCounts", honesty.neverScoredCounts));
|
|
626
|
-
lines.push(...counterLines("skippedCounts", honesty.skippedCounts));
|
|
627
|
-
lines.push(...counterLines("judge invoked", honesty.judge.invoked));
|
|
628
|
-
lines.push(...counterLines("judge settled", honesty.judge.settled));
|
|
629
|
-
lines.push(...counterLines("judge abstained", honesty.judge.abstained));
|
|
630
|
-
lines.push(...counterLines("judge errors", honesty.judge.errors));
|
|
631
|
-
lines.push(...counterLines("judge notInvoked", honesty.judge.notInvoked));
|
|
632
|
-
lines.push("### judge cost", "");
|
|
633
|
-
lines.push(`- calls: ${honesty.judge.cost.calls}`);
|
|
634
|
-
lines.push(`- inputTokens: ${honesty.judge.cost.inputTokens}`);
|
|
635
|
-
lines.push(`- outputTokens: ${honesty.judge.cost.outputTokens}`);
|
|
636
|
-
lines.push(`- outcomesMissingUsage: ${honesty.judge.cost.outcomesMissingUsage}`);
|
|
637
|
-
lines.push("");
|
|
638
|
-
lines.push("## Axis Distributions", "");
|
|
639
|
-
lines.push(...counterLines("elementIdentity", report.axisDistributions.elementIdentity));
|
|
640
|
-
lines.push(...counterLines("value", report.axisDistributions.value));
|
|
641
|
-
lines.push(...counterLines("opClass", report.axisDistributions.opClass));
|
|
642
|
-
lines.push(...counterLines("sideEffect", report.axisDistributions.sideEffect));
|
|
643
|
-
lines.push(...counterLines("loop", report.axisDistributions.loop));
|
|
644
|
-
const recovery = report.recovery;
|
|
645
|
-
lines.push("## Recovery", "");
|
|
646
|
-
lines.push(`- selfHealPairCount: ${recovery.selfHealPairCount}`);
|
|
647
|
-
lines.push(`- recoveredPositiveCount: ${recovery.recoveredPositiveCount}`);
|
|
648
|
-
lines.push(`- eligiblePositiveCount: ${recovery.eligiblePositiveCount}`);
|
|
649
|
-
lines.push(
|
|
650
|
-
`- recoveryAccuracy: ${recovery.recoveryAccuracy === void 0 ? "n/a (no eligible positives)" : recovery.recoveryAccuracy}`
|
|
651
|
-
);
|
|
652
|
-
lines.push(`- negativeDecisionCount: ${recovery.negativeDecisionCount}`);
|
|
653
|
-
lines.push(`- negativeFullMatchCount: ${recovery.negativeFullMatchCount}`);
|
|
654
|
-
lines.push("");
|
|
655
|
-
return lines.join("\n");
|
|
656
|
-
}
|
|
657
|
-
var notSupplied;
|
|
658
|
-
var init_eval_report_render = __esm({
|
|
659
|
-
"src/eval-report-render.ts"() {
|
|
660
|
-
"use strict";
|
|
661
|
-
notSupplied = "unavailable (not supplied)";
|
|
662
|
-
}
|
|
663
|
-
});
|
|
664
|
-
|
|
665
|
-
// src/model-defaults.ts
|
|
666
|
-
var DEFAULT_ANTHROPIC_MODEL;
|
|
667
|
-
var init_model_defaults = __esm({
|
|
668
|
-
"src/model-defaults.ts"() {
|
|
669
|
-
"use strict";
|
|
670
|
-
DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-5-5";
|
|
671
|
-
}
|
|
672
|
-
});
|
|
673
|
-
|
|
674
|
-
// src/llm-utils.ts
|
|
675
|
-
var llm_utils_exports = {};
|
|
676
|
-
__export(llm_utils_exports, {
|
|
677
|
-
applyResolvedAuthToModelConfig: () => applyResolvedAuthToModelConfig,
|
|
678
|
-
buildAnalysisTokenBudget: () => buildAnalysisTokenBudget,
|
|
679
|
-
buildJobModelCapability: () => buildJobModelCapability,
|
|
680
|
-
loadAuthPluginsForRawConfig: () => loadAuthPluginsForRawConfig,
|
|
681
|
-
loadProviderPluginsForRawConfig: () => loadProviderPluginsForRawConfig,
|
|
682
|
-
resolveCliAnalysisTokenBudget: () => resolveCliAnalysisTokenBudget,
|
|
683
|
-
resolveCliChatCapabilities: () => resolveCliChatCapabilities,
|
|
684
|
-
resolveCliJobModel: () => resolveCliJobModel,
|
|
685
|
-
resolveCredentials: () => resolveCredentials,
|
|
686
|
-
resolveLLMModels: () => resolveLLMModels,
|
|
687
|
-
resolveModelAuth: () => resolveModelAuth,
|
|
688
|
-
resolveNamedConfig: () => resolveNamedConfig,
|
|
689
|
-
toModelProfileAndAuth: () => toModelProfileAndAuth
|
|
690
|
-
});
|
|
691
|
-
function normalizeLLMConfig(raw) {
|
|
692
|
-
return {
|
|
693
|
-
name: raw.name,
|
|
694
|
-
provider: raw.provider,
|
|
695
|
-
model: raw.model,
|
|
696
|
-
...raw.baseURL !== void 0 ? { baseURL: raw.baseURL } : {},
|
|
697
|
-
...raw.providerHeaders !== void 0 ? { providerHeaders: raw.providerHeaders } : {},
|
|
698
|
-
...raw.vision !== void 0 ? { vision: raw.vision } : {},
|
|
699
|
-
...raw.contextWindow !== void 0 ? { contextWindow: raw.contextWindow } : {}
|
|
700
|
-
};
|
|
701
|
-
}
|
|
702
|
-
async function loadAuthPluginsForRawConfig(raw, configPath = "agent-qa.config.yaml") {
|
|
703
|
-
const declarations = raw?.plugins?.auth;
|
|
704
|
-
if (!Array.isArray(declarations) || declarations.length === 0) return;
|
|
705
|
-
const { loadLLMAuthPlugins: loadLLMAuthPlugins2 } = await import("@vostride/agent-qa-core");
|
|
706
|
-
await loadLLMAuthPlugins2(declarations, {
|
|
707
|
-
baseDir: (0, import_node_path11.dirname)((0, import_node_path11.resolve)(configPath))
|
|
708
|
-
});
|
|
709
|
-
}
|
|
710
|
-
async function resolveNamedConfig(configName, configPath = "agent-qa.config.yaml", options = {}) {
|
|
711
|
-
const { loadConfigFile: loadConfigFile2 } = await Promise.resolve().then(() => (init_config(), config_exports));
|
|
712
|
-
const raw = await loadConfigFile2(configPath);
|
|
713
|
-
if (options.keyOnly !== true) {
|
|
714
|
-
await loadAuthPluginsForRawConfig(raw, configPath);
|
|
715
|
-
await loadProviderPluginsForRawConfig(raw, configPath);
|
|
716
|
-
}
|
|
717
|
-
const llms = raw?.registry?.llms?.map(normalizeLLMConfig);
|
|
718
|
-
const defaultLLM = raw?.use?.llm;
|
|
719
|
-
if (!llms?.length) {
|
|
720
|
-
throw new Error("No LLM configs found. Run `agent-qa init` to set up your config.");
|
|
721
|
-
}
|
|
722
|
-
const targetName = configName ?? defaultLLM;
|
|
723
|
-
if (!targetName) {
|
|
724
|
-
throw new Error("No config name specified and no use.llm set in config.");
|
|
725
|
-
}
|
|
726
|
-
const found = llms.find((c) => c.name === targetName);
|
|
727
|
-
if (!found) {
|
|
728
|
-
throw new Error(`Config "${targetName}" not found. Available: ${llms.map((c) => c.name).join(", ")}`);
|
|
729
|
-
}
|
|
730
|
-
return { config: found, allConfigs: llms, defaultName: defaultLLM ?? "" };
|
|
731
|
-
}
|
|
732
|
-
function resolveLLMModels(config) {
|
|
733
|
-
const defaults = { provider: "anthropic-subscription", model: DEFAULT_ANTHROPIC_MODEL };
|
|
734
|
-
if (!config.registry?.llms || !config.use?.llm) {
|
|
735
|
-
return { planner: defaults, verifier: defaults, configName: "" };
|
|
736
|
-
}
|
|
737
|
-
const found = config.registry.llms.find((c) => c.name === config.use.llm);
|
|
738
|
-
if (!found) {
|
|
739
|
-
return { planner: defaults, verifier: defaults, configName: "" };
|
|
740
|
-
}
|
|
741
|
-
const { name, ...modelFields } = normalizeLLMConfig(found);
|
|
742
|
-
return { planner: modelFields, verifier: modelFields, configName: name };
|
|
743
|
-
}
|
|
744
|
-
async function resolveModelAuth(configName, llmConfig) {
|
|
745
|
-
const { resolveLLMAuth } = await import("@vostride/agent-qa-core");
|
|
746
|
-
return resolveLLMAuth(configName, llmConfig);
|
|
747
|
-
}
|
|
748
|
-
function applyResolvedAuthToModelConfig(llmConfig, auth) {
|
|
749
|
-
switch (auth.kind) {
|
|
750
|
-
case "api-key":
|
|
751
|
-
return { ...llmConfig, apiKey: auth.apiKey };
|
|
752
|
-
case "bearer-token":
|
|
753
|
-
return { ...llmConfig, authToken: auth.token };
|
|
754
|
-
case "auth-fetch":
|
|
755
|
-
return { ...llmConfig, fetch: auth.fetch, modelAdapter: auth.modelAdapter };
|
|
756
|
-
case "unauthenticated":
|
|
757
|
-
return { ...llmConfig };
|
|
758
|
-
case "missing":
|
|
759
|
-
throw new Error(auth.message);
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
function toModelProfileAndAuth(llmConfig, resolved) {
|
|
763
|
-
const providerOptions = (0, import_agent_qa_model2.getProviderOptions)({
|
|
764
|
-
provider: llmConfig.provider,
|
|
765
|
-
modelAdapter: resolved.kind === "auth-fetch" ? resolved.modelAdapter : void 0,
|
|
766
|
-
fetch: resolved.kind === "auth-fetch" ? resolved.fetch : void 0
|
|
767
|
-
});
|
|
768
|
-
const profile = {
|
|
769
|
-
provider: llmConfig.provider,
|
|
770
|
-
model: llmConfig.model,
|
|
771
|
-
...llmConfig.baseURL !== void 0 ? { baseURL: llmConfig.baseURL } : {},
|
|
772
|
-
...llmConfig.providerHeaders !== void 0 ? { headers: llmConfig.providerHeaders } : {},
|
|
773
|
-
...providerOptions ? { providerOptions } : {},
|
|
774
|
-
auth: toAuthDeclaration(resolved)
|
|
775
|
-
};
|
|
776
|
-
switch (resolved.kind) {
|
|
777
|
-
case "api-key":
|
|
778
|
-
return { profile, auth: { kind: "api-key", apiKey: resolved.apiKey } };
|
|
779
|
-
case "bearer-token":
|
|
780
|
-
return { profile, auth: { kind: "bearer-token", token: resolved.token } };
|
|
781
|
-
case "auth-fetch":
|
|
782
|
-
return { profile, auth: { kind: "auth-fetch", modelAdapter: resolved.modelAdapter, fetch: resolved.fetch } };
|
|
783
|
-
case "unauthenticated":
|
|
784
|
-
return { profile, auth: { kind: "none" } };
|
|
785
|
-
case "missing":
|
|
786
|
-
throw new Error(resolved.message);
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
function toAuthDeclaration(resolved) {
|
|
790
|
-
switch (resolved.kind) {
|
|
791
|
-
case "api-key":
|
|
792
|
-
return { mode: "api-key" };
|
|
793
|
-
case "bearer-token":
|
|
794
|
-
return { mode: "bearer-token" };
|
|
795
|
-
case "auth-fetch":
|
|
796
|
-
return { mode: "subscription", plugin: resolved.provider };
|
|
797
|
-
case "unauthenticated":
|
|
798
|
-
case "missing":
|
|
799
|
-
return { mode: "none" };
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
function buildJobModelCapability(profile, auth, mask, deps = {}) {
|
|
803
|
-
const model = (deps.createModelImpl ?? import_agent_qa_model2.createModel)(profile, auth, mask);
|
|
804
|
-
const accounting = (deps.createTokenAccountingImpl ?? import_agent_qa_model2.createTokenAccounting)({
|
|
805
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? import_agent_qa_model2.resolveCapabilities)(profile)
|
|
806
|
-
});
|
|
807
|
-
const generate = deps.generateTextImpl ?? import_ai2.generateText;
|
|
808
|
-
const vision = (deps.resolveCapabilitiesImpl ?? import_agent_qa_model2.resolveCapabilities)(profile).vision;
|
|
809
|
-
return {
|
|
810
|
-
modelId: `${profile.provider}:${profile.model}`,
|
|
811
|
-
vision,
|
|
812
|
-
async generate(request2) {
|
|
813
|
-
const base = {
|
|
814
|
-
model,
|
|
815
|
-
maxRetries: 0,
|
|
816
|
-
temperature: 0,
|
|
817
|
-
maxOutputTokens: accounting.getCeilings().maxOutput,
|
|
818
|
-
...request2.system === void 0 ? {} : { system: request2.system },
|
|
819
|
-
...request2.signal === void 0 ? {} : { abortSignal: request2.signal }
|
|
820
|
-
};
|
|
821
|
-
if (request2.images !== void 0 && request2.images.length > 0 && (0, import_agent_qa_model2.resolveVisionGate)({ vision }).attachImage) {
|
|
822
|
-
const result2 = await generate({
|
|
823
|
-
...base,
|
|
824
|
-
messages: [
|
|
825
|
-
{
|
|
826
|
-
role: "user",
|
|
827
|
-
content: [
|
|
828
|
-
{ type: "text", text: request2.prompt },
|
|
829
|
-
...request2.images.map((image) => ({ type: "image", image }))
|
|
830
|
-
]
|
|
831
|
-
}
|
|
832
|
-
]
|
|
833
|
-
});
|
|
834
|
-
return { text: result2.text };
|
|
835
|
-
}
|
|
836
|
-
const result = await generate({
|
|
837
|
-
...base,
|
|
838
|
-
prompt: request2.prompt
|
|
839
|
-
});
|
|
840
|
-
return { text: result.text };
|
|
841
|
-
}
|
|
842
|
-
};
|
|
843
|
-
}
|
|
844
|
-
function buildAnalysisTokenBudget(profile, deps = {}) {
|
|
845
|
-
const accounting = (deps.createTokenAccountingImpl ?? import_agent_qa_model2.createTokenAccounting)({
|
|
846
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? import_agent_qa_model2.resolveCapabilities)(profile)
|
|
847
|
-
});
|
|
848
|
-
const ceilings = accounting.getCeilings();
|
|
849
|
-
return {
|
|
850
|
-
estimate: (text) => accounting.estimate([{ role: "user", content: text }]),
|
|
851
|
-
inputBudget: Math.max(0, ceilings.contextWindow - ceilings.maxOutput)
|
|
852
|
-
};
|
|
853
|
-
}
|
|
854
|
-
async function resolveCliAnalysisTokenBudget(configPath, deps = {}) {
|
|
855
|
-
try {
|
|
856
|
-
const { config } = await resolveNamedConfig(void 0, configPath);
|
|
857
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
858
|
-
const { profile } = toModelProfileAndAuth(config, resolved);
|
|
859
|
-
return buildAnalysisTokenBudget(profile, deps);
|
|
860
|
-
} catch {
|
|
861
|
-
return void 0;
|
|
862
|
-
}
|
|
863
|
-
}
|
|
864
|
-
async function resolveCliJobModel(configPath, deps = {}) {
|
|
865
|
-
const { config } = await resolveNamedConfig(void 0, configPath);
|
|
866
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
867
|
-
const { profile, auth } = toModelProfileAndAuth(config, resolved);
|
|
868
|
-
return buildJobModelCapability(profile, auth, createDeclaredSecretMask(configPath).mask, deps);
|
|
869
|
-
}
|
|
870
|
-
async function resolveCliChatCapabilities(configPath, deps = {}) {
|
|
871
|
-
const { mask } = createDeclaredSecretMask(configPath);
|
|
872
|
-
const build = async (configName) => {
|
|
873
|
-
const { config } = await resolveNamedConfig(configName, configPath);
|
|
874
|
-
const resolved = await resolveModelAuth(config.name, config);
|
|
875
|
-
const { profile, auth } = toModelProfileAndAuth(config, resolved);
|
|
876
|
-
return {
|
|
877
|
-
model: (deps.createModelImpl ?? import_agent_qa_model2.createModel)(profile, auth, mask),
|
|
878
|
-
accounting: (deps.createTokenAccountingImpl ?? import_agent_qa_model2.createTokenAccounting)({
|
|
879
|
-
getCapabilities: () => (deps.resolveCapabilitiesImpl ?? import_agent_qa_model2.resolveCapabilities)(profile)
|
|
880
|
-
})
|
|
881
|
-
};
|
|
882
|
-
};
|
|
883
|
-
try {
|
|
884
|
-
const opened = await build();
|
|
885
|
-
return {
|
|
886
|
-
model: opened.model,
|
|
887
|
-
accounting: opened.accounting,
|
|
888
|
-
// A THROW rather than a degrade on this arm: `/model nonsense` is a person naming something,
|
|
889
|
-
// and the honest answer is the refusal that says which name failed. The turn engine's one
|
|
890
|
-
// refusal path carries it, leaving the same countable row any other refusal leaves.
|
|
891
|
-
modelFor: async (name) => (await build(name)).model
|
|
892
|
-
};
|
|
893
|
-
} catch {
|
|
894
|
-
return void 0;
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
async function resolveCredentials(configName, llmConfig) {
|
|
898
|
-
try {
|
|
899
|
-
const auth = await resolveModelAuth(configName, llmConfig);
|
|
900
|
-
if (auth.kind === "api-key") return { apiKey: auth.apiKey };
|
|
901
|
-
if (auth.kind === "bearer-token") return { authToken: auth.token };
|
|
902
|
-
if (auth.kind === "auth-fetch") return { authFetch: auth.fetch };
|
|
903
|
-
return {};
|
|
904
|
-
} catch {
|
|
905
|
-
return {};
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
var import_node_path11, import_agent_qa_model2, import_ai2;
|
|
909
|
-
var init_llm_utils = __esm({
|
|
910
|
-
"src/llm-utils.ts"() {
|
|
911
|
-
"use strict";
|
|
912
|
-
import_node_path11 = require("path");
|
|
913
|
-
import_agent_qa_model2 = require("@vostride/agent-qa-model");
|
|
914
|
-
import_ai2 = require("ai");
|
|
915
|
-
init_declared_secrets();
|
|
916
|
-
init_model_defaults();
|
|
917
|
-
init_provider_plugins();
|
|
918
|
-
}
|
|
919
|
-
});
|
|
920
|
-
|
|
921
|
-
// src/judge-panel-preflight.ts
|
|
922
|
-
async function preflightJudgePanel(session, options = {}, deps = {}) {
|
|
923
|
-
const minN = (0, import_agent_qa_eval5.resolveJudgeCalibrationMinimum)(options.minN);
|
|
924
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
925
|
-
if (names.length < 2) return {
|
|
926
|
-
kind: "capability-gap",
|
|
927
|
-
status: "capability-gap",
|
|
928
|
-
reason: "panel-not-configured",
|
|
929
|
-
minN,
|
|
930
|
-
entries: []
|
|
931
|
-
};
|
|
932
|
-
const named = deps.resolveNamedConfig ?? resolveNamedConfig;
|
|
933
|
-
const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
|
|
934
|
-
const rows = [];
|
|
935
|
-
for (const [index, name] of names.entries()) {
|
|
936
|
-
const diagnostic = {
|
|
937
|
-
index,
|
|
938
|
-
name,
|
|
939
|
-
families: [],
|
|
940
|
-
credential: "resolution-failed",
|
|
941
|
-
failures: []
|
|
942
|
-
};
|
|
943
|
-
const row = { diagnostic };
|
|
944
|
-
try {
|
|
945
|
-
const { config } = await named(name, session.configPath, { keyOnly: true });
|
|
946
|
-
row.config = config;
|
|
947
|
-
diagnostic.modelId = config.model;
|
|
948
|
-
diagnostic.families = [...(0, import_agent_qa_eval5.resolveJudgeFamilies)(config.model)].sort();
|
|
949
|
-
} catch {
|
|
950
|
-
diagnostic.failures = ["config-unresolved"];
|
|
951
|
-
}
|
|
952
|
-
if (row.config) {
|
|
953
|
-
try {
|
|
954
|
-
const auth = await resolveAuth(row.config.name, row.config);
|
|
955
|
-
row.auth = auth;
|
|
956
|
-
diagnostic.credential = auth.kind === "missing" || auth.kind === "unauthenticated" ? auth.kind : "present";
|
|
957
|
-
} catch {
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
if (diagnostic.credential !== "present") diagnostic.failures = [...diagnostic.failures, "no-credential"];
|
|
961
|
-
rows.push(row);
|
|
962
|
-
}
|
|
963
|
-
let replayProfile;
|
|
964
|
-
try {
|
|
965
|
-
replayProfile = await resolveReplayProfileDeclaration(session, named);
|
|
966
|
-
} catch {
|
|
967
|
-
throw new Error("Cannot resolve the replay profile for judge panel preflight. Check use.llm and its registry.llms entry.");
|
|
968
|
-
}
|
|
969
|
-
const replayFamilies = (0, import_agent_qa_eval5.resolveJudgeFamilies)(replayProfile.model);
|
|
970
|
-
const sets = rows.map((row) => (0, import_agent_qa_eval5.resolveJudgeFamilies)(row.config?.model ?? ""));
|
|
971
|
-
for (const [index, row] of rows.entries()) {
|
|
972
|
-
const failures = [...row.diagnostic.failures];
|
|
973
|
-
if (row.config && (sets[index].size === 0 || replayFamilies.size === 0)) failures.push("unknown-family");
|
|
974
|
-
if ((0, import_agent_qa_eval5.judgeFamiliesIntersect)(sets[index], replayFamilies)) failures.push("replay-family-collision");
|
|
975
|
-
if (sets.some((set, other) => other !== index && (0, import_agent_qa_eval5.judgeFamiliesIntersect)(sets[index], set))) failures.push("member-family-collision");
|
|
976
|
-
row.diagnostic.failures = failures;
|
|
977
|
-
}
|
|
978
|
-
const entries = rows.map(({ diagnostic }) => ({
|
|
979
|
-
...diagnostic,
|
|
980
|
-
name: session.mask(diagnostic.name),
|
|
981
|
-
...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
|
|
982
|
-
}));
|
|
983
|
-
const modelIds = rows.flatMap((row) => row.config ? [row.config.model] : []);
|
|
984
|
-
const complete = modelIds.length === names.length;
|
|
985
|
-
const identity = complete ? (0, import_agent_qa_eval.createPanelIdentity)(modelIds) : void 0;
|
|
986
|
-
const safeIdentity = [replayProfile.model, ...modelIds].every((id) => session.mask(id) === id) ? identity : void 0;
|
|
987
|
-
const gap = (reason) => ({
|
|
988
|
-
kind: "capability-gap",
|
|
989
|
-
status: "capability-gap",
|
|
990
|
-
reason,
|
|
991
|
-
minN,
|
|
992
|
-
entries,
|
|
993
|
-
...safeIdentity === void 0 ? {} : { identity: safeIdentity }
|
|
994
|
-
});
|
|
995
|
-
if (rows.some((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
|
|
996
|
-
const eligibility = (0, import_agent_qa_eval5.assessPanelEligibility)({ memberModelIds: modelIds, replayedModelId: replayProfile.model });
|
|
997
|
-
if (!eligibility.eligible) return gap("single-family-host");
|
|
998
|
-
return {
|
|
999
|
-
kind: "eligible",
|
|
1000
|
-
status: "uncalibrated",
|
|
1001
|
-
n: 0,
|
|
1002
|
-
minN,
|
|
1003
|
-
entries,
|
|
1004
|
-
identity,
|
|
1005
|
-
replayProfile,
|
|
1006
|
-
members: rows.map((row) => toModelProfileAndAuth(row.config, row.auth))
|
|
1007
|
-
};
|
|
1008
|
-
}
|
|
1009
|
-
var init_judge_panel_preflight = __esm({
|
|
1010
|
-
"src/judge-panel-preflight.ts"() {
|
|
1011
|
-
"use strict";
|
|
1012
|
-
init_eval_session_composition();
|
|
1013
|
-
init_llm_utils();
|
|
1014
|
-
}
|
|
1015
|
-
});
|
|
1016
|
-
|
|
1017
|
-
// src/adversarial-probe-preflight.ts
|
|
1018
|
-
var adversarial_probe_preflight_exports = {};
|
|
1019
|
-
__export(adversarial_probe_preflight_exports, {
|
|
1020
|
-
resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability
|
|
1021
|
-
});
|
|
1022
|
-
function authoringFamily(modelId) {
|
|
1023
|
-
return [...(0, import_agent_qa_eval5.resolveJudgeFamilies)(modelId)].sort().join(",");
|
|
1024
|
-
}
|
|
1025
|
-
async function resolveAdversarialProbeCapability(session, deps = {}) {
|
|
1026
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
1027
|
-
if (names.length === 0) {
|
|
1028
|
-
return { kind: "no-probe-model", reason: "panel-not-configured", entries: [] };
|
|
1029
|
-
}
|
|
1030
|
-
const named = deps.resolveNamedConfig ?? resolveNamedConfig;
|
|
1031
|
-
const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
|
|
1032
|
-
const rows = [];
|
|
1033
|
-
for (const [index, name] of names.entries()) {
|
|
1034
|
-
const diagnostic = {
|
|
1035
|
-
index,
|
|
1036
|
-
name,
|
|
1037
|
-
families: [],
|
|
1038
|
-
credential: "resolution-failed",
|
|
1039
|
-
failures: []
|
|
1040
|
-
};
|
|
1041
|
-
const row = { diagnostic };
|
|
1042
|
-
try {
|
|
1043
|
-
const { config } = await named(name, session.configPath, { keyOnly: true });
|
|
1044
|
-
row.config = config;
|
|
1045
|
-
diagnostic.modelId = config.model;
|
|
1046
|
-
diagnostic.families = [...(0, import_agent_qa_eval5.resolveJudgeFamilies)(config.model)].sort();
|
|
1047
|
-
} catch {
|
|
1048
|
-
diagnostic.failures = ["config-unresolved"];
|
|
1049
|
-
}
|
|
1050
|
-
if (row.config) {
|
|
1051
|
-
try {
|
|
1052
|
-
const auth2 = await resolveAuth(row.config.name, row.config);
|
|
1053
|
-
row.auth = auth2;
|
|
1054
|
-
diagnostic.credential = auth2.kind === "missing" || auth2.kind === "unauthenticated" ? auth2.kind : "present";
|
|
1055
|
-
} catch {
|
|
1056
|
-
}
|
|
1057
|
-
}
|
|
1058
|
-
if (diagnostic.credential !== "present") {
|
|
1059
|
-
diagnostic.failures = [...diagnostic.failures, "no-credential"];
|
|
1060
|
-
}
|
|
1061
|
-
rows.push(row);
|
|
1062
|
-
}
|
|
1063
|
-
let replayProfile;
|
|
1064
|
-
try {
|
|
1065
|
-
replayProfile = await resolveReplayProfileDeclaration(session, named);
|
|
1066
|
-
} catch {
|
|
1067
|
-
throw new Error("Cannot resolve the replay profile for adversarial probe preflight. Check use.llm and its registry.llms entry.");
|
|
1068
|
-
}
|
|
1069
|
-
const replayFamilies = (0, import_agent_qa_eval5.resolveJudgeFamilies)(replayProfile.model);
|
|
1070
|
-
for (const row of rows) {
|
|
1071
|
-
if (!row.config) continue;
|
|
1072
|
-
const families = (0, import_agent_qa_eval5.resolveJudgeFamilies)(row.config.model);
|
|
1073
|
-
row.eligible = (0, import_agent_qa_eval5.assessPanelEligibility)({
|
|
1074
|
-
memberModelIds: [row.config.model],
|
|
1075
|
-
replayedModelId: replayProfile.model
|
|
1076
|
-
}).eligible;
|
|
1077
|
-
const failures = [...row.diagnostic.failures];
|
|
1078
|
-
if (families.size === 0 || replayFamilies.size === 0) failures.push("unknown-family");
|
|
1079
|
-
else if (!row.eligible) failures.push("replay-family-collision");
|
|
1080
|
-
row.diagnostic.failures = failures;
|
|
1081
|
-
}
|
|
1082
|
-
const entries = rows.map(({ diagnostic }) => ({
|
|
1083
|
-
...diagnostic,
|
|
1084
|
-
name: session.mask(diagnostic.name),
|
|
1085
|
-
...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
|
|
1086
|
-
}));
|
|
1087
|
-
const gap = (reason) => ({ kind: "no-probe-model", reason, entries });
|
|
1088
|
-
if (rows.every((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
|
|
1089
|
-
const selected = rows.find((row) => row.diagnostic.credential === "present" && row.config !== void 0 && row.eligible === true);
|
|
1090
|
-
if (!selected) return gap("single-family-host");
|
|
1091
|
-
const { profile, auth } = toModelProfileAndAuth(selected.config, selected.auth);
|
|
1092
|
-
return {
|
|
1093
|
-
kind: "probe-author",
|
|
1094
|
-
entries,
|
|
1095
|
-
author: {
|
|
1096
|
-
index: selected.diagnostic.index,
|
|
1097
|
-
name: selected.config.name,
|
|
1098
|
-
authoringModel: { id: profile.model, family: authoringFamily(profile.model) },
|
|
1099
|
-
profile,
|
|
1100
|
-
auth
|
|
1101
|
-
}
|
|
1102
|
-
};
|
|
1103
|
-
}
|
|
1104
|
-
var init_adversarial_probe_preflight = __esm({
|
|
1105
|
-
"src/adversarial-probe-preflight.ts"() {
|
|
1106
|
-
"use strict";
|
|
1107
|
-
init_eval_session_composition();
|
|
1108
|
-
init_llm_utils();
|
|
1109
|
-
}
|
|
1110
|
-
});
|
|
1111
|
-
|
|
1112
|
-
// src/adversarial-probe-author.ts
|
|
1113
|
-
var adversarial_probe_author_exports = {};
|
|
1114
|
-
__export(adversarial_probe_author_exports, {
|
|
1115
|
-
PROBE_AUTHOR_MAX_OUTPUT_TOKENS: () => PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
|
|
1116
|
-
PROBE_AUTHOR_TIMEOUT_MS: () => PROBE_AUTHOR_TIMEOUT_MS,
|
|
1117
|
-
createAdversarialProbeAuthor: () => createAdversarialProbeAuthor
|
|
1118
|
-
});
|
|
1119
|
-
function createAdversarialProbeAuthor(input) {
|
|
1120
|
-
const { author } = input;
|
|
1121
|
-
const create = input.createModelImpl ?? import_agent_qa_model3.createModel;
|
|
1122
|
-
const generate = input.generateTextImpl ?? import_ai3.generateText;
|
|
1123
|
-
return {
|
|
1124
|
-
async author(request2) {
|
|
1125
|
-
if (request2.authoringModel.id !== author.authoringModel.id) {
|
|
1126
|
-
throw new Error("This probe author is bound to a different authoring model than the request names.");
|
|
1127
|
-
}
|
|
1128
|
-
await input.onCallAttempted?.({
|
|
1129
|
-
seedCassetteId: request2.seedCassetteId,
|
|
1130
|
-
kind: request2.kind,
|
|
1131
|
-
promptVersion: request2.promptVersion,
|
|
1132
|
-
authoringModelId: author.authoringModel.id
|
|
1133
|
-
});
|
|
1134
|
-
const profile = {
|
|
1135
|
-
...author.profile,
|
|
1136
|
-
...author.profile.capabilities === void 0 ? {} : { capabilities: { ...author.profile.capabilities } }
|
|
1137
|
-
};
|
|
1138
|
-
return await (0, import_agent_qa_safety_local2.runSafetyScope)({ executionContext: input.executionContext }, async () => {
|
|
1139
|
-
const model = create(profile, author.auth, input.mask);
|
|
1140
|
-
const prompt = (0, import_agent_qa_eval8.renderAdversarialProbePrompt)(request2);
|
|
1141
|
-
const result = await generate({
|
|
1142
|
-
model,
|
|
1143
|
-
maxRetries: 0,
|
|
1144
|
-
temperature: 0,
|
|
1145
|
-
maxOutputTokens: PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
|
|
1146
|
-
abortSignal: AbortSignal.timeout(PROBE_AUTHOR_TIMEOUT_MS),
|
|
1147
|
-
system: prompt.system,
|
|
1148
|
-
prompt: prompt.user
|
|
1149
|
-
});
|
|
1150
|
-
const parsed = AUTHORED_NAME.safeParse(parseJson2(result.text));
|
|
1151
|
-
if (!parsed.success) {
|
|
1152
|
-
throw new Error("The probe author returned a response that is not the single-field name object the frozen prompt requires.");
|
|
1153
|
-
}
|
|
1154
|
-
return {
|
|
1155
|
-
text: parsed.data.name,
|
|
1156
|
-
// Reported spend, carried as reported -- and ABSENT when the provider reported
|
|
1157
|
-
// nothing, never zero. A zero here would be a measured claim about a call whose
|
|
1158
|
-
// cost we do not know, and the disclosure that reads this cannot tell the two
|
|
1159
|
-
// apart once they are the same number. The probe budget is denominated in CALLS
|
|
1160
|
-
// for exactly this reason, so an unreported token still never buys an extra call.
|
|
1161
|
-
usage: {
|
|
1162
|
-
...reportedCount(result.usage?.inputTokens) === void 0 ? {} : { promptTokens: reportedCount(result.usage?.inputTokens) },
|
|
1163
|
-
...reportedCount(result.usage?.outputTokens) === void 0 ? {} : { completionTokens: reportedCount(result.usage?.outputTokens) }
|
|
1164
|
-
}
|
|
1165
|
-
};
|
|
1166
|
-
});
|
|
1167
|
-
}
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
function parseJson2(text) {
|
|
1171
|
-
try {
|
|
1172
|
-
return JSON.parse(text);
|
|
1173
|
-
} catch {
|
|
1174
|
-
return void 0;
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
function reportedCount(value) {
|
|
1178
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : void 0;
|
|
1179
|
-
}
|
|
1180
|
-
var import_ai3, import_zod7, PROBE_AUTHOR_MAX_OUTPUT_TOKENS, PROBE_AUTHOR_TIMEOUT_MS, AUTHORED_NAME;
|
|
1181
|
-
var init_adversarial_probe_author = __esm({
|
|
1182
|
-
"src/adversarial-probe-author.ts"() {
|
|
1183
|
-
"use strict";
|
|
1184
|
-
import_ai3 = require("ai");
|
|
1185
|
-
import_zod7 = require("zod");
|
|
1186
|
-
init_eval_session_composition();
|
|
1187
|
-
PROBE_AUTHOR_MAX_OUTPUT_TOKENS = 200;
|
|
1188
|
-
PROBE_AUTHOR_TIMEOUT_MS = 2e4;
|
|
1189
|
-
AUTHORED_NAME = import_zod7.z.object({ name: import_zod7.z.string() }).strict();
|
|
1190
|
-
}
|
|
1191
|
-
});
|
|
1192
|
-
|
|
1193
|
-
// src/curation/trust-priors.ts
|
|
1194
|
-
function buildTrustPriorsResolver(input) {
|
|
1195
|
-
const configDir = (0, import_node_path12.dirname)((0, import_node_path12.resolve)(input.configPath));
|
|
1196
|
-
const committedRoot = (0, import_node_path12.resolve)(configDir, "agent-qa-eval");
|
|
1197
|
-
const read = (0, import_agent_qa_eval_local3.readLabelCalibration)(committedRoot);
|
|
1198
|
-
if (read.kind === "refused") return Object.freeze({ kind: "refused", reason: REFUSAL });
|
|
1199
|
-
const table = {};
|
|
1200
|
-
const nByOrigin = { "agent-verified": 0, "agent-observed": 0 };
|
|
1201
|
-
if (read.kind === "loaded") {
|
|
1202
|
-
for (const [origin, channel] of [["agent-verified", "recorded-verdict"], ["agent-observed", "self-heal-pair"]]) {
|
|
1203
|
-
nByOrigin[origin] = read.artifact.channels[channel].n;
|
|
1204
|
-
const prior = read.artifact.priors[origin];
|
|
1205
|
-
if (prior !== void 0) table[origin] = Object.freeze({ alpha: prior.alpha, beta: prior.beta });
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
const priors = Object.keys(table).length === 0 ? void 0 : Object.freeze(table);
|
|
1209
|
-
const disclosure = Object.freeze({
|
|
1210
|
-
status: priors === void 0 ? "provisional" : "calibrated",
|
|
1211
|
-
...read.kind === "loaded" ? { digest: read.artifact.generationId } : {},
|
|
1212
|
-
nByOrigin: Object.freeze(nByOrigin)
|
|
1213
|
-
});
|
|
1214
|
-
let calibration;
|
|
1215
|
-
if (input.inspectCalibration) {
|
|
1216
|
-
if (read.kind === "absent") calibration = "label calibration: absent";
|
|
1217
|
-
else {
|
|
1218
|
-
const identity = (0, import_agent_qa_eval_local3.readCalibrationGoldIdentity)({ committedRoot, agentQaRoot: (0, import_node_path12.resolve)(configDir, ".agent-qa"), calibrationRead: read });
|
|
1219
|
-
const freshness = identity.kind === "unavailable" ? "unavailable" : identity.goldSetDigest === read.artifact.goldSetDigest ? "current" : "stale";
|
|
1220
|
-
const channels = ["recorded-verdict", "self-heal-pair"].map((channel) => {
|
|
1221
|
-
const row = read.artifact.channels[channel];
|
|
1222
|
-
return `${channel} ${row.status} n=${row.n}`;
|
|
1223
|
-
}).join(", ");
|
|
1224
|
-
calibration = `label calibration: ${channels}; goldDigest ${freshness}`;
|
|
1225
|
-
}
|
|
1226
|
-
}
|
|
1227
|
-
return Object.freeze({
|
|
1228
|
-
kind: "ready",
|
|
1229
|
-
resolve: () => priors,
|
|
1230
|
-
disclosure,
|
|
1231
|
-
...calibration === void 0 ? {} : { calibration }
|
|
1232
|
-
});
|
|
1233
|
-
}
|
|
1234
|
-
function trustPriorsForHost(input) {
|
|
1235
|
-
const result = buildTrustPriorsResolver(input);
|
|
1236
|
-
if (result.kind === "refused") throw new TrustPriorsRefusedError();
|
|
1237
|
-
return result;
|
|
1238
|
-
}
|
|
1239
|
-
var import_node_path12, REFUSAL, TrustPriorsRefusedError;
|
|
1240
|
-
var init_trust_priors = __esm({
|
|
1241
|
-
"src/curation/trust-priors.ts"() {
|
|
1242
|
-
"use strict";
|
|
1243
|
-
import_node_path12 = require("path");
|
|
1244
|
-
init_eval_session_composition();
|
|
1245
|
-
REFUSAL = "Trust prior calibration is invalid or unavailable; inspect the selected config's committed calibration marker and referenced revisions, following https://vostride.com/docs/agent-qa/self-improvement/evals#if-calibration-is-refused before composing this host.";
|
|
1246
|
-
TrustPriorsRefusedError = class extends Error {
|
|
1247
|
-
kind = "trust-priors-refused";
|
|
1248
|
-
constructor() {
|
|
1249
|
-
super(REFUSAL);
|
|
1250
|
-
this.name = "TrustPriorsRefusedError";
|
|
1251
|
-
}
|
|
1252
|
-
};
|
|
1253
|
-
}
|
|
1254
|
-
});
|
|
1255
|
-
|
|
1256
|
-
// src/gates/replay-gate-host.ts
|
|
1257
|
-
var replay_gate_host_exports = {};
|
|
1258
|
-
__export(replay_gate_host_exports, {
|
|
1259
|
-
createReplayGateHost: () => createReplayGateHost,
|
|
1260
|
-
createReplayGatePortForHost: () => createReplayGatePortForHost
|
|
1261
|
-
});
|
|
1262
|
-
function createReplayGatePortForHost(input) {
|
|
1263
|
-
const resolveTrustPriors = input.resolveTrustPriors ?? trustPriorsForHost(input).resolve;
|
|
1264
|
-
return {
|
|
1265
|
-
async runReplayGate(candidateId) {
|
|
1266
|
-
const { resolveEvalSession: resolveEvalSession2, runReplayGateVerb: runReplayGateVerb2 } = await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports));
|
|
1267
|
-
const session = await resolveEvalSession2({ configPath: input.configPath, keyOnly: true });
|
|
1268
|
-
const borrowed = input.resolveClaimAuthority?.();
|
|
1269
|
-
const result = await runReplayGateVerb2(session, candidateId, {
|
|
1270
|
-
resolveTrustPriors,
|
|
1271
|
-
agentQaRoot: input.agentQaRoot,
|
|
1272
|
-
onProgress: () => void 0,
|
|
1273
|
-
...borrowed === void 0 ? {} : { claimAuthority: borrowed }
|
|
1274
|
-
});
|
|
1275
|
-
if (result.kind === "refused") throw new Error(result.refusal.reason);
|
|
1276
|
-
return checkedReceipt(acknowledged(result, candidateId), candidateId);
|
|
1277
|
-
}
|
|
1278
|
-
};
|
|
1279
|
-
}
|
|
1280
|
-
function acknowledged(result, candidateId) {
|
|
1281
|
-
if (!ACKNOWLEDGED_RECORDINGS.includes(result.recorded)) {
|
|
1282
|
-
throw new import_agent_qa_agent3.Gate1ReceiptNotPersistedError(candidateId, result.recorded);
|
|
1283
|
-
}
|
|
1284
|
-
return result.receipt;
|
|
1285
|
-
}
|
|
1286
|
-
function checkedReceipt(value, candidateId) {
|
|
1287
|
-
const receipt = import_agent_qa_schema8.GateReceiptSchema.parse(value);
|
|
1288
|
-
if (receipt.gate !== 1 || receipt.candidateId !== candidateId || receipt.receiptDigest !== (0, import_agent_qa_schema8.receiptDigestOf)(receipt))
|
|
1289
|
-
throw new Error("Gate 1 returned a foreign candidate, wrong gate or invalid receipt digest");
|
|
1290
|
-
return receipt;
|
|
1291
|
-
}
|
|
1292
|
-
function createReplayGateHost(deps) {
|
|
1293
|
-
const resolveTrustPriors = deps.resolveTrustPriors ?? trustPriorsForHost(deps).resolve;
|
|
1294
|
-
const configDir = (0, import_node_path13.dirname)((0, import_node_path13.resolve)(deps.configPath));
|
|
1295
|
-
const knowledgeRoot = resolveConfiguredKnowledgeRoot(deps.session.config, configDir);
|
|
1296
|
-
const reader = deps.reader ?? (0, import_agent_qa_agent_local3.createKnowledgeReader)(() => knowledgeRoot, void 0, void 0, resolveTrustPriors);
|
|
1297
|
-
const curation = deps.curation ?? import_agent_qa_agent_local3.LOCAL_CURATION_CAPABILITIES;
|
|
1298
|
-
async function run(candidateId) {
|
|
1299
|
-
const read = reader.candidateInjection;
|
|
1300
|
-
if (read === void 0) throw new Error("knowledge reader binds no candidateInjection");
|
|
1301
|
-
const found = await read.call(reader, { candidateId });
|
|
1302
|
-
if (found.kind === "not-found")
|
|
1303
|
-
return {
|
|
1304
|
-
kind: "refused",
|
|
1305
|
-
refusal: {
|
|
1306
|
-
path: knowledgeRoot,
|
|
1307
|
-
reason: `candidate ${candidateId} not found`
|
|
1308
|
-
},
|
|
1309
|
-
notices: [],
|
|
1310
|
-
exit: 2
|
|
1311
|
-
};
|
|
1312
|
-
if (found.injection.candidateId !== candidateId)
|
|
1313
|
-
throw new Error("Gate 1 reader returned a foreign candidate");
|
|
1314
|
-
const runGate = deps.runReplayGate ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).runReplayGate;
|
|
1315
|
-
const result = await runGate({
|
|
1316
|
-
...deps.composition,
|
|
1317
|
-
injection: found.injection,
|
|
1318
|
-
committedRoot: (0, import_node_path13.resolve)(configDir, "agent-qa-eval"),
|
|
1319
|
-
agentQaRoot: deps.agentQaRoot,
|
|
1320
|
-
mask: deps.session.mask,
|
|
1321
|
-
producedBy: { opId: "eval.replay-gate" }
|
|
1322
|
-
});
|
|
1323
|
-
if (result.kind === "refused") return { ...result, exit: 2 };
|
|
1324
|
-
const receipt = checkedReceipt(result.receipt, candidateId);
|
|
1325
|
-
const answer = {
|
|
1326
|
-
kind: "receipt",
|
|
1327
|
-
receipt,
|
|
1328
|
-
lines: result.lines,
|
|
1329
|
-
notices: [...result.notices],
|
|
1330
|
-
recorded: "dry-run",
|
|
1331
|
-
exit: receipt.verdict === "pass" ? 0 : receipt.verdict === "fail" ? 1 : 2
|
|
1332
|
-
};
|
|
1333
|
-
if (!deps.record) return answer;
|
|
1334
|
-
const borrowed = deps.resolveClaimAuthority?.();
|
|
1335
|
-
if (borrowed !== void 0) {
|
|
1336
|
-
const outcome = await curation.recordGateReceipt({
|
|
1337
|
-
resolveTrustPriors,
|
|
1338
|
-
receipt,
|
|
1339
|
-
lease: { leaseId: borrowed.leaseId, epoch: borrowed.epoch },
|
|
1340
|
-
knowledgeRoot,
|
|
1341
|
-
executionContext: deps.session.executionContext,
|
|
1342
|
-
sourceBuildOrdinal: receipt.producedAtOrdinal,
|
|
1343
|
-
...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
|
|
1344
|
-
});
|
|
1345
|
-
answer.recorded = outcome.kind;
|
|
1346
|
-
return answer;
|
|
1347
|
-
}
|
|
1348
|
-
const lease = curation.maintenance.acquireWriterLease(knowledgeRoot);
|
|
1349
|
-
if (lease.kind !== "acquired") {
|
|
1350
|
-
answer.recorded = lease.kind === "held" ? "lease-held" : "lease-refused";
|
|
1351
|
-
answer.exit = 2;
|
|
1352
|
-
answer.notices.push(
|
|
1353
|
-
lease.kind === "held" ? "writer lease held by another process; receipt printed, not recorded \u2014 re-run when the daemon releases it" : `writer lease refused: ${lease.reason}; receipt printed, not recorded`
|
|
1354
|
-
);
|
|
1355
|
-
return answer;
|
|
1356
|
-
}
|
|
1357
|
-
try {
|
|
1358
|
-
const outcome = await curation.recordGateReceipt({
|
|
1359
|
-
resolveTrustPriors,
|
|
1360
|
-
receipt,
|
|
1361
|
-
lease: { leaseId: lease.leaseId, epoch: lease.epoch },
|
|
1362
|
-
knowledgeRoot,
|
|
1363
|
-
executionContext: deps.session.executionContext,
|
|
1364
|
-
sourceBuildOrdinal: receipt.producedAtOrdinal,
|
|
1365
|
-
...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
|
|
1366
|
-
});
|
|
1367
|
-
answer.recorded = outcome.kind;
|
|
1368
|
-
return answer;
|
|
1369
|
-
} finally {
|
|
1370
|
-
curation.maintenance.releaseLease({ knowledgeRoot, leaseId: lease.leaseId });
|
|
1371
|
-
}
|
|
1372
|
-
}
|
|
1373
|
-
return {
|
|
1374
|
-
run,
|
|
1375
|
-
port: {
|
|
1376
|
-
async runReplayGate(id) {
|
|
1377
|
-
const result = await run(id);
|
|
1378
|
-
if (result.kind === "refused") throw new Error(result.refusal.reason);
|
|
1379
|
-
return acknowledged(result, id);
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
};
|
|
1383
|
-
}
|
|
1384
|
-
var import_node_path13, import_agent_qa_agent3, import_agent_qa_agent_local3, import_agent_qa_schema8, ACKNOWLEDGED_RECORDINGS;
|
|
1385
|
-
var init_replay_gate_host = __esm({
|
|
1386
|
-
"src/gates/replay-gate-host.ts"() {
|
|
1387
|
-
"use strict";
|
|
1388
|
-
import_node_path13 = require("path");
|
|
1389
|
-
init_trust_priors();
|
|
1390
|
-
import_agent_qa_agent3 = require("@vostride/agent-qa-agent");
|
|
1391
|
-
import_agent_qa_agent_local3 = require("@vostride/agent-qa-agent-local");
|
|
1392
|
-
import_agent_qa_schema8 = require("@vostride/agent-qa-schema");
|
|
1393
|
-
init_config();
|
|
1394
|
-
ACKNOWLEDGED_RECORDINGS = ["appended", "duplicate"];
|
|
1395
|
-
}
|
|
1396
|
-
});
|
|
1397
|
-
|
|
1398
|
-
// src/eval-session-composition.ts
|
|
1399
|
-
var eval_session_composition_exports = {};
|
|
1400
|
-
__export(eval_session_composition_exports, {
|
|
1401
|
-
ADVERSARIAL_STAGING_SUBDIR: () => import_agent_qa_eval_local.ADVERSARIAL_STAGING_SUBDIR,
|
|
1402
|
-
ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS: () => ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS,
|
|
1403
|
-
EvalJudgeDeclarationError: () => EvalJudgeDeclarationError,
|
|
1404
|
-
EvalNamedInputError: () => EvalNamedInputError,
|
|
1405
|
-
EvalPartitionFlagsRequiredError: () => EvalPartitionFlagsRequiredError,
|
|
1406
|
-
EvalPartitionManifestConflictError: () => EvalPartitionManifestConflictError,
|
|
1407
|
-
FLOOR_DEFAULT_BOOTSTRAP_SEED: () => FLOOR_DEFAULT_BOOTSTRAP_SEED,
|
|
1408
|
-
FLOOR_DEFAULT_REPEATS: () => FLOOR_DEFAULT_REPEATS,
|
|
1409
|
-
FLOOR_DEFAULT_SEEDS: () => FLOOR_DEFAULT_SEEDS,
|
|
1410
|
-
FsHandoffSource: () => import_agent_qa_eval_local5.FsHandoffSource,
|
|
1411
|
-
FsMarkLedger: () => import_agent_qa_eval_local5.FsMarkLedger,
|
|
1412
|
-
GATE1_DEFAULT_K: () => GATE1_DEFAULT_K,
|
|
1413
|
-
GATE1_DEFAULT_MAX_SCREENS: () => GATE1_DEFAULT_MAX_SCREENS,
|
|
1414
|
-
GATE2_DEFAULT_CANARY_HORIZON_BUILDS: () => GATE2_DEFAULT_CANARY_HORIZON_BUILDS,
|
|
1415
|
-
GATE2_DEFAULT_MAX_ABLATIONS: () => GATE2_DEFAULT_MAX_ABLATIONS,
|
|
1416
|
-
HANDOFF_FILE: () => import_agent_qa_runner_local3.HANDOFF_FILE,
|
|
1417
|
-
LABEL_CALIBRATION_MIN_GOLD: () => LABEL_CALIBRATION_MIN_GOLD,
|
|
1418
|
-
MARK_INTENT_FILE: () => import_agent_qa_eval_local5.MARK_INTENT_FILE,
|
|
1419
|
-
MarkLedgerCorruptError: () => import_agent_qa_eval_local5.MarkLedgerCorruptError,
|
|
1420
|
-
NO_PROBE_MODEL_REASONS: () => import_agent_qa_eval8.NO_PROBE_MODEL_REASONS,
|
|
1421
|
-
RUNS_SUBDIR: () => import_agent_qa_runner_local3.RUNS_SUBDIR,
|
|
1422
|
-
SEED_POLICIES: () => import_agent_qa_eval2.SEED_POLICIES,
|
|
1423
|
-
adversarialHostOptions: () => adversarialHostOptions,
|
|
1424
|
-
assessPanelEligibility: () => import_agent_qa_eval5.assessPanelEligibility,
|
|
1425
|
-
canonicalJson: () => import_agent_qa_eval7.canonicalJson,
|
|
1426
|
-
computeCorpusDigest: () => import_agent_qa_eval3.computeCorpusDigest,
|
|
1427
|
-
computeReplayProfileFingerprintFor: () => import_agent_qa_eval_local.computeReplayProfileFingerprintFor,
|
|
1428
|
-
createAdversarialProbeAuthor: () => createAdversarialProbeAuthor,
|
|
1429
|
-
createModel: () => import_agent_qa_model3.createModel,
|
|
1430
|
-
createPanelIdentity: () => import_agent_qa_eval.createPanelIdentity,
|
|
1431
|
-
feedbackSeedHostOptions: () => feedbackSeedHostOptions,
|
|
1432
|
-
freezeGoldenTrajectory: () => import_agent_qa_eval_local5.freezeGoldenTrajectory,
|
|
1433
|
-
isFeedbackStateNovel: () => isFeedbackStateNovel,
|
|
1434
|
-
judgeCapabilityOf: () => import_agent_qa_eval4.judgeCapabilityOf,
|
|
1435
|
-
judgeFamiliesIntersect: () => import_agent_qa_eval5.judgeFamiliesIntersect,
|
|
1436
|
-
latestMarkReceiptFor: () => import_agent_qa_eval_local5.latestMarkReceiptFor,
|
|
1437
|
-
listAdversarialBatches: () => listAdversarialBatches,
|
|
1438
|
-
listStagedAdversarial: () => import_agent_qa_eval_local4.listStagedAdversarial,
|
|
1439
|
-
locateMisdirectionDecision: () => locateMisdirectionDecision,
|
|
1440
|
-
markedDecisionCount: () => import_agent_qa_eval7.markedDecisionCount,
|
|
1441
|
-
preflightJudgePanel: () => preflightJudgePanel,
|
|
1442
|
-
previewMarkLabels: () => import_agent_qa_eval7.previewMarkLabels,
|
|
1443
|
-
projectMatcherConfigContract: () => import_agent_qa_eval6.projectMatcherConfigContract,
|
|
1444
|
-
readActiveCommittedCorpus: () => import_agent_qa_eval_local2.readActiveCommittedCorpus,
|
|
1445
|
-
readAdversarialRefreshRecords: () => import_agent_qa_eval_local4.readAdversarialRefreshRecords,
|
|
1446
|
-
readCalibrationGoldIdentity: () => import_agent_qa_eval_local3.readCalibrationGoldIdentity,
|
|
1447
|
-
readCanaryCorpusDigest: () => readCanaryCorpusDigest,
|
|
1448
|
-
readCommittedCorpus: () => import_agent_qa_eval_local2.readCommittedCorpus,
|
|
1449
|
-
readGatePostureFor: () => import_agent_qa_eval_local2.readGatePostureFor,
|
|
1450
|
-
readJudgeCalibration: () => import_agent_qa_eval_local.readJudgeCalibration,
|
|
1451
|
-
readLabelCalibration: () => import_agent_qa_eval_local3.readLabelCalibration,
|
|
1452
|
-
readMisdirectionRootPosture: () => readMisdirectionRootPosture,
|
|
1453
|
-
renderAdversarialProbePrompt: () => import_agent_qa_eval8.renderAdversarialProbePrompt,
|
|
1454
|
-
renderBaselineComparatorRefusal: () => import_agent_qa_eval2.renderBaselineComparatorRefusal,
|
|
1455
|
-
renderCensusMarkdown: () => import_agent_qa_eval.renderCensusMarkdown,
|
|
1456
|
-
renderJudgeTargetedCalibration: () => renderJudgeTargetedCalibration,
|
|
1457
|
-
resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability,
|
|
1458
|
-
resolveEvalJudge: () => resolveEvalJudge,
|
|
1459
|
-
resolveEvalSession: () => resolveEvalSession,
|
|
1460
|
-
resolveGate2Caps: () => resolveGate2Caps,
|
|
1461
|
-
resolveGateProfileIdentity: () => resolveGateProfileIdentity,
|
|
1462
|
-
resolveJudgeCalibrationMinimum: () => import_agent_qa_eval5.resolveJudgeCalibrationMinimum,
|
|
1463
|
-
resolveJudgeFamilies: () => import_agent_qa_eval5.resolveJudgeFamilies,
|
|
1464
|
-
resolveMarkLabelTargets: () => import_agent_qa_eval7.resolveMarkLabelTargets,
|
|
1465
|
-
resolveMarkedDecisions: () => import_agent_qa_eval7.resolveMarkedDecisions,
|
|
1466
|
-
resolveMinimumGateSet: () => import_agent_qa_eval_local2.resolveMinimumGateSet,
|
|
1467
|
-
resolveMisdirectionCaps: () => resolveMisdirectionCaps,
|
|
1468
|
-
resolveProductPartition: () => import_agent_qa_eval7.resolveProductPartition,
|
|
1469
|
-
resolveReplayPlanner: () => resolveReplayPlanner,
|
|
1470
|
-
resolveReplayProfileDeclaration: () => resolveReplayProfileDeclaration,
|
|
1471
|
-
resolveReplayProfileFingerprint: () => resolveReplayProfileFingerprint,
|
|
1472
|
-
resolveTargetEnvironmentForProduct: () => resolveTargetEnvironmentForProduct,
|
|
1473
|
-
runAdversarialVerb: () => runAdversarialVerb,
|
|
1474
|
-
runCalibrateVerb: () => runCalibrateVerb,
|
|
1475
|
-
runCanaryAblation: () => runCanaryAblation,
|
|
1476
|
-
runCensusVerb: () => runCensusVerb,
|
|
1477
|
-
runExtractVerb: () => runExtractVerb,
|
|
1478
|
-
runFeedbackStaging: () => runFeedbackStaging,
|
|
1479
|
-
runFloorVerb: () => runFloorVerb,
|
|
1480
|
-
runGateVerb: () => runGateVerb,
|
|
1481
|
-
runJudgeCalibrateVerb: () => runJudgeCalibrateVerb,
|
|
1482
|
-
runMisdirectionCheck: () => runMisdirectionCheck,
|
|
1483
|
-
runReplayGate: () => import_agent_qa_eval_local3.runReplayGate,
|
|
1484
|
-
runReplayGateVerb: () => runReplayGateVerb,
|
|
1485
|
-
runReplayVerb: () => runReplayVerb,
|
|
1486
|
-
runReportVerb: () => runReportVerb,
|
|
1487
|
-
runRetireAdversarialVerb: () => runRetireAdversarialVerb,
|
|
1488
|
-
runSafetyScope: () => import_agent_qa_safety_local2.runSafetyScope,
|
|
1489
|
-
runSeedFromFeedbackVerb: () => runSeedFromFeedbackVerb,
|
|
1490
|
-
runSeedVerb: () => runSeedVerb,
|
|
1491
|
-
selectGateSet: () => import_agent_qa_eval3.selectGateSet,
|
|
1492
|
-
selectedEvalJudgeNames: () => selectedEvalJudgeNames,
|
|
1493
|
-
serializeReplayGateReceipt: () => import_agent_qa_eval_local3.serializeReplayGateReceipt,
|
|
1494
|
-
slugBranch: () => import_agent_qa_runner_local3.slugBranch
|
|
1495
|
-
});
|
|
1496
|
-
function selectedEvalJudgeNames(config) {
|
|
1497
|
-
if (config.judge !== void 0 && config.judges !== void 0) {
|
|
1498
|
-
throw new EvalJudgeDeclarationError("Set either eval.judge or eval.judges, never both; remove one declaration.");
|
|
1499
|
-
}
|
|
1500
|
-
if (config.judge !== void 0 && typeof config.judge !== "string") {
|
|
1501
|
-
throw new EvalJudgeDeclarationError("eval.judge must be one nonempty registry name; use eval.judges for a list.");
|
|
1502
|
-
}
|
|
1503
|
-
const names = config.judge !== void 0 ? [config.judge] : config.judges === void 0 ? [] : config.judges;
|
|
1504
|
-
if (!Array.isArray(names) || Array.from(names).some((name) => typeof name !== "string" || name.trim().length === 0)) {
|
|
1505
|
-
throw new EvalJudgeDeclarationError("eval.judges must be an array of nonempty registry names.");
|
|
1506
|
-
}
|
|
1507
|
-
if (names.length > import_agent_qa_eval.JUDGE_CONFIG.maxPanelMembers) {
|
|
1508
|
-
throw new EvalJudgeDeclarationError(`Choose at most ${import_agent_qa_eval.JUDGE_CONFIG.maxPanelMembers} explicit judge names.`);
|
|
1509
|
-
}
|
|
1510
|
-
return [...names];
|
|
1511
|
-
}
|
|
1512
|
-
function withJudgeOverrides(judge, modelOverrides) {
|
|
1513
|
-
if (judge === void 0 || modelOverrides === void 0) return judge;
|
|
1514
|
-
return "members" in judge ? { ...judge, members: judge.members.map((member) => ({ ...member, modelOverrides })) } : { ...judge, modelOverrides };
|
|
1515
|
-
}
|
|
1516
|
-
async function resolveEvalSession(input = {}) {
|
|
1517
|
-
const configPath = input.configPath ?? "agent-qa.config.yaml";
|
|
1518
|
-
const configDir = (0, import_node_path14.dirname)((0, import_node_path14.resolve)(configPath));
|
|
1519
|
-
const raw = await loadConfigFile(configPath);
|
|
1520
|
-
if (raw?.eval && typeof raw.eval === "object") selectedEvalJudgeNames(raw.eval);
|
|
1521
|
-
const config = await resolveConfig({ configPath, loadAuthPlugins: input.keyOnly !== true });
|
|
1522
|
-
const evalConfig = mergeEvalConfig(config.eval, input.flags);
|
|
1523
|
-
if (input.keyOnly === true) {
|
|
1524
|
-
return {
|
|
1525
|
-
config,
|
|
1526
|
-
configPath,
|
|
1527
|
-
configDir,
|
|
1528
|
-
evalConfig,
|
|
1529
|
-
mask: () => {
|
|
1530
|
-
throw new Error("A key-only eval session has no declared secrets to mask with");
|
|
1531
|
-
},
|
|
1532
|
-
executionContext: (0, import_agent_qa_agent_local4.resolveRunExecutionContext)()
|
|
1533
|
-
};
|
|
1534
|
-
}
|
|
1535
|
-
const secretStore = await loadEvalSecretStore(config, configDir);
|
|
1536
|
-
const { SecretRedactor: SecretRedactor2 } = await import("@vostride/agent-qa-runner");
|
|
1537
|
-
const redactor = new SecretRedactor2(secretStore);
|
|
1538
|
-
return {
|
|
1539
|
-
config,
|
|
1540
|
-
configPath,
|
|
1541
|
-
configDir,
|
|
1542
|
-
evalConfig,
|
|
1543
|
-
mask: (text) => redactor.redactString(text),
|
|
1544
|
-
executionContext: (0, import_agent_qa_agent_local4.resolveRunExecutionContext)()
|
|
1545
|
-
};
|
|
1546
|
-
}
|
|
1547
|
-
function committedCorpusMask(session) {
|
|
1548
|
-
const declared = createDeclaredSecretMask(session.configPath);
|
|
1549
|
-
declared.redactor();
|
|
1550
|
-
return declared.mask;
|
|
1551
|
-
}
|
|
1552
|
-
async function runExtractVerb(session, filters = {}) {
|
|
1553
|
-
return await (0, import_agent_qa_safety_local.runSafetyScope)(
|
|
1554
|
-
{ executionContext: session.executionContext },
|
|
1555
|
-
async () => sweepAndExtract((0, import_node_path14.resolve)(session.configDir, ".agent-qa"), filters)
|
|
1556
|
-
);
|
|
1557
|
-
}
|
|
1558
|
-
function sweepAndExtract(agentQaRoot, filters) {
|
|
1559
|
-
const runsRoot = (0, import_node_path14.resolve)(agentQaRoot, import_agent_qa_runner_local2.RUNS_SUBDIR);
|
|
1560
|
-
let runSlugs = listRunSlugsWithHandoff(runsRoot);
|
|
1561
|
-
if (filters.run !== void 0) {
|
|
1562
|
-
const named = filters.run;
|
|
1563
|
-
runSlugs = runSlugs.filter((slug) => slug === named || slug === (0, import_agent_qa_runner_local2.slugBranch)(named));
|
|
1564
|
-
if (runSlugs.length === 0) {
|
|
1565
|
-
throw new EvalNamedInputError("run", named, runsRoot);
|
|
1566
|
-
}
|
|
1567
|
-
}
|
|
1568
|
-
const store = new import_agent_qa_eval_local.FsCassetteStore({ root: agentQaRoot });
|
|
1569
|
-
const runsSkipped = [];
|
|
1570
|
-
const completeness = {};
|
|
1571
|
-
let runsFiltered = 0;
|
|
1572
|
-
let cassettesWritten = 0;
|
|
1573
|
-
for (const runSlug of runSlugs) {
|
|
1574
|
-
let extraction;
|
|
1575
|
-
try {
|
|
1576
|
-
const handoff = (0, import_agent_qa_runner_local2.loadCompletedRunHandoff)((0, import_node_path14.resolve)(runsRoot, runSlug, import_agent_qa_runner_local2.HANDOFF_FILE));
|
|
1577
|
-
if (filters.product !== void 0 && (0, import_agent_qa_eval.resolveProductPartition)(handoff.scope) !== filters.product) {
|
|
1578
|
-
runsFiltered++;
|
|
1579
|
-
continue;
|
|
1580
|
-
}
|
|
1581
|
-
extraction = (0, import_agent_qa_eval.extractRunCassettes)({
|
|
1582
|
-
handoff,
|
|
1583
|
-
artifacts: new import_agent_qa_eval_local.FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
|
|
1584
|
-
});
|
|
1585
|
-
} catch (err) {
|
|
1586
|
-
runsSkipped.push({ runSlug, reason: err instanceof Error ? err.message : String(err) });
|
|
1587
|
-
continue;
|
|
1588
|
-
}
|
|
1589
|
-
for (const extracted of extraction.cassettes) {
|
|
1590
|
-
store.write(extracted);
|
|
1591
|
-
cassettesWritten++;
|
|
1592
|
-
}
|
|
1593
|
-
for (const emitted of extraction.report.emitted) {
|
|
1594
|
-
bumpCounter(completeness, emitted.completenessStatus);
|
|
1595
|
-
}
|
|
1596
|
-
for (const skip of extraction.report.skips) {
|
|
1597
|
-
bumpCounter(completeness, skip.reason);
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
return {
|
|
1601
|
-
runsSwept: runSlugs.length,
|
|
1602
|
-
runsSkipped,
|
|
1603
|
-
runsFiltered,
|
|
1604
|
-
cassettesWritten,
|
|
1605
|
-
completeness,
|
|
1606
|
-
cassettesRoot: (0, import_node_path14.resolve)(agentQaRoot, import_agent_qa_eval_local.DEFAULT_EVAL_CASSETTES_SUBDIR)
|
|
1607
|
-
};
|
|
1608
|
-
}
|
|
1609
|
-
function listRunSlugsWithHandoff(runsRoot) {
|
|
1610
|
-
let entries;
|
|
1611
|
-
try {
|
|
1612
|
-
entries = (0, import_node_fs7.readdirSync)(runsRoot, { withFileTypes: true });
|
|
1613
|
-
} catch (err) {
|
|
1614
|
-
if (err.code === "ENOENT") return [];
|
|
1615
|
-
throw err;
|
|
1616
|
-
}
|
|
1617
|
-
return entries.filter((entry) => entry.isDirectory() && (0, import_node_fs7.existsSync)((0, import_node_path14.resolve)(runsRoot, entry.name, import_agent_qa_runner_local2.HANDOFF_FILE))).map((entry) => entry.name).sort();
|
|
1618
|
-
}
|
|
1619
|
-
function bumpCounter(counters, key) {
|
|
1620
|
-
counters[key] = (counters[key] ?? 0) + 1;
|
|
1621
|
-
}
|
|
1622
|
-
function resolveCassettesRoot(session, source) {
|
|
1623
|
-
if (source === "committed") {
|
|
1624
|
-
const cassettesRoot = (0, import_node_path14.join)(
|
|
1625
|
-
(0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
1626
|
-
import_agent_qa_eval_local.COMMITTED_CASSETTES_DIR
|
|
1627
|
-
);
|
|
1628
|
-
return { cassettesRoot, notice: `cassette source: committed root ${cassettesRoot}` };
|
|
1629
|
-
}
|
|
1630
|
-
return {
|
|
1631
|
-
cassettesRoot: (0, import_node_path14.join)((0, import_node_path14.resolve)(session.configDir, ".agent-qa"), import_agent_qa_eval_local.DEFAULT_EVAL_CASSETTES_SUBDIR)
|
|
1632
|
-
};
|
|
1633
|
-
}
|
|
1634
|
-
function replayInvocationSpend(entries, seed, modelCalls, started, calibration) {
|
|
1635
|
-
const report = (0, import_agent_qa_eval.buildEvalReport)(entries, void 0, void 0, { declaredSeed: seed, calibration });
|
|
1636
|
-
const cost = report.totalCost;
|
|
1637
|
-
const judgeCost = report.honesty.judge.cost;
|
|
1638
|
-
return {
|
|
1639
|
-
seed: seed ?? null,
|
|
1640
|
-
seedHonored: cost.sampleCount === 0 ? null : report.seedHonored ?? null,
|
|
1641
|
-
decisions: report.headline.perDecisionCost.decisionCount,
|
|
1642
|
-
modelCalls,
|
|
1643
|
-
promptTokens: cost.promptTokens + judgeCost.inputTokens,
|
|
1644
|
-
completionTokens: cost.completionTokens + judgeCost.outputTokens,
|
|
1645
|
-
wallClockMs: Math.max(0, Math.round(performance.now() - started)),
|
|
1646
|
-
sampleCount: cost.sampleCount,
|
|
1647
|
-
samplesMissingTokenUsage: cost.samplesMissingTokenUsage,
|
|
1648
|
-
judgeCost
|
|
1649
|
-
};
|
|
1650
|
-
}
|
|
1651
|
-
async function runReplayVerb(session, filters = {}, opts = {}) {
|
|
1652
|
-
const started = performance.now();
|
|
1653
|
-
const agentQaRoot = (0, import_node_path14.resolve)(session.configDir, ".agent-qa");
|
|
1654
|
-
const source = opts.source ?? "staged";
|
|
1655
|
-
const sourceRoot = resolveCassettesRoot(session, source);
|
|
1656
|
-
const cassettesRoot = sourceRoot.cassettesRoot;
|
|
1657
|
-
const replaysRoot = (0, import_node_path14.join)(agentQaRoot, import_agent_qa_eval_local.DEFAULT_EVAL_REPLAYS_SUBDIR);
|
|
1658
|
-
const { swept, loaded, loadRefusals } = (0, import_agent_qa_eval_local.loadCassetteBundles)(cassettesRoot);
|
|
1659
|
-
const activeCorpus = (0, import_agent_qa_eval_local.readActiveCommittedCorpus)((0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"), agentQaRoot);
|
|
1660
|
-
const activeIds = source !== "committed" ? void 0 : new Set(activeCorpus.snapshot.entries.map((entry) => entry.cassetteId));
|
|
1661
|
-
let selected = activeIds === void 0 ? loaded : loaded.filter((entry) => activeIds.has(entry.cassette.id));
|
|
1662
|
-
if (filters.cassette !== void 0) {
|
|
1663
|
-
const named = filters.cassette;
|
|
1664
|
-
selected = selected.filter((entry) => entry.cassette.id === named);
|
|
1665
|
-
if (selected.length === 0) {
|
|
1666
|
-
throw new EvalNamedInputError("cassette", named, cassettesRoot, "replay");
|
|
1667
|
-
}
|
|
1668
|
-
}
|
|
1669
|
-
if (filters.run !== void 0) {
|
|
1670
|
-
const named = filters.run;
|
|
1671
|
-
selected = selected.filter((entry) => {
|
|
1672
|
-
const head = entry.cassette.id.split("/")[0];
|
|
1673
|
-
return head === named || head === (0, import_agent_qa_runner_local2.slugBranch)(named);
|
|
1674
|
-
});
|
|
1675
|
-
if (selected.length === 0) {
|
|
1676
|
-
throw new EvalNamedInputError("run", named, cassettesRoot, "replay");
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
if (filters.product !== void 0) {
|
|
1680
|
-
const named = filters.product;
|
|
1681
|
-
selected = selected.filter((entry) => (0, import_agent_qa_eval_local.cassetteProductPartition)(entry.cassette) === named);
|
|
1682
|
-
}
|
|
1683
|
-
selected.sort((a, b) => a.cassette.id < b.cassette.id ? -1 : a.cassette.id > b.cassette.id ? 1 : 0);
|
|
1684
|
-
const notices = [];
|
|
1685
|
-
if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
|
|
1686
|
-
notices.push(...activeCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
|
|
1687
|
-
const zeroJudge = (configured) => ({
|
|
1688
|
-
configured,
|
|
1689
|
-
invoked: 0,
|
|
1690
|
-
settled: 0,
|
|
1691
|
-
abstained: 0,
|
|
1692
|
-
errors: 0,
|
|
1693
|
-
notInvoked: 0
|
|
1694
|
-
});
|
|
1695
|
-
if (selected.length === 0) {
|
|
1696
|
-
notices.push(`empty sweep \u2014 no ${source} cassette bundles matched`);
|
|
1697
|
-
return {
|
|
1698
|
-
batchKey: "empty",
|
|
1699
|
-
swept,
|
|
1700
|
-
replayed: 0,
|
|
1701
|
-
loadRefusals,
|
|
1702
|
-
replayRefusals: {},
|
|
1703
|
-
scored: 0,
|
|
1704
|
-
judge: zeroJudge(selectedEvalJudgeNames(session.evalConfig).length > 0),
|
|
1705
|
-
replaysRoot,
|
|
1706
|
-
source,
|
|
1707
|
-
notices,
|
|
1708
|
-
spend: replayInvocationSpend([], session.evalConfig.seed, 0, started, activeCorpus?.calibration)
|
|
1709
|
-
};
|
|
1710
|
-
}
|
|
1711
|
-
const batchKey = (0, import_agent_qa_eval.computeBatchKey)(selected.map((entry) => entry.cassette.id));
|
|
1712
|
-
const planner = await resolveReplayPlanner(session);
|
|
1713
|
-
const judgeResolution = await resolveEvalJudge(session);
|
|
1714
|
-
if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
|
|
1715
|
-
const recordStore = new import_agent_qa_eval_local.FsReplayRecordStore({ root: agentQaRoot });
|
|
1716
|
-
const scoreStore = new import_agent_qa_eval_local.FsScoreStore({ root: agentQaRoot });
|
|
1717
|
-
let modelCalls = 0;
|
|
1718
|
-
const { replayed, replayRefusals, replayedDirs, notices: replayNotices } = await (0, import_agent_qa_eval_local.sweepCassetteReplays)({
|
|
1719
|
-
selected,
|
|
1720
|
-
labelRecords: activeCorpus?.snapshot.labelsById,
|
|
1721
|
-
profile: planner.profile,
|
|
1722
|
-
auth: planner.auth,
|
|
1723
|
-
modelCallObservation: { onProviderCall: () => {
|
|
1724
|
-
modelCalls++;
|
|
1725
|
-
} },
|
|
1726
|
-
...opts.plannerOverrides !== void 0 ? { modelOverrides: opts.plannerOverrides } : {},
|
|
1727
|
-
...session.evalConfig.k !== void 0 ? { k: session.evalConfig.k } : {},
|
|
1728
|
-
...session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed } : {},
|
|
1729
|
-
...judgeResolution.judge !== void 0 ? {
|
|
1730
|
-
judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides)
|
|
1731
|
-
} : {},
|
|
1732
|
-
mask: session.mask,
|
|
1733
|
-
recordStore,
|
|
1734
|
-
scoreStore,
|
|
1735
|
-
replaysRoot,
|
|
1736
|
-
// Progress ALWAYS rides stderr — stdout stays reserved for the rendered DTO.
|
|
1737
|
-
onProgress: (id) => process.stderr.write(`replay: ${id}
|
|
1738
|
-
`)
|
|
1739
|
-
});
|
|
1740
|
-
notices.push(...replayNotices);
|
|
1741
|
-
const judge = zeroJudge(judgeResolution.judge !== void 0);
|
|
1742
|
-
let scored = 0;
|
|
1743
|
-
for (const recordDir of replayedDirs) {
|
|
1744
|
-
const replay = (0, import_agent_qa_eval_local.loadReplayRecord)(recordDir);
|
|
1745
|
-
const load = (0, import_agent_qa_eval_local.loadScoreRecord)(recordDir);
|
|
1746
|
-
if (replay.kind !== "loaded") {
|
|
1747
|
-
notices.push(`spend replay read refused under ${recordDir}: ${replay.refusal.detail}`);
|
|
1748
|
-
}
|
|
1749
|
-
if (load.kind === "absent") continue;
|
|
1750
|
-
if (load.kind === "refused") {
|
|
1751
|
-
notices.push(`score read refused under ${recordDir}: ${load.refusal.detail}`);
|
|
1752
|
-
continue;
|
|
1753
|
-
}
|
|
1754
|
-
scored++;
|
|
1755
|
-
foldJudgeDispositions(load.record.score, judge);
|
|
1756
|
-
}
|
|
1757
|
-
const spendBatch = (0, import_agent_qa_eval_local.assembleScoredBatch)({
|
|
1758
|
-
replaysRoot,
|
|
1759
|
-
cassettesRoot,
|
|
1760
|
-
source,
|
|
1761
|
-
recordDirs: replayedDirs,
|
|
1762
|
-
labelRecords: activeCorpus?.snapshot.labelsById,
|
|
1763
|
-
corpusEntriesById: source === "committed" ? activeCorpus.corpusEntriesById : void 0,
|
|
1764
|
-
excludedLabelIds: activeCorpus?.snapshot.excludedLabelIds,
|
|
1765
|
-
calibration: activeCorpus?.calibration
|
|
1766
|
-
});
|
|
1767
|
-
notices.push(...spendBatch.notices);
|
|
1768
|
-
return {
|
|
1769
|
-
batchKey,
|
|
1770
|
-
swept,
|
|
1771
|
-
replayed,
|
|
1772
|
-
loadRefusals,
|
|
1773
|
-
replayRefusals,
|
|
1774
|
-
scored,
|
|
1775
|
-
judge,
|
|
1776
|
-
replaysRoot,
|
|
1777
|
-
source,
|
|
1778
|
-
notices,
|
|
1779
|
-
spend: replayInvocationSpend(spendBatch.entries, session.evalConfig.seed, modelCalls, started, spendBatch.calibration)
|
|
1780
|
-
};
|
|
1781
|
-
}
|
|
1782
|
-
function foldJudgeDispositions(score, judge) {
|
|
1783
|
-
const entries = score.kind === "leaf" ? [score.entry] : score.kind === "procedure" ? score.entries : [];
|
|
1784
|
-
for (const entry of entries) {
|
|
1785
|
-
if (entry.kind !== "decision") continue;
|
|
1786
|
-
const decision = entry.score;
|
|
1787
|
-
if (decision.kind === "unresolved-tie") {
|
|
1788
|
-
const outcome = decision.judgeOutcome;
|
|
1789
|
-
if (outcome === void 0) continue;
|
|
1790
|
-
if (outcome.kind === "abstained") {
|
|
1791
|
-
judge.abstained++;
|
|
1792
|
-
judge.invoked++;
|
|
1793
|
-
} else if (outcome.kind === "judge-error") {
|
|
1794
|
-
judge.errors++;
|
|
1795
|
-
judge.invoked++;
|
|
1796
|
-
} else {
|
|
1797
|
-
judge.notInvoked++;
|
|
1798
|
-
}
|
|
1799
|
-
} else if (decision.kind === "scored" && decision.judgeSettled !== void 0) {
|
|
1800
|
-
judge.settled++;
|
|
1801
|
-
judge.invoked++;
|
|
1802
|
-
}
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
async function resolveEvalJudge(session) {
|
|
1806
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
1807
|
-
if (names.length === 0) {
|
|
1808
|
-
return {
|
|
1809
|
-
notice: "no judge configured \u2014 unresolved ties will record not-invoked{no-judge-configured}"
|
|
1810
|
-
};
|
|
1811
|
-
}
|
|
1812
|
-
if (names.length > 1) {
|
|
1813
|
-
const members = [];
|
|
1814
|
-
for (const name of names) members.push(await resolveNamedProfile(name, session.configPath));
|
|
1815
|
-
return { judge: { members }, ...session.evalConfig.judgeAllowSameFamily === true ? { notice: "eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint." } : {} };
|
|
1816
|
-
}
|
|
1817
|
-
const resolved = await resolveNamedProfile(names[0], session.configPath);
|
|
1818
|
-
const planner = await resolveReplayPlanner(session);
|
|
1819
|
-
const judgeFamily = (0, import_agent_qa_eval.resolveJudgeFamily)(resolved.profile.model);
|
|
1820
|
-
const plannerFamily = (0, import_agent_qa_eval.resolveJudgeFamily)(planner.profile.model);
|
|
1821
|
-
const overrideActive = session.evalConfig.judgeAllowSameFamily === true;
|
|
1822
|
-
const judge = overrideActive ? { ...resolved, allowSameFamily: true } : resolved;
|
|
1823
|
-
if (judgeFamily !== void 0 && plannerFamily !== void 0 && judgeFamily === plannerFamily && overrideActive) {
|
|
1824
|
-
return {
|
|
1825
|
-
judge,
|
|
1826
|
-
notice: `D-10 override active (eval.judgeAllowSameFamily, owner decision 2026-07-12): judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" share the judge family '${judgeFamily}' \u2014 same-family judging runs and every scored entry discloses familyOverride: same-family-allowed`
|
|
1827
|
-
};
|
|
1828
|
-
}
|
|
1829
|
-
if (judgeFamily === void 0 || plannerFamily === void 0 || judgeFamily === plannerFamily) {
|
|
1830
|
-
return {
|
|
1831
|
-
judge,
|
|
1832
|
-
notice: `judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" resolve to the same or unknown family \u2014 per-cassette judge eligibility will refuse self-judging and ties may record not-invoked`
|
|
1833
|
-
};
|
|
1834
|
-
}
|
|
1835
|
-
return { judge };
|
|
1836
|
-
}
|
|
1837
|
-
async function resolveReplayPlanner(session) {
|
|
1838
|
-
const plannerName = session.config.use?.llm;
|
|
1839
|
-
if (!plannerName) {
|
|
1840
|
-
throw new Error(
|
|
1841
|
-
"use.llm is not set in the config \u2014 the replay planner is the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
|
|
1842
|
-
);
|
|
1843
|
-
}
|
|
1844
|
-
return resolveNamedProfile(plannerName, session.configPath);
|
|
1845
|
-
}
|
|
1846
|
-
async function runJudgeCalibrateVerb(session, opts = {}) {
|
|
1847
|
-
if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
|
|
1848
|
-
const preflight = await preflightJudgePanel(session, { minN: opts.minN });
|
|
1849
|
-
if (preflight.kind === "capability-gap" && !preflight.identity) return {
|
|
1850
|
-
...await (0, import_agent_qa_eval_local.runJudgeCalibration)({
|
|
1851
|
-
kind: "capability-gap",
|
|
1852
|
-
reason: preflight.reason,
|
|
1853
|
-
minN: preflight.minN,
|
|
1854
|
-
commit: opts.commit,
|
|
1855
|
-
replace: opts.replace
|
|
1856
|
-
}),
|
|
1857
|
-
entries: preflight.entries
|
|
1858
|
-
};
|
|
1859
|
-
const replayProfile = preflight.kind === "eligible" ? preflight.replayProfile : await resolveReplayProfileDeclaration(session);
|
|
1860
|
-
const fingerprint = (0, import_agent_qa_eval_local.computeReplayProfileFingerprintFor)({
|
|
1861
|
-
profile: replayProfile,
|
|
1862
|
-
seedPolicy: "declared"
|
|
1863
|
-
});
|
|
1864
|
-
const context = {
|
|
1865
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
1866
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
1867
|
-
// Saved evidence must prove this actual planner/profile; judging does not rerun a planner.
|
|
1868
|
-
profile: {
|
|
1869
|
-
plannerModelId: replayProfile.model,
|
|
1870
|
-
fingerprint,
|
|
1871
|
-
matcherVersion: (await import("@vostride/agent-qa-eval")).projectMatcherConfigContract().version,
|
|
1872
|
-
judgeModelId: preflight.identity.judgeModelId,
|
|
1873
|
-
judgeVersion: preflight.identity.judgeVersion
|
|
1874
|
-
},
|
|
1875
|
-
identity: preflight.identity,
|
|
1876
|
-
minN: preflight.minN
|
|
1877
|
-
};
|
|
1878
|
-
const result = await (0, import_agent_qa_eval_local.runJudgeCalibration)(preflight.kind === "eligible" ? { kind: "probe", commit: opts.commit, replace: opts.replace, probe: {
|
|
1879
|
-
...context,
|
|
1880
|
-
replayProfile,
|
|
1881
|
-
members: preflight.members.map((member) => ({ ...member, modelOverrides: opts.judgeOverrides })),
|
|
1882
|
-
mask: session.mask
|
|
1883
|
-
} } : {
|
|
1884
|
-
kind: "capability-gap",
|
|
1885
|
-
reason: preflight.reason,
|
|
1886
|
-
minN: preflight.minN,
|
|
1887
|
-
publication: context,
|
|
1888
|
-
commit: opts.commit,
|
|
1889
|
-
replace: opts.replace
|
|
1890
|
-
});
|
|
1891
|
-
return (0, import_agent_qa_eval_local.maskJudgeCalibrationText)({ ...result, entries: preflight.entries }, session.mask);
|
|
1892
|
-
}
|
|
1893
|
-
function renderJudgeTargetedCalibration(disclosure) {
|
|
1894
|
-
if (disclosure === void 0) {
|
|
1895
|
-
return ["judge-targeted adversarial bait: not recorded (this artifact predates the disclosure)"];
|
|
1896
|
-
}
|
|
1897
|
-
if (disclosure.source === "unavailable") {
|
|
1898
|
-
return [`judge-targeted adversarial bait: not determined (${disclosure.reason})`];
|
|
1899
|
-
}
|
|
1900
|
-
const lines = [
|
|
1901
|
-
`judge-targeted adversarial bait: judge-targeted=${disclosure.judgeTargeted} self-authored-by-this-panel=${disclosure.selfAuthored} observed-in-measured-population=${disclosure.observedInCalibration}`,
|
|
1902
|
-
`adversarial provenance: model-authored=${disclosure.byProvenance.probe} table-derived=${disclosure.byProvenance.table} legacy-unattributed=${disclosure.byProvenance.absent} malformed=${disclosure.byProvenance.malformed} unresolved=${disclosure.metadataUnavailable} of ${disclosure.total} committed members (arms are counted apart, never summed)`
|
|
1903
|
-
];
|
|
1904
|
-
for (const row of disclosure.byAuthoringModel) {
|
|
1905
|
-
lines.push(`author ${row.authoringModel} (${row.authoringFamily})${row.panelMember ? " [panel member \u2014 self-authored]" : ""}: judge-targeted=${row.judgeTargeted} authored=${row.probeAuthored}`);
|
|
1906
|
-
}
|
|
1907
|
-
lines.push("Bait counts above are RAW: the agreement, interval, per-member rows and minN are not adjusted for them.");
|
|
1908
|
-
return lines;
|
|
1909
|
-
}
|
|
1910
|
-
async function runCalibrateVerb(session, opts = {}) {
|
|
1911
|
-
const minGold = opts.minGold ?? LABEL_CALIBRATION_MIN_GOLD;
|
|
1912
|
-
if (!Number.isSafeInteger(minGold) || minGold < 2) {
|
|
1913
|
-
return { kind: "refused", reason: "--min-gold must be a safe integer of at least 2." };
|
|
1914
|
-
}
|
|
1915
|
-
if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
|
|
1916
|
-
const result = await (0, import_agent_qa_eval_local.runLabelCalibration)({
|
|
1917
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
1918
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
1919
|
-
commit: opts.commit,
|
|
1920
|
-
replace: opts.replace,
|
|
1921
|
-
minGold: { value: minGold, source: opts.minGold === void 0 ? "default" : "cli" }
|
|
1922
|
-
});
|
|
1923
|
-
if ("artifact" in result) return result;
|
|
1924
|
-
const reasons = {
|
|
1925
|
-
refused: "Calibration inputs unavailable or invalid; inspect committed labels and run evidence.",
|
|
1926
|
-
busy: "Another calibration publisher holds the lock; retry after it finishes.",
|
|
1927
|
-
"input-drift": "Calibration inputs changed; retry from the current snapshot.",
|
|
1928
|
-
"publication-failed": "Calibration publication interrupted; retry the identical command.",
|
|
1929
|
-
"write-conflict": "Calibration bytes conflict; review the dry-run, then use --commit --replace to replace the marker. Immutable revision conflicts require inspection."
|
|
1930
|
-
};
|
|
1931
|
-
return { kind: result.kind, reason: reasons[result.kind] };
|
|
1932
|
-
}
|
|
1933
|
-
function resolveGate2Caps(session, opts = {}) {
|
|
1934
|
-
const maxAblations = opts.maxAblations ?? session.evalConfig.gate2?.maxAblations ?? GATE2_DEFAULT_MAX_ABLATIONS;
|
|
1935
|
-
if (!Number.isSafeInteger(maxAblations) || maxAblations < 1) {
|
|
1936
|
-
throw new TypeError("--max-ablations must be a positive integer");
|
|
1937
|
-
}
|
|
1938
|
-
const maxAblationsSource = opts.maxAblations !== void 0 ? "flag" : session.evalConfig.gate2?.maxAblations !== void 0 ? "config" : "default";
|
|
1939
|
-
const canaryHorizonBuilds = session.evalConfig.gate2?.canaryHorizonBuilds ?? GATE2_DEFAULT_CANARY_HORIZON_BUILDS;
|
|
1940
|
-
const maxLowerTrustPerScope = session.evalConfig.gate2?.maxLowerTrustPerScope ?? import_agent_qa_schema9.GATE2_DEFAULT_MAX_LOWER_TRUST_PER_SCOPE;
|
|
1941
|
-
for (const [name, value] of Object.entries({ canaryHorizonBuilds, maxLowerTrustPerScope })) {
|
|
1942
|
-
if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`eval.gate2.${name} must be a positive integer`);
|
|
1943
|
-
}
|
|
1944
|
-
return {
|
|
1945
|
-
maxAblations,
|
|
1946
|
-
maxAblationsSource,
|
|
1947
|
-
canaryHorizonBuilds,
|
|
1948
|
-
maxLowerTrustPerScope,
|
|
1949
|
-
canaryHorizonBuildsSource: session.evalConfig.gate2?.canaryHorizonBuilds === void 0 ? "default" : "config",
|
|
1950
|
-
maxLowerTrustPerScopeSource: session.evalConfig.gate2?.maxLowerTrustPerScope === void 0 ? "default" : "config"
|
|
1951
|
-
};
|
|
1952
|
-
}
|
|
1953
|
-
function readCanaryCorpusDigest(session) {
|
|
1954
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
1955
|
-
try {
|
|
1956
|
-
(0, import_node_fs7.lstatSync)(committedRoot);
|
|
1957
|
-
} catch (error) {
|
|
1958
|
-
return error.code === "ENOENT" ? { kind: "no-eval-root" } : { kind: "unavailable", reason: error instanceof Error ? error.message : String(error) };
|
|
1959
|
-
}
|
|
1960
|
-
return (0, import_agent_qa_eval_local.readCommittedCorpusDigest)({ committedRoot });
|
|
1961
|
-
}
|
|
1962
|
-
async function runCanaryAblation(session, input) {
|
|
1963
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
1964
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
1965
|
-
if (digest2.kind !== "digest") return digest2;
|
|
1966
|
-
const k = input.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
1967
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
1968
|
-
throw new TypeError("--k must be an odd integer of at least 3");
|
|
1969
|
-
}
|
|
1970
|
-
const identity = await resolveGateProfileIdentity(session, input.plannerOverrides);
|
|
1971
|
-
const floor = (0, import_agent_qa_eval_local.listNoiseFloorArtefacts)(committedRoot).artefacts.find(
|
|
1972
|
-
(entry) => entry.artefact.profileKey === identity.profileKey
|
|
1973
|
-
)?.artefact;
|
|
1974
|
-
const baseSeed = input.seed ?? session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
|
|
1975
|
-
const execution = await resolveEvalSession({ configPath: session.configPath });
|
|
1976
|
-
const planner = await resolveReplayPlanner(execution);
|
|
1977
|
-
return (0, import_agent_qa_eval_local.ablateCanaryStep)({
|
|
1978
|
-
agentQaRoot: input.agentQaRoot,
|
|
1979
|
-
committedRoot,
|
|
1980
|
-
recordId: input.recordId,
|
|
1981
|
-
target: input.target,
|
|
1982
|
-
attributedObservationId: input.attributedObservationId,
|
|
1983
|
-
k,
|
|
1984
|
-
baseSeed,
|
|
1985
|
-
replay: {
|
|
1986
|
-
profile: planner.profile,
|
|
1987
|
-
auth: planner.auth,
|
|
1988
|
-
modelOverrides: input.plannerOverrides,
|
|
1989
|
-
mask: execution.mask
|
|
1990
|
-
}
|
|
1991
|
-
});
|
|
1992
|
-
}
|
|
1993
|
-
async function readMisdirectionRootPosture(session) {
|
|
1994
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
1995
|
-
const floorless = !hasNoiseFloorArtefacts((0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"));
|
|
1996
|
-
if (digest2.kind === "no-eval-root" || floorless) return { kind: "no-eval-root" };
|
|
1997
|
-
if (digest2.kind !== "digest") return { kind: "broken-eval-root", reason: "eval-root-unreadable" };
|
|
1998
|
-
const identity = await resolveGateProfileIdentity(session);
|
|
1999
|
-
return {
|
|
2000
|
-
kind: "eval-root",
|
|
2001
|
-
corpusDigest: digest2.corpusDigest,
|
|
2002
|
-
profileDigest: (0, import_agent_qa_schema10.sha256hex)(identity.profileKey)
|
|
2003
|
-
};
|
|
2004
|
-
}
|
|
2005
|
-
function resolveMisdirectionCaps(_session, opts = {}) {
|
|
2006
|
-
const maxChecks = opts.maxChecks ?? ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS;
|
|
2007
|
-
if (!Number.isSafeInteger(maxChecks) || maxChecks < 0) {
|
|
2008
|
-
throw new TypeError("--max-misdirection-checks must be a non-negative integer");
|
|
2009
|
-
}
|
|
2010
|
-
return { maxChecks, source: opts.maxChecks === void 0 ? "default" : "flag" };
|
|
2011
|
-
}
|
|
2012
|
-
async function locateMisdirectionDecision(session, target) {
|
|
2013
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2014
|
-
const corpus = (0, import_agent_qa_eval_local.readCommittedCorpus)(committedRoot);
|
|
2015
|
-
const located = await (0, import_agent_qa_eval_local.locateTargetCassettes)({
|
|
2016
|
-
committedRoot,
|
|
2017
|
-
agentQaRoot: target.agentQaRoot,
|
|
2018
|
-
gate1Dir: misdirectionAuditDir(target),
|
|
2019
|
-
corpusEntries: corpus.snapshot.entries,
|
|
2020
|
-
target: { runId: target.runId, testId: target.testId, stepIndex: target.stepIndex },
|
|
2021
|
-
source: "handoff"
|
|
2022
|
-
});
|
|
2023
|
-
if (located.kind === "unavailable") {
|
|
2024
|
-
return { kind: "unavailable", reason: located.detail.reason ?? "target-unavailable" };
|
|
2025
|
-
}
|
|
2026
|
-
if (located.cassettes.length !== 1) {
|
|
2027
|
-
return { kind: "unavailable", reason: "ambiguous live step: multiple recorded decisions" };
|
|
2028
|
-
}
|
|
2029
|
-
return { kind: "located", decisionOrdinal: located.cassettes[0].cassette.decisionOrdinal };
|
|
2030
|
-
}
|
|
2031
|
-
async function runMisdirectionCheck(session, input) {
|
|
2032
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2033
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2034
|
-
if (digest2.kind !== "digest") {
|
|
2035
|
-
return { kind: "unavailable", reason: digest2.kind === "no-eval-root" ? "no-eval-root" : digest2.reason };
|
|
2036
|
-
}
|
|
2037
|
-
const k = session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
2038
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
2039
|
-
throw new TypeError("eval.k must be an odd integer of at least 3");
|
|
2040
|
-
}
|
|
2041
|
-
const identity = await resolveGateProfileIdentity(session);
|
|
2042
|
-
const floor = (0, import_agent_qa_eval_local.listNoiseFloorArtefacts)(committedRoot).artefacts.find(
|
|
2043
|
-
(entry) => entry.artefact.profileKey === identity.profileKey
|
|
2044
|
-
)?.artefact;
|
|
2045
|
-
const baseSeed = session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
|
|
2046
|
-
const execution = await resolveEvalSession({ configPath: session.configPath });
|
|
2047
|
-
const planner = await resolveReplayPlanner(execution);
|
|
2048
|
-
const result = await (0, import_agent_qa_eval_local.ablateCanaryStep)({
|
|
2049
|
-
agentQaRoot: input.agentQaRoot,
|
|
2050
|
-
committedRoot,
|
|
2051
|
-
recordId: input.recordId,
|
|
2052
|
-
target: { runId: input.runId, testId: input.testId, stepIndex: input.stepIndex },
|
|
2053
|
-
attributedObservationId: misdirectionAuditKey(input),
|
|
2054
|
-
k,
|
|
2055
|
-
baseSeed,
|
|
2056
|
-
admission: { failingPathCandidate: input.authorization },
|
|
2057
|
-
replay: {
|
|
2058
|
-
profile: planner.profile,
|
|
2059
|
-
auth: planner.auth,
|
|
2060
|
-
mask: execution.mask
|
|
2061
|
-
}
|
|
2062
|
-
});
|
|
2063
|
-
if (result.kind !== "ablated") return { kind: "unavailable", reason: result.reason };
|
|
2064
|
-
return {
|
|
2065
|
-
kind: "checked",
|
|
2066
|
-
crossCheck: result.crossCheck,
|
|
2067
|
-
seedHonored: result.seedHonored,
|
|
2068
|
-
samples: { k: result.k, seeds: [...result.seeds], modelCalls: result.spend.modelCalls }
|
|
2069
|
-
};
|
|
2070
|
-
}
|
|
2071
|
-
function misdirectionAuditKey(target) {
|
|
2072
|
-
return `misdirection/${(0, import_agent_qa_runner_local2.slugBranch)(target.runId)}/${(0, import_agent_qa_runner_local2.slugBranch)(target.testId)}/${String(target.stepIndex)}${target.recordId === void 0 ? "" : `/${(0, import_agent_qa_runner_local2.slugBranch)(target.recordId)}`}`;
|
|
2073
|
-
}
|
|
2074
|
-
function misdirectionAuditDir(target) {
|
|
2075
|
-
return (0, import_node_path14.join)(target.agentQaRoot, import_agent_qa_eval_local.GATE2_AUDIT_SUBDIR, (0, import_agent_qa_runner_local2.slugBranch)(misdirectionAuditKey(target)));
|
|
2076
|
-
}
|
|
2077
|
-
function isFeedbackStateNovel(session, input) {
|
|
2078
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2079
|
-
if (digest2.kind === "no-eval-root") return false;
|
|
2080
|
-
if (digest2.kind !== "digest") throw new TypeError("feedback corpus unavailable");
|
|
2081
|
-
return (0, import_agent_qa_eval_local.isNovelState)({ ...input, committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval") });
|
|
2082
|
-
}
|
|
2083
|
-
async function runFeedbackStaging(session, input) {
|
|
2084
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2085
|
-
if (digest2.kind !== "digest") return digest2;
|
|
2086
|
-
return (0, import_agent_qa_eval_local.stageFeedbackEntry)({
|
|
2087
|
-
...input,
|
|
2088
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
2089
|
-
mask: committedCorpusMask(session)
|
|
2090
|
-
});
|
|
2091
|
-
}
|
|
2092
|
-
async function runSeedFromFeedbackVerb(session, opts) {
|
|
2093
|
-
const stagingDir = (0, import_node_path14.join)(opts.agentQaRoot, import_agent_qa_eval_local.FEEDBACK_STAGING_SUBDIR);
|
|
2094
|
-
const configArg = `'${(0, import_node_path14.resolve)(session.configPath).replaceAll("'", `'"'"'`)}'`;
|
|
2095
|
-
const floorCommand = `agent-qa --config ${configArg} eval floor`;
|
|
2096
|
-
const base = { entries: [], refusals: [], staleReceipts: [], writesStarted: false, failures: [], stagingDir, floorCommand };
|
|
2097
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2098
|
-
if (digest2.kind !== "digest") {
|
|
2099
|
-
const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
|
|
2100
|
-
return { ...base, kind: "refused", reason, failures: [{ phase: "before-write", reason }] };
|
|
2101
|
-
}
|
|
2102
|
-
try {
|
|
2103
|
-
const receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
|
|
2104
|
-
const result = await (0, import_agent_qa_eval_local.admitStagedFeedback)({
|
|
2105
|
-
...opts,
|
|
2106
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
2107
|
-
mask: committedCorpusMask(session)
|
|
2108
|
-
});
|
|
2109
|
-
const selected = opts.dryRun ? result.entries : result.admitted;
|
|
2110
|
-
const entries = selected.map((entry) => ({
|
|
2111
|
-
cassetteId: entry.extracted.cassette.id,
|
|
2112
|
-
product: entry.product,
|
|
2113
|
-
split: "natural",
|
|
2114
|
-
source: "canary-feedback",
|
|
2115
|
-
benefit: entry.benefit,
|
|
2116
|
-
attributedObservationId: entry.attributedObservationId,
|
|
2117
|
-
runId: entry.runId,
|
|
2118
|
-
stepIndex: entry.stepIndex,
|
|
2119
|
-
captured_at_ordinal: entry.extracted.cassette.receipt.buildOrdinal
|
|
2120
|
-
}));
|
|
2121
|
-
const moves = opts.dryRun ? entries.length > 0 : result.nextCorpusDigest === void 0 ? void 0 : result.nextCorpusDigest !== result.previousCorpusDigest;
|
|
2122
|
-
const refused = result.refusals.length > 0 || result.failures.length > 0;
|
|
2123
|
-
return {
|
|
2124
|
-
kind: refused ? "refused" : result.entries.length === 0 ? "nothing-staged" : opts.dryRun ? "dry-run" : "admitted",
|
|
2125
|
-
...refused ? { reason: "admission-refused" } : {},
|
|
2126
|
-
entries,
|
|
2127
|
-
refusals: result.refusals,
|
|
2128
|
-
writesStarted: result.writesStarted,
|
|
2129
|
-
failures: result.failures,
|
|
2130
|
-
staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === result.previousCorpusDigest) : [],
|
|
2131
|
-
previousCorpusDigest: result.previousCorpusDigest,
|
|
2132
|
-
...result.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: result.nextCorpusDigest },
|
|
2133
|
-
stagingDir,
|
|
2134
|
-
floorCommand
|
|
2135
|
-
};
|
|
2136
|
-
} catch {
|
|
2137
|
-
return {
|
|
2138
|
-
...base,
|
|
2139
|
-
kind: "refused",
|
|
2140
|
-
reason: "feedback-unavailable",
|
|
2141
|
-
failures: [{ phase: "before-write", reason: "feedback-unavailable" }],
|
|
2142
|
-
previousCorpusDigest: digest2.corpusDigest
|
|
2143
|
-
};
|
|
2144
|
-
}
|
|
2145
|
-
}
|
|
2146
|
-
function authoringModelDisclosureOf(model) {
|
|
2147
|
-
return {
|
|
2148
|
-
idDigest: (0, import_node_crypto5.createHash)("sha256").update(model.id, "utf8").digest("hex"),
|
|
2149
|
-
family: model.family
|
|
2150
|
-
};
|
|
2151
|
-
}
|
|
2152
|
-
function adversarialHostOptions(session) {
|
|
2153
|
-
return {
|
|
2154
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
2155
|
-
knowledgeRoot: (0, import_agent_qa_core4.resolveKnowledgeRoot)(session.config.knowledge, session.configDir),
|
|
2156
|
-
readStandingGateReceipts: import_agent_qa_agent_local4.LOCAL_CURATION_CAPABILITIES.readStandingGateReceipts
|
|
2157
|
-
};
|
|
2158
|
-
}
|
|
2159
|
-
function runRetireAdversarialVerb(session, opts) {
|
|
2160
|
-
const result = (0, import_agent_qa_eval_local.retireStagedAdversarialBatch)({
|
|
2161
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
2162
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
2163
|
-
product: opts.product,
|
|
2164
|
-
batchDigest: opts.batch
|
|
2165
|
-
});
|
|
2166
|
-
return {
|
|
2167
|
-
...result,
|
|
2168
|
-
modelCalls: 0,
|
|
2169
|
-
admitsNothing: true,
|
|
2170
|
-
settledWorkPreserved: true,
|
|
2171
|
-
newMeasurementStaleness: false
|
|
2172
|
-
};
|
|
2173
|
-
}
|
|
2174
|
-
async function runAdversarialVerb(session, opts) {
|
|
2175
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2176
|
-
const product = opts.product ?? "default";
|
|
2177
|
-
const stagingDir = (0, import_node_path14.join)(opts.agentQaRoot, import_agent_qa_eval_local.ADVERSARIAL_STAGING_SUBDIR, (0, import_agent_qa_runner_local2.slugBranch)(product));
|
|
2178
|
-
const configArg = `'${(0, import_node_path14.resolve)(session.configPath).replaceAll("'", `'"'"'`)}'`;
|
|
2179
|
-
const floorCommand = `agent-qa --config ${configArg} eval floor`;
|
|
2180
|
-
const cap = (0, import_agent_qa_eval_local.resolveAdversarialSeedCap)(
|
|
2181
|
-
opts.maxSeeds !== void 0 ? { value: opts.maxSeeds, source: "flag" } : session.evalConfig.adversarial?.maxSeedsPerRefresh !== void 0 ? { value: session.evalConfig.adversarial.maxSeedsPerRefresh, source: "config" } : {}
|
|
2182
|
-
);
|
|
2183
|
-
const probeCap = (0, import_agent_qa_eval_local.resolveAdversarialProbeCallCap)(
|
|
2184
|
-
opts.maxProbeCalls !== void 0 ? { value: opts.maxProbeCalls, source: "flag" } : session.evalConfig.adversarial?.maxProbeCalls !== void 0 ? { value: session.evalConfig.adversarial.maxProbeCalls, source: "config" } : {}
|
|
2185
|
-
);
|
|
2186
|
-
const base = {
|
|
2187
|
-
kind: "refused",
|
|
2188
|
-
product,
|
|
2189
|
-
stagingDir,
|
|
2190
|
-
staged: [],
|
|
2191
|
-
admitted: [],
|
|
2192
|
-
unchanged: [],
|
|
2193
|
-
remaining: [],
|
|
2194
|
-
refusals: [],
|
|
2195
|
-
settledRefusals: [],
|
|
2196
|
-
stagingSkips: [],
|
|
2197
|
-
skips: [],
|
|
2198
|
-
seedsConsidered: 0,
|
|
2199
|
-
seedsDeferred: 0,
|
|
2200
|
-
eligibleSeeds: 0,
|
|
2201
|
-
cap,
|
|
2202
|
-
writesStarted: false,
|
|
2203
|
-
staleReceipts: [],
|
|
2204
|
-
floorCommand,
|
|
2205
|
-
probeCap,
|
|
2206
|
-
spend: { modelCalls: 0, promptTokens: 0, completionTokens: 0, usageKnown: true },
|
|
2207
|
-
probeSkips: [],
|
|
2208
|
-
probeRefusals: []
|
|
2209
|
-
};
|
|
2210
|
-
const digest2 = readCanaryCorpusDigest(session);
|
|
2211
|
-
if (digest2.kind !== "digest") {
|
|
2212
|
-
const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
|
|
2213
|
-
return { ...base, kind: "refused", reason };
|
|
2214
|
-
}
|
|
2215
|
-
if (opts.admit === true) {
|
|
2216
|
-
if (opts.batch === void 0) {
|
|
2217
|
-
return { ...base, kind: "refused", reason: "admit-requires-batch" };
|
|
2218
|
-
}
|
|
2219
|
-
let receipts = [];
|
|
2220
|
-
const refreshed = (0, import_agent_qa_eval_local.refreshAdversarialSplit)({
|
|
2221
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2222
|
-
committedRoot,
|
|
2223
|
-
product,
|
|
2224
|
-
batchDigest: opts.batch,
|
|
2225
|
-
beforeWrites: () => {
|
|
2226
|
-
receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
|
|
2227
|
-
},
|
|
2228
|
-
...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun },
|
|
2229
|
-
mask: committedCorpusMask(session)
|
|
2230
|
-
});
|
|
2231
|
-
const moves = refreshed.nextCorpusDigest === void 0 ? refreshed.writesStarted ? void 0 : false : refreshed.nextCorpusDigest !== refreshed.previousCorpusDigest;
|
|
2232
|
-
return {
|
|
2233
|
-
...base,
|
|
2234
|
-
kind: refreshed.kind,
|
|
2235
|
-
...refreshed.reason === void 0 ? {} : { reason: refreshed.reason },
|
|
2236
|
-
batchDigest: refreshed.batchDigest,
|
|
2237
|
-
admitted: refreshed.admitted,
|
|
2238
|
-
unchanged: refreshed.unchanged,
|
|
2239
|
-
remaining: refreshed.remaining,
|
|
2240
|
-
refusals: refreshed.refusals,
|
|
2241
|
-
stagingSkips: refreshed.stagingSkips,
|
|
2242
|
-
writesStarted: refreshed.writesStarted,
|
|
2243
|
-
...refreshed.retirement === void 0 ? {} : { retirement: refreshed.retirement },
|
|
2244
|
-
...refreshed.lifecycleBusy === void 0 ? {} : { lifecycleBusy: refreshed.lifecycleBusy },
|
|
2245
|
-
staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === refreshed.previousCorpusDigest) : [],
|
|
2246
|
-
...refreshed.previousCorpusDigest === void 0 ? {} : { previousCorpusDigest: refreshed.previousCorpusDigest },
|
|
2247
|
-
...refreshed.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: refreshed.nextCorpusDigest }
|
|
2248
|
-
};
|
|
2249
|
-
}
|
|
2250
|
-
const probe = await resolveProbeBinding(session, opts, product);
|
|
2251
|
-
const result = await (0, import_agent_qa_eval_local.stageAdversarialProposal)({
|
|
2252
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2253
|
-
committedRoot,
|
|
2254
|
-
product,
|
|
2255
|
-
...opts.mutators === void 0 ? {} : { mutators: opts.mutators },
|
|
2256
|
-
// Only a NON-default cap is pushed down: the composition owns the recorded default, and
|
|
2257
|
-
// handing it back an unattributed value would have it record an operator choice nobody made.
|
|
2258
|
-
...cap.source === "default" ? {} : { maxSeeds: cap.value, maxSeedsSource: cap.source },
|
|
2259
|
-
...probeCap.source === "default" ? {} : { maxProbeCalls: probeCap.value, maxProbeCallsSource: probeCap.source },
|
|
2260
|
-
...probe.noProbeModel === void 0 ? {} : { noProbeModel: probe.noProbeModel },
|
|
2261
|
-
// A DRY RUN authors nothing. A preview that spends money is not a preview, and there is no
|
|
2262
|
-
// way to un-spend it once the batch it previewed is discarded.
|
|
2263
|
-
...probe.author === void 0 || opts.dryRun === true ? {} : { probeAuthor: probe.author, authoringModel: probe.authoringModel },
|
|
2264
|
-
...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun }
|
|
2265
|
-
});
|
|
2266
|
-
return {
|
|
2267
|
-
...base,
|
|
2268
|
-
kind: result.kind,
|
|
2269
|
-
...result.batchDigest === void 0 ? {} : { batchDigest: result.batchDigest },
|
|
2270
|
-
staged: result.staged,
|
|
2271
|
-
refusals: result.refusals,
|
|
2272
|
-
settledRefusals: result.settledRefusals,
|
|
2273
|
-
stagingSkips: result.stagingSkips,
|
|
2274
|
-
skips: result.skips,
|
|
2275
|
-
seedsConsidered: result.seedsConsidered,
|
|
2276
|
-
seedsDeferred: result.seedsDeferred,
|
|
2277
|
-
eligibleSeeds: result.eligibleSeeds,
|
|
2278
|
-
cap: result.cap,
|
|
2279
|
-
tableVersion: result.tableVersion,
|
|
2280
|
-
tableHash: result.tableHash,
|
|
2281
|
-
previousCorpusDigest: digest2.corpusDigest,
|
|
2282
|
-
// Staging commits nothing, so nothing it does can stale a standing receipt.
|
|
2283
|
-
staleReceipts: [],
|
|
2284
|
-
...result.noProbeModel === void 0 ? {} : { noProbeModel: result.noProbeModel },
|
|
2285
|
-
// The raw id stays on the binding, where derivation and the committed stamp need it. What
|
|
2286
|
-
// leaves on the result is the family and a digest -- see AdversarialVerbResult.
|
|
2287
|
-
...result.authoringModel === void 0 ? {} : { authoringModel: authoringModelDisclosureOf(result.authoringModel) },
|
|
2288
|
-
probeCap: result.probeCap,
|
|
2289
|
-
spend: result.spend,
|
|
2290
|
-
probeSkips: result.probeSkips,
|
|
2291
|
-
probeRefusals: result.probeRefusals
|
|
2292
|
-
};
|
|
2293
|
-
}
|
|
2294
|
-
async function resolveProbeBinding(session, opts, product) {
|
|
2295
|
-
const capability = opts.probeCapability ?? await (await Promise.resolve().then(() => (init_adversarial_probe_preflight(), adversarial_probe_preflight_exports))).resolveAdversarialProbeCapability(session);
|
|
2296
|
-
if (capability.kind !== "probe-author") {
|
|
2297
|
-
return { noProbeModel: { kind: "no-probe-model", reason: capability.reason } };
|
|
2298
|
-
}
|
|
2299
|
-
const bind = opts.bindProbeAuthor ?? await defaultProbeAuthorBinder(session);
|
|
2300
|
-
const binding = {
|
|
2301
|
-
onCallAttempted: (attempt) => {
|
|
2302
|
-
const state = (0, import_agent_qa_eval_local.readProbeAttempt)({
|
|
2303
|
-
agentQaRoot: opts.agentQaRoot,
|
|
2304
|
-
key: {
|
|
2305
|
-
product,
|
|
2306
|
-
seedCassetteId: attempt.seedCassetteId,
|
|
2307
|
-
kind: attempt.kind,
|
|
2308
|
-
promptVersion: attempt.promptVersion
|
|
2309
|
-
}
|
|
2310
|
-
});
|
|
2311
|
-
if (state.kind === "unattempted") {
|
|
2312
|
-
throw new Error("This probe call holds no durable reservation. Nothing reaches a provider before the spend is recorded.");
|
|
2313
|
-
}
|
|
2314
|
-
}
|
|
2315
|
-
};
|
|
2316
|
-
return { author: bind(capability.author, binding), authoringModel: capability.author.authoringModel };
|
|
2317
|
-
}
|
|
2318
|
-
async function defaultProbeAuthorBinder(session) {
|
|
2319
|
-
const { createAdversarialProbeAuthor: createAdversarialProbeAuthor2 } = await Promise.resolve().then(() => (init_adversarial_probe_author(), adversarial_probe_author_exports));
|
|
2320
|
-
return (author, binding) => createAdversarialProbeAuthor2({
|
|
2321
|
-
author,
|
|
2322
|
-
executionContext: session.executionContext,
|
|
2323
|
-
mask: session.mask,
|
|
2324
|
-
onCallAttempted: binding.onCallAttempted
|
|
2325
|
-
});
|
|
2326
|
-
}
|
|
2327
|
-
function listAdversarialBatches(session, opts) {
|
|
2328
|
-
void session;
|
|
2329
|
-
return (0, import_agent_qa_eval_local.listStagedAdversarial)(opts).map((batch) => ({
|
|
2330
|
-
dir: batch.dir,
|
|
2331
|
-
batchDigest: batch.manifest.batchDigest,
|
|
2332
|
-
product: batch.manifest.product,
|
|
2333
|
-
tableVersion: batch.manifest.tableVersion,
|
|
2334
|
-
tableHash: batch.manifest.tableHash,
|
|
2335
|
-
stagedAt: batch.manifest.stagedAt,
|
|
2336
|
-
cases: batch.manifest.cases.map((row) => ({
|
|
2337
|
-
cassetteId: row.cassetteId,
|
|
2338
|
-
seedCassetteId: row.seedCassetteId,
|
|
2339
|
-
mutator: row.mutator,
|
|
2340
|
-
kind: row.kind,
|
|
2341
|
-
tableEntryId: row.tableEntryId
|
|
2342
|
-
}))
|
|
2343
|
-
}));
|
|
2344
|
-
}
|
|
2345
|
-
function feedbackSeedHostOptions(session) {
|
|
2346
|
-
return {
|
|
2347
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
2348
|
-
knowledgeRoot: (0, import_agent_qa_core4.resolveKnowledgeRoot)(session.config.knowledge, session.configDir),
|
|
2349
|
-
readStandingGateReceipts: import_agent_qa_agent_local4.LOCAL_CURATION_CAPABILITIES.readStandingGateReceipts
|
|
2350
|
-
};
|
|
2351
|
-
}
|
|
2352
|
-
async function runReplayGateVerb(session, candidateId, opts = {}) {
|
|
2353
|
-
const k = opts.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
|
|
2354
|
-
if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
|
|
2355
|
-
throw new TypeError("--k must be an odd integer of at least 3");
|
|
2356
|
-
}
|
|
2357
|
-
const kSource = opts.k !== void 0 ? "flag" : session.evalConfig.k !== void 0 ? "config" : "default";
|
|
2358
|
-
const maxScreens = opts.maxScreens ?? session.evalConfig.gate1?.maxScreens ?? GATE1_DEFAULT_MAX_SCREENS;
|
|
2359
|
-
const maxScreensSource = opts.maxScreens !== void 0 ? "flag" : session.evalConfig.gate1?.maxScreens !== void 0 ? "config" : "default";
|
|
2360
|
-
if (!Number.isSafeInteger(maxScreens) || maxScreens < 1) {
|
|
2361
|
-
throw new TypeError("--max-screens must be a positive integer");
|
|
2362
|
-
}
|
|
2363
|
-
const identity = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2364
|
-
const profile = await resolveReplayProfileDeclaration(session);
|
|
2365
|
-
let execution;
|
|
2366
|
-
let mask = session.mask;
|
|
2367
|
-
const names = selectedEvalJudgeNames(session.evalConfig);
|
|
2368
|
-
const prepare = () => execution ??= resolveEvalSession({
|
|
2369
|
-
configPath: session.configPath,
|
|
2370
|
-
flags: { judge: names }
|
|
2371
|
-
}).then(async (full) => {
|
|
2372
|
-
if ((await resolveGateProfileIdentity(full, opts.plannerOverrides)).profileKey !== identity.profileKey) {
|
|
2373
|
-
throw new EvalJudgeDeclarationError("Gate profile identity changed before execution; re-run eval replay-gate.");
|
|
2374
|
-
}
|
|
2375
|
-
mask = full.mask;
|
|
2376
|
-
return full;
|
|
2377
|
-
});
|
|
2378
|
-
const declarations = await resolveEvalJudgeDeclarations(session);
|
|
2379
|
-
const lazyJudges = declarations.map(({ name, profile: profile2 }) => ({
|
|
2380
|
-
profile: profile2,
|
|
2381
|
-
modelOverrides: opts.judgeOverrides,
|
|
2382
|
-
resolveAuth: async () => {
|
|
2383
|
-
const full = await prepare();
|
|
2384
|
-
return (await resolveNamedProfile(name, full.configPath)).auth;
|
|
2385
|
-
}
|
|
2386
|
-
}));
|
|
2387
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2388
|
-
const floor = (0, import_agent_qa_eval_local.listNoiseFloorArtefacts)(committedRoot).artefacts.find(
|
|
2389
|
-
(entry) => entry.path === (0, import_node_path14.join)(committedRoot, (0, import_agent_qa_eval_local.committedNoiseFloorPath)(identity.profileKey)) && entry.artefact.profileKey === identity.profileKey
|
|
2390
|
-
)?.artefact;
|
|
2391
|
-
const { createReplayGateHost: createReplayGateHost2 } = await Promise.resolve().then(() => (init_replay_gate_host(), replay_gate_host_exports));
|
|
2392
|
-
const result = await createReplayGateHost2({
|
|
2393
|
-
...opts.resolveTrustPriors === void 0 ? {} : { resolveTrustPriors: opts.resolveTrustPriors },
|
|
2394
|
-
// 422 D-14: the lease the caller lent this verb, re-presented to the bridge as a resolver
|
|
2395
|
-
// over the value this call already holds. The bridge calls it once, at its own claim.
|
|
2396
|
-
...opts.claimAuthority === void 0 ? {} : { resolveClaimAuthority: () => opts.claimAuthority },
|
|
2397
|
-
configPath: session.configPath,
|
|
2398
|
-
agentQaRoot: opts.agentQaRoot ?? (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
2399
|
-
record: opts.dryRun !== true,
|
|
2400
|
-
session: {
|
|
2401
|
-
...session,
|
|
2402
|
-
mask: (text) => mask(text)
|
|
2403
|
-
},
|
|
2404
|
-
composition: {
|
|
2405
|
-
identity,
|
|
2406
|
-
baselineWarmup: {
|
|
2407
|
-
configPath: session.configPath,
|
|
2408
|
-
...names.length > 1 ? { judges: names } : { judge: names[0] }
|
|
2409
|
-
},
|
|
2410
|
-
k,
|
|
2411
|
-
kSource,
|
|
2412
|
-
maxScreens,
|
|
2413
|
-
planner: {
|
|
2414
|
-
profile,
|
|
2415
|
-
modelOverrides: opts.plannerOverrides,
|
|
2416
|
-
resolveAuth: async () => (await resolveReplayPlanner(await prepare())).auth
|
|
2417
|
-
},
|
|
2418
|
-
...lazyJudges.length > 1 ? { judgeMembers: lazyJudges } : { judge: lazyJudges[0] },
|
|
2419
|
-
judgeOverrideConfigured: names.length <= 1 && session.evalConfig.judgeAllowSameFamily === true,
|
|
2420
|
-
baselineSeed: session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed, source: "config" } : { floorSeeds: floor?.seeds ?? [] },
|
|
2421
|
-
clock: opts.clock,
|
|
2422
|
-
onProgress: opts.onProgress
|
|
2423
|
-
}
|
|
2424
|
-
}).run(candidateId);
|
|
2425
|
-
const capNotice = `cap: maxScreens=${maxScreens} (${maxScreensSource})`;
|
|
2426
|
-
return {
|
|
2427
|
-
...result,
|
|
2428
|
-
maxScreens,
|
|
2429
|
-
maxScreensSource,
|
|
2430
|
-
k,
|
|
2431
|
-
kSource,
|
|
2432
|
-
notices: [...result.notices, ...names.length > 1 && session.evalConfig.judgeAllowSameFamily === true ? ["eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint."] : [], capNotice],
|
|
2433
|
-
...result.kind === "receipt" ? { lines: [...result.lines, capNotice] } : {}
|
|
2434
|
-
};
|
|
2435
|
-
}
|
|
2436
|
-
async function runFloorVerb(session, opts = {}) {
|
|
2437
|
-
const repeats = opts.repeats ?? FLOOR_DEFAULT_REPEATS;
|
|
2438
|
-
const seeds = [...new Set(opts.seeds ?? FLOOR_DEFAULT_SEEDS)];
|
|
2439
|
-
const bootstrapSeed = opts.bootstrapSeed ?? FLOOR_DEFAULT_BOOTSTRAP_SEED;
|
|
2440
|
-
if (!Number.isSafeInteger(repeats) || repeats < import_agent_qa_eval.FLOOR_PROTOCOL_MINIMUM.repeats || seeds.length < import_agent_qa_eval.FLOOR_PROTOCOL_MINIMUM.seeds || !seeds.every(Number.isSafeInteger)) {
|
|
2441
|
-
throw new import_agent_qa_eval_local.FloorProtocolError();
|
|
2442
|
-
}
|
|
2443
|
-
if (!Number.isSafeInteger(bootstrapSeed)) throw new TypeError("--bootstrap-seed must be a safe integer");
|
|
2444
|
-
let classifierRepeatability;
|
|
2445
|
-
if (opts.classifierRepeatability !== void 0) {
|
|
2446
|
-
const path = (0, import_node_path14.resolve)(session.configDir, opts.classifierRepeatability);
|
|
2447
|
-
let value;
|
|
2448
|
-
try {
|
|
2449
|
-
value = JSON.parse(await (0, import_promises3.readFile)(path, "utf8"));
|
|
2450
|
-
} catch (error) {
|
|
2451
|
-
throw new TypeError(`classifier repeatability unreadable at ${path}: ${error.message}`);
|
|
2452
|
-
}
|
|
2453
|
-
const guarded = (0, import_agent_qa_eval.guardClassifierRepeatability)(value);
|
|
2454
|
-
if (guarded.kind === "rejected") {
|
|
2455
|
-
throw new TypeError(`classifier repeatability rejected at ${path}: ${guarded.reason}`);
|
|
2456
|
-
}
|
|
2457
|
-
classifierRepeatability = guarded.repeatability;
|
|
2458
|
-
}
|
|
2459
|
-
const planner = opts.fromMeasured === void 0 ? await resolveReplayPlanner(session) : { profile: await resolveReplayProfileDeclaration(session) };
|
|
2460
|
-
const judgeResolution = opts.fromMeasured === void 0 ? await resolveEvalJudge(session) : {};
|
|
2461
|
-
const judge = withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides);
|
|
2462
|
-
const { plannerModelId, matcherVersion, judgeModelId, judgeVersion } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2463
|
-
if (opts.fromMeasured === void 0 && (0, import_agent_qa_eval_local.replayJudgeRequest)(judge)?.judgeModelId !== judgeModelId) {
|
|
2464
|
-
throw new EvalJudgeDeclarationError("Floor judge identity changed before execution; re-run eval floor.");
|
|
2465
|
-
}
|
|
2466
|
-
const profileKeyInputs = {
|
|
2467
|
-
plannerModelId,
|
|
2468
|
-
matcherVersion,
|
|
2469
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
|
|
2470
|
-
};
|
|
2471
|
-
const costLines = [];
|
|
2472
|
-
const result = await (0, import_agent_qa_eval_local.runNoiseFloorMeasurement)({
|
|
2473
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
2474
|
-
cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
|
|
2475
|
-
agentQaRoot: (0, import_node_path14.resolve)(session.configDir, ".agent-qa"),
|
|
2476
|
-
...planner,
|
|
2477
|
-
...opts.fromMeasured === void 0 ? {} : { fromMeasured: (0, import_node_path14.resolve)(session.configDir, opts.fromMeasured) },
|
|
2478
|
-
modelOverrides: opts.plannerOverrides,
|
|
2479
|
-
judge,
|
|
2480
|
-
mask: session.mask,
|
|
2481
|
-
profileKeyInputs,
|
|
2482
|
-
seeds,
|
|
2483
|
-
repeats,
|
|
2484
|
-
bootstrapSeed,
|
|
2485
|
-
classifierRepeatability,
|
|
2486
|
-
explanation: opts.explanation,
|
|
2487
|
-
commit: opts.commit === true,
|
|
2488
|
-
onProgress: (line) => {
|
|
2489
|
-
if (/^floor r\d+ seed=/.test(line)) costLines.push(line);
|
|
2490
|
-
process.stderr.write(line + "\n");
|
|
2491
|
-
}
|
|
2492
|
-
});
|
|
2493
|
-
const notices = [...result.notices];
|
|
2494
|
-
if (judgeResolution.notice !== void 0) notices.unshift(judgeResolution.notice);
|
|
2495
|
-
if (result.committed?.kind === "refused") notices.push((0, import_agent_qa_eval.renderFloorRefusal)(result.committed.refusal));
|
|
2496
|
-
return {
|
|
2497
|
-
...result,
|
|
2498
|
-
notices,
|
|
2499
|
-
repeats,
|
|
2500
|
-
seeds,
|
|
2501
|
-
bootstrapSeed,
|
|
2502
|
-
costLines,
|
|
2503
|
-
recordedDefaults: opts.repeats === void 0 && opts.seeds === void 0 && opts.bootstrapSeed === void 0,
|
|
2504
|
-
corpusDigest: result.artefact.corpusDigest
|
|
2505
|
-
};
|
|
2506
|
-
}
|
|
2507
|
-
async function runGateVerb(session, opts = {}) {
|
|
2508
|
-
if (opts.stagedDir !== void 0 && opts.report === void 0) {
|
|
2509
|
-
throw new TypeError("--staged-dir requires --report <batchKey>");
|
|
2510
|
-
}
|
|
2511
|
-
const declaredSeed = opts.seed ?? session.evalConfig.seed;
|
|
2512
|
-
if (declaredSeed !== void 0 && !Number.isSafeInteger(declaredSeed)) {
|
|
2513
|
-
throw new TypeError("--seed must be a safe integer");
|
|
2514
|
-
}
|
|
2515
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2516
|
-
const agentQaRoot = (0, import_node_path14.resolve)(session.configDir, ".agent-qa");
|
|
2517
|
-
const { plannerModelId, judgeModelId, judgeVersion, fingerprint, matcherVersion, profileKey } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
|
|
2518
|
-
const profileKeyInputs = {
|
|
2519
|
-
plannerModelId,
|
|
2520
|
-
matcherVersion,
|
|
2521
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
|
|
2522
|
-
};
|
|
2523
|
-
const floorPath = (0, import_node_path14.join)(committedRoot, (0, import_agent_qa_eval_local.committedNoiseFloorPath)(profileKey));
|
|
2524
|
-
const floorSeed = declaredSeed === void 0 ? (0, import_agent_qa_eval_local.listNoiseFloorArtefacts)(committedRoot).artefacts.find((entry) => entry.path === floorPath && entry.profileKey === profileKey)?.artefact.seeds[0] : void 0;
|
|
2525
|
-
const seed = declaredSeed ?? floorSeed ?? FLOOR_DEFAULT_SEEDS[0];
|
|
2526
|
-
const seedSource = opts.seed !== void 0 ? "flag" : session.evalConfig.seed !== void 0 ? "config" : floorSeed !== void 0 ? "floor" : "default";
|
|
2527
|
-
const probedDirs = [];
|
|
2528
|
-
const notices = [];
|
|
2529
|
-
let mode;
|
|
2530
|
-
if (opts.report !== void 0) {
|
|
2531
|
-
const choices = opts.stagedDir !== void 0 ? [{ dir: (0, import_node_path14.resolve)(session.configDir, opts.stagedDir), label: "--staged-dir" }] : [
|
|
2532
|
-
{ dir: (0, import_node_path14.join)(agentQaRoot, "eval", "gate", (0, import_agent_qa_runner_local2.slugBranch)(profileKey), "reports"), label: "gate root" },
|
|
2533
|
-
{ dir: (0, import_node_path14.join)(agentQaRoot, import_agent_qa_eval_local.DEFAULT_EVAL_REPORTS_SUBDIR), label: "default staging root" }
|
|
2534
|
-
];
|
|
2535
|
-
const files = [];
|
|
2536
|
-
let stagedOutputDir;
|
|
2537
|
-
for (const choice of choices) {
|
|
2538
|
-
probedDirs.push(choice.dir);
|
|
2539
|
-
const file = (0, import_node_path14.join)(choice.dir, `${(0, import_agent_qa_runner_local2.slugBranch)(opts.report)}.json`);
|
|
2540
|
-
files.push(file);
|
|
2541
|
-
try {
|
|
2542
|
-
(0, import_node_fs7.lstatSync)(file);
|
|
2543
|
-
} catch (error) {
|
|
2544
|
-
if (error.code === "ENOENT") continue;
|
|
2545
|
-
}
|
|
2546
|
-
stagedOutputDir = choice.dir;
|
|
2547
|
-
notices.push(`gate --report: reading ${file} (${choice.label})`);
|
|
2548
|
-
break;
|
|
2549
|
-
}
|
|
2550
|
-
if (stagedOutputDir === void 0) {
|
|
2551
|
-
return {
|
|
2552
|
-
kind: "refused",
|
|
2553
|
-
refusal: {
|
|
2554
|
-
path: files[files.length - 1],
|
|
2555
|
-
reason: `staged report absent; probed ${files.join(", ")}`
|
|
2556
|
-
},
|
|
2557
|
-
notices,
|
|
2558
|
-
seed,
|
|
2559
|
-
seedSource,
|
|
2560
|
-
probedDirs
|
|
2561
|
-
};
|
|
2562
|
-
}
|
|
2563
|
-
mode = {
|
|
2564
|
-
kind: "report",
|
|
2565
|
-
batchKey: opts.report,
|
|
2566
|
-
stagedOutputDir,
|
|
2567
|
-
profile: await resolveReplayProfileDeclaration(session),
|
|
2568
|
-
modelOverrides: opts.plannerOverrides
|
|
2569
|
-
};
|
|
2570
|
-
} else {
|
|
2571
|
-
const planner = await resolveReplayPlanner(session);
|
|
2572
|
-
const judgeResolution = await resolveEvalJudge(session);
|
|
2573
|
-
if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
|
|
2574
|
-
mode = {
|
|
2575
|
-
kind: "fresh",
|
|
2576
|
-
...planner,
|
|
2577
|
-
modelOverrides: opts.plannerOverrides,
|
|
2578
|
-
judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides),
|
|
2579
|
-
mask: session.mask,
|
|
2580
|
-
seed,
|
|
2581
|
-
cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
|
|
2582
|
-
onProgress: (line) => process.stderr.write(line + "\n")
|
|
2583
|
-
};
|
|
2584
|
-
}
|
|
2585
|
-
const result = await (0, import_agent_qa_eval_local.runReleaseGate)({ committedRoot, agentQaRoot, profileKeyInputs, mode });
|
|
2586
|
-
return { ...result, notices: [...notices, ...result.notices], seed, seedSource, probedDirs };
|
|
2587
|
-
}
|
|
2588
|
-
function isExclusionVerdict(verdict) {
|
|
2589
|
-
return EXCLUDED_VERDICT_CLASSES.has(verdict.class);
|
|
2590
|
-
}
|
|
2591
|
-
async function runSeedVerb(session, filters = {}, params = {}) {
|
|
2592
|
-
const agentQaRoot = (0, import_node_path14.resolve)(session.configDir, ".agent-qa");
|
|
2593
|
-
const runsRoot = (0, import_node_path14.resolve)(agentQaRoot, import_agent_qa_runner_local2.RUNS_SUBDIR);
|
|
2594
|
-
let runSlugs = listRunSlugsWithHandoff(runsRoot);
|
|
2595
|
-
if (filters.run !== void 0) {
|
|
2596
|
-
const named = filters.run;
|
|
2597
|
-
runSlugs = runSlugs.filter((slug) => slug === named || slug === (0, import_agent_qa_runner_local2.slugBranch)(named));
|
|
2598
|
-
if (runSlugs.length === 0) {
|
|
2599
|
-
throw new EvalNamedInputError("run", named, runsRoot, "seed");
|
|
2600
|
-
}
|
|
2601
|
-
}
|
|
2602
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2603
|
-
const silverPerTestCap = params.silverPerTestCap ?? session.evalConfig.seeding?.silverPerTestCap;
|
|
2604
|
-
const result = {
|
|
2605
|
-
membersExcluded: [],
|
|
2606
|
-
admissionConflicts: [],
|
|
2607
|
-
silverExclusionRows: [],
|
|
2608
|
-
supersededOriginalCount: 0,
|
|
2609
|
-
retirementCounts: { missing: 0, conflict: 0 },
|
|
2610
|
-
runsSwept: runSlugs.length,
|
|
2611
|
-
runsSkipped: [],
|
|
2612
|
-
runsFiltered: 0,
|
|
2613
|
-
runsExcluded: [],
|
|
2614
|
-
runsAdmitted: [],
|
|
2615
|
-
notices: [],
|
|
2616
|
-
handoffsSeeded: 0,
|
|
2617
|
-
seededCount: 0,
|
|
2618
|
-
unchangedCount: 0,
|
|
2619
|
-
refusals: [],
|
|
2620
|
-
pairSkips: 0,
|
|
2621
|
-
silverExclusions: 0,
|
|
2622
|
-
pairCounts: { total: 0, sameState: 0, differingState: 0 },
|
|
2623
|
-
contracts: {},
|
|
2624
|
-
scaffold: {},
|
|
2625
|
-
partitions: [],
|
|
2626
|
-
exclusionsWritten: [],
|
|
2627
|
-
committedRoot
|
|
2628
|
-
};
|
|
2629
|
-
const admitted = [];
|
|
2630
|
-
const initialCommitted = (0, import_agent_qa_eval_local.readCommittedCorpus)(committedRoot);
|
|
2631
|
-
for (const runSlug of runSlugs) {
|
|
2632
|
-
try {
|
|
2633
|
-
const handoffPath = (0, import_node_path14.resolve)(runsRoot, runSlug, import_agent_qa_runner_local2.HANDOFF_FILE);
|
|
2634
|
-
const located = (0, import_node_path14.relative)((0, import_node_fs7.realpathSync)(agentQaRoot), (0, import_node_fs7.realpathSync)(handoffPath));
|
|
2635
|
-
if (located === ".." || located.startsWith("../") || located.startsWith("..\\") || (0, import_node_path14.isAbsolute)(located)) {
|
|
2636
|
-
throw new Error("seed handoff path is outside the operator run root");
|
|
2637
|
-
}
|
|
2638
|
-
const handoff = (0, import_agent_qa_runner_local2.loadCompletedRunHandoff)(handoffPath);
|
|
2639
|
-
if ((0, import_agent_qa_runner_local2.slugBranch)(handoff.runId) !== runSlug) throw new Error("seed handoff run identity does not match its directory");
|
|
2640
|
-
const product = (0, import_agent_qa_eval.resolveProductPartition)(handoff.scope);
|
|
2641
|
-
if (filters.product !== void 0 && product !== filters.product) {
|
|
2642
|
-
result.runsFiltered++;
|
|
2643
|
-
continue;
|
|
2644
|
-
}
|
|
2645
|
-
const verified = (0, import_agent_qa_eval_local.loadLabelEvidence)({ snapshot: initialCommitted.snapshot, agentQaRoot, incoming: handoff });
|
|
2646
|
-
if (!verified.handoffs.some((value) => value.runId === handoff.runId)) {
|
|
2647
|
-
throw new Error("seed incoming handoff evidence could not be positively verified before admission");
|
|
2648
|
-
}
|
|
2649
|
-
const analysis = readRunAnalysisArtifact(runsRoot, runSlug, handoff.runId, result.notices);
|
|
2650
|
-
const row = {
|
|
2651
|
-
runSlug,
|
|
2652
|
-
product,
|
|
2653
|
-
verdicts: resolveSeedVerdictRows(analysis.artifact, handoff)
|
|
2654
|
-
};
|
|
2655
|
-
const excluding = row.verdicts.filter(isExclusionVerdict);
|
|
2656
|
-
const memberVerdicts = resolveSeedMemberVerdicts(analysis.artifact, handoff);
|
|
2657
|
-
const eligible = handoff.tests.filter((member) => (0, import_agent_qa_eval.admitSeedMember)(member, memberVerdicts.get(member.testIndex)));
|
|
2658
|
-
const hasEligibleRerun = eligible.some((member) => member.isRerun);
|
|
2659
|
-
if (excluding.length > 0 && !hasEligibleRerun) {
|
|
2660
|
-
result.runsExcluded.push(row);
|
|
2661
|
-
const record = (0, import_agent_qa_eval.projectRunExclusion)({
|
|
2662
|
-
runId: handoff.runId,
|
|
2663
|
-
runSlug,
|
|
2664
|
-
product,
|
|
2665
|
-
verdicts: excluding,
|
|
2666
|
-
analysisDigest: analysis.text === void 0 ? (0, import_agent_qa_schema10.sha256hex)((0, import_agent_qa_eval.canonicalJson)(row.verdicts)) : (0, import_agent_qa_schema10.sha256hex)(analysis.text)
|
|
2667
|
-
});
|
|
2668
|
-
const outcome = (0, import_agent_qa_eval_local.writeRunExclusion)({
|
|
2669
|
-
record,
|
|
2670
|
-
committedRoot
|
|
2671
|
-
});
|
|
2672
|
-
result.exclusionsWritten.push({
|
|
2673
|
-
runSlug,
|
|
2674
|
-
product,
|
|
2675
|
-
path: (0, import_agent_qa_eval_local.committedExclusionPath)(product, runSlug),
|
|
2676
|
-
outcome: outcome.kind
|
|
2677
|
-
});
|
|
2678
|
-
continue;
|
|
2679
|
-
}
|
|
2680
|
-
const exclusionPath = (0, import_agent_qa_eval_local.committedExclusionPath)(product, runSlug);
|
|
2681
|
-
if ((0, import_node_fs7.existsSync)((0, import_node_path14.join)(committedRoot, exclusionPath))) {
|
|
2682
|
-
result.admissionConflicts.push({ runSlug, product, path: exclusionPath, reason: "existing-run-exclusion" });
|
|
2683
|
-
result.notices.push(`seed ${runSlug}: existing run exclusion requires the reviewed exclusion path before admission`);
|
|
2684
|
-
continue;
|
|
2685
|
-
}
|
|
2686
|
-
const testIndexes = new Set(eligible.map((member) => member.testIndex));
|
|
2687
|
-
for (const member of handoff.tests) {
|
|
2688
|
-
const verdict = memberVerdicts.get(member.testIndex);
|
|
2689
|
-
if (!testIndexes.has(member.testIndex) && verdict) {
|
|
2690
|
-
result.membersExcluded.push({ runSlug, product, testIndex: member.testIndex, verdict });
|
|
2691
|
-
}
|
|
2692
|
-
}
|
|
2693
|
-
result.runsAdmitted.push(row);
|
|
2694
|
-
admitted.push({ runSlug, handoff, product, testIndexes });
|
|
2695
|
-
} catch (err) {
|
|
2696
|
-
result.runsSkipped.push({
|
|
2697
|
-
runSlug,
|
|
2698
|
-
reason: err instanceof Error ? err.message : String(err)
|
|
2699
|
-
});
|
|
2700
|
-
continue;
|
|
2701
|
-
}
|
|
2702
|
-
}
|
|
2703
|
-
const manifests = ensurePartitionManifests(session, admitted, committedRoot, result, params);
|
|
2704
|
-
for (const { runSlug, handoff, product, testIndexes } of admitted) {
|
|
2705
|
-
let report;
|
|
2706
|
-
try {
|
|
2707
|
-
const committed = (0, import_agent_qa_eval_local.readCommittedCorpus)(committedRoot);
|
|
2708
|
-
const evidence = (0, import_agent_qa_eval_local.loadLabelEvidence)({ snapshot: committed.snapshot, agentQaRoot, incoming: handoff });
|
|
2709
|
-
for (const refusal of evidence.refusals) result.notices.push(`SOFT evidence ${refusal.runId}: ${refusal.reason}`);
|
|
2710
|
-
if (!evidence.handoffs.some((value) => value.runId === handoff.runId)) {
|
|
2711
|
-
throw new Error("seed incoming handoff evidence could not be positively verified");
|
|
2712
|
-
}
|
|
2713
|
-
const mask = committedCorpusMask(session);
|
|
2714
|
-
const admittedHandoff = (0, import_agent_qa_eval.maskCorpusText)(
|
|
2715
|
-
{ ...handoff, tests: handoff.tests.filter((member) => testIndexes.has(member.testIndex)) },
|
|
2716
|
-
mask
|
|
2717
|
-
);
|
|
2718
|
-
const populationHandoffs = evidence.handoffs.map((source) => {
|
|
2719
|
-
if (source.runId === handoff.runId) return admittedHandoff;
|
|
2720
|
-
const sourceAnalysis = readRunAnalysisArtifact(runsRoot, (0, import_agent_qa_runner_local2.slugBranch)(source.runId), source.runId, result.notices);
|
|
2721
|
-
const verdicts = resolveSeedMemberVerdicts(sourceAnalysis.artifact, source);
|
|
2722
|
-
return (0, import_agent_qa_eval.maskCorpusText)(
|
|
2723
|
-
{ ...source, tests: source.tests.filter((member) => (0, import_agent_qa_eval.admitSeedMember)(member, verdicts.get(member.testIndex))) },
|
|
2724
|
-
mask
|
|
2725
|
-
);
|
|
2726
|
-
});
|
|
2727
|
-
const manifest = manifests.get(product);
|
|
2728
|
-
if (manifest === void 0) {
|
|
2729
|
-
throw new Error(`no committed partition manifest for product ${product}`);
|
|
2730
|
-
}
|
|
2731
|
-
const extraction = (0, import_agent_qa_eval.extractRunCassettes)({
|
|
2732
|
-
handoff: admittedHandoff,
|
|
2733
|
-
artifacts: new import_agent_qa_eval_local.FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
|
|
2734
|
-
});
|
|
2735
|
-
const targetEnvironment = resolveTargetEnvironmentForProduct(session.config, product);
|
|
2736
|
-
const plan = (0, import_agent_qa_eval.buildSeedingPlan)({
|
|
2737
|
-
handoff: admittedHandoff,
|
|
2738
|
-
populationContext: { snapshot: committed.snapshot, handoffs: populationHandoffs, marks: evidence.marks },
|
|
2739
|
-
cassettes: extraction.cassettes,
|
|
2740
|
-
// 414 D-07: the split comes from the product's COMMITTED manifest params — the cli adds
|
|
2741
|
-
// no selection or relabel logic of its own.
|
|
2742
|
-
partition: (0, import_agent_qa_eval.partitionParamsOf)(manifest),
|
|
2743
|
-
capturedAt: new Date(handoff.endedAt).toISOString(),
|
|
2744
|
-
...targetEnvironment === void 0 ? {} : { targetEnvironment },
|
|
2745
|
-
params: {
|
|
2746
|
-
...silverPerTestCap !== void 0 ? { silverPerTestCap } : {}
|
|
2747
|
-
},
|
|
2748
|
-
mask
|
|
2749
|
-
});
|
|
2750
|
-
report = (0, import_agent_qa_eval_local.seedCommittedCorpus)({
|
|
2751
|
-
plan,
|
|
2752
|
-
cassettes: extraction.cassettes,
|
|
2753
|
-
committedRoot,
|
|
2754
|
-
mask
|
|
2755
|
-
});
|
|
2756
|
-
for (const skip of report.softEvidence?.skips ?? []) {
|
|
2757
|
-
result.notices.push(`SOFT evidence ${skip.runId}/${skip.testIndex}/${skip.stepIndex}/${skip.attempt}: ${skip.reason}`);
|
|
2758
|
-
}
|
|
2759
|
-
} catch (err) {
|
|
2760
|
-
result.runsSkipped.push({
|
|
2761
|
-
runSlug,
|
|
2762
|
-
reason: err instanceof Error ? err.message : String(err)
|
|
2763
|
-
});
|
|
2764
|
-
continue;
|
|
2765
|
-
}
|
|
2766
|
-
result.handoffsSeeded++;
|
|
2767
|
-
result.seededCount += report.seededCount;
|
|
2768
|
-
result.unchangedCount += report.unchangedCount;
|
|
2769
|
-
result.supersededOriginalCount += report.supersededOriginalCount;
|
|
2770
|
-
result.retirementCounts.missing += report.retirementCounts.missing;
|
|
2771
|
-
result.retirementCounts.conflict += report.retirementCounts.conflict;
|
|
2772
|
-
result.silverExclusionRows.push(...report.silverExclusions.map((row) => ({ ...row, runSlug, product })));
|
|
2773
|
-
result.refusals.push(...report.refusals);
|
|
2774
|
-
result.pairSkips += report.pairSkips.length;
|
|
2775
|
-
result.silverExclusions += report.silverExclusions.length;
|
|
2776
|
-
result.pairCounts.total += report.pairCounts.total;
|
|
2777
|
-
result.pairCounts.sameState += report.pairCounts.sameState;
|
|
2778
|
-
result.pairCounts.differingState += report.pairCounts.differingState;
|
|
2779
|
-
for (const contract of report.contracts) {
|
|
2780
|
-
bumpCounter(result.contracts, contract.outcome);
|
|
2781
|
-
}
|
|
2782
|
-
bumpCounter(result.scaffold, report.scaffold.gitignore);
|
|
2783
|
-
bumpCounter(result.scaffold, report.scaffold.marker);
|
|
2784
|
-
}
|
|
2785
|
-
return result;
|
|
2786
|
-
}
|
|
2787
|
-
function ensurePartitionManifests(session, admitted, committedRoot, result, params) {
|
|
2788
|
-
const byProduct = /* @__PURE__ */ new Map();
|
|
2789
|
-
const read = (0, import_agent_qa_eval_local.readPartitionManifests)(committedRoot);
|
|
2790
|
-
for (const manifest of read.manifests) byProduct.set(manifest.product, manifest);
|
|
2791
|
-
result.notices.push(...read.notices);
|
|
2792
|
-
for (const refusal of read.refusals) {
|
|
2793
|
-
result.notices.push(`partition manifest refused at ${refusal.path}: ${refusal.reason}`);
|
|
2794
|
-
}
|
|
2795
|
-
const products = [...new Set(admitted.map((entry) => entry.product))].sort();
|
|
2796
|
-
const { heldOutFraction, privateFraction } = params;
|
|
2797
|
-
const unpartitioned = products.filter((product) => !byProduct.has(product));
|
|
2798
|
-
if (unpartitioned.length > 0 && (heldOutFraction === void 0 || privateFraction === void 0)) {
|
|
2799
|
-
throw new EvalPartitionFlagsRequiredError(unpartitioned);
|
|
2800
|
-
}
|
|
2801
|
-
for (const product of products) {
|
|
2802
|
-
const committed = byProduct.get(product);
|
|
2803
|
-
const relPath = (0, import_agent_qa_eval_local.committedPartitionManifestPath)(product);
|
|
2804
|
-
if (committed === void 0) {
|
|
2805
|
-
const saltHex = (0, import_node_crypto5.randomBytes)(32).toString("hex");
|
|
2806
|
-
const manifest = (0, import_agent_qa_eval.projectPartitionManifest)({
|
|
2807
|
-
product,
|
|
2808
|
-
saltHex,
|
|
2809
|
-
heldOutFraction,
|
|
2810
|
-
privateFraction
|
|
2811
|
-
});
|
|
2812
|
-
const outcome2 = (0, import_agent_qa_eval_local.writePartitionManifest)({
|
|
2813
|
-
manifest,
|
|
2814
|
-
committedRoot
|
|
2815
|
-
});
|
|
2816
|
-
byProduct.set(product, manifest);
|
|
2817
|
-
result.partitions.push(partitionRow(manifest, relPath, requireManifestWritten(outcome2, product, relPath)));
|
|
2818
|
-
continue;
|
|
2819
|
-
}
|
|
2820
|
-
if (heldOutFraction === void 0 || privateFraction === void 0) {
|
|
2821
|
-
result.partitions.push(partitionRow(committed, relPath, "unchanged"));
|
|
2822
|
-
continue;
|
|
2823
|
-
}
|
|
2824
|
-
const reprojected = (0, import_agent_qa_eval.projectPartitionManifest)({
|
|
2825
|
-
product,
|
|
2826
|
-
saltHex: committed.salt,
|
|
2827
|
-
heldOutFraction,
|
|
2828
|
-
privateFraction
|
|
2829
|
-
});
|
|
2830
|
-
const outcome = (0, import_agent_qa_eval_local.writePartitionManifest)({
|
|
2831
|
-
manifest: reprojected,
|
|
2832
|
-
committedRoot
|
|
2833
|
-
});
|
|
2834
|
-
result.partitions.push(
|
|
2835
|
-
partitionRow(committed, relPath, requireManifestWritten(outcome, product, relPath))
|
|
2836
|
-
);
|
|
2837
|
-
}
|
|
2838
|
-
return byProduct;
|
|
2839
|
-
}
|
|
2840
|
-
function partitionRow(manifest, path, outcome) {
|
|
2841
|
-
return {
|
|
2842
|
-
product: manifest.product,
|
|
2843
|
-
path,
|
|
2844
|
-
outcome,
|
|
2845
|
-
saltDigest: (0, import_agent_qa_schema10.sha256hex)(manifest.salt),
|
|
2846
|
-
heldOutFraction: manifest.heldOutFraction,
|
|
2847
|
-
privateFraction: manifest.privateFraction,
|
|
2848
|
-
partitionVersion: manifest.partitionVersion
|
|
2849
|
-
};
|
|
2850
|
-
}
|
|
2851
|
-
function resolveTargetEnvironmentForProduct(config, product) {
|
|
2852
|
-
const declared = /* @__PURE__ */ new Set();
|
|
2853
|
-
for (const summary of listTargets(config)) {
|
|
2854
|
-
const target = resolveTarget(config, summary.name);
|
|
2855
|
-
if (target.product !== product) continue;
|
|
2856
|
-
if (target.environment === void 0) continue;
|
|
2857
|
-
declared.add(target.environment);
|
|
2858
|
-
}
|
|
2859
|
-
return declared.size === 1 ? [...declared][0] : void 0;
|
|
2860
|
-
}
|
|
2861
|
-
function requireManifestWritten(outcome, product, path) {
|
|
2862
|
-
if (outcome.kind === "write-conflict") {
|
|
2863
|
-
throw new EvalPartitionManifestConflictError(product, path);
|
|
2864
|
-
}
|
|
2865
|
-
return outcome.kind;
|
|
2866
|
-
}
|
|
2867
|
-
function readRunAnalysisArtifact(runsRoot, runSlug, runId, notices) {
|
|
2868
|
-
const path = (0, import_node_path14.join)(runsRoot, runSlug, import_agent_qa_schema10.ANALYSIS_FILE);
|
|
2869
|
-
if (!(0, import_node_fs7.existsSync)(path)) return {};
|
|
2870
|
-
try {
|
|
2871
|
-
const text = (0, import_node_fs7.readFileSync)(path, "utf8");
|
|
2872
|
-
const admission = (0, import_agent_qa_schema10.admitAnalysisArtifactForRun)(
|
|
2873
|
-
(0, import_agent_qa_schema10.guardRunAnalysisArtifact)(JSON.parse(text)),
|
|
2874
|
-
runId
|
|
2875
|
-
);
|
|
2876
|
-
if (admission.kind === "admitted") return { artifact: admission.artifact, text };
|
|
2877
|
-
notices.push(
|
|
2878
|
-
`${import_agent_qa_schema10.ANALYSIS_FILE} for ${runSlug} records run ${admission.recordedRunId}, not ${admission.resolvingRunId} \u2014 deterministic floor used`
|
|
2879
|
-
);
|
|
2880
|
-
return {};
|
|
2881
|
-
} catch (err) {
|
|
2882
|
-
notices.push(
|
|
2883
|
-
`${import_agent_qa_schema10.ANALYSIS_FILE} unreadable for ${runSlug}: ${err instanceof Error ? err.message : String(err)} \u2014 deterministic floor used`
|
|
2884
|
-
);
|
|
2885
|
-
return {};
|
|
2886
|
-
}
|
|
2887
|
-
}
|
|
2888
|
-
function resolveSeedVerdictRows(artifact, handoff) {
|
|
2889
|
-
return [...resolveSeedMemberVerdicts(artifact, handoff).entries()].sort(([first], [second]) => first - second).map(([, verdict]) => verdict);
|
|
2890
|
-
}
|
|
2891
|
-
function resolveSeedMemberVerdicts(artifact, handoff) {
|
|
2892
|
-
const byPerTest = /* @__PURE__ */ new Map();
|
|
2893
|
-
for (const member of handoff.tests) {
|
|
2894
|
-
if (member.rootCause === void 0) continue;
|
|
2895
|
-
byPerTest.set(member.testIndex, {
|
|
2896
|
-
class: member.rootCause.class,
|
|
2897
|
-
producer: member.rootCause.producer
|
|
2898
|
-
});
|
|
2899
|
-
}
|
|
2900
|
-
for (const verdict of (0, import_agent_qa_schema10.resolveRunVerdicts)(artifact, handoff)) {
|
|
2901
|
-
byPerTest.set(verdict.perTest, { class: verdict.class, producer: verdict.producer });
|
|
2902
|
-
}
|
|
2903
|
-
return byPerTest;
|
|
2904
|
-
}
|
|
2905
|
-
async function runReportVerb(session, filters = {}, opts = {}) {
|
|
2906
|
-
if (opts.fromCommitted !== void 0) {
|
|
2907
|
-
if (opts.baseline !== true) throw new TypeError("--from-committed requires --baseline");
|
|
2908
|
-
if (opts.seedPolicy === void 0) {
|
|
2909
|
-
throw new TypeError("--from-committed requires --seed-policy <none|declared>");
|
|
2910
|
-
}
|
|
2911
|
-
if (opts.commit === true) throw new TypeError("--from-committed cannot be combined with --commit");
|
|
2912
|
-
if (!/^[0-9a-f]{64}$/.test(opts.fromCommitted)) {
|
|
2913
|
-
throw new TypeError("--from-committed requires a batchKey of 64 lowercase hexadecimal characters");
|
|
2914
|
-
}
|
|
2915
|
-
if (!import_agent_qa_eval.SEED_POLICIES.includes(opts.seedPolicy)) {
|
|
2916
|
-
throw new TypeError('--seed-policy must be "none" or "declared"');
|
|
2917
|
-
}
|
|
2918
|
-
const committedRoot2 = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
2919
|
-
const reportList = (0, import_agent_qa_eval_local.listCommittedReports)(committedRoot2);
|
|
2920
|
-
const requestedFile = (0, import_node_path14.join)(committedRoot2, "reports", opts.fromCommitted, "report.json");
|
|
2921
|
-
const entry = reportList.reports.find((candidate) => candidate.batchKey === opts.fromCommitted);
|
|
2922
|
-
if (entry === void 0) {
|
|
2923
|
-
const refusal = reportList.refusals.find((candidate) => candidate.path === requestedFile);
|
|
2924
|
-
throw new EvalNamedInputError(
|
|
2925
|
-
"batchKey",
|
|
2926
|
-
opts.fromCommitted,
|
|
2927
|
-
refusal?.path ?? (0, import_node_path14.join)(committedRoot2, "reports"),
|
|
2928
|
-
"designate",
|
|
2929
|
-
refusal?.reason
|
|
2930
|
-
);
|
|
2931
|
-
}
|
|
2932
|
-
if (entry.path !== requestedFile || entry.report.batchKey !== opts.fromCommitted) {
|
|
2933
|
-
throw new EvalNamedInputError(
|
|
2934
|
-
"batchKey",
|
|
2935
|
-
opts.fromCommitted,
|
|
2936
|
-
entry.path,
|
|
2937
|
-
"designate",
|
|
2938
|
-
`report batchKey "${entry.report.batchKey}" does not match the requested directory`
|
|
2939
|
-
);
|
|
2940
|
-
}
|
|
2941
|
-
const notices2 = reportList.refusals.map(
|
|
2942
|
-
(refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`
|
|
2943
|
-
);
|
|
2944
|
-
const designation = await designateReportBaseline(
|
|
2945
|
-
session,
|
|
2946
|
-
entry.report,
|
|
2947
|
-
opts.seedPolicy,
|
|
2948
|
-
opts.plannerOverrides,
|
|
2949
|
-
notices2
|
|
2950
|
-
);
|
|
2951
|
-
return {
|
|
2952
|
-
batchKey: entry.report.batchKey,
|
|
2953
|
-
entries: 0,
|
|
2954
|
-
report: entry.report,
|
|
2955
|
-
committedFile: entry.path,
|
|
2956
|
-
judgeIdentity: entry.report.judgeIdentity === "mixed" ? "mixed" : entry.report.judge === void 0 ? "absent" : "present",
|
|
2957
|
-
source: "committed",
|
|
2958
|
-
...designation,
|
|
2959
|
-
notices: notices2
|
|
2960
|
-
};
|
|
2961
|
-
}
|
|
2962
|
-
if (opts.baseline === true && opts.commit !== true) {
|
|
2963
|
-
throw new Error(
|
|
2964
|
-
"--baseline requires --commit: the pointer designates a COMMITTED report (413 D-19)"
|
|
2965
|
-
);
|
|
2966
|
-
}
|
|
2967
|
-
const agentQaRoot = (0, import_node_path14.resolve)(session.configDir, ".agent-qa");
|
|
2968
|
-
const replaysRoot = (0, import_node_path14.join)(agentQaRoot, import_agent_qa_eval_local.DEFAULT_EVAL_REPLAYS_SUBDIR);
|
|
2969
|
-
const source = opts.source ?? "staged";
|
|
2970
|
-
const sourceRoot = resolveCassettesRoot(session, source);
|
|
2971
|
-
const cassettesRoot = sourceRoot.cassettesRoot;
|
|
2972
|
-
const notices = [];
|
|
2973
|
-
if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
|
|
2974
|
-
const labelCorpus = (0, import_agent_qa_eval_local.readActiveCommittedCorpus)((0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"), agentQaRoot);
|
|
2975
|
-
const committedCorpus = source === "committed" ? labelCorpus : void 0;
|
|
2976
|
-
notices.push(...labelCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
|
|
2977
|
-
const {
|
|
2978
|
-
entries: assembled,
|
|
2979
|
-
judgeBlocks,
|
|
2980
|
-
partitionsById,
|
|
2981
|
-
missingBundleCount,
|
|
2982
|
-
notices: assemblyNotices
|
|
2983
|
-
} = (0, import_agent_qa_eval_local.assembleScoredBatch)({
|
|
2984
|
-
replaysRoot,
|
|
2985
|
-
cassettesRoot,
|
|
2986
|
-
source,
|
|
2987
|
-
labelRecords: labelCorpus.snapshot.labelsById,
|
|
2988
|
-
excludedLabelIds: labelCorpus.snapshot.excludedLabelIds,
|
|
2989
|
-
...committedCorpus === void 0 ? {} : {
|
|
2990
|
-
productByCassetteId: new Map(committedCorpus.snapshot.entries.map((entry) => [entry.cassetteId, entry.product])),
|
|
2991
|
-
eligibleCassetteIds: new Set(committedCorpus.snapshot.entries.map((entry) => entry.cassetteId)),
|
|
2992
|
-
corpusEntriesById: committedCorpus.corpusEntriesById
|
|
2993
|
-
}
|
|
2994
|
-
});
|
|
2995
|
-
notices.push(...assemblyNotices);
|
|
2996
|
-
if (assembled.some((entry) => entry.split === void 0)) {
|
|
2997
|
-
notices.push("corpus split ownership unknown; entries without corpus metadata are unassigned");
|
|
2998
|
-
}
|
|
2999
|
-
if (missingBundleCount > 0) {
|
|
3000
|
-
notices.push(
|
|
3001
|
-
`${missingBundleCount} replay record(s) have no ${source} cassette bundle \u2014 label legs absent`
|
|
3002
|
-
);
|
|
3003
|
-
}
|
|
3004
|
-
let entries = assembled;
|
|
3005
|
-
if (filters.cassette !== void 0) {
|
|
3006
|
-
const named = filters.cassette;
|
|
3007
|
-
entries = entries.filter((entry) => entry.cassetteId === named);
|
|
3008
|
-
if (entries.length === 0) {
|
|
3009
|
-
throw new EvalNamedInputError("cassette", named, replaysRoot, "report");
|
|
3010
|
-
}
|
|
3011
|
-
}
|
|
3012
|
-
if (filters.run !== void 0) {
|
|
3013
|
-
const named = filters.run;
|
|
3014
|
-
entries = entries.filter((entry) => {
|
|
3015
|
-
const head = entry.cassetteId.split("/")[0];
|
|
3016
|
-
return head === named || head === (0, import_agent_qa_runner_local2.slugBranch)(named);
|
|
3017
|
-
});
|
|
3018
|
-
if (entries.length === 0) {
|
|
3019
|
-
throw new EvalNamedInputError("run", named, replaysRoot, "report");
|
|
3020
|
-
}
|
|
3021
|
-
}
|
|
3022
|
-
if (filters.product !== void 0) {
|
|
3023
|
-
const named = filters.product;
|
|
3024
|
-
entries = entries.filter((entry) => partitionsById.get(entry.cassetteId) === named);
|
|
3025
|
-
}
|
|
3026
|
-
entries.sort((a, b) => a.cassetteId < b.cassetteId ? -1 : a.cassetteId > b.cassetteId ? 1 : 0);
|
|
3027
|
-
let judgeIdentity;
|
|
3028
|
-
let judgeForReport;
|
|
3029
|
-
let mixedIdentity;
|
|
3030
|
-
if (judgeBlocks.length === 0) {
|
|
3031
|
-
judgeIdentity = "absent";
|
|
3032
|
-
notices.push("no judge identity on any persisted score record \u2014 the report carries no judge block");
|
|
3033
|
-
} else {
|
|
3034
|
-
const identities = judgeBlocks.map((block) => import_agent_qa_eval.judgeIdentitySchema.strip().parse(block));
|
|
3035
|
-
const first = (0, import_agent_qa_eval.canonicalJson)(identities[0]);
|
|
3036
|
-
if (identities.every((block) => (0, import_agent_qa_eval.canonicalJson)(block) === first)) {
|
|
3037
|
-
judgeIdentity = "present";
|
|
3038
|
-
judgeForReport = identities[0];
|
|
3039
|
-
} else {
|
|
3040
|
-
judgeIdentity = "mixed";
|
|
3041
|
-
mixedIdentity = {
|
|
3042
|
-
judgeIdentity: "mixed",
|
|
3043
|
-
judgeModelIds: [...new Set(judgeBlocks.map((block) => block.judgeModelId))].sort()
|
|
3044
|
-
};
|
|
3045
|
-
notices.push(
|
|
3046
|
-
"mixed judge identities across persisted score records \u2014 the report carries no judge block (never fabricated)"
|
|
3047
|
-
);
|
|
3048
|
-
}
|
|
3049
|
-
}
|
|
3050
|
-
const report = (0, import_agent_qa_eval.buildEvalReport)(entries, judgeForReport, mixedIdentity, { calibration: labelCorpus.calibration });
|
|
3051
|
-
const reportSeedPolicy = session.evalConfig.seed !== void 0 ? "declared" : "none";
|
|
3052
|
-
let reportProfileProof;
|
|
3053
|
-
if (report.judge?.panelSize !== void 0) {
|
|
3054
|
-
let calibrationRead;
|
|
3055
|
-
try {
|
|
3056
|
-
const replayProfile = await resolveReplayProfileDeclaration(session);
|
|
3057
|
-
const actualModels = entries.flatMap((entry) => {
|
|
3058
|
-
if ("kind" in entry.outcome) return [];
|
|
3059
|
-
const results = entry.outcome.granularity === "leaf" ? [entry.outcome] : entry.outcome.steps.flatMap((step) => step.kind === "decision" ? [step.result] : []);
|
|
3060
|
-
return results.map((result) => result.replayedModel.modelId);
|
|
3061
|
-
});
|
|
3062
|
-
if (actualModels.length === 0 || actualModels.some((model) => model !== replayProfile.model)) {
|
|
3063
|
-
throw new Error("report replay profile unavailable");
|
|
3064
|
-
}
|
|
3065
|
-
const fingerprint = (0, import_agent_qa_eval_local.computeReplayProfileFingerprintFor)({
|
|
3066
|
-
profile: replayProfile,
|
|
3067
|
-
seedPolicy: reportSeedPolicy,
|
|
3068
|
-
modelOverrides: opts.plannerOverrides
|
|
3069
|
-
});
|
|
3070
|
-
reportProfileProof = (0, import_agent_qa_eval_local.readReportProfileProof)({
|
|
3071
|
-
agentQaRoot,
|
|
3072
|
-
replaysRoot,
|
|
3073
|
-
report,
|
|
3074
|
-
entries,
|
|
3075
|
-
fingerprint,
|
|
3076
|
-
corpusDigest: (0, import_agent_qa_eval.computeCorpusDigest)(labelCorpus.snapshot.entries),
|
|
3077
|
-
seed: session.evalConfig.seed
|
|
3078
|
-
});
|
|
3079
|
-
if (!reportProfileProof) throw new Error("report full profile proof unavailable");
|
|
3080
|
-
calibrationRead = (0, import_agent_qa_eval_local.readJudgeCalibration)({
|
|
3081
|
-
committedRoot: (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval"),
|
|
3082
|
-
agentQaRoot,
|
|
3083
|
-
identity: report.judge,
|
|
3084
|
-
profile: {
|
|
3085
|
-
plannerModelId: replayProfile.model,
|
|
3086
|
-
fingerprint,
|
|
3087
|
-
matcherVersion: report.matcherVersion,
|
|
3088
|
-
judgeModelId: report.judge.judgeModelId,
|
|
3089
|
-
judgeVersion: report.judge.judgeVersion
|
|
3090
|
-
}
|
|
3091
|
-
});
|
|
3092
|
-
} catch {
|
|
3093
|
-
}
|
|
3094
|
-
(0, import_agent_qa_eval_local.copyJudgeCalibrationToReport)(report, calibrationRead);
|
|
3095
|
-
if (calibrationRead?.kind !== "loaded") notices.push("judge calibration unavailable: no matching guarded artifact");
|
|
3096
|
-
}
|
|
3097
|
-
const vectors = (0, import_agent_qa_eval.decisionCorrectnessVectors)(entries, partitionsById);
|
|
3098
|
-
const decisions = (0, import_agent_qa_eval.projectDecisionVectorsDocument)({
|
|
3099
|
-
batchKey: report.batchKey,
|
|
3100
|
-
matcherVersion: report.matcherVersion,
|
|
3101
|
-
vectors
|
|
3102
|
-
});
|
|
3103
|
-
if (reportProfileProof) report.runIdentity = reportProfileProof.runIdentity;
|
|
3104
|
-
const staged = new import_agent_qa_eval_local.FsReportStore({ root: agentQaRoot }).save(report, decisions, reportProfileProof?.replayGeneration);
|
|
3105
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
3106
|
-
let committed;
|
|
3107
|
-
if (opts.commit === true) {
|
|
3108
|
-
committed = (0, import_agent_qa_eval_local.commitEvalReport)({
|
|
3109
|
-
report,
|
|
3110
|
-
decisions,
|
|
3111
|
-
markdown: renderEvalReportMarkdown(report),
|
|
3112
|
-
committedRoot
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
let baseline;
|
|
3116
|
-
let profileKey;
|
|
3117
|
-
if (opts.baseline === true) {
|
|
3118
|
-
if (committed?.kind !== "committed") {
|
|
3119
|
-
notices.push(
|
|
3120
|
-
`baseline pointer skipped: report commit outcome was ${committed?.kind ?? "absent"}`
|
|
3121
|
-
);
|
|
3122
|
-
} else {
|
|
3123
|
-
const designation = await designateReportBaseline(
|
|
3124
|
-
session,
|
|
3125
|
-
report,
|
|
3126
|
-
reportSeedPolicy,
|
|
3127
|
-
opts.plannerOverrides,
|
|
3128
|
-
notices
|
|
3129
|
-
);
|
|
3130
|
-
profileKey = designation.profileKey;
|
|
3131
|
-
baseline = designation.baseline;
|
|
3132
|
-
}
|
|
3133
|
-
}
|
|
3134
|
-
return {
|
|
3135
|
-
batchKey: report.batchKey,
|
|
3136
|
-
entries: entries.length,
|
|
3137
|
-
report,
|
|
3138
|
-
stagedFile: staged.file,
|
|
3139
|
-
decisionVectorCount: decisions.vectors.length,
|
|
3140
|
-
...committed !== void 0 ? { committed } : {},
|
|
3141
|
-
judgeIdentity,
|
|
3142
|
-
source,
|
|
3143
|
-
...baseline !== void 0 ? { baseline } : {},
|
|
3144
|
-
...profileKey !== void 0 ? { profileKey } : {},
|
|
3145
|
-
notices
|
|
3146
|
-
};
|
|
3147
|
-
}
|
|
3148
|
-
async function designateReportBaseline(session, report, seedPolicy, plannerOverrides, notices) {
|
|
3149
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
3150
|
-
const read = (0, import_agent_qa_eval_local.readActiveCommittedCorpus)(committedRoot, (0, import_node_path14.resolve)(session.configDir, ".agent-qa"));
|
|
3151
|
-
notices.push(...read.notices);
|
|
3152
|
-
for (const refusal of read.refusals) {
|
|
3153
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3154
|
-
}
|
|
3155
|
-
const pointer = (0, import_agent_qa_eval.projectBaselinePointer)({
|
|
3156
|
-
plannerModelId: await resolvePlannerModelId(session),
|
|
3157
|
-
fingerprint: await resolveReplayProfileFingerprint(session, seedPolicy, plannerOverrides),
|
|
3158
|
-
corpusDigest: (0, import_agent_qa_eval.computeCorpusDigest)(read.snapshot.entries),
|
|
3159
|
-
matcherVersion: report.matcherVersion,
|
|
3160
|
-
...report.judge === void 0 ? {} : {
|
|
3161
|
-
judgeModelId: report.judge.judgeModelId,
|
|
3162
|
-
...report.judge.judgeVersion === void 0 ? {} : { judgeVersion: report.judge.judgeVersion }
|
|
3163
|
-
},
|
|
3164
|
-
batchKey: report.batchKey
|
|
3165
|
-
});
|
|
3166
|
-
const baseline = (0, import_agent_qa_eval_local.writeBaselinePointer)({ report, pointer, committedRoot });
|
|
3167
|
-
if (baseline.kind === "refused") notices.push((0, import_agent_qa_eval.renderBaselineComparatorRefusal)(baseline.refusal));
|
|
3168
|
-
if (seedPolicy === "none" && (baseline.kind === "written" || baseline.kind === "unchanged")) {
|
|
3169
|
-
notices.push(
|
|
3170
|
-
"baseline designated under seed policy none \u2014 eval gate compares against a declared-policy pointer; re-designate with eval report --baseline --from-committed <batchKey> --seed-policy declared"
|
|
3171
|
-
);
|
|
3172
|
-
}
|
|
3173
|
-
return { profileKey: pointer.profileKey, baseline };
|
|
3174
|
-
}
|
|
3175
|
-
async function resolvePlannerModelId(session) {
|
|
3176
|
-
return (await resolveBaselinePlannerConfig(session)).model;
|
|
3177
|
-
}
|
|
3178
|
-
async function resolveGateProfileIdentity(session, modelOverrides) {
|
|
3179
|
-
const plannerModelId = await resolvePlannerModelId(session);
|
|
3180
|
-
const fingerprint = await resolveReplayProfileFingerprint(session, "declared", modelOverrides);
|
|
3181
|
-
const judgeProfiles = await resolveEvalJudgeDeclarations(session);
|
|
3182
|
-
const judgeMemberModelIds = judgeProfiles.map((member) => member.profile.model);
|
|
3183
|
-
const judgeIdentity = judgeMemberModelIds.length === 0 ? void 0 : (0, import_agent_qa_eval.createPanelIdentity)(judgeMemberModelIds);
|
|
3184
|
-
const judgeModelId = judgeIdentity?.judgeModelId;
|
|
3185
|
-
const identity = {
|
|
3186
|
-
plannerModelId,
|
|
3187
|
-
fingerprint,
|
|
3188
|
-
matcherVersion: (0, import_agent_qa_eval.buildEvalReport)([]).matcherVersion,
|
|
3189
|
-
...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion: import_agent_qa_eval.JUDGE_CONFIG.judgeVersion }
|
|
3190
|
-
};
|
|
3191
|
-
return {
|
|
3192
|
-
...identity,
|
|
3193
|
-
profileKey: (0, import_agent_qa_eval.composeBaselineProfileKey)(identity),
|
|
3194
|
-
...judgeMemberModelIds.length > 1 ? { judgeMemberModelIds } : {}
|
|
3195
|
-
};
|
|
3196
|
-
}
|
|
3197
|
-
async function resolveEvalJudgeDeclarations(session) {
|
|
3198
|
-
const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3199
|
-
const members = [];
|
|
3200
|
-
for (const name of selectedEvalJudgeNames(session.evalConfig)) {
|
|
3201
|
-
const { config } = await resolveNamedConfig2(name, session.configPath, { keyOnly: true });
|
|
3202
|
-
members.push({ name, profile: {
|
|
3203
|
-
provider: config.provider,
|
|
3204
|
-
model: config.model,
|
|
3205
|
-
auth: { mode: "none" },
|
|
3206
|
-
...config.baseURL === void 0 ? {} : { baseURL: config.baseURL },
|
|
3207
|
-
...config.providerHeaders === void 0 ? {} : { headers: config.providerHeaders }
|
|
3208
|
-
} });
|
|
3209
|
-
}
|
|
3210
|
-
return members;
|
|
3211
|
-
}
|
|
3212
|
-
async function resolveReplayProfileFingerprint(session, seedPolicy, modelOverrides) {
|
|
3213
|
-
const profile = await resolveReplayProfileDeclaration(session);
|
|
3214
|
-
return (0, import_agent_qa_eval_local.computeReplayProfileFingerprintFor)({ profile, seedPolicy, modelOverrides });
|
|
3215
|
-
}
|
|
3216
|
-
async function resolveReplayProfileDeclaration(session, namedResolver) {
|
|
3217
|
-
const config = await resolveBaselinePlannerConfig(session, namedResolver);
|
|
3218
|
-
return {
|
|
3219
|
-
provider: config.provider,
|
|
3220
|
-
model: config.model,
|
|
3221
|
-
...config.baseURL !== void 0 ? { baseURL: config.baseURL } : {},
|
|
3222
|
-
...config.providerHeaders !== void 0 ? { headers: config.providerHeaders } : {},
|
|
3223
|
-
auth: { mode: "none" }
|
|
3224
|
-
};
|
|
3225
|
-
}
|
|
3226
|
-
async function resolveBaselinePlannerConfig(session, namedResolver) {
|
|
3227
|
-
const plannerName = session.config.use?.llm;
|
|
3228
|
-
if (!plannerName) {
|
|
3229
|
-
throw new Error(
|
|
3230
|
-
"use.llm is not set in the config \u2014 the baseline profile key is keyed on the replay planner, the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
|
|
3231
|
-
);
|
|
3232
|
-
}
|
|
3233
|
-
const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3234
|
-
const { config } = await (namedResolver ?? resolveNamedConfig2)(plannerName, session.configPath, { keyOnly: true });
|
|
3235
|
-
return config;
|
|
3236
|
-
}
|
|
3237
|
-
async function runCensusVerb(session, opts = {}) {
|
|
3238
|
-
const committedRoot = (0, import_node_path14.resolve)(session.configDir, "agent-qa-eval");
|
|
3239
|
-
const notices = [];
|
|
3240
|
-
const read = (0, import_agent_qa_eval_local.readActiveCommittedCorpus)(committedRoot, (0, import_node_path14.resolve)(session.configDir, ".agent-qa"));
|
|
3241
|
-
const manifestRead = (0, import_agent_qa_eval_local.readPartitionManifests)(committedRoot);
|
|
3242
|
-
const exclusionRead = (0, import_agent_qa_eval_local.readRunExclusions)(committedRoot);
|
|
3243
|
-
notices.push(...read.notices, ...manifestRead.notices);
|
|
3244
|
-
for (const refusal of [...read.refusals, ...manifestRead.refusals, ...exclusionRead.refusals]) {
|
|
3245
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3246
|
-
}
|
|
3247
|
-
let campaign;
|
|
3248
|
-
let campaignManifest;
|
|
3249
|
-
if (opts.campaign !== void 0) {
|
|
3250
|
-
campaignManifest = (0, import_node_path14.resolve)(process.cwd(), opts.campaign);
|
|
3251
|
-
const manifest = (0, import_agent_qa_eval.guardCampaignManifest)(JSON.parse((0, import_node_fs7.readFileSync)(campaignManifest, "utf8")));
|
|
3252
|
-
campaign = { legs: manifest.legs, manifestDigest: (0, import_agent_qa_schema10.sha256hex)((0, import_agent_qa_eval.canonicalJson)(manifest)) };
|
|
3253
|
-
}
|
|
3254
|
-
const reportList = (0, import_agent_qa_eval_local.listCommittedReports)(committedRoot);
|
|
3255
|
-
const pointerList = (0, import_agent_qa_eval_local.listBaselinePointers)(committedRoot);
|
|
3256
|
-
for (const refusal of [...reportList.refusals, ...pointerList.refusals]) {
|
|
3257
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3258
|
-
}
|
|
3259
|
-
const reports = [];
|
|
3260
|
-
for (const entry of reportList.reports) {
|
|
3261
|
-
const comparator = (0, import_agent_qa_eval.assertBaselineComparator)(entry.report);
|
|
3262
|
-
const designatedBy = pointerList.pointers.filter((pointer) => pointer.batchKey === entry.batchKey).map((pointer) => pointer.profileKey).sort();
|
|
3263
|
-
notices.push(
|
|
3264
|
-
`cross-check ${entry.batchKey}: report decisions ${entry.report.headline.perDecisionCost.decisionCount} vs corpus cassettes ${read.snapshot.entries.length} \u2014 batch membership differs (batchKey only)`
|
|
3265
|
-
);
|
|
3266
|
-
if (!comparator.ok) notices.push((0, import_agent_qa_eval.renderBaselineComparatorRefusal)(comparator.refusal));
|
|
3267
|
-
reports.push({ report: entry.report, comparator, designatedBy });
|
|
3268
|
-
}
|
|
3269
|
-
const floorList = (0, import_agent_qa_eval_local.listNoiseFloorArtefacts)(committedRoot);
|
|
3270
|
-
for (const refusal of floorList.refusals) {
|
|
3271
|
-
notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
|
|
3272
|
-
}
|
|
3273
|
-
const noiseFloor = floorList.artefacts.map(({ profileKey, path, artefact }) => {
|
|
3274
|
-
const repeatability = artefact.classifierRepeatability;
|
|
3275
|
-
return {
|
|
3276
|
-
profileKey,
|
|
3277
|
-
path,
|
|
3278
|
-
...artefact.minimumGateSet.value === void 0 ? {} : { minimumGateSet: artefact.minimumGateSet.value },
|
|
3279
|
-
minimumGateSetDerivation: artefact.minimumGateSet.derivation,
|
|
3280
|
-
...repeatability === void 0 ? {} : {
|
|
3281
|
-
classifierRepeatability: {
|
|
3282
|
-
n: repeatability.n,
|
|
3283
|
-
k: repeatability.k,
|
|
3284
|
-
...repeatability.classFlipRate === void 0 ? {} : { classFlipRate: repeatability.classFlipRate },
|
|
3285
|
-
...repeatability.admitExcludeFlipRate === void 0 ? {} : { admitExcludeFlipRate: repeatability.admitExcludeFlipRate }
|
|
3286
|
-
}
|
|
3287
|
-
}
|
|
3288
|
-
};
|
|
3289
|
-
});
|
|
3290
|
-
const census = (0, import_agent_qa_eval.deriveCensus)({
|
|
3291
|
-
snapshot: read.snapshot,
|
|
3292
|
-
...read.corpusEntriesById === void 0 ? {} : { corpusEntriesById: read.corpusEntriesById },
|
|
3293
|
-
...read.retiredMarkedCassetteIds === void 0 ? {} : { retiredMarkedCassetteIds: read.retiredMarkedCassetteIds },
|
|
3294
|
-
...campaign === void 0 ? {} : { campaign },
|
|
3295
|
-
reports,
|
|
3296
|
-
manifests: manifestRead.manifests,
|
|
3297
|
-
exclusions: exclusionRead.exclusions,
|
|
3298
|
-
schemaVersionsSeen: read.schemaVersionsSeen
|
|
3299
|
-
});
|
|
3300
|
-
let committed;
|
|
3301
|
-
if (opts.commit === true) {
|
|
3302
|
-
committed = (0, import_agent_qa_eval_local.commitCensus)({
|
|
3303
|
-
census,
|
|
3304
|
-
markdown: (0, import_agent_qa_eval.renderCensusMarkdown)(census),
|
|
3305
|
-
committedRoot
|
|
3306
|
-
});
|
|
3307
|
-
}
|
|
3308
|
-
return {
|
|
3309
|
-
corpusDigest: census.corpusDigest,
|
|
3310
|
-
entryCount: census.entryCount,
|
|
3311
|
-
productsSeen: read.productsSeen,
|
|
3312
|
-
census,
|
|
3313
|
-
...campaignManifest !== void 0 ? { campaignManifest } : {},
|
|
3314
|
-
...committed !== void 0 ? { committed } : {},
|
|
3315
|
-
...noiseFloor.length === 0 ? {} : { noiseFloor },
|
|
3316
|
-
notices
|
|
3317
|
-
};
|
|
3318
|
-
}
|
|
3319
|
-
async function resolveNamedProfile(name, configPath) {
|
|
3320
|
-
const { resolveNamedConfig: resolveNamedConfig2, resolveModelAuth: resolveModelAuth2, toModelProfileAndAuth: toModelProfileAndAuth2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
|
|
3321
|
-
const { config } = await resolveNamedConfig2(name, configPath);
|
|
3322
|
-
const auth = await resolveModelAuth2(config.name, config);
|
|
3323
|
-
if (auth.kind === "missing") {
|
|
3324
|
-
throw new Error(
|
|
3325
|
-
`No credentials for LLM config "${config.name}": ${auth.message} Run \`agent-qa auth login\` to authenticate.`
|
|
3326
|
-
);
|
|
3327
|
-
}
|
|
3328
|
-
return toModelProfileAndAuth2(config, auth);
|
|
3329
|
-
}
|
|
3330
|
-
async function loadEvalSecretStore(config, configDir) {
|
|
3331
|
-
const secretsPath = (0, import_node_path14.resolve)(configDir, config.workspace.secretsFile);
|
|
3332
|
-
let content;
|
|
3333
|
-
try {
|
|
3334
|
-
content = await (0, import_promises3.readFile)(secretsPath, "utf-8");
|
|
3335
|
-
} catch {
|
|
3336
|
-
throw new Error(
|
|
3337
|
-
`Secrets file not found or unreadable: ${secretsPath}. Create the file before running eval verbs (it may be empty, but it must exist).`
|
|
3338
|
-
);
|
|
3339
|
-
}
|
|
3340
|
-
try {
|
|
3341
|
-
const { SecretStore: SecretStore2 } = await import("@vostride/agent-qa-runner");
|
|
3342
|
-
return SecretStore2.fromEnvContent(content);
|
|
3343
|
-
} catch (err) {
|
|
3344
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3345
|
-
throw new Error(`Secrets file is invalid: ${secretsPath}: ${message}`);
|
|
3346
|
-
}
|
|
3347
|
-
}
|
|
3348
|
-
function mergeEvalConfig(fromConfig, flags) {
|
|
3349
|
-
const configured = selectedEvalJudgeNames(fromConfig ?? {});
|
|
3350
|
-
const names = flags?.judge === void 0 ? configured : selectedEvalJudgeNames(typeof flags.judge === "string" ? { judge: flags.judge } : { judges: flags.judge });
|
|
3351
|
-
const judgeAllowSameFamily = fromConfig?.judgeAllowSameFamily;
|
|
3352
|
-
const k = flags?.k ?? fromConfig?.k;
|
|
3353
|
-
const seed = flags?.seed ?? fromConfig?.seed;
|
|
3354
|
-
const silverPerTestCap = flags?.silverCap ?? fromConfig?.seeding?.silverPerTestCap;
|
|
3355
|
-
return {
|
|
3356
|
-
...names.length === 1 ? { judge: names[0] } : names.length > 1 ? { judges: names } : {},
|
|
3357
|
-
...judgeAllowSameFamily !== void 0 ? { judgeAllowSameFamily } : {},
|
|
3358
|
-
...k !== void 0 ? { k } : {},
|
|
3359
|
-
...seed !== void 0 ? { seed } : {},
|
|
3360
|
-
...fromConfig?.gate2 === void 0 ? {} : { gate2: fromConfig.gate2 },
|
|
3361
|
-
...fromConfig?.adversarial === void 0 ? {} : { adversarial: fromConfig.adversarial },
|
|
3362
|
-
...silverPerTestCap !== void 0 ? { seeding: { silverPerTestCap } } : {},
|
|
3363
|
-
// 422-09: forwarded WHOLE, exactly as `gate2` and `adversarial` above are. The field-by-field
|
|
3364
|
-
// rebuild this replaces silently dropped every member the schema gained after it was written.
|
|
3365
|
-
...fromConfig?.gate1 === void 0 ? {} : { gate1: fromConfig.gate1 }
|
|
3366
|
-
};
|
|
3367
|
-
}
|
|
3368
|
-
var import_agent_qa_schema9, import_agent_qa_core4, import_node_crypto5, import_node_fs7, import_promises3, import_node_path14, import_agent_qa_runner_local2, import_agent_qa_eval, import_agent_qa_eval_local, import_agent_qa_safety_local, import_agent_qa_schema10, import_agent_qa_agent_local4, import_agent_qa_eval2, import_agent_qa_eval3, import_agent_qa_eval4, import_agent_qa_eval_local2, import_agent_qa_eval5, import_agent_qa_eval6, import_agent_qa_eval_local3, import_agent_qa_eval_local4, import_agent_qa_eval7, import_agent_qa_eval_local5, import_agent_qa_runner_local3, import_agent_qa_safety_local2, import_agent_qa_model3, import_agent_qa_eval8, EvalJudgeDeclarationError, EvalNamedInputError, FLOOR_DEFAULT_REPEATS, FLOOR_DEFAULT_SEEDS, FLOOR_DEFAULT_BOOTSTRAP_SEED, LABEL_CALIBRATION_MIN_GOLD, GATE1_DEFAULT_MAX_SCREENS, GATE1_DEFAULT_K, GATE2_DEFAULT_MAX_ABLATIONS, GATE2_DEFAULT_CANARY_HORIZON_BUILDS, ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS, EXCLUDED_VERDICT_CLASSES, EvalPartitionFlagsRequiredError, EvalPartitionManifestConflictError;
|
|
3369
|
-
var init_eval_session_composition = __esm({
|
|
3370
|
-
"src/eval-session-composition.ts"() {
|
|
3371
|
-
"use strict";
|
|
3372
|
-
import_agent_qa_schema9 = require("@vostride/agent-qa-schema");
|
|
3373
|
-
import_agent_qa_core4 = require("@vostride/agent-qa-core");
|
|
3374
|
-
import_node_crypto5 = require("crypto");
|
|
3375
|
-
import_node_fs7 = require("fs");
|
|
3376
|
-
import_promises3 = require("fs/promises");
|
|
3377
|
-
import_node_path14 = require("path");
|
|
3378
|
-
import_agent_qa_runner_local2 = require("@vostride/agent-qa-runner-local");
|
|
3379
|
-
import_agent_qa_eval = require("@vostride/agent-qa-eval");
|
|
3380
|
-
import_agent_qa_eval_local = require("@vostride/agent-qa-eval-local");
|
|
3381
|
-
import_agent_qa_safety_local = require("@vostride/agent-qa-safety-local");
|
|
3382
|
-
import_agent_qa_schema10 = require("@vostride/agent-qa-schema");
|
|
3383
|
-
init_config();
|
|
3384
|
-
init_declared_secrets();
|
|
3385
|
-
init_targets();
|
|
3386
|
-
init_promotion_posture();
|
|
3387
|
-
init_eval_report_render();
|
|
3388
|
-
import_agent_qa_agent_local4 = require("@vostride/agent-qa-agent-local");
|
|
3389
|
-
import_agent_qa_eval2 = require("@vostride/agent-qa-eval");
|
|
3390
|
-
import_agent_qa_eval3 = require("@vostride/agent-qa-eval");
|
|
3391
|
-
import_agent_qa_eval4 = require("@vostride/agent-qa-eval");
|
|
3392
|
-
import_agent_qa_eval_local2 = require("@vostride/agent-qa-eval-local");
|
|
3393
|
-
init_judge_panel_preflight();
|
|
3394
|
-
init_judge_panel_preflight();
|
|
3395
|
-
import_agent_qa_eval5 = require("@vostride/agent-qa-eval");
|
|
3396
|
-
import_agent_qa_eval6 = require("@vostride/agent-qa-eval");
|
|
3397
|
-
import_agent_qa_eval_local3 = require("@vostride/agent-qa-eval-local");
|
|
3398
|
-
import_agent_qa_eval_local4 = require("@vostride/agent-qa-eval-local");
|
|
3399
|
-
import_agent_qa_eval7 = require("@vostride/agent-qa-eval");
|
|
3400
|
-
import_agent_qa_eval_local5 = require("@vostride/agent-qa-eval-local");
|
|
3401
|
-
import_agent_qa_runner_local3 = require("@vostride/agent-qa-runner-local");
|
|
3402
|
-
import_agent_qa_safety_local2 = require("@vostride/agent-qa-safety-local");
|
|
3403
|
-
import_agent_qa_model3 = require("@vostride/agent-qa-model");
|
|
3404
|
-
import_agent_qa_eval8 = require("@vostride/agent-qa-eval");
|
|
3405
|
-
init_adversarial_probe_preflight();
|
|
3406
|
-
init_adversarial_probe_author();
|
|
3407
|
-
EvalJudgeDeclarationError = class extends TypeError {
|
|
3408
|
-
constructor(message) {
|
|
3409
|
-
super(message);
|
|
3410
|
-
this.name = "EvalJudgeDeclarationError";
|
|
3411
|
-
}
|
|
3412
|
-
};
|
|
3413
|
-
EvalNamedInputError = class extends Error {
|
|
3414
|
-
inputKind;
|
|
3415
|
-
inputName;
|
|
3416
|
-
searchRoot;
|
|
3417
|
-
constructor(inputKind, inputName, searchRoot, verb = "extract", detail) {
|
|
3418
|
-
super(
|
|
3419
|
-
detail === void 0 ? `No recorded ${inputKind} named "${inputName}" under ${searchRoot} \u2014 nothing to ${verb}.` : `Cannot ${verb} ${inputKind} "${inputName}" at ${searchRoot}: ${detail}`
|
|
3420
|
-
);
|
|
3421
|
-
this.name = "EvalNamedInputError";
|
|
3422
|
-
this.inputKind = inputKind;
|
|
3423
|
-
this.inputName = inputName;
|
|
3424
|
-
this.searchRoot = searchRoot;
|
|
3425
|
-
}
|
|
3426
|
-
};
|
|
3427
|
-
FLOOR_DEFAULT_REPEATS = 3;
|
|
3428
|
-
FLOOR_DEFAULT_SEEDS = [11, 29];
|
|
3429
|
-
FLOOR_DEFAULT_BOOTSTRAP_SEED = 7;
|
|
3430
|
-
LABEL_CALIBRATION_MIN_GOLD = 20;
|
|
3431
|
-
GATE1_DEFAULT_MAX_SCREENS = 64;
|
|
3432
|
-
GATE1_DEFAULT_K = 3;
|
|
3433
|
-
GATE2_DEFAULT_MAX_ABLATIONS = 4;
|
|
3434
|
-
GATE2_DEFAULT_CANARY_HORIZON_BUILDS = 5;
|
|
3435
|
-
ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS = 4;
|
|
3436
|
-
EXCLUDED_VERDICT_CLASSES = new Set(
|
|
3437
|
-
// 414 D-02: the local literal triplet is gone. `EXCLUSION_CLASSES` is the SAME closed
|
|
3438
|
-
// vocabulary, and it now also decides what the committed exclusion record admits — a second
|
|
3439
|
-
// spelling here could let the seeder drop a run the record schema would refuse to describe.
|
|
3440
|
-
import_agent_qa_eval.EXCLUSION_CLASSES
|
|
3441
|
-
);
|
|
3442
|
-
EvalPartitionFlagsRequiredError = class extends Error {
|
|
3443
|
-
products;
|
|
3444
|
-
constructor(products) {
|
|
3445
|
-
super(
|
|
3446
|
-
`partition of ${products.join(", ")} requires --held-out-fraction and --private-fraction on its first seed (414 D-07); no default exists`
|
|
3447
|
-
);
|
|
3448
|
-
this.name = "EvalPartitionFlagsRequiredError";
|
|
3449
|
-
this.products = products;
|
|
3450
|
-
}
|
|
3451
|
-
};
|
|
3452
|
-
EvalPartitionManifestConflictError = class extends Error {
|
|
3453
|
-
product;
|
|
3454
|
-
path;
|
|
3455
|
-
constructor(product, path) {
|
|
3456
|
-
super(
|
|
3457
|
-
`partition manifest for ${product} at ${path}: write-conflict \u2014 the committed manifest disagrees with the supplied fractions; changing a manifest is a reviewed git edit and a named migration (414 D-05)`
|
|
3458
|
-
);
|
|
3459
|
-
this.name = "EvalPartitionManifestConflictError";
|
|
3460
|
-
this.product = product;
|
|
3461
|
-
this.path = path;
|
|
3462
|
-
}
|
|
3463
|
-
};
|
|
3464
|
-
}
|
|
3465
|
-
});
|
|
3466
|
-
|
|
3467
|
-
// src/index.ts
|
|
3468
|
-
var index_exports = {};
|
|
3469
|
-
__export(index_exports, {
|
|
3470
|
-
CURATION_TAIL_RECEIPT_FILE: () => CURATION_TAIL_RECEIPT_FILE,
|
|
3471
|
-
CURATION_TAIL_RECEIPT_SCHEMA_VERSION: () => CURATION_TAIL_RECEIPT_SCHEMA_VERSION,
|
|
3472
|
-
CURATOR_MAX_NEIGHBORS: () => CURATOR_MAX_NEIGHBORS,
|
|
3473
|
-
CURATOR_RETRY_CAP: () => CURATOR_RETRY_CAP,
|
|
3474
|
-
ConfigFileNotFoundError: () => ConfigFileNotFoundError,
|
|
3475
|
-
ConfigFileNotYamlError: () => ConfigFileNotYamlError,
|
|
3476
|
-
VERSION: () => VERSION,
|
|
3477
|
-
buildCuratorAuthor: () => buildCuratorAuthor,
|
|
3478
|
-
curationTailHandoffDigest: () => curationTailHandoffDigest,
|
|
3479
|
-
deepMerge: () => deepMerge,
|
|
3480
|
-
emitCanaryOutcomes: () => emitCanaryOutcomes,
|
|
3481
|
-
formatConfigDebug: () => formatConfigDebug,
|
|
3482
|
-
listTargets: () => listTargets,
|
|
3483
|
-
loadConfigFile: () => loadConfigFile,
|
|
3484
|
-
loadConfigPlugins: () => loadConfigPlugins,
|
|
3485
|
-
loadEnvOverrides: () => loadEnvOverrides,
|
|
3486
|
-
loadRequiredConfigFile: () => loadRequiredConfigFile,
|
|
3487
|
-
mergeConfigs: () => mergeConfigs,
|
|
3488
|
-
mergeUseBlocks: () => mergeUseBlocks,
|
|
3489
|
-
mergeWithTestConfig: () => mergeWithTestConfig,
|
|
3490
|
-
normalizeConfigForValidation: () => normalizeConfigForValidation,
|
|
3491
|
-
projectCanaryOutcomes: () => projectCanaryOutcomes,
|
|
3492
|
-
purgeAll: () => purgeAll,
|
|
3493
|
-
purgeTest: () => purgeTest,
|
|
3494
|
-
readAnalyticsPrivacyDeclaration: () => readAnalyticsPrivacyDeclaration,
|
|
3495
|
-
resolveConfig: () => resolveConfig,
|
|
3496
|
-
resolveConfiguredKnowledgeRoot: () => resolveConfiguredKnowledgeRoot,
|
|
3497
|
-
resolveRawConfigKnowledgeRoot: () => resolveRawConfigKnowledgeRoot,
|
|
3498
|
-
resolveTarget: () => resolveTarget,
|
|
3499
|
-
runCurationTail: () => runCurationTail,
|
|
3500
|
-
runVerdictFirstCurationTail: () => runVerdictFirstCurationTail
|
|
3501
|
-
});
|
|
3502
|
-
module.exports = __toCommonJS(index_exports);
|
|
3503
|
-
var import_agent_qa_core6 = require("@vostride/agent-qa-core");
|
|
3504
|
-
__reExport(index_exports, require("@vostride/agent-qa-core"), module.exports);
|
|
3505
|
-
init_config();
|
|
3506
|
-
init_targets();
|
|
3507
|
-
|
|
3508
|
-
// src/commands/cache.ts
|
|
3509
|
-
var import_commander = require("commander");
|
|
3510
|
-
var import_picocolors2 = __toESM(require("picocolors"), 1);
|
|
3511
|
-
var import_promises2 = require("fs/promises");
|
|
3512
|
-
var import_node_path3 = require("path");
|
|
3513
|
-
var import_agent_qa_core2 = require("@vostride/agent-qa-core");
|
|
3514
|
-
init_config();
|
|
3515
|
-
async function removeCacheEntries(cacheRoot) {
|
|
3516
|
-
let branchDirs;
|
|
3517
|
-
try {
|
|
3518
|
-
branchDirs = await (0, import_promises2.readdir)(cacheRoot);
|
|
3519
|
-
} catch (err) {
|
|
3520
|
-
if (err.code === "ENOENT") return 0;
|
|
3521
|
-
throw err;
|
|
3522
|
-
}
|
|
3523
|
-
let purged = 0;
|
|
3524
|
-
for (const branch of branchDirs) {
|
|
3525
|
-
const branchPath = (0, import_node_path3.join)(cacheRoot, branch);
|
|
3526
|
-
let entryStat;
|
|
3527
|
-
try {
|
|
3528
|
-
entryStat = await (0, import_promises2.stat)(branchPath);
|
|
3529
|
-
} catch {
|
|
3530
|
-
continue;
|
|
3531
|
-
}
|
|
3532
|
-
if (!entryStat.isDirectory()) {
|
|
3533
|
-
await (0, import_promises2.rm)(branchPath, { force: true });
|
|
3534
|
-
continue;
|
|
3535
|
-
}
|
|
3536
|
-
let files;
|
|
3537
|
-
try {
|
|
3538
|
-
files = await (0, import_promises2.readdir)(branchPath);
|
|
3539
|
-
} catch {
|
|
3540
|
-
continue;
|
|
3541
|
-
}
|
|
3542
|
-
for (const file of files) {
|
|
3543
|
-
if (!file.endsWith(".json")) continue;
|
|
3544
|
-
await (0, import_promises2.rm)((0, import_node_path3.join)(branchPath, file), { force: true });
|
|
3545
|
-
purged++;
|
|
3546
|
-
}
|
|
3547
|
-
await (0, import_promises2.rm)(branchPath, { recursive: true, force: true }).catch(() => {
|
|
3548
|
-
});
|
|
3549
|
-
}
|
|
3550
|
-
return purged;
|
|
359
|
+
async function loadConfigPlugins(config, configPath) {
|
|
360
|
+
await (0, import_agent_qa_core.loadLLMAuthPlugins)(config.plugins?.auth, {
|
|
361
|
+
baseDir: (0, import_node_path2.dirname)((0, import_node_path2.resolve)(configPath))
|
|
362
|
+
});
|
|
363
|
+
const { loadProviderPluginsForRawConfig: loadProviderPluginsForRawConfig2 } = await Promise.resolve().then(() => (init_provider_plugins(), provider_plugins_exports));
|
|
364
|
+
await loadProviderPluginsForRawConfig2(config, configPath);
|
|
365
|
+
}
|
|
366
|
+
function resolveConfiguredKnowledgeRoot(config, configDir) {
|
|
367
|
+
return (0, import_agent_qa_core.resolveKnowledgeRoot)(config.knowledge, configDir);
|
|
3551
368
|
}
|
|
3552
|
-
|
|
3553
|
-
|
|
369
|
+
function resolveRawConfigKnowledgeRoot(config, configPath) {
|
|
370
|
+
if (config === null) return null;
|
|
3554
371
|
try {
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
372
|
+
return (0, import_agent_qa_core.resolveKnowledgeRoot)(
|
|
373
|
+
import_agent_qa_core.KnowledgeConfigSchema.parse(config.knowledge ?? {}),
|
|
374
|
+
(0, import_node_path2.dirname)((0, import_node_path2.resolve)(configPath))
|
|
375
|
+
);
|
|
376
|
+
} catch {
|
|
377
|
+
return null;
|
|
3559
378
|
}
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// src/targets.ts
|
|
382
|
+
function resolveTarget(config, targetName) {
|
|
383
|
+
const targets = config.registry?.targets;
|
|
384
|
+
if (!targets || !(targetName in targets)) {
|
|
385
|
+
throw new Error(`Target "${targetName}" not found in config. Available targets: ${targets ? Object.keys(targets).join(", ") : "none"}`);
|
|
3567
386
|
}
|
|
3568
|
-
|
|
387
|
+
const target = targets[targetName];
|
|
388
|
+
return {
|
|
389
|
+
name: targetName,
|
|
390
|
+
product: target.product ?? targetName,
|
|
391
|
+
platform: target.platform,
|
|
392
|
+
url: target.url,
|
|
393
|
+
bundleId: target.bundleId,
|
|
394
|
+
appPackage: target.appPackage,
|
|
395
|
+
appActivity: target.appActivity,
|
|
396
|
+
device: target.device,
|
|
397
|
+
// Carried unchanged, `undefined` included: an undeclared target gets no filled-in class.
|
|
398
|
+
environment: target.environment
|
|
399
|
+
};
|
|
3569
400
|
}
|
|
3570
|
-
|
|
3571
|
-
|
|
401
|
+
function listTargets(config) {
|
|
402
|
+
const targets = config.registry?.targets;
|
|
403
|
+
if (!targets) return [];
|
|
404
|
+
return Object.entries(targets).map(([name, entry]) => {
|
|
405
|
+
const typedEntry = entry;
|
|
406
|
+
return {
|
|
407
|
+
name,
|
|
408
|
+
platform: typedEntry.platform
|
|
409
|
+
};
|
|
410
|
+
});
|
|
3572
411
|
}
|
|
412
|
+
|
|
413
|
+
// src/commands/cache.ts
|
|
414
|
+
var import_commander = require("commander");
|
|
415
|
+
var import_picocolors2 = __toESM(require("picocolors"), 1);
|
|
416
|
+
var import_node_path3 = require("path");
|
|
417
|
+
var import_agent_qa_agent_local = require("@vostride/agent-qa-agent-local");
|
|
418
|
+
var import_agent_qa_core2 = require("@vostride/agent-qa-core");
|
|
3573
419
|
async function purgeAll(cacheDir) {
|
|
3574
|
-
|
|
3575
|
-
const total = await countCacheEntries(resolvedDir);
|
|
3576
|
-
if (total === 0) return 0;
|
|
3577
|
-
return removeCacheEntries(resolvedDir);
|
|
420
|
+
return (0, import_agent_qa_agent_local.purgeActionCache)((0, import_node_path3.resolve)(cacheDir));
|
|
3578
421
|
}
|
|
3579
422
|
|
|
3580
423
|
// src/curation/curator-author.ts
|
|
@@ -3586,7 +429,7 @@ var import_agent_qa_schema4 = require("@vostride/agent-qa-schema");
|
|
|
3586
429
|
// src/curation/emit-candidates.ts
|
|
3587
430
|
var import_node_path4 = require("path");
|
|
3588
431
|
var import_zod = require("zod");
|
|
3589
|
-
var
|
|
432
|
+
var import_agent_qa_agent_local2 = require("@vostride/agent-qa-agent-local");
|
|
3590
433
|
var import_agent_qa_runner_local = require("@vostride/agent-qa-runner-local");
|
|
3591
434
|
var import_agent_qa_schema3 = require("@vostride/agent-qa-schema");
|
|
3592
435
|
|
|
@@ -3877,9 +720,7 @@ var trajectoryCompositeValueSchema = import_zod.z.union([
|
|
|
3877
720
|
import_zod.z.record(import_zod.z.string(), import_agent_qa_schema3.procedureJsonValueSchema)
|
|
3878
721
|
]);
|
|
3879
722
|
var trajectoryValueSlotSchema = import_zod.z.object({
|
|
3880
|
-
//
|
|
3881
|
-
// fill.submit, scroll.maxSwipes/retryOpposite, durations, or scale here would author a
|
|
3882
|
-
// procedure whose behavior differs from the verified trajectory.
|
|
723
|
+
// Boolean and numeric action fields are replay semantics; dropping one changes the replay.
|
|
3883
724
|
observed: import_zod.z.union([
|
|
3884
725
|
import_zod.z.string(),
|
|
3885
726
|
import_zod.z.number(),
|
|
@@ -3887,11 +728,7 @@ var trajectoryValueSlotSchema = import_zod.z.object({
|
|
|
3887
728
|
trajectoryCompositeValueSchema
|
|
3888
729
|
]),
|
|
3889
730
|
source: import_zod.z.enum(["literal", "test-variable", "masked-secret"]),
|
|
3890
|
-
//
|
|
3891
|
-
// a test variable matched against Plan 381-41's interpolation provenance, or a secret
|
|
3892
|
-
// recovered from its recorded masked form. `curator-author` forces the covering
|
|
3893
|
-
// parameter to exactly this name so consult-time fillability (D-08, which matches on
|
|
3894
|
-
// binding NAMES) succeeds. Absent when the source is an anonymous literal.
|
|
731
|
+
// The binding name behind this slot; `curator-author` names the covering parameter after it.
|
|
3895
732
|
sourceName: import_zod.z.string().min(1).optional()
|
|
3896
733
|
}).strict();
|
|
3897
734
|
var trajectoryStepSchema = import_zod.z.object({
|
|
@@ -4189,7 +1026,7 @@ function emitCurationCandidates(opts) {
|
|
|
4189
1026
|
const artifactSources = bindArtifactSources(projected, opts.runDir);
|
|
4190
1027
|
let outcome;
|
|
4191
1028
|
try {
|
|
4192
|
-
outcome =
|
|
1029
|
+
outcome = import_agent_qa_agent_local2.LOCAL_CURATION_CAPABILITIES.enqueueCurationCandidateProjection({
|
|
4193
1030
|
knowledgeRoot: opts.knowledgeRoot,
|
|
4194
1031
|
projection: projected,
|
|
4195
1032
|
artifactSources,
|
|
@@ -4361,9 +1198,9 @@ function buildPrompt(job, neighbors, accounting) {
|
|
|
4361
1198
|
const { contextWindow, maxOutput } = accounting.getCeilings();
|
|
4362
1199
|
const inputBudget = contextWindow - maxOutput;
|
|
4363
1200
|
if (inputBudget <= 0) return void 0;
|
|
4364
|
-
for (let
|
|
4365
|
-
const presented = limited.slice(0,
|
|
4366
|
-
const messages = renderPrompt(job, presented, neighbors.length -
|
|
1201
|
+
for (let count = limited.length; count >= 0; count -= 1) {
|
|
1202
|
+
const presented = limited.slice(0, count);
|
|
1203
|
+
const messages = renderPrompt(job, presented, neighbors.length - count);
|
|
4367
1204
|
if (accounting.estimate(messages) <= inputBudget) {
|
|
4368
1205
|
return {
|
|
4369
1206
|
messages,
|
|
@@ -5170,14 +2007,14 @@ function buildCuratorAuthor(deps) {
|
|
|
5170
2007
|
}
|
|
5171
2008
|
|
|
5172
2009
|
// src/curation/curation-tail.ts
|
|
5173
|
-
var
|
|
5174
|
-
var
|
|
5175
|
-
var
|
|
5176
|
-
var
|
|
5177
|
-
var
|
|
5178
|
-
var
|
|
5179
|
-
var
|
|
5180
|
-
var
|
|
2010
|
+
var import_node_crypto4 = require("crypto");
|
|
2011
|
+
var import_node_fs6 = require("fs");
|
|
2012
|
+
var import_node_path10 = require("path");
|
|
2013
|
+
var import_agent_qa_core4 = require("@vostride/agent-qa-core");
|
|
2014
|
+
var import_agent_qa_agent_local5 = require("@vostride/agent-qa-agent-local");
|
|
2015
|
+
var import_agent_qa_runner_local2 = require("@vostride/agent-qa-runner-local");
|
|
2016
|
+
var import_agent_qa_safety_local = require("@vostride/agent-qa-safety-local");
|
|
2017
|
+
var import_agent_qa_schema6 = require("@vostride/agent-qa-schema");
|
|
5181
2018
|
|
|
5182
2019
|
// src/daemon/client.ts
|
|
5183
2020
|
var import_node_http2 = require("http");
|
|
@@ -5189,7 +2026,7 @@ var import_agent_qa_analytics2 = require("@vostride/agent-qa-analytics");
|
|
|
5189
2026
|
var import_node_crypto2 = require("crypto");
|
|
5190
2027
|
var import_node_fs2 = require("fs");
|
|
5191
2028
|
var import_node_path6 = require("path");
|
|
5192
|
-
var
|
|
2029
|
+
var import_agent_qa_agent_local3 = require("@vostride/agent-qa-agent-local");
|
|
5193
2030
|
var import_zod4 = require("zod");
|
|
5194
2031
|
|
|
5195
2032
|
// src/daemon/server.ts
|
|
@@ -5351,8 +2188,8 @@ var DaemonIdentityHandshakeTimeoutError = class extends Error {
|
|
|
5351
2188
|
};
|
|
5352
2189
|
var DEFAULT_CLASSIFICATION_OPERATIONS = {
|
|
5353
2190
|
fetchIdentity: (registration) => fetchDaemonIdentity(registration.socketPath),
|
|
5354
|
-
readWriterLease:
|
|
5355
|
-
deriveWriterLeaseFreshness:
|
|
2191
|
+
readWriterLease: import_agent_qa_agent_local3.LOCAL_DAEMON_CAPABILITIES.readWriterLease,
|
|
2192
|
+
deriveWriterLeaseFreshness: import_agent_qa_agent_local3.LOCAL_DAEMON_CAPABILITIES.deriveWriterLeaseFreshness,
|
|
5356
2193
|
now: () => Date.now(),
|
|
5357
2194
|
setTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
|
|
5358
2195
|
clearTimeout: (timer) => clearTimeout(timer)
|
|
@@ -5424,15 +2261,7 @@ async function classifyDaemonRegistration(input) {
|
|
|
5424
2261
|
);
|
|
5425
2262
|
switch (verdict.kind) {
|
|
5426
2263
|
case "fresh":
|
|
5427
|
-
|
|
5428
|
-
* CR-01: `unknown` answers held, with the same holder payload as `fresh`.
|
|
5429
|
-
*
|
|
5430
|
-
* This verdict selects registration deletion and socket unlink, and the two preconditions
|
|
5431
|
-
* are correlated rather than independent — fd exhaustion produces both the failed identity
|
|
5432
|
-
* handshake and the unreadable epoch at once. Treating "I could not read the authority
|
|
5433
|
-
* state" as "nobody holds it" therefore de-registers a live, healthy daemon precisely when
|
|
5434
|
-
* the host is under stress. The refusal is the only recoverable answer.
|
|
5435
|
-
*/
|
|
2264
|
+
// `unknown` answers held: fd exhaustion fails both checks at once, so unknown is not free.
|
|
5436
2265
|
case "unknown":
|
|
5437
2266
|
return {
|
|
5438
2267
|
kind: "unreachable-lease-fresh",
|
|
@@ -5546,369 +2375,29 @@ async function nudgeDaemon(agentQaRoot, options = {}) {
|
|
|
5546
2375
|
}
|
|
5547
2376
|
|
|
5548
2377
|
// src/curation/emit-canary-outcomes.ts
|
|
5549
|
-
var import_node_fs8 = require("fs");
|
|
5550
|
-
var import_node_path15 = require("path");
|
|
5551
|
-
var import_agent_qa_agent_local5 = require("@vostride/agent-qa-agent-local");
|
|
5552
|
-
var import_agent_qa_schema11 = require("@vostride/agent-qa-schema");
|
|
5553
|
-
|
|
5554
|
-
// src/curation/canary-run-state.ts
|
|
5555
2378
|
var import_node_fs3 = require("fs");
|
|
5556
|
-
var import_node_crypto3 = require("crypto");
|
|
5557
2379
|
var import_node_path8 = require("path");
|
|
5558
|
-
var
|
|
2380
|
+
var import_agent_qa_agent_local4 = require("@vostride/agent-qa-agent-local");
|
|
5559
2381
|
var import_agent_qa_schema5 = require("@vostride/agent-qa-schema");
|
|
5560
|
-
var POLICY_FILE = "canary-policy.json";
|
|
5561
|
-
function readRunCanaryPolicy(runDir, runId) {
|
|
5562
|
-
let text;
|
|
5563
|
-
try {
|
|
5564
|
-
text = (0, import_node_fs3.readFileSync)((0, import_node_path8.join)(runDir, POLICY_FILE), "utf8");
|
|
5565
|
-
} catch (error) {
|
|
5566
|
-
if (error.code === "ENOENT") return void 0;
|
|
5567
|
-
throw new Error("canary policy unavailable");
|
|
5568
|
-
}
|
|
5569
|
-
const policy = JSON.parse(text);
|
|
5570
|
-
if (typeof policy !== "object" || policy === null || Array.isArray(policy) || Object.keys(policy).sort().join(",") !== "maxAblations,runId,schemaVersion,source" || !("schemaVersion" in policy) || policy.schemaVersion !== 2 || !("runId" in policy) || policy.runId !== runId || !("maxAblations" in policy) || typeof policy.maxAblations !== "number" || !Number.isSafeInteger(policy.maxAblations) || policy.maxAblations < 1 || !("source" in policy) || typeof policy.source !== "string" || !["flag", "config", "default"].includes(policy.source)) {
|
|
5571
|
-
throw new Error("canary policy invalid");
|
|
5572
|
-
}
|
|
5573
|
-
return policy;
|
|
5574
|
-
}
|
|
5575
|
-
var count = import_zod5.z.number().int().nonnegative();
|
|
5576
|
-
var digest = import_zod5.z.string().regex(/^[a-f0-9]{64}$/);
|
|
5577
|
-
var completionSchema = import_zod5.z.object({
|
|
5578
|
-
schemaVersion: import_zod5.z.literal(2),
|
|
5579
|
-
runId: import_zod5.z.string().min(1),
|
|
5580
|
-
candidateId: import_zod5.z.string().min(1),
|
|
5581
|
-
attributedObservationId: import_zod5.z.string().min(1),
|
|
5582
|
-
observedStateDigest: digest,
|
|
5583
|
-
result: import_zod5.z.union([
|
|
5584
|
-
import_zod5.z.object({
|
|
5585
|
-
kind: import_zod5.z.literal("ablated"),
|
|
5586
|
-
crossCheck: import_zod5.z.enum(import_agent_qa_schema5.CROSS_CHECKS),
|
|
5587
|
-
corpusDigest: digest,
|
|
5588
|
-
k: import_zod5.z.number().int().min(3).refine((k) => k % 2 === 1),
|
|
5589
|
-
baseSeed: import_zod5.z.number().int(),
|
|
5590
|
-
seeds: import_zod5.z.array(import_zod5.z.number().int()),
|
|
5591
|
-
seedHonored: import_zod5.z.boolean(),
|
|
5592
|
-
spend: import_zod5.z.object({ modelCalls: count, screens: count, promptTokens: count, completionTokens: count }).strict(),
|
|
5593
|
-
gate2Dir: import_zod5.z.string().min(1)
|
|
5594
|
-
}).strict().refine((result) => result.seeds.length === result.k && result.seeds[0] === result.baseSeed),
|
|
5595
|
-
import_zod5.z.object({ kind: import_zod5.z.literal("error"), message: import_zod5.z.literal("ablation-error") }).strict(),
|
|
5596
|
-
import_zod5.z.object({ kind: import_zod5.z.literal("unavailable"), reason: import_zod5.z.literal("ablation-unavailable") }).strict(),
|
|
5597
|
-
import_zod5.z.object({ kind: import_zod5.z.literal("no-eval-root") }).strict()
|
|
5598
|
-
])
|
|
5599
|
-
}).strict();
|
|
5600
|
-
var COMPLETIONS_DIR = "canary-completions";
|
|
5601
|
-
function readCompletedCanaryAblations(runDir, runId) {
|
|
5602
|
-
const dir = (0, import_node_path8.join)(runDir, COMPLETIONS_DIR);
|
|
5603
|
-
let names;
|
|
5604
|
-
try {
|
|
5605
|
-
names = (0, import_node_fs3.readdirSync)(dir).sort();
|
|
5606
|
-
} catch (error) {
|
|
5607
|
-
if (error.code === "ENOENT") return [];
|
|
5608
|
-
throw new Error("completed canary evidence unavailable");
|
|
5609
|
-
}
|
|
5610
|
-
return names.filter((name) => !name.startsWith(".")).map((name) => {
|
|
5611
|
-
let value;
|
|
5612
|
-
try {
|
|
5613
|
-
value = completionSchema.parse(JSON.parse((0, import_node_fs3.readFileSync)((0, import_node_path8.join)(dir, name), "utf8")));
|
|
5614
|
-
} catch {
|
|
5615
|
-
throw new Error("completed canary evidence invalid");
|
|
5616
|
-
}
|
|
5617
|
-
if (value.runId !== runId || name !== `${(0, import_agent_qa_schema5.sha256hex)(value.candidateId)}.json`) {
|
|
5618
|
-
throw new Error("completed canary evidence identity mismatch");
|
|
5619
|
-
}
|
|
5620
|
-
return value;
|
|
5621
|
-
});
|
|
5622
|
-
}
|
|
5623
|
-
function saveCompletedCanaryAblation(runDir, input) {
|
|
5624
|
-
const value = completionSchema.parse(input);
|
|
5625
|
-
const dir = (0, import_node_path8.join)(runDir, COMPLETIONS_DIR);
|
|
5626
|
-
(0, import_node_fs3.mkdirSync)(dir, { recursive: true });
|
|
5627
|
-
const file = (0, import_node_path8.join)(dir, `${(0, import_agent_qa_schema5.sha256hex)(value.candidateId)}.json`);
|
|
5628
|
-
const temporary = (0, import_node_path8.join)(dir, `.${(0, import_node_crypto3.randomUUID)()}.tmp`);
|
|
5629
|
-
const text = (0, import_agent_qa_schema5.canonicalJson)(value) + "\n";
|
|
5630
|
-
try {
|
|
5631
|
-
(0, import_node_fs3.writeFileSync)(temporary, text, { flag: "wx", mode: 384 });
|
|
5632
|
-
const fd = (0, import_node_fs3.openSync)(temporary, "r");
|
|
5633
|
-
try {
|
|
5634
|
-
(0, import_node_fs3.fsyncSync)(fd);
|
|
5635
|
-
} finally {
|
|
5636
|
-
(0, import_node_fs3.closeSync)(fd);
|
|
5637
|
-
}
|
|
5638
|
-
try {
|
|
5639
|
-
(0, import_node_fs3.linkSync)(temporary, file);
|
|
5640
|
-
} catch (error) {
|
|
5641
|
-
if (error.code !== "EEXIST" || (0, import_node_fs3.readFileSync)(file, "utf8") !== text) {
|
|
5642
|
-
throw new Error("completed canary evidence conflict");
|
|
5643
|
-
}
|
|
5644
|
-
}
|
|
5645
|
-
const directory = (0, import_node_fs3.openSync)(dir, "r");
|
|
5646
|
-
try {
|
|
5647
|
-
(0, import_node_fs3.fsyncSync)(directory);
|
|
5648
|
-
} finally {
|
|
5649
|
-
(0, import_node_fs3.closeSync)(directory);
|
|
5650
|
-
}
|
|
5651
|
-
} finally {
|
|
5652
|
-
(0, import_node_fs3.rmSync)(temporary, { force: true });
|
|
5653
|
-
}
|
|
5654
|
-
}
|
|
5655
|
-
|
|
5656
|
-
// src/curation/misdirection-run-state.ts
|
|
5657
|
-
var import_node_fs4 = require("fs");
|
|
5658
|
-
var import_node_crypto4 = require("crypto");
|
|
5659
|
-
var import_node_path9 = require("path");
|
|
5660
|
-
var import_zod6 = require("zod");
|
|
5661
|
-
var import_agent_qa_schema6 = require("@vostride/agent-qa-schema");
|
|
5662
|
-
var MISDIRECTION_COMPLETIONS_DIR = "misdirection-completions";
|
|
5663
|
-
function misdirectionCheckKeyDigest(key) {
|
|
5664
|
-
return (0, import_agent_qa_schema6.sha256hex)(
|
|
5665
|
-
(0, import_agent_qa_schema6.canonicalJson)({
|
|
5666
|
-
runId: key.runId,
|
|
5667
|
-
perTest: key.perTest,
|
|
5668
|
-
stepIndex: key.stepIndex,
|
|
5669
|
-
decisionOrdinal: key.decisionOrdinal,
|
|
5670
|
-
recordId: key.recordId
|
|
5671
|
-
})
|
|
5672
|
-
);
|
|
5673
|
-
}
|
|
5674
|
-
var policySchema = import_zod6.z.object({
|
|
5675
|
-
schemaVersion: import_zod6.z.literal(2),
|
|
5676
|
-
runId: import_zod6.z.string().min(1),
|
|
5677
|
-
maxChecks: import_zod6.z.number().int().nonnegative(),
|
|
5678
|
-
source: import_zod6.z.enum(["flag", "default"])
|
|
5679
|
-
}).strict();
|
|
5680
|
-
var reservationSchema = import_zod6.z.object({
|
|
5681
|
-
schemaVersion: import_zod6.z.literal(2),
|
|
5682
|
-
runId: import_zod6.z.string().min(1),
|
|
5683
|
-
perTest: import_zod6.z.number().int().nonnegative(),
|
|
5684
|
-
stepIndex: import_zod6.z.number().int().nonnegative(),
|
|
5685
|
-
decisionOrdinal: import_zod6.z.number().int().nonnegative(),
|
|
5686
|
-
recordId: import_zod6.z.string().min(1),
|
|
5687
|
-
handoffBaseDigest: import_zod6.z.string().min(1),
|
|
5688
|
-
briefDigest: import_zod6.z.string().min(1),
|
|
5689
|
-
corpusDigest: import_zod6.z.string().min(1),
|
|
5690
|
-
profileDigest: import_zod6.z.string().min(1)
|
|
5691
|
-
}).strict();
|
|
5692
|
-
function readDocuments(dir, unavailable) {
|
|
5693
|
-
let names;
|
|
5694
|
-
try {
|
|
5695
|
-
names = (0, import_node_fs4.readdirSync)(dir).sort();
|
|
5696
|
-
} catch (error) {
|
|
5697
|
-
if (error.code === "ENOENT") return [];
|
|
5698
|
-
throw new Error(unavailable);
|
|
5699
|
-
}
|
|
5700
|
-
return names.filter((name) => !name.startsWith(".")).map((name) => ({ name, text: (0, import_node_fs4.readFileSync)((0, import_node_path9.join)(dir, name), "utf8") }));
|
|
5701
|
-
}
|
|
5702
|
-
function readCompletedMisdirectionChecks(runDir, runId) {
|
|
5703
|
-
return readDocuments(
|
|
5704
|
-
(0, import_node_path9.join)(runDir, MISDIRECTION_COMPLETIONS_DIR),
|
|
5705
|
-
"completed misdirection evidence unavailable"
|
|
5706
|
-
).map(({ name, text }) => {
|
|
5707
|
-
let value;
|
|
5708
|
-
try {
|
|
5709
|
-
value = import_agent_qa_schema6.MisdirectionRecordResultSchema.parse(JSON.parse(text));
|
|
5710
|
-
} catch {
|
|
5711
|
-
throw new Error("completed misdirection evidence invalid");
|
|
5712
|
-
}
|
|
5713
|
-
if (value.runId !== runId || name !== `${misdirectionCheckKeyDigest(value)}.json`) {
|
|
5714
|
-
throw new Error("completed misdirection evidence identity mismatch");
|
|
5715
|
-
}
|
|
5716
|
-
return value;
|
|
5717
|
-
});
|
|
5718
|
-
}
|
|
5719
|
-
var postureSchema = import_zod6.z.object({
|
|
5720
|
-
schemaVersion: import_zod6.z.literal(2),
|
|
5721
|
-
runId: import_zod6.z.string().min(1),
|
|
5722
|
-
kind: import_zod6.z.enum(["eval-root", "no-eval-root", "broken-eval-root"]),
|
|
5723
|
-
corpusDigest: import_zod6.z.string().min(1).optional(),
|
|
5724
|
-
profileDigest: import_zod6.z.string().min(1).optional(),
|
|
5725
|
-
reason: import_zod6.z.string().min(1).optional()
|
|
5726
|
-
}).strict();
|
|
5727
2382
|
|
|
5728
2383
|
// src/curation/canary-credit.ts
|
|
5729
|
-
var import_agent_qa_schema7 = require("@vostride/agent-qa-schema");
|
|
5730
2384
|
function emptyCanaryCreditCounters() {
|
|
5731
|
-
return {
|
|
5732
|
-
credited: 0,
|
|
5733
|
-
rideAlongUncredited: 0,
|
|
5734
|
-
ablationCapExceeded: 0,
|
|
5735
|
-
ablationHeld: 0,
|
|
5736
|
-
ablationInconclusive: 0,
|
|
5737
|
-
ablationError: 0,
|
|
5738
|
-
rerunExcluded: 0,
|
|
5739
|
-
nonSignal: { environment: 0, timeout: 0, flake: 0 },
|
|
5740
|
-
staged: 0,
|
|
5741
|
-
stagingRefused: 0
|
|
5742
|
-
};
|
|
2385
|
+
return { rideAlongUncredited: 0, rerunExcluded: 0, nonSignal: { environment: 0, timeout: 0, flake: 0 } };
|
|
5743
2386
|
}
|
|
5744
|
-
|
|
5745
|
-
if (!Number.isSafeInteger(input.maxAblations) || input.maxAblations < 1) {
|
|
5746
|
-
throw new TypeError("--max-ablations must be a positive integer");
|
|
5747
|
-
}
|
|
2387
|
+
function creditCanaryAttributions(input) {
|
|
5748
2388
|
const counters = emptyCanaryCreditCounters();
|
|
5749
2389
|
const attributions = [];
|
|
5750
|
-
let digestRead = false;
|
|
5751
|
-
let corpusDigest;
|
|
5752
|
-
const ablatedCandidates = /* @__PURE__ */ new Set();
|
|
5753
|
-
const completedByCandidate = new Map(input.completedAblations.map((entry) => [entry.candidateId, entry]));
|
|
5754
|
-
let ablationsRun = completedByCandidate.size;
|
|
5755
2390
|
for (const row of input.attributions) {
|
|
5756
2391
|
if (row.kind !== "outcome") {
|
|
5757
2392
|
attributions.push(row);
|
|
5758
2393
|
continue;
|
|
5759
2394
|
}
|
|
5760
|
-
const served = row.source === "served-brief";
|
|
5761
|
-
if (served && input.ablation !== void 0 && !digestRead) {
|
|
5762
|
-
digestRead = true;
|
|
5763
|
-
try {
|
|
5764
|
-
const digest2 = await input.ablation.corpusDigest();
|
|
5765
|
-
if (digest2.kind === "digest") corpusDigest = digest2.corpusDigest;
|
|
5766
|
-
} catch {
|
|
5767
|
-
}
|
|
5768
|
-
}
|
|
5769
2395
|
const outcome = {
|
|
5770
2396
|
...row.outcome,
|
|
5771
|
-
|
|
5772
|
-
benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along",
|
|
5773
|
-
...served && corpusDigest !== void 0 ? { corpusDigest } : {}
|
|
2397
|
+
benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along"
|
|
5774
2398
|
};
|
|
5775
|
-
if (
|
|
5776
|
-
|
|
5777
|
-
delete outcome.ablation;
|
|
5778
|
-
if (served) {
|
|
5779
|
-
const step = input.handoff.tests.find((test) => test.testIndex === outcome.testIndex && test.testId === outcome.testId)?.steps.find((candidate) => candidate.stepIndex === outcome.stepIndex);
|
|
5780
|
-
const live = step?.subActions?.filter((action2) => action2.exchanges !== void 0) ?? [];
|
|
5781
|
-
const action = live.length === 1 ? live[0] : void 0;
|
|
5782
|
-
const acted = action === void 0 ? void 0 : actedSubAction(action);
|
|
5783
|
-
const observed = step !== void 0 && acted !== void 0 && (0, import_agent_qa_schema7.exercisedAnchors)({ ...step, subActions: [acted] }).some((anchor) => (0, import_agent_qa_schema7.canonicalJson)((0, import_agent_qa_schema7.normalizeAnchor)(anchor)) === (0, import_agent_qa_schema7.canonicalJson)((0, import_agent_qa_schema7.normalizeAnchor)(outcome.exercisedAnchor))) ? (0, import_agent_qa_schema7.actedPair)(acted).stateBefore : void 0;
|
|
5784
|
-
if (observed !== void 0) outcome.observedStateDigest = (0, import_agent_qa_schema7.distilledScreenStateDigest)(observed);
|
|
5785
|
-
const bundle = input.readBundleStateDigests({
|
|
5786
|
-
knowledgeRoot: input.knowledgeRoot,
|
|
5787
|
-
candidateId: outcome.candidateId
|
|
5788
|
-
});
|
|
5789
|
-
if (outcome.observedStateDigest !== void 0 && bundle.kind === "found" && bundle.digests.length > 0) {
|
|
5790
|
-
outcome.stateValidity = bundle.digests.includes(outcome.observedStateDigest) ? "valid" : "stale";
|
|
5791
|
-
}
|
|
5792
|
-
}
|
|
5793
|
-
let crossCheck = "not-run";
|
|
5794
|
-
let corpusDigestMoved;
|
|
5795
|
-
const completed = completedByCandidate.get(outcome.candidateId);
|
|
5796
|
-
if (outcome.runOutcome === "positive") {
|
|
5797
|
-
if (!served || input.ablation === void 0) {
|
|
5798
|
-
counters.rideAlongUncredited += 1;
|
|
5799
|
-
} else if (outcome.observedStateDigest === void 0) {
|
|
5800
|
-
counters.ablationInconclusive += 1;
|
|
5801
|
-
} else if (ablatedCandidates.has(outcome.candidateId)) {
|
|
5802
|
-
counters.rideAlongUncredited += 1;
|
|
5803
|
-
} else if (completed !== void 0 && (completed.attributedObservationId !== input.observationIdOf(outcome) || completed.observedStateDigest !== outcome.observedStateDigest)) {
|
|
5804
|
-
counters.rideAlongUncredited += 1;
|
|
5805
|
-
} else if (completed === void 0 && ablationsRun >= input.maxAblations) {
|
|
5806
|
-
counters.ablationCapExceeded += 1;
|
|
5807
|
-
} else {
|
|
5808
|
-
ablatedCandidates.add(outcome.candidateId);
|
|
5809
|
-
let result;
|
|
5810
|
-
if (completed !== void 0) result = completed.result;
|
|
5811
|
-
else {
|
|
5812
|
-
ablationsRun += 1;
|
|
5813
|
-
try {
|
|
5814
|
-
result = await input.ablation.ablate({
|
|
5815
|
-
handoff: input.handoff,
|
|
5816
|
-
runDir: input.runDir,
|
|
5817
|
-
candidateId: outcome.candidateId,
|
|
5818
|
-
runId: outcome.runId,
|
|
5819
|
-
testId: outcome.testId,
|
|
5820
|
-
testIndex: outcome.testIndex,
|
|
5821
|
-
stepIndex: outcome.stepIndex,
|
|
5822
|
-
attributedObservationId: input.observationIdOf(outcome)
|
|
5823
|
-
});
|
|
5824
|
-
} catch (error) {
|
|
5825
|
-
result = { kind: "error", message: error instanceof Error ? error.message : String(error) };
|
|
5826
|
-
}
|
|
5827
|
-
input.saveCompletedAblation({
|
|
5828
|
-
schemaVersion: 2,
|
|
5829
|
-
runId: outcome.runId,
|
|
5830
|
-
candidateId: outcome.candidateId,
|
|
5831
|
-
attributedObservationId: input.observationIdOf(outcome),
|
|
5832
|
-
observedStateDigest: outcome.observedStateDigest,
|
|
5833
|
-
result: result.kind === "ablated" ? {
|
|
5834
|
-
kind: "ablated",
|
|
5835
|
-
crossCheck: result.crossCheck,
|
|
5836
|
-
corpusDigest: result.corpusDigest,
|
|
5837
|
-
k: result.k,
|
|
5838
|
-
baseSeed: result.baseSeed,
|
|
5839
|
-
seeds: result.seeds,
|
|
5840
|
-
seedHonored: result.seedHonored,
|
|
5841
|
-
spend: result.spend,
|
|
5842
|
-
gate2Dir: result.gate2Dir
|
|
5843
|
-
} : result.kind === "error" ? { kind: "error", message: "ablation-error" } : result.kind === "unavailable" ? { kind: "unavailable", reason: "ablation-unavailable" } : { kind: "no-eval-root" }
|
|
5844
|
-
});
|
|
5845
|
-
}
|
|
5846
|
-
switch (result.kind) {
|
|
5847
|
-
case "ablated":
|
|
5848
|
-
crossCheck = result.crossCheck;
|
|
5849
|
-
if (corpusDigest !== void 0 && corpusDigest !== result.corpusDigest) corpusDigestMoved = true;
|
|
5850
|
-
outcome.corpusDigest = result.corpusDigest;
|
|
5851
|
-
outcome.ablation = {
|
|
5852
|
-
crossCheck,
|
|
5853
|
-
k: result.k,
|
|
5854
|
-
baseSeed: result.baseSeed,
|
|
5855
|
-
seeds: result.seeds,
|
|
5856
|
-
seedHonored: result.seedHonored,
|
|
5857
|
-
spend: result.spend
|
|
5858
|
-
};
|
|
5859
|
-
if (crossCheck === "flipped") {
|
|
5860
|
-
outcome.benefit = "exercised-benefited";
|
|
5861
|
-
counters.credited += 1;
|
|
5862
|
-
} else if (crossCheck === "held") counters.ablationHeld += 1;
|
|
5863
|
-
else counters.ablationInconclusive += 1;
|
|
5864
|
-
break;
|
|
5865
|
-
case "no-eval-root":
|
|
5866
|
-
counters.rideAlongUncredited += 1;
|
|
5867
|
-
break;
|
|
5868
|
-
case "unavailable":
|
|
5869
|
-
counters.ablationInconclusive += 1;
|
|
5870
|
-
break;
|
|
5871
|
-
case "error":
|
|
5872
|
-
counters.ablationError += 1;
|
|
5873
|
-
break;
|
|
5874
|
-
}
|
|
5875
|
-
}
|
|
5876
|
-
}
|
|
5877
|
-
let staged;
|
|
5878
|
-
if (served && input.ablation !== void 0 && outcome.observedStateDigest !== void 0) {
|
|
5879
|
-
try {
|
|
5880
|
-
const product = input.handoff.scope.tier === "product" && input.handoff.scope.key !== "" && input.handoff.scope.key !== "unknown" ? input.handoff.scope.key : "default";
|
|
5881
|
-
const stateDigest = outcome.observedStateDigest;
|
|
5882
|
-
if (await input.ablation.isNovel({ product, stateDigest })) {
|
|
5883
|
-
const stage = await input.ablation.stage({
|
|
5884
|
-
handoff: input.handoff,
|
|
5885
|
-
runDir: input.runDir,
|
|
5886
|
-
candidateId: outcome.candidateId,
|
|
5887
|
-
runId: outcome.runId,
|
|
5888
|
-
testId: outcome.testId,
|
|
5889
|
-
testIndex: outcome.testIndex,
|
|
5890
|
-
stepIndex: outcome.stepIndex,
|
|
5891
|
-
attributedObservationId: input.observationIdOf(outcome),
|
|
5892
|
-
product,
|
|
5893
|
-
stateDigest,
|
|
5894
|
-
benefit: outcome.benefit
|
|
5895
|
-
});
|
|
5896
|
-
if (stage.kind === "staged") {
|
|
5897
|
-
counters.staged += 1;
|
|
5898
|
-
staged = true;
|
|
5899
|
-
} else if (!["already-staged", "not-novel", "no-eval-root"].includes(stage.kind)) counters.stagingRefused += 1;
|
|
5900
|
-
}
|
|
5901
|
-
} catch {
|
|
5902
|
-
counters.stagingRefused += 1;
|
|
5903
|
-
}
|
|
5904
|
-
}
|
|
5905
|
-
attributions.push({
|
|
5906
|
-
...row,
|
|
5907
|
-
outcome,
|
|
5908
|
-
crossCheck,
|
|
5909
|
-
...staged === void 0 ? {} : { staged },
|
|
5910
|
-
...corpusDigestMoved === void 0 ? {} : { corpusDigestMoved }
|
|
5911
|
-
});
|
|
2399
|
+
if (outcome.runOutcome === "positive") counters.rideAlongUncredited += 1;
|
|
2400
|
+
attributions.push({ ...row, outcome });
|
|
5912
2401
|
}
|
|
5913
2402
|
return { attributions, counters };
|
|
5914
2403
|
}
|
|
@@ -5924,7 +2413,7 @@ function resolveAttributionClasses(artifact, handoff) {
|
|
|
5924
2413
|
for (const member of handoff.tests) {
|
|
5925
2414
|
if (member.rootCause !== void 0) classes.set(member.testIndex, member.rootCause.class);
|
|
5926
2415
|
}
|
|
5927
|
-
for (const verdict of (0,
|
|
2416
|
+
for (const verdict of (0, import_agent_qa_schema5.resolveRunVerdicts)(artifact, handoff)) {
|
|
5928
2417
|
classes.set(verdict.perTest, verdict.class);
|
|
5929
2418
|
}
|
|
5930
2419
|
return classes;
|
|
@@ -5932,13 +2421,13 @@ function resolveAttributionClasses(artifact, handoff) {
|
|
|
5932
2421
|
function loadRunAnalysisArtifact(runDir, runId) {
|
|
5933
2422
|
let bytes;
|
|
5934
2423
|
try {
|
|
5935
|
-
bytes = (0,
|
|
2424
|
+
bytes = (0, import_node_fs3.readFileSync)((0, import_node_path8.join)(runDir, import_agent_qa_schema5.ANALYSIS_FILE), "utf8");
|
|
5936
2425
|
} catch (error) {
|
|
5937
2426
|
if (error.code === "ENOENT") return void 0;
|
|
5938
2427
|
throw error;
|
|
5939
2428
|
}
|
|
5940
|
-
const admission = (0,
|
|
5941
|
-
(0,
|
|
2429
|
+
const admission = (0, import_agent_qa_schema5.admitAnalysisArtifactForRun)(
|
|
2430
|
+
(0, import_agent_qa_schema5.guardRunAnalysisArtifact)(JSON.parse(bytes)),
|
|
5942
2431
|
runId
|
|
5943
2432
|
);
|
|
5944
2433
|
return admission.kind === "admitted" ? admission.artifact : void 0;
|
|
@@ -5949,7 +2438,7 @@ var STALENESS_MISS_REASONS = /* @__PURE__ */ new Set([
|
|
|
5949
2438
|
"page-anchor-mismatch"
|
|
5950
2439
|
]);
|
|
5951
2440
|
function normalizedKey(anchor) {
|
|
5952
|
-
return (0,
|
|
2441
|
+
return (0, import_agent_qa_schema5.canonicalJson)((0, import_agent_qa_schema5.normalizeAnchor)(anchor));
|
|
5953
2442
|
}
|
|
5954
2443
|
function nonAttributableFailure(test, resolved) {
|
|
5955
2444
|
const resolvedClass = resolved.get(test.testIndex);
|
|
@@ -5960,45 +2449,6 @@ function attributedOutcome(test, step, resolved) {
|
|
|
5960
2449
|
if (nonAttributableFailure(test, resolved)) return void 0;
|
|
5961
2450
|
return "negative";
|
|
5962
2451
|
}
|
|
5963
|
-
var EXPOSURE_SEPARATOR = "";
|
|
5964
|
-
function exposureKey(perTest, stepIndex, recordId) {
|
|
5965
|
-
return [perTest, stepIndex, recordId].join(EXPOSURE_SEPARATOR);
|
|
5966
|
-
}
|
|
5967
|
-
function flippedExposures(handoff, evidence) {
|
|
5968
|
-
if (evidence === void 0 || evidence.length === 0) return /* @__PURE__ */ new Set();
|
|
5969
|
-
const runId = String(handoff.runId);
|
|
5970
|
-
const baseDigest = (0, import_agent_qa_schema11.misdirectionSourceDigest)(handoff);
|
|
5971
|
-
const exposed = /* @__PURE__ */ new Set();
|
|
5972
|
-
for (const test of handoff.tests) {
|
|
5973
|
-
for (const site of (0, import_agent_qa_schema11.deriveMemoryOnPathSites)(test.steps)) {
|
|
5974
|
-
exposed.add(exposureKey(test.testIndex, site.stepIndex, site.recordId));
|
|
5975
|
-
}
|
|
5976
|
-
}
|
|
5977
|
-
const flipped = /* @__PURE__ */ new Set();
|
|
5978
|
-
const answered = /* @__PURE__ */ new Set();
|
|
5979
|
-
for (const row of evidence) {
|
|
5980
|
-
if (row.runId !== runId) continue;
|
|
5981
|
-
if (row.handoffBaseDigest !== baseDigest) continue;
|
|
5982
|
-
const key = exposureKey(row.perTest, row.stepIndex, row.recordId);
|
|
5983
|
-
if (!exposed.has(key)) continue;
|
|
5984
|
-
if (answered.has(key)) {
|
|
5985
|
-
flipped.delete(key);
|
|
5986
|
-
continue;
|
|
5987
|
-
}
|
|
5988
|
-
answered.add(key);
|
|
5989
|
-
if (row.crossCheck === "flipped") flipped.add(key);
|
|
5990
|
-
}
|
|
5991
|
-
return flipped;
|
|
5992
|
-
}
|
|
5993
|
-
function causalRecordsOn(test, step, flipped) {
|
|
5994
|
-
if (flipped.size === 0 || step.verifierVerdict.pass) return [];
|
|
5995
|
-
const ids = [];
|
|
5996
|
-
for (const site of (0, import_agent_qa_schema11.deriveMemoryOnPathSites)([step])) {
|
|
5997
|
-
if (!flipped.has(exposureKey(test.testIndex, site.stepIndex, site.recordId))) continue;
|
|
5998
|
-
if (!ids.includes(site.recordId)) ids.push(site.recordId);
|
|
5999
|
-
}
|
|
6000
|
-
return ids;
|
|
6001
|
-
}
|
|
6002
2452
|
function procedureReplayOutcome(test, step, resolved) {
|
|
6003
2453
|
const source = step.planSource;
|
|
6004
2454
|
if (source === void 0 || source.kind !== "procedure-replay") return void 0;
|
|
@@ -6034,7 +2484,7 @@ function servedCanaryIds(step) {
|
|
|
6034
2484
|
}
|
|
6035
2485
|
function firstMatchingAnchor(candidateAnchors, stepAnchors) {
|
|
6036
2486
|
const exercisedByKey = new Map(
|
|
6037
|
-
stepAnchors.map((anchor) => [normalizedKey(anchor), (0,
|
|
2487
|
+
stepAnchors.map((anchor) => [normalizedKey(anchor), (0, import_agent_qa_schema5.normalizeAnchor)(anchor)])
|
|
6038
2488
|
);
|
|
6039
2489
|
for (const candidateAnchor of candidateAnchors) {
|
|
6040
2490
|
const match = exercisedByKey.get(normalizedKey(candidateAnchor));
|
|
@@ -6059,10 +2509,6 @@ function projectCanaryAttributions(input) {
|
|
|
6059
2509
|
return resolutionByCandidate.get(id);
|
|
6060
2510
|
};
|
|
6061
2511
|
const resolved = resolveAttributionClasses(input.artifact, input.handoff);
|
|
6062
|
-
const flipped = flippedExposures(
|
|
6063
|
-
input.handoff,
|
|
6064
|
-
input.misdirectionEvidence ?? input.handoff.misdirectionEvidence
|
|
6065
|
-
);
|
|
6066
2512
|
const projected = [];
|
|
6067
2513
|
for (const test of input.handoff.tests) {
|
|
6068
2514
|
if (test.isRerun === true) {
|
|
@@ -6070,45 +2516,33 @@ function projectCanaryAttributions(input) {
|
|
|
6070
2516
|
continue;
|
|
6071
2517
|
}
|
|
6072
2518
|
for (const step of test.steps) {
|
|
6073
|
-
projectBriefAttributions(input, test, step, resolveAnchors, resolved,
|
|
2519
|
+
projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected);
|
|
6074
2520
|
projectProcedureAttribution(input, test, step, resolveAnchors, resolved, projected);
|
|
6075
2521
|
}
|
|
6076
2522
|
}
|
|
6077
2523
|
return projected;
|
|
6078
2524
|
}
|
|
6079
|
-
function projectBriefAttributions(input, test, step, resolveAnchors, resolved,
|
|
2525
|
+
function projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected) {
|
|
6080
2526
|
const runOutcome = attributedOutcome(test, step, resolved);
|
|
6081
2527
|
const resolvedClass = resolved.get(test.testIndex);
|
|
6082
|
-
|
|
6083
|
-
const causalSet = new Set(causal);
|
|
6084
|
-
const nonSignal = (candidateId) => {
|
|
2528
|
+
if (runOutcome === void 0) {
|
|
6085
2529
|
if (resolvedClass === void 0 || !isNonSignalClass(resolvedClass)) return;
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
for (const candidateId of servedCanaryIds(step)) nonSignal(candidateId);
|
|
2530
|
+
for (const candidateId of servedCanaryIds(step)) {
|
|
2531
|
+
projected.push({
|
|
2532
|
+
kind: "non-signal",
|
|
2533
|
+
source: "served-brief",
|
|
2534
|
+
class: resolvedClass,
|
|
2535
|
+
candidateId,
|
|
2536
|
+
testId: test.testId,
|
|
2537
|
+
testIndex: test.testIndex,
|
|
2538
|
+
stepIndex: step.stepIndex
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
6098
2541
|
return;
|
|
6099
2542
|
}
|
|
6100
2543
|
const stepAnchors = (0, import_agent_qa_schema2.exercisedAnchors)(step);
|
|
6101
|
-
if (stepAnchors.length === 0
|
|
6102
|
-
const
|
|
6103
|
-
const candidateIds = [...served, ...causal.filter((id) => !served.includes(id))];
|
|
6104
|
-
for (const candidateId of candidateIds) {
|
|
6105
|
-
const causalRow = causalSet.has(candidateId);
|
|
6106
|
-
const rowOutcome = causalRow ? "negative" : runOutcome;
|
|
6107
|
-
if (rowOutcome === void 0) {
|
|
6108
|
-
nonSignal(candidateId);
|
|
6109
|
-
continue;
|
|
6110
|
-
}
|
|
6111
|
-
if (!causalRow && stepAnchors.length === 0) continue;
|
|
2544
|
+
if (stepAnchors.length === 0) return;
|
|
2545
|
+
for (const candidateId of servedCanaryIds(step)) {
|
|
6112
2546
|
const resolution = resolveAnchors(candidateId);
|
|
6113
2547
|
if (resolution.kind === "unresolved") {
|
|
6114
2548
|
projected.push({
|
|
@@ -6123,20 +2557,16 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
|
|
|
6123
2557
|
});
|
|
6124
2558
|
continue;
|
|
6125
2559
|
}
|
|
6126
|
-
const
|
|
6127
|
-
const fallback = resolution.anchors[0];
|
|
6128
|
-
const exercisedAnchor = contacted ?? (causalRow && fallback !== void 0 ? (0, import_agent_qa_schema11.normalizeAnchor)(fallback) : void 0);
|
|
2560
|
+
const exercisedAnchor = firstMatchingAnchor(resolution.anchors, stepAnchors);
|
|
6129
2561
|
if (exercisedAnchor === void 0) continue;
|
|
6130
2562
|
projected.push({
|
|
6131
2563
|
kind: "outcome",
|
|
6132
2564
|
source: "served-brief",
|
|
6133
|
-
...causalRow ? { misdirectionFlipped: true } : {},
|
|
6134
2565
|
outcome: {
|
|
6135
2566
|
candidateId,
|
|
6136
|
-
runOutcome
|
|
6137
|
-
//
|
|
2567
|
+
runOutcome,
|
|
2568
|
+
// The credit pass sets the benefit from the run outcome.
|
|
6138
2569
|
benefit: "rode-along",
|
|
6139
|
-
stateValidity: "unknown",
|
|
6140
2570
|
runId: String(input.handoff.runId),
|
|
6141
2571
|
testId: test.testId,
|
|
6142
2572
|
testIndex: test.testIndex,
|
|
@@ -6145,10 +2575,9 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
|
|
|
6145
2575
|
briefDigest: step.servedBrief.briefDigest,
|
|
6146
2576
|
attribution: [
|
|
6147
2577
|
`Candidate ${candidateId}`,
|
|
6148
|
-
`matched ${(0,
|
|
2578
|
+
`matched ${(0, import_agent_qa_schema5.canonicalJson)(exercisedAnchor)}`,
|
|
6149
2579
|
`on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
|
|
6150
|
-
`with ${
|
|
6151
|
-
...causalRow ? ["after the misdirection cross-check flipped"] : []
|
|
2580
|
+
`with ${runOutcome} verdict`
|
|
6152
2581
|
].join(" "),
|
|
6153
2582
|
exercisedAnchor
|
|
6154
2583
|
}
|
|
@@ -6201,9 +2630,8 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
|
|
|
6201
2630
|
outcome: {
|
|
6202
2631
|
candidateId: source.procedureId,
|
|
6203
2632
|
runOutcome,
|
|
6204
|
-
//
|
|
2633
|
+
// The credit pass sets the benefit from the run outcome.
|
|
6205
2634
|
benefit: "rode-along",
|
|
6206
|
-
stateValidity: "unknown",
|
|
6207
2635
|
runId: String(input.handoff.runId),
|
|
6208
2636
|
testId: test.testId,
|
|
6209
2637
|
testIndex: test.testIndex,
|
|
@@ -6215,7 +2643,7 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
|
|
|
6215
2643
|
`on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
|
|
6216
2644
|
attributionDetail
|
|
6217
2645
|
].join(" "),
|
|
6218
|
-
exercisedAnchor: (0,
|
|
2646
|
+
exercisedAnchor: (0, import_agent_qa_schema5.normalizeAnchor)(anchor)
|
|
6219
2647
|
}
|
|
6220
2648
|
});
|
|
6221
2649
|
}
|
|
@@ -6225,45 +2653,27 @@ function projectCanaryOutcomes(input) {
|
|
|
6225
2653
|
);
|
|
6226
2654
|
}
|
|
6227
2655
|
async function emitCanaryOutcomes(opts) {
|
|
6228
|
-
if (!(0,
|
|
6229
|
-
if (!(0,
|
|
2656
|
+
if (!(0, import_node_path8.isAbsolute)(opts.knowledgeRoot)) throw new TypeError("knowledgeRoot must be absolute");
|
|
2657
|
+
if (!(0, import_node_path8.isAbsolute)(opts.runDir)) throw new TypeError("runDir must be absolute");
|
|
6230
2658
|
const artifact = loadRunAnalysisArtifact(opts.runDir, String(opts.handoff.runId));
|
|
6231
|
-
const recovered = opts.handoff.misdirectionEvidence === void 0 ? readCompletedMisdirectionChecks(opts.runDir, String(opts.handoff.runId)) : void 0;
|
|
6232
2659
|
const projected = projectCanaryAttributions({
|
|
6233
2660
|
handoff: opts.handoff,
|
|
6234
2661
|
...artifact === void 0 ? {} : { artifact },
|
|
6235
|
-
|
|
6236
|
-
resolveCandidateAnchors: (id) => import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.readCanonicalBundleAnchors({
|
|
2662
|
+
resolveCandidateAnchors: (id) => import_agent_qa_agent_local4.LOCAL_CURATION_CAPABILITIES.readCanonicalBundleAnchors({
|
|
6237
2663
|
knowledgeRoot: opts.knowledgeRoot,
|
|
6238
2664
|
bundleId: id
|
|
6239
2665
|
})
|
|
6240
2666
|
});
|
|
6241
|
-
const credit =
|
|
6242
|
-
completedAblations: readCompletedCanaryAblations(opts.runDir, String(opts.handoff.runId)),
|
|
6243
|
-
saveCompletedAblation: (entry) => saveCompletedCanaryAblation(opts.runDir, entry),
|
|
6244
|
-
attributions: projected,
|
|
6245
|
-
handoff: opts.handoff,
|
|
6246
|
-
runDir: opts.runDir,
|
|
6247
|
-
...opts.credit,
|
|
6248
|
-
knowledgeRoot: opts.knowledgeRoot,
|
|
6249
|
-
observationIdOf: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.canaryOutcomeObservationId,
|
|
6250
|
-
maxAblations: opts.credit.maxAblations ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
|
|
6251
|
-
readBundleStateDigests: opts.credit.readBundleStateDigests ?? import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.readCanonicalBundleStateDigests
|
|
6252
|
-
});
|
|
6253
|
-
const canary = credit?.counters ?? emptyCanaryCreditCounters();
|
|
6254
|
-
if (credit === void 0) {
|
|
6255
|
-
canary.rideAlongUncredited = projected.filter((row) => row.kind === "outcome" && row.outcome.runOutcome === "positive").length;
|
|
6256
|
-
}
|
|
2667
|
+
const credit = creditCanaryAttributions({ attributions: projected });
|
|
6257
2668
|
const report = {
|
|
6258
|
-
canary,
|
|
2669
|
+
canary: credit.counters,
|
|
6259
2670
|
emitted: 0,
|
|
6260
2671
|
refused: 0,
|
|
6261
2672
|
unresolved: 0,
|
|
6262
2673
|
procedure: { emitted: 0, refused: 0, unresolved: 0 },
|
|
6263
2674
|
details: []
|
|
6264
2675
|
};
|
|
6265
|
-
const
|
|
6266
|
-
for (const attribution of attributions) {
|
|
2676
|
+
for (const attribution of credit.attributions) {
|
|
6267
2677
|
if (attribution.kind === "rerun-excluded") {
|
|
6268
2678
|
report.canary.rerunExcluded += 1;
|
|
6269
2679
|
report.details.push({
|
|
@@ -6303,7 +2713,6 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6303
2713
|
});
|
|
6304
2714
|
continue;
|
|
6305
2715
|
}
|
|
6306
|
-
const causal = attribution.misdirectionFlipped === true;
|
|
6307
2716
|
let outcome;
|
|
6308
2717
|
try {
|
|
6309
2718
|
opts.emit(attribution.outcome);
|
|
@@ -6318,10 +2727,6 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6318
2727
|
}
|
|
6319
2728
|
report.details.push({
|
|
6320
2729
|
benefit: attribution.outcome.benefit,
|
|
6321
|
-
crossCheck: attribution.crossCheck ?? "not-run",
|
|
6322
|
-
...attribution.staged === void 0 ? {} : { staged: attribution.staged },
|
|
6323
|
-
...attribution.corpusDigestMoved === void 0 ? {} : { corpusDigestMoved: attribution.corpusDigestMoved },
|
|
6324
|
-
...causal ? { misdirectionFlipped: true } : {},
|
|
6325
2730
|
candidateId: attribution.outcome.candidateId,
|
|
6326
2731
|
testId: attribution.outcome.testId,
|
|
6327
2732
|
testIndex: attribution.outcome.testIndex,
|
|
@@ -6332,33 +2737,117 @@ async function emitCanaryOutcomes(opts) {
|
|
|
6332
2737
|
return report;
|
|
6333
2738
|
}
|
|
6334
2739
|
|
|
6335
|
-
// src/curation/
|
|
6336
|
-
|
|
2740
|
+
// src/curation/promotion-line.ts
|
|
2741
|
+
function renderPromotionLine(entry) {
|
|
2742
|
+
return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
// src/curation/run-curation-context.ts
|
|
2746
|
+
var import_node_crypto3 = require("crypto");
|
|
2747
|
+
var import_node_fs5 = require("fs");
|
|
2748
|
+
var import_node_path9 = require("path");
|
|
2749
|
+
var import_zod5 = require("zod");
|
|
2750
|
+
|
|
2751
|
+
// src/composition/declared-secrets.ts
|
|
2752
|
+
var import_node_fs4 = require("fs");
|
|
2753
|
+
var import_yaml2 = require("yaml");
|
|
2754
|
+
var import_agent_qa_core3 = require("@vostride/agent-qa-core");
|
|
2755
|
+
var import_agent_qa_runner = require("@vostride/agent-qa-runner");
|
|
2756
|
+
var DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
|
|
2757
|
+
lines;
|
|
2758
|
+
constructor(lines) {
|
|
2759
|
+
super(lines.join(" "));
|
|
2760
|
+
this.name = "DeclaredSecretsUnavailableError";
|
|
2761
|
+
this.lines = lines;
|
|
2762
|
+
}
|
|
2763
|
+
static required() {
|
|
2764
|
+
return new _DeclaredSecretsUnavailableError([
|
|
2765
|
+
"Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
|
|
2766
|
+
]);
|
|
2767
|
+
}
|
|
2768
|
+
};
|
|
2769
|
+
function readDeclaredSecrets(secretsPath) {
|
|
2770
|
+
if (!(0, import_node_fs4.existsSync)(secretsPath)) {
|
|
2771
|
+
throw new DeclaredSecretsUnavailableError([
|
|
2772
|
+
`Error: Secrets file not found: ${secretsPath}`,
|
|
2773
|
+
"Create the file before running tests. It may be empty, but it must exist."
|
|
2774
|
+
]);
|
|
2775
|
+
}
|
|
2776
|
+
let content;
|
|
2777
|
+
try {
|
|
2778
|
+
content = (0, import_node_fs4.readFileSync)(secretsPath, "utf-8");
|
|
2779
|
+
} catch {
|
|
2780
|
+
throw new DeclaredSecretsUnavailableError([
|
|
2781
|
+
`Error: Secrets file could not be read: ${secretsPath}`,
|
|
2782
|
+
"Check the path and file permissions before running tests."
|
|
2783
|
+
]);
|
|
2784
|
+
}
|
|
2785
|
+
try {
|
|
2786
|
+
return import_agent_qa_runner.SecretStore.fromEnvContent(content);
|
|
2787
|
+
} catch (err) {
|
|
2788
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2789
|
+
throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
function resolveSecretsPath(configPath) {
|
|
2793
|
+
const file = (0, import_yaml2.parse)((0, import_node_fs4.readFileSync)(configPath, "utf-8")) ?? {};
|
|
2794
|
+
const config = mergeConfigs(file, loadEnvOverrides(), {});
|
|
2795
|
+
const configured = config?.workspace?.secretsFile;
|
|
2796
|
+
if (typeof configured !== "string" || configured.trim().length === 0) {
|
|
2797
|
+
throw DeclaredSecretsUnavailableError.required();
|
|
2798
|
+
}
|
|
2799
|
+
const workspace = (0, import_agent_qa_core3.resolveWorkspacePaths)({ config, configPath });
|
|
2800
|
+
return workspace.secretsFile.absolutePath;
|
|
2801
|
+
}
|
|
2802
|
+
function statKey(path) {
|
|
2803
|
+
const stat = (0, import_node_fs4.statSync)(path);
|
|
2804
|
+
return `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
|
|
2805
|
+
}
|
|
2806
|
+
function createDeclaredSecretMask(configPath) {
|
|
2807
|
+
let resolved;
|
|
2808
|
+
let cached;
|
|
2809
|
+
const currentSecretsPath = () => {
|
|
2810
|
+
const configKey = statKey(configPath);
|
|
2811
|
+
if (resolved?.configKey !== configKey) {
|
|
2812
|
+
resolved = void 0;
|
|
2813
|
+
resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
|
|
2814
|
+
}
|
|
2815
|
+
return resolved.secretsPath;
|
|
2816
|
+
};
|
|
2817
|
+
const redactor = () => {
|
|
2818
|
+
const secretsPath = currentSecretsPath();
|
|
2819
|
+
let key;
|
|
2820
|
+
try {
|
|
2821
|
+
key = `${secretsPath}\0${statKey(secretsPath)}`;
|
|
2822
|
+
} catch {
|
|
2823
|
+
key = void 0;
|
|
2824
|
+
}
|
|
2825
|
+
if (key !== void 0 && cached?.key === key) return cached.redactor;
|
|
2826
|
+
const current = new import_agent_qa_runner.SecretRedactor(readDeclaredSecrets(secretsPath));
|
|
2827
|
+
cached = key === void 0 ? void 0 : { key, redactor: current };
|
|
2828
|
+
return current;
|
|
2829
|
+
};
|
|
2830
|
+
return { mask: (text) => redactor().redactString(text), redactor };
|
|
2831
|
+
}
|
|
6337
2832
|
|
|
6338
2833
|
// src/curation/run-curation-context.ts
|
|
6339
|
-
var import_node_crypto6 = require("crypto");
|
|
6340
|
-
var import_node_fs9 = require("fs");
|
|
6341
|
-
var import_node_path16 = require("path");
|
|
6342
|
-
var import_zod8 = require("zod");
|
|
6343
|
-
init_declared_secrets();
|
|
6344
|
-
init_config();
|
|
6345
2834
|
var RUN_CONFIG_RECORD_FILE = "run-config.json";
|
|
6346
2835
|
var RUN_CONFIG_RECORD_SCHEMA_VERSION = 2;
|
|
6347
|
-
var runConfigRecordSchema =
|
|
6348
|
-
schemaVersion:
|
|
6349
|
-
runId:
|
|
6350
|
-
configPath:
|
|
2836
|
+
var runConfigRecordSchema = import_zod5.z.object({
|
|
2837
|
+
schemaVersion: import_zod5.z.literal(RUN_CONFIG_RECORD_SCHEMA_VERSION),
|
|
2838
|
+
runId: import_zod5.z.string().min(1),
|
|
2839
|
+
configPath: import_zod5.z.string().min(1)
|
|
6351
2840
|
}).strict();
|
|
6352
2841
|
function runConfigRecordPath(agentQaRoot, runId) {
|
|
6353
|
-
return (0,
|
|
2842
|
+
return (0, import_node_path9.join)(completedRunDir(agentQaRoot, runId), RUN_CONFIG_RECORD_FILE);
|
|
6354
2843
|
}
|
|
6355
2844
|
function readRunConfigRecord(agentQaRoot, runId) {
|
|
6356
2845
|
const path = runConfigRecordPath(agentQaRoot, runId);
|
|
6357
2846
|
try {
|
|
6358
|
-
if (!(0,
|
|
6359
|
-
const parsed = runConfigRecordSchema.safeParse(JSON.parse((0,
|
|
2847
|
+
if (!(0, import_node_fs5.lstatSync)(path).isFile()) return void 0;
|
|
2848
|
+
const parsed = runConfigRecordSchema.safeParse(JSON.parse((0, import_node_fs5.readFileSync)(path, "utf8")));
|
|
6360
2849
|
if (!parsed.success) return void 0;
|
|
6361
|
-
if (parsed.data.runId !== runId || !(0,
|
|
2850
|
+
if (parsed.data.runId !== runId || !(0, import_node_path9.isAbsolute)(parsed.data.configPath)) return void 0;
|
|
6362
2851
|
return parsed.data;
|
|
6363
2852
|
} catch {
|
|
6364
2853
|
return void 0;
|
|
@@ -6386,13 +2875,13 @@ async function resolveRunCurationContext(input) {
|
|
|
6386
2875
|
let knowledgeRoot;
|
|
6387
2876
|
try {
|
|
6388
2877
|
const config = await resolveConfig({ configPath, loadAuthPlugins: false });
|
|
6389
|
-
knowledgeRoot = resolveConfiguredKnowledgeRoot(config, (0,
|
|
2878
|
+
knowledgeRoot = resolveConfiguredKnowledgeRoot(config, (0, import_node_path9.dirname)(configPath));
|
|
6390
2879
|
} catch {
|
|
6391
2880
|
return {
|
|
6392
2881
|
handBack: `Run ${runId} was recorded under ${configPath}, which cannot be read as an agent-qa config, so it is left for a run that can read it.`
|
|
6393
2882
|
};
|
|
6394
2883
|
}
|
|
6395
|
-
if ((0,
|
|
2884
|
+
if ((0, import_node_path9.resolve)(knowledgeRoot) !== (0, import_node_path9.resolve)(input.drainerKnowledgeRoot)) {
|
|
6396
2885
|
return {
|
|
6397
2886
|
handBack: `Run ${runId} keeps its knowledge in ${knowledgeRoot}; it is left for a run of its own project.`
|
|
6398
2887
|
};
|
|
@@ -6420,18 +2909,18 @@ var CURATION_TAIL_RECEIPT_FILE = "curation-tail-receipt.json";
|
|
|
6420
2909
|
var CURATION_TAIL_RECEIPT_SCHEMA_VERSION = 2;
|
|
6421
2910
|
function defaultOperations() {
|
|
6422
2911
|
return {
|
|
6423
|
-
loadCompletedRunHandoff:
|
|
6424
|
-
resolveKnowledgeRoot:
|
|
6425
|
-
runSafetyScope:
|
|
2912
|
+
loadCompletedRunHandoff: import_agent_qa_runner_local2.loadCompletedRunHandoff,
|
|
2913
|
+
resolveKnowledgeRoot: import_agent_qa_core4.resolveKnowledgeRoot,
|
|
2914
|
+
runSafetyScope: import_agent_qa_safety_local.runSafetyScope,
|
|
6426
2915
|
emitCurationCandidates,
|
|
6427
|
-
openKnowledgeReader:
|
|
2916
|
+
openKnowledgeReader: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.openKnowledgeReader,
|
|
6428
2917
|
emitCanaryOutcomes,
|
|
6429
|
-
foldRunOutcomes:
|
|
6430
|
-
reconcileRunOutcomesProjection:
|
|
6431
|
-
acquireWriterLease:
|
|
6432
|
-
releaseLease:
|
|
2918
|
+
foldRunOutcomes: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.foldRunOutcomes,
|
|
2919
|
+
reconcileRunOutcomesProjection: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.reconcileRunOutcomesProjection,
|
|
2920
|
+
acquireWriterLease: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.maintenance.acquireWriterLease,
|
|
2921
|
+
releaseLease: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.maintenance.releaseLease,
|
|
6433
2922
|
buildCuratorAuthor,
|
|
6434
|
-
drainQueue:
|
|
2923
|
+
drainQueue: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.drainQueue,
|
|
6435
2924
|
listCompletedRunDirs,
|
|
6436
2925
|
readCurationTailReceipt,
|
|
6437
2926
|
writeCurationTailReceipt,
|
|
@@ -6440,15 +2929,15 @@ function defaultOperations() {
|
|
|
6440
2929
|
};
|
|
6441
2930
|
}
|
|
6442
2931
|
function completedRunDir(agentQaRoot, runId) {
|
|
6443
|
-
return (0,
|
|
2932
|
+
return (0, import_node_path10.join)(agentQaRoot, import_agent_qa_runner_local2.RUNS_SUBDIR, (0, import_agent_qa_runner_local2.slugBranch)(runId));
|
|
6444
2933
|
}
|
|
6445
2934
|
function completedRunsRoot(agentQaRoot) {
|
|
6446
|
-
return (0,
|
|
2935
|
+
return (0, import_node_path10.join)(agentQaRoot, import_agent_qa_runner_local2.RUNS_SUBDIR);
|
|
6447
2936
|
}
|
|
6448
2937
|
function listCompletedRunDirs(agentQaRoot) {
|
|
6449
2938
|
const runsRoot = completedRunsRoot(agentQaRoot);
|
|
6450
|
-
if (!(0,
|
|
6451
|
-
return (0,
|
|
2939
|
+
if (!(0, import_node_fs6.existsSync)(runsRoot)) return [];
|
|
2940
|
+
return (0, import_node_fs6.readdirSync)(runsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => (0, import_node_path10.join)(runsRoot, entry.name)).filter((runDir) => (0, import_node_fs6.existsSync)((0, import_node_path10.join)(runDir, import_agent_qa_runner_local2.HANDOFF_FILE))).sort();
|
|
6452
2941
|
}
|
|
6453
2942
|
function isSortedCandidateIds(value) {
|
|
6454
2943
|
if (!Array.isArray(value)) return false;
|
|
@@ -6460,23 +2949,23 @@ function isSortedCandidateIds(value) {
|
|
|
6460
2949
|
function isCanaryCreditCounters(value) {
|
|
6461
2950
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
6462
2951
|
const record = value;
|
|
6463
|
-
if (Object.keys(record).sort().join(",") !== "
|
|
6464
|
-
const
|
|
6465
|
-
if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" ||
|
|
2952
|
+
if (Object.keys(record).sort().join(",") !== "nonSignal,rerunExcluded,rideAlongUncredited") return false;
|
|
2953
|
+
const count = (entry) => typeof entry === "number" && Number.isInteger(entry) && entry >= 0;
|
|
2954
|
+
if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" || count(entry))) return false;
|
|
6466
2955
|
const nonSignal = record.nonSignal;
|
|
6467
2956
|
if (typeof nonSignal !== "object" || nonSignal === null || Array.isArray(nonSignal)) return false;
|
|
6468
|
-
return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(
|
|
2957
|
+
return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(count);
|
|
6469
2958
|
}
|
|
6470
2959
|
function isCurationTailReceipt(value) {
|
|
6471
2960
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
6472
2961
|
const record = value;
|
|
6473
2962
|
const optional = /* @__PURE__ */ new Set(["snapshot", "curated", "deprecated"]);
|
|
6474
|
-
return Object.keys(record).filter((key) => !optional.has(key)).sort().join(",") === "canary,candidateIds,drainKind,handoffDigest,handoffHolderPid,projectionVersion,runId,schemaVersion" && ["curated", "deprecated"].every((key) => record[key] === void 0 || typeof record[key] === "number" && Number.isInteger(record[key]) && record[key] >= 0) && (record.snapshot === void 0 || record.drainKind === "drained" &&
|
|
2963
|
+
return Object.keys(record).filter((key) => !optional.has(key)).sort().join(",") === "canary,candidateIds,drainKind,handoffDigest,handoffHolderPid,projectionVersion,runId,schemaVersion" && ["curated", "deprecated"].every((key) => record[key] === void 0 || typeof record[key] === "number" && Number.isInteger(record[key]) && record[key] >= 0) && (record.snapshot === void 0 || record.drainKind === "drained" && import_agent_qa_schema6.CurationSnapshotSchema.safeParse(record.snapshot).success) && isCanaryCreditCounters(record.canary) && record.schemaVersion === CURATION_TAIL_RECEIPT_SCHEMA_VERSION && typeof record.runId === "string" && record.runId.length > 0 && typeof record.handoffDigest === "string" && /^[0-9a-f]{64}$/u.test(record.handoffDigest) && typeof record.projectionVersion === "number" && Number.isInteger(record.projectionVersion) && record.projectionVersion > 0 && (record.drainKind === "drained" || record.drainKind === "handed-off") && (record.drainKind === "drained" && record.handoffHolderPid === null || record.drainKind === "handed-off" && typeof record.handoffHolderPid === "number" && Number.isInteger(record.handoffHolderPid) && record.handoffHolderPid > 0) && isSortedCandidateIds(record.candidateIds);
|
|
6475
2964
|
}
|
|
6476
2965
|
function readCurationTailReceipt(runDir) {
|
|
6477
2966
|
try {
|
|
6478
2967
|
const parsed = JSON.parse(
|
|
6479
|
-
(0,
|
|
2968
|
+
(0, import_node_fs6.readFileSync)((0, import_node_path10.join)(runDir, CURATION_TAIL_RECEIPT_FILE), "utf8")
|
|
6480
2969
|
);
|
|
6481
2970
|
return isCurationTailReceipt(parsed) ? parsed : void 0;
|
|
6482
2971
|
} catch {
|
|
@@ -6484,36 +2973,36 @@ function readCurationTailReceipt(runDir) {
|
|
|
6484
2973
|
}
|
|
6485
2974
|
}
|
|
6486
2975
|
function fsyncFile(file) {
|
|
6487
|
-
const fd = (0,
|
|
2976
|
+
const fd = (0, import_node_fs6.openSync)(file, "r");
|
|
6488
2977
|
try {
|
|
6489
|
-
(0,
|
|
2978
|
+
(0, import_node_fs6.fsyncSync)(fd);
|
|
6490
2979
|
} finally {
|
|
6491
|
-
(0,
|
|
2980
|
+
(0, import_node_fs6.closeSync)(fd);
|
|
6492
2981
|
}
|
|
6493
2982
|
}
|
|
6494
2983
|
function writeCurationTailReceipt(runDir, receipt) {
|
|
6495
2984
|
if (!isCurationTailReceipt(receipt)) throw new TypeError("invalid curation tail receipt");
|
|
6496
|
-
const file = (0,
|
|
6497
|
-
const staged = (0,
|
|
2985
|
+
const file = (0, import_node_path10.join)(runDir, CURATION_TAIL_RECEIPT_FILE);
|
|
2986
|
+
const staged = (0, import_node_path10.join)(runDir, `.${CURATION_TAIL_RECEIPT_FILE}.${(0, import_node_crypto4.randomUUID)()}.tmp`);
|
|
6498
2987
|
try {
|
|
6499
|
-
(0,
|
|
2988
|
+
(0, import_node_fs6.writeFileSync)(staged, `${(0, import_agent_qa_schema6.canonicalJson)(receipt)}
|
|
6500
2989
|
`, { encoding: "utf8", mode: 384 });
|
|
6501
2990
|
fsyncFile(staged);
|
|
6502
|
-
(0,
|
|
2991
|
+
(0, import_node_fs6.renameSync)(staged, file);
|
|
6503
2992
|
fsyncFile(runDir);
|
|
6504
2993
|
} finally {
|
|
6505
|
-
(0,
|
|
2994
|
+
(0, import_node_fs6.rmSync)(staged, { force: true });
|
|
6506
2995
|
}
|
|
6507
2996
|
}
|
|
6508
2997
|
function curationTailHandoffDigest(handoff) {
|
|
6509
|
-
return (0,
|
|
2998
|
+
return (0, import_agent_qa_schema6.sha256hex)((0, import_agent_qa_schema6.canonicalJson)(handoff));
|
|
6510
2999
|
}
|
|
6511
3000
|
function receiptIdentityForHandoff(handoff) {
|
|
6512
3001
|
return {
|
|
6513
3002
|
schemaVersion: CURATION_TAIL_RECEIPT_SCHEMA_VERSION,
|
|
6514
3003
|
runId: String(handoff.runId),
|
|
6515
3004
|
handoffDigest: curationTailHandoffDigest(handoff),
|
|
6516
|
-
projectionVersion:
|
|
3005
|
+
projectionVersion: import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.runOutcomesProjectionVersion
|
|
6517
3006
|
};
|
|
6518
3007
|
}
|
|
6519
3008
|
function receiptMatches(receipt, expected) {
|
|
@@ -6538,12 +3027,6 @@ function candidateSummaryLine(runId, report) {
|
|
|
6538
3027
|
});
|
|
6539
3028
|
return `Knowledge curation candidates for ${runId}: ${segments.join("; ")}.`;
|
|
6540
3029
|
}
|
|
6541
|
-
function misdirectionHarmLine(runId, rows) {
|
|
6542
|
-
const landed = rows.filter((row) => row.outcome.kind === "emitted");
|
|
6543
|
-
const refused = rows.length - landed.length;
|
|
6544
|
-
const named = rows.map((row) => row.candidateId).join(", ");
|
|
6545
|
-
return `Misdirection harm for ${runId}: ${landed.length} harmed outcome(s) emitted from flipped cross-check evidence regardless of verdict class (${named})${refused === 0 ? "" : `; ${refused} refused by the queue`}.`;
|
|
6546
|
-
}
|
|
6547
3030
|
function curationOutcomeLine(input) {
|
|
6548
3031
|
if (input.committed <= 0) return void 0;
|
|
6549
3032
|
const proposed = Math.min(Math.max(input.downgraded, 0), input.committed);
|
|
@@ -6586,8 +3069,8 @@ function logQuarantineDisposition(runId, outcome, log) {
|
|
|
6586
3069
|
}
|
|
6587
3070
|
for (const sighting of outcome.escalatedNotQuarantined) {
|
|
6588
3071
|
const authored = sighting.producer === "analyze-run";
|
|
6589
|
-
const disposition =
|
|
6590
|
-
const floorEligibility =
|
|
3072
|
+
const disposition = import_agent_qa_agent_local5.QUARANTINE_CLASS_DISPOSITIONS[sighting.verdictClass];
|
|
3073
|
+
const floorEligibility = import_agent_qa_agent_local5.QUARANTINE_FLOOR_ELIGIBILITY[sighting.verdictClass];
|
|
6591
3074
|
log.warn(
|
|
6592
3075
|
`Quarantine withheld for ${sighting.testName} (${sighting.filePath}): ${sighting.verdictClass} recurred at ${String(sighting.occurrenceOrdinals)} of the last ${String(sighting.windowOrdinals)} failing builds and will keep failing this build. Verdict producer: ${sighting.producer} \u2014 ` + remedyFor({ authored, disposition, floorEligibility })
|
|
6593
3076
|
);
|
|
@@ -6611,7 +3094,6 @@ async function curateCompletedRun(input) {
|
|
|
6611
3094
|
deprecated: previous.deprecated ?? 0,
|
|
6612
3095
|
complete: true,
|
|
6613
3096
|
drainKind: previous.drainKind,
|
|
6614
|
-
candidateIds: previous.candidateIds,
|
|
6615
3097
|
snapshot: previous.snapshot
|
|
6616
3098
|
};
|
|
6617
3099
|
}
|
|
@@ -6626,7 +3108,7 @@ async function curateCompletedRun(input) {
|
|
|
6626
3108
|
const budgetMs = input.deadlineMs === void 0 ? void 0 : Math.max(0, input.deadlineMs - tailStartedAtMs);
|
|
6627
3109
|
await operations.runSafetyScope(
|
|
6628
3110
|
{
|
|
6629
|
-
//
|
|
3111
|
+
// Posture is persisted run truth, never an ambient re-detection.
|
|
6630
3112
|
executionContext: input.handoff.executionContext
|
|
6631
3113
|
},
|
|
6632
3114
|
async () => {
|
|
@@ -6637,29 +3119,13 @@ async function curateCompletedRun(input) {
|
|
|
6637
3119
|
...input.mask === void 0 ? {} : { mask: input.mask }
|
|
6638
3120
|
});
|
|
6639
3121
|
input.log.info?.(candidateSummaryLine(receiptIdentity.runId, candidates));
|
|
6640
|
-
const reader = operations.openKnowledgeReader({
|
|
6641
|
-
knowledgeRoot: input.knowledgeRoot,
|
|
6642
|
-
priors: input.resolveTrustPriors?.()
|
|
6643
|
-
});
|
|
6644
|
-
const ablation = operations.canaryAblation;
|
|
6645
|
-
const recordedPolicy = readRunCanaryPolicy(input.runDir, String(input.handoff.runId));
|
|
6646
|
-
const caps = ablation === void 0 ? void 0 : recordedPolicy ?? (input.maxAblations !== void 0 ? { maxAblations: input.maxAblations, source: "flag" } : await ablation.resolveCaps?.() ?? {
|
|
6647
|
-
maxAblations: (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
|
|
6648
|
-
source: "default"
|
|
6649
|
-
});
|
|
6650
|
-
if (ablation !== void 0 && recordedPolicy === void 0) input.log.info?.("canary cap: legacy run without recorded policy; using explicit run override or current config/default");
|
|
6651
|
-
if (caps !== void 0) input.log.info?.(`canary cap: maxAblations=${caps.maxAblations} (${caps.source})`);
|
|
3122
|
+
const reader = operations.openKnowledgeReader({ knowledgeRoot: input.knowledgeRoot });
|
|
6652
3123
|
const canaries = await operations.emitCanaryOutcomes({
|
|
6653
3124
|
handoff: input.handoff,
|
|
6654
3125
|
knowledgeRoot: input.knowledgeRoot,
|
|
6655
3126
|
runDir: input.runDir,
|
|
6656
|
-
emit: reader.emitCanaryOutcome.bind(reader)
|
|
6657
|
-
...operations.canaryAblation === void 0 ? {} : { credit: { ablation: operations.canaryAblation, maxAblations: caps.maxAblations, readBundleStateDigests: import_agent_qa_agent_local6.LOCAL_CURATION_CAPABILITIES.readCanonicalBundleStateDigests } }
|
|
3127
|
+
emit: reader.emitCanaryOutcome.bind(reader)
|
|
6658
3128
|
});
|
|
6659
|
-
const causalHarm = (canaries.details ?? []).filter((row) => row.misdirectionFlipped === true);
|
|
6660
|
-
if (causalHarm.length > 0) {
|
|
6661
|
-
input.log.info?.(misdirectionHarmLine(receiptIdentity.runId, causalHarm));
|
|
6662
|
-
}
|
|
6663
3129
|
try {
|
|
6664
3130
|
const fold = operations.foldRunOutcomes({
|
|
6665
3131
|
knowledgeRoot: input.knowledgeRoot,
|
|
@@ -6684,15 +3150,13 @@ async function curateCompletedRun(input) {
|
|
|
6684
3150
|
}));
|
|
6685
3151
|
const drain = await operations.drainQueue({
|
|
6686
3152
|
sourceRunId: String(input.handoff.runId),
|
|
6687
|
-
observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [
|
|
3153
|
+
observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.canaryOutcomeObservationId({
|
|
6688
3154
|
candidateId: row.candidateId,
|
|
6689
3155
|
runId: input.handoff.runId,
|
|
6690
3156
|
testId: row.testId,
|
|
6691
3157
|
testIndex: row.testIndex,
|
|
6692
3158
|
stepIndex: row.stepIndex
|
|
6693
3159
|
})]),
|
|
6694
|
-
priors: input.resolveTrustPriors?.(),
|
|
6695
|
-
...input.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: input.resolvePromotionPosture },
|
|
6696
3160
|
knowledgeRoot: input.knowledgeRoot,
|
|
6697
3161
|
author,
|
|
6698
3162
|
executionContext: input.handoff.executionContext,
|
|
@@ -6724,7 +3188,7 @@ async function curateCompletedRun(input) {
|
|
|
6724
3188
|
}
|
|
6725
3189
|
complete = candidates.refused === 0 && candidates.capExceeded === 0 && canaries.refused === 0 && canaries.unresolved === 0 && !projectionFailed && (drain.kind === "drained" || drain.kind === "handed-off") && drain.failures === 0 && drain.requeued === 0 && (drain.kind === "handed-off" || drain.remaining === void 0 || drain.remaining === 0);
|
|
6726
3190
|
if (complete && (drain.kind === "drained" || drain.kind === "handed-off")) {
|
|
6727
|
-
const validated = drain.kind !== "drained" || drain.snapshot === void 0 ? void 0 :
|
|
3191
|
+
const validated = drain.kind !== "drained" || drain.snapshot === void 0 ? void 0 : import_agent_qa_schema6.CurationSnapshotSchema.safeParse(drain.snapshot);
|
|
6728
3192
|
if (drain.kind === "drained" && drain.snapshotFoldFailed === true || validated !== void 0 && !validated.success) {
|
|
6729
3193
|
input.log.warn(SNAPSHOT_NOT_RECORDED_WARNING);
|
|
6730
3194
|
}
|
|
@@ -6734,8 +3198,7 @@ async function curateCompletedRun(input) {
|
|
|
6734
3198
|
drainKind: drain.kind,
|
|
6735
3199
|
handoffHolderPid: drain.kind === "handed-off" ? drain.holder.pid : null,
|
|
6736
3200
|
candidateIds: sortedCandidateIds(candidates),
|
|
6737
|
-
//
|
|
6738
|
-
// answers from this receipt; a handed-off receipt has no counts of its own to record.
|
|
3201
|
+
// Counts travel with the snapshot, when the rerun short-circuit answers from the receipt.
|
|
6739
3202
|
...validated?.success === true ? { snapshot: validated.data, curated, deprecated } : {}
|
|
6740
3203
|
};
|
|
6741
3204
|
}
|
|
@@ -6767,7 +3230,7 @@ async function curateCompletedRun(input) {
|
|
|
6767
3230
|
operations.writeCurationTailReceipt(input.runDir, completedReceipt);
|
|
6768
3231
|
if (completedReceipt.snapshot !== void 0) {
|
|
6769
3232
|
const breakdown = completedReceipt.snapshot.breakdown;
|
|
6770
|
-
input.log.info?.(`curation: ${
|
|
3233
|
+
input.log.info?.(`curation: ${import_agent_qa_schema6.CURATION_BREAKDOWN_BUCKETS.map((bucket) => `${bucket} ${breakdown[bucket]}`).join(", ")}`);
|
|
6771
3234
|
}
|
|
6772
3235
|
if (completedReceipt.drainKind === "handed-off") {
|
|
6773
3236
|
void operations.nudgeDaemon(
|
|
@@ -6780,7 +3243,6 @@ async function curateCompletedRun(input) {
|
|
|
6780
3243
|
deprecated,
|
|
6781
3244
|
complete: true,
|
|
6782
3245
|
drainKind,
|
|
6783
|
-
candidateIds: completedReceipt.candidateIds,
|
|
6784
3246
|
...completedReceipt.snapshot === void 0 ? {} : { snapshot: completedReceipt.snapshot }
|
|
6785
3247
|
};
|
|
6786
3248
|
}
|
|
@@ -6793,7 +3255,7 @@ async function runCurationTail(opts) {
|
|
|
6793
3255
|
let currentHandoff;
|
|
6794
3256
|
try {
|
|
6795
3257
|
currentHandoff = operations.loadCompletedRunHandoff(
|
|
6796
|
-
(0,
|
|
3258
|
+
(0, import_node_path10.join)(currentRunDir, import_agent_qa_runner_local2.HANDOFF_FILE)
|
|
6797
3259
|
);
|
|
6798
3260
|
} catch {
|
|
6799
3261
|
opts.log.warn(
|
|
@@ -6835,7 +3297,7 @@ async function runCurationTail(opts) {
|
|
|
6835
3297
|
opts.log.warn("curation: run-outcome projection not reconciled (no writer authority)");
|
|
6836
3298
|
} else if (reconcile.kind === "rebuilt") {
|
|
6837
3299
|
opts.log.info?.(
|
|
6838
|
-
`curation: run-outcome projection rebuilt at version ${
|
|
3300
|
+
`curation: run-outcome projection rebuilt at version ${import_agent_qa_agent_local5.LOCAL_CURATION_CAPABILITIES.runOutcomesProjectionVersion} (folded ${reconcile.report.folded}, skipped ${reconcile.report.skippedHandoffs} handoff, ${reconcile.report.skippedArtifacts} artifact)`
|
|
6839
3301
|
);
|
|
6840
3302
|
}
|
|
6841
3303
|
} catch (error) {
|
|
@@ -6857,7 +3319,7 @@ async function runCurationTail(opts) {
|
|
|
6857
3319
|
for (const runDir of runDirs) {
|
|
6858
3320
|
let handoff;
|
|
6859
3321
|
try {
|
|
6860
|
-
handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff((0,
|
|
3322
|
+
handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff((0, import_node_path10.join)(runDir, import_agent_qa_runner_local2.HANDOFF_FILE));
|
|
6861
3323
|
} catch {
|
|
6862
3324
|
opts.log.warn(`Skipping post-verdict knowledge curation for ${runDir}: handoff is malformed.`);
|
|
6863
3325
|
continue;
|
|
@@ -6879,11 +3341,7 @@ async function runCurationTail(opts) {
|
|
|
6879
3341
|
}
|
|
6880
3342
|
runMask = context.mask;
|
|
6881
3343
|
}
|
|
6882
|
-
const resolvePromotionPosture = opts.resolvePromotionPostureForRun === void 0 ? opts.resolvePromotionPosture : await opts.resolvePromotionPostureForRun(handoff.buildOrdinal, handoff.scope);
|
|
6883
3344
|
const folded = await curateCompletedRun({
|
|
6884
|
-
resolveTrustPriors: opts.resolveTrustPriors,
|
|
6885
|
-
...runDir !== currentRunDir || opts.maxAblations === void 0 ? {} : { maxAblations: opts.maxAblations },
|
|
6886
|
-
...resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture },
|
|
6887
3345
|
runDir,
|
|
6888
3346
|
handoff,
|
|
6889
3347
|
knowledgeRoot,
|
|
@@ -6940,7 +3398,7 @@ async function runVerdictFirstCurationTail(opts) {
|
|
|
6940
3398
|
}
|
|
6941
3399
|
|
|
6942
3400
|
// src/index.ts
|
|
6943
|
-
var VERSION = (0,
|
|
3401
|
+
var VERSION = (0, import_agent_qa_core5.getAgentQaVersion)();
|
|
6944
3402
|
// Annotate the CommonJS export names for ESM import in node:
|
|
6945
3403
|
0 && (module.exports = {
|
|
6946
3404
|
CURATION_TAIL_RECEIPT_FILE,
|
|
@@ -6966,7 +3424,6 @@ var VERSION = (0, import_agent_qa_core6.getAgentQaVersion)();
|
|
|
6966
3424
|
normalizeConfigForValidation,
|
|
6967
3425
|
projectCanaryOutcomes,
|
|
6968
3426
|
purgeAll,
|
|
6969
|
-
purgeTest,
|
|
6970
3427
|
readAnalyticsPrivacyDeclaration,
|
|
6971
3428
|
resolveConfig,
|
|
6972
3429
|
resolveConfiguredKnowledgeRoot,
|