agent-qa 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -9,29 +9,6 @@ var __export = (target, all) => {
9
9
  __defProp(target, name, { get: all[name], enumerable: true });
10
10
  };
11
11
 
12
- // src/hard-cut.ts
13
- function INVALID_CONFIG_FIX(configPath) {
14
- return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
15
- }
16
- function isRetiredKeyIssue(issue) {
17
- return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.retired === true;
18
- }
19
- var RE_INITIALIZE_URL, HARD_CUT_STEPS;
20
- var init_hard_cut = __esm({
21
- "src/hard-cut.ts"() {
22
- "use strict";
23
- RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
24
- HARD_CUT_STEPS = [
25
- "This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
26
- " 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
27
- " 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
28
- " 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
29
- " 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
30
- `What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
31
- ].join("\n");
32
- }
33
- });
34
-
35
12
  // src/provider-plugins.ts
36
13
  var provider_plugins_exports = {};
37
14
  __export(provider_plugins_exports, {
@@ -56,26 +33,11 @@ var init_provider_plugins = __esm({
56
33
  }
57
34
  });
58
35
 
36
+ // src/index.ts
37
+ import { getAgentQaVersion } from "@vostride/agent-qa-core";
38
+ export * from "@vostride/agent-qa-core";
39
+
59
40
  // src/config.ts
60
- var config_exports = {};
61
- __export(config_exports, {
62
- ConfigFileNotFoundError: () => ConfigFileNotFoundError,
63
- ConfigFileNotYamlError: () => ConfigFileNotYamlError,
64
- deepMerge: () => deepMerge,
65
- formatConfigDebug: () => formatConfigDebug,
66
- loadConfigFile: () => loadConfigFile,
67
- loadConfigPlugins: () => loadConfigPlugins,
68
- loadEnvOverrides: () => loadEnvOverrides,
69
- loadRequiredConfigFile: () => loadRequiredConfigFile,
70
- mergeConfigs: () => mergeConfigs,
71
- mergeUseBlocks: () => mergeUseBlocks,
72
- mergeWithTestConfig: () => mergeWithTestConfig,
73
- normalizeConfigForValidation: () => normalizeConfigForValidation,
74
- readAnalyticsPrivacyDeclaration: () => readAnalyticsPrivacyDeclaration,
75
- resolveConfig: () => resolveConfig,
76
- resolveConfiguredKnowledgeRoot: () => resolveConfiguredKnowledgeRoot,
77
- resolveRawConfigKnowledgeRoot: () => resolveRawConfigKnowledgeRoot
78
- });
79
41
  import { readFile } from "fs/promises";
80
42
  import { dirname as dirname2, resolve as resolvePath2 } from "path";
81
43
  import { parse as parseYaml } from "yaml";
@@ -87,6 +49,32 @@ import {
87
49
  resolveKnowledgeRoot,
88
50
  yamlParseFailureMessage
89
51
  } from "@vostride/agent-qa-core";
52
+
53
+ // src/hard-cut.ts
54
+ var RE_INITIALIZE_URL = "https://vostride.com/docs/agent-qa/migrate/v0-1-to-v0-2";
55
+ var HARD_CUT_STEPS = [
56
+ "This config was written for agent-qa before v2. v2 does not upgrade a workspace; re-create it:",
57
+ " 1. Stop agent-qa in this project: the dashboard, the daemon (agent-qa daemon stop) and any MCP server.",
58
+ " 2. Copy aside rules you wrote: agent-qa-knowledge/policies/*.md and the agent-rules.md file your config names.",
59
+ " 3. Delete .agent-qa/, agent-qa-knowledge/ and agent-qa-memory/.",
60
+ " 4. Run agent-qa init --force. Your own tests, .env and .env.secrets.local are kept.",
61
+ `What re-initializing loses, and every step: ${RE_INITIALIZE_URL}`
62
+ ].join("\n");
63
+ function INVALID_CONFIG_FIX(configPath) {
64
+ return `Fix or remove the key named above in ${configPath}, then run agent-qa validate. To start over, agent-qa init --force rewrites agent-qa.config.yaml.`;
65
+ }
66
+ function isHardCutIssue(issue) {
67
+ return issue.code === "custom" && typeof issue.params === "object" && issue.params !== null && issue.params.hardCut === true;
68
+ }
69
+
70
+ // src/config.ts
71
+ var ENV_MAPPING = {
72
+ AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
73
+ AGENT_QA_MCP_PORT: "services.mcp.port",
74
+ AGENT_QA_CACHE_DIR: "services.cache.dir",
75
+ AGENT_QA_LOG_LEVEL: "services.logging.level",
76
+ AGENT_QA_HEADLESS: "use.browser.headless"
77
+ };
90
78
  function setNestedValue(obj, path, value) {
91
79
  const parts = path.split(".");
92
80
  let current = obj;
@@ -111,6 +99,15 @@ function parseEnvValue(key, value) {
111
99
  }
112
100
  return value;
113
101
  }
102
+ var ConfigFileNotYamlError = class extends Error {
103
+ configPath;
104
+ constructor(configPath, cause) {
105
+ const positionOnly = `${configPath}: ${yamlParseFailureMessage(cause)}`;
106
+ super(positionOnly, { cause: new Error(positionOnly) });
107
+ this.name = "ConfigFileNotYamlError";
108
+ this.configPath = configPath;
109
+ }
110
+ };
114
111
  async function loadConfigFile(configPath) {
115
112
  let content;
116
113
  try {
@@ -125,6 +122,14 @@ async function loadConfigFile(configPath) {
125
122
  throw new ConfigFileNotYamlError(configPath, err);
126
123
  }
127
124
  }
125
+ var ConfigFileNotFoundError = class extends Error {
126
+ configPath;
127
+ constructor(configPath) {
128
+ super(`Config file not found: ${configPath}`);
129
+ this.name = "ConfigFileNotFoundError";
130
+ this.configPath = configPath;
131
+ }
132
+ };
128
133
  async function loadRequiredConfigFile(configPath) {
129
134
  let content;
130
135
  try {
@@ -190,6 +195,7 @@ function mergeUseBlocks(globalUse, suiteUse, testUse, cliFlags) {
190
195
  cliFlags
191
196
  );
192
197
  }
198
+ var CIRCULAR_MARKER = "<circular>";
193
199
  function flattenObject(obj, prefix = "", ancestors = []) {
194
200
  const result = {};
195
201
  if (typeof obj !== "object" || obj === null) {
@@ -287,7 +293,7 @@ Hint: services.dashboard.enabled was removed. Dashboard behavior is enabled by t
287
293
  const baseUrlHint = baseUrlRefused ? `
288
294
 
289
295
  Hint: services.dashboard.baseUrl must now carry a scheme \u2014 write an absolute URL such as http://127.0.0.1:3470 rather than a bare host like localhost:3470, or remove the key entirely. An empty string is not an absence: absence is a legal and deliberate posture here \u2014 a standalone \`agent-qa mcp\` with no dashboard has no base, and the carrier tool result then names that absence rather than fabricating a URL. The value is refused because it is assigned to an href inside the served App document and appended verbatim to every carrier tool result, so a schemeless or empty base composes a link that resolves against nothing.` : "";
290
- const closing = result.error.issues.some(isRetiredKeyIssue) ? HARD_CUT_STEPS : INVALID_CONFIG_FIX(configPath);
296
+ const closing = result.error.issues.some(isHardCutIssue) ? HARD_CUT_STEPS : INVALID_CONFIG_FIX(configPath);
291
297
  throw new Error(
292
298
  `Config validation failed:
293
299
  ${messages.join("\n")}
@@ -319,39 +325,6 @@ function resolveRawConfigKnowledgeRoot(config, configPath) {
319
325
  return null;
320
326
  }
321
327
  }
322
- var ENV_MAPPING, ConfigFileNotYamlError, ConfigFileNotFoundError, CIRCULAR_MARKER;
323
- var init_config = __esm({
324
- "src/config.ts"() {
325
- "use strict";
326
- init_hard_cut();
327
- ENV_MAPPING = {
328
- AGENT_QA_DASHBOARD_PORT: "services.dashboard.port",
329
- AGENT_QA_MCP_PORT: "services.mcp.port",
330
- AGENT_QA_CACHE_DIR: "services.cache.dir",
331
- AGENT_QA_CACHE_TTL: "services.cache.ttl",
332
- AGENT_QA_LOG_LEVEL: "services.logging.level",
333
- AGENT_QA_HEADLESS: "use.browser.headless"
334
- };
335
- ConfigFileNotYamlError = class extends Error {
336
- configPath;
337
- constructor(configPath, cause) {
338
- const positionOnly = `${configPath}: ${yamlParseFailureMessage(cause)}`;
339
- super(positionOnly, { cause: new Error(positionOnly) });
340
- this.name = "ConfigFileNotYamlError";
341
- this.configPath = configPath;
342
- }
343
- };
344
- ConfigFileNotFoundError = class extends Error {
345
- configPath;
346
- constructor(configPath) {
347
- super(`Config file not found: ${configPath}`);
348
- this.name = "ConfigFileNotFoundError";
349
- this.configPath = configPath;
350
- }
351
- };
352
- CIRCULAR_MARKER = "<circular>";
353
- }
354
- });
355
328
 
356
329
  // src/targets.ts
357
330
  function resolveTarget(config, targetName) {
@@ -369,8 +342,7 @@ function resolveTarget(config, targetName) {
369
342
  appPackage: target.appPackage,
370
343
  appActivity: target.appActivity,
371
344
  device: target.device,
372
- // Carried through UNCHANGED, undefined included: the resolver does not fill in a class for a
373
- // target whose author did not declare one (D-06).
345
+ // Carried unchanged, `undefined` included: an undeclared target gets no filled-in class.
374
346
  environment: target.environment
375
347
  };
376
348
  }
@@ -381,3261 +353,19 @@ function listTargets(config) {
381
353
  const typedEntry = entry;
382
354
  return {
383
355
  name,
384
- platform: typedEntry.platform
385
- };
386
- });
387
- }
388
- var init_targets = __esm({
389
- "src/targets.ts"() {
390
- "use strict";
391
- }
392
- });
393
-
394
- // src/composition/declared-secrets.ts
395
- import { existsSync as existsSync2, readFileSync as readFileSync4, statSync as statSync2 } from "fs";
396
- import { parse as parseYaml2 } from "yaml";
397
- import { resolveWorkspacePaths } from "@vostride/agent-qa-core";
398
- import { SecretRedactor, SecretStore } from "@vostride/agent-qa-runner";
399
- function readDeclaredSecrets(secretsPath) {
400
- if (!existsSync2(secretsPath)) {
401
- throw new DeclaredSecretsUnavailableError([
402
- `Error: Secrets file not found: ${secretsPath}`,
403
- "Create the file before running tests. It may be empty, but it must exist."
404
- ]);
405
- }
406
- let content;
407
- try {
408
- content = readFileSync4(secretsPath, "utf-8");
409
- } catch {
410
- throw new DeclaredSecretsUnavailableError([
411
- `Error: Secrets file could not be read: ${secretsPath}`,
412
- "Check the path and file permissions before running tests."
413
- ]);
414
- }
415
- try {
416
- return SecretStore.fromEnvContent(content);
417
- } catch (err) {
418
- const message = err instanceof Error ? err.message : String(err);
419
- throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
420
- }
421
- }
422
- function resolveSecretsPath(configPath) {
423
- const file = parseYaml2(readFileSync4(configPath, "utf-8")) ?? {};
424
- const config = mergeConfigs(file, loadEnvOverrides(), {});
425
- const configured = config?.workspace?.secretsFile;
426
- if (typeof configured !== "string" || configured.trim().length === 0) {
427
- throw DeclaredSecretsUnavailableError.required();
428
- }
429
- const workspace = resolveWorkspacePaths({ config, configPath });
430
- return workspace.secretsFile.absolutePath;
431
- }
432
- function statKey(path) {
433
- const stat2 = statSync2(path);
434
- return `${stat2.mtimeMs}:${stat2.size}:${stat2.ino}`;
435
- }
436
- function createDeclaredSecretMask(configPath) {
437
- let resolved;
438
- let cached;
439
- const currentSecretsPath = () => {
440
- const configKey = statKey(configPath);
441
- if (resolved?.configKey !== configKey) {
442
- resolved = void 0;
443
- resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
444
- }
445
- return resolved.secretsPath;
446
- };
447
- const redactor = () => {
448
- const secretsPath = currentSecretsPath();
449
- let key;
450
- try {
451
- key = `${secretsPath}\0${statKey(secretsPath)}`;
452
- } catch {
453
- key = void 0;
454
- }
455
- if (key !== void 0 && cached?.key === key) return cached.redactor;
456
- const current = new SecretRedactor(readDeclaredSecrets(secretsPath));
457
- cached = key === void 0 ? void 0 : { key, redactor: current };
458
- return current;
459
- };
460
- return { mask: (text) => redactor().redactString(text), redactor };
461
- }
462
- var DeclaredSecretsUnavailableError;
463
- var init_declared_secrets = __esm({
464
- "src/composition/declared-secrets.ts"() {
465
- "use strict";
466
- init_config();
467
- DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
468
- lines;
469
- constructor(lines) {
470
- super(lines.join(" "));
471
- this.name = "DeclaredSecretsUnavailableError";
472
- this.lines = lines;
473
- }
474
- static required() {
475
- return new _DeclaredSecretsUnavailableError([
476
- "Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
477
- ]);
478
- }
479
- };
480
- }
481
- });
482
-
483
- // src/curation/promotion-posture.ts
484
- import { existsSync as existsSync3, readdirSync as readdirSync3 } from "fs";
485
- import { dirname as dirname7, join as join8, resolve as resolve3 } from "path";
486
- function hasNoiseFloorArtefacts(committedRoot) {
487
- if (existsSync3(join8(committedRoot, "reports", "noise-floor.json"))) return true;
488
- try {
489
- return readdirSync3(join8(committedRoot, "reports", "noise-floor")).some((name) => name.endsWith(".json"));
490
- } catch (error) {
491
- return error.code !== "ENOENT";
492
- }
493
- }
494
- function renderPromotionLine(entry) {
495
- return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
496
- }
497
- var init_promotion_posture = __esm({
498
- "src/curation/promotion-posture.ts"() {
499
- "use strict";
500
- }
501
- });
502
-
503
- // src/eval-report-render.ts
504
- function counterLines(title, counts) {
505
- const lines = [`### ${title}`, ""];
506
- const keys = Object.keys(counts).sort();
507
- if (keys.length === 0) {
508
- lines.push("- none");
509
- } else {
510
- for (const key of keys) lines.push(`- ${key}: ${counts[key]}`);
511
- }
512
- lines.push("");
513
- return lines;
514
- }
515
- function rateLine(name, metric) {
516
- const rate = metric.rate === void 0 ? "n/a (zero denominator)" : `${metric.rate}`;
517
- return `- ${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
518
- }
519
- function metricLine(name, metric) {
520
- if (metric === void 0) return `${name}: ${notSupplied}`;
521
- const rate = metric.denominator === 0 ? "unavailable (zero denominator)" : metric.rate ?? notSupplied;
522
- return `${name}: ${metric.numerator}/${metric.denominator} (rate: ${rate})`;
523
- }
524
- function recoveryLines(name, recovery) {
525
- const weightSum = recovery?.weightSum;
526
- const weighted = weightSum === 0 ? "unavailable (zero weightSum)" : recovery?.weightedAccuracy ?? notSupplied;
527
- return [
528
- metricLine(`${name}.recoveryAccuracy`, recovery === void 0 ? void 0 : {
529
- numerator: recovery.recoveredPositiveCount,
530
- denominator: recovery.eligiblePositiveCount,
531
- ...recovery.recoveryAccuracy === void 0 ? {} : { rate: recovery.recoveryAccuracy }
532
- }),
533
- `${name}.weightedAccuracy: ${weighted} (weightSum: ${weightSum ?? notSupplied})`,
534
- `${name}.pairsWithoutWeight: ${recovery?.pairsWithoutWeight ?? notSupplied}`
535
- ];
536
- }
537
- function splitLines(name, cell) {
538
- const cost = cell?.perDecisionCost;
539
- return [
540
- metricLine(`${name}.firstAttemptDecisionAccuracy`, cell?.firstAttemptDecisionAccuracy),
541
- metricLine(`${name}.verifierFalsePassRate`, cell?.verifierFalsePassRate),
542
- ...recoveryLines(`${name}.recovery`, cell?.recovery),
543
- `${name}.perDecisionCost: ` + (cost === void 0 ? notSupplied : `decisionCount=${cost.decisionCount} promptTokens=${cost.promptTokens} completionTokens=${cost.completionTokens} totalTokens=${cost.totalTokens} durationMs=${cost.durationMs}`)
544
- ];
545
- }
546
- function renderReportLabelMetricLines(report) {
547
- const calibration = report.calibration;
548
- return [
549
- metricLine("verifierFalsePassRate", report.headline.verifierFalsePassRate),
550
- `calibration.status: ${calibration?.status ?? notSupplied}`,
551
- `calibration.n_gold: ${calibration?.n_gold ?? notSupplied}`,
552
- `calibration.precision: ${calibration?.precision ?? notSupplied}`,
553
- metricLine("calibration.verifierFalsePassRate", calibration?.verifierFalsePassRate),
554
- ...recoveryLines("recovery", report.recovery),
555
- ...splitLines("perSplit.natural", report.perSplit?.natural),
556
- ...splitLines("perSplit.held-out.shared", report.perSplit?.["held-out"]?.shared),
557
- ...splitLines("perSplit.held-out.private", report.perSplit?.["held-out"]?.private),
558
- ...splitLines("perSplit.adversarial", report.perSplit?.adversarial),
559
- `perSplit.unassignedCount: ${report.perSplit?.unassignedCount ?? notSupplied}`
560
- ];
561
- }
562
- function renderEvalReportMarkdown(report) {
563
- const lines = ["# Eval Baseline Report", ""];
564
- lines.push("## Provenance", "");
565
- lines.push(`- batchKey: ${report.batchKey}`);
566
- lines.push(`- matcherVersion: ${report.matcherVersion}`);
567
- if (report.judgeIdentity === "mixed") {
568
- lines.push(
569
- `- judge: mixed identities across score records (${(report.mixedJudgeModelIds ?? []).join(", ")}) \u2014 no judge block, not a gate comparator`
570
- );
571
- } else if (report.judge === void 0) {
572
- lines.push("- judge: no judge invoked");
573
- } else {
574
- lines.push(
575
- `- judge: judgeModelId=${report.judge.judgeModelId} judgeFamily=${report.judge.judgeFamily} judgeVersion=${report.judge.judgeVersion} confidenceThreshold=${report.judge.confidenceThreshold}`
576
- );
577
- if (report.judge.familyOverride !== void 0) {
578
- lines.push(
579
- `- judge family override: ${report.judge.familyOverride} (self-preference bias condition disclosed; owner override of D-10, 2026-07-12)`
580
- );
581
- }
582
- }
583
- const calibration = report.judge?.calibration;
584
- if (calibration !== void 0) {
585
- lines.push("- judge calibration: " + (calibration.status === "capability-gap" ? `capability-gap reason=${calibration.reason}` : calibration.status === "uncalibrated" ? `uncalibrated n=${calibration.n}` + (calibration.provenance === "unavailable" ? " (unavailable)" : " (copied)") : `calibrated n=${calibration.n} pooledAgreement=${calibration.pooledAgreement} wilson95=[${calibration.wilson95.lower},${calibration.wilson95.upper}] halfWidth=${calibration.wilson95.halfWidth} (copied)`));
586
- }
587
- lines.push("");
588
- const headline = report.headline;
589
- lines.push("## Headline", "");
590
- lines.push(rateLine("firstAttemptDecisionAccuracy", headline.firstAttemptDecisionAccuracy));
591
- lines.push("- perDecisionCost:");
592
- lines.push(` - decisionCount: ${headline.perDecisionCost.decisionCount}`);
593
- lines.push(` - promptTokens: ${headline.perDecisionCost.promptTokens}`);
594
- lines.push(` - completionTokens: ${headline.perDecisionCost.completionTokens}`);
595
- lines.push(` - totalTokens: ${headline.perDecisionCost.totalTokens}`);
596
- lines.push(` - durationMs: ${headline.perDecisionCost.durationMs}`);
597
- lines.push("");
598
- lines.push("### Label metrics", "", ...renderReportLabelMetricLines(report).map((line) => `- ${line}`), "");
599
- const honesty = report.honesty;
600
- lines.push("## Honesty Counters", "");
601
- lines.push(...counterLines("tieCounts", honesty.tieCounts));
602
- lines.push(...counterLines("unscorableIdentityCounts", honesty.unscorableIdentityCounts));
603
- lines.push(...counterLines("refusalCounts", honesty.refusalCounts));
604
- lines.push(...counterLines("missByFailureCounts", honesty.missByFailureCounts));
605
- lines.push(`- harnessErrorCount: ${honesty.harnessErrorCount}`, "");
606
- lines.push(...counterLines("neverScoredCounts", honesty.neverScoredCounts));
607
- lines.push(...counterLines("skippedCounts", honesty.skippedCounts));
608
- lines.push(...counterLines("judge invoked", honesty.judge.invoked));
609
- lines.push(...counterLines("judge settled", honesty.judge.settled));
610
- lines.push(...counterLines("judge abstained", honesty.judge.abstained));
611
- lines.push(...counterLines("judge errors", honesty.judge.errors));
612
- lines.push(...counterLines("judge notInvoked", honesty.judge.notInvoked));
613
- lines.push("### judge cost", "");
614
- lines.push(`- calls: ${honesty.judge.cost.calls}`);
615
- lines.push(`- inputTokens: ${honesty.judge.cost.inputTokens}`);
616
- lines.push(`- outputTokens: ${honesty.judge.cost.outputTokens}`);
617
- lines.push(`- outcomesMissingUsage: ${honesty.judge.cost.outcomesMissingUsage}`);
618
- lines.push("");
619
- lines.push("## Axis Distributions", "");
620
- lines.push(...counterLines("elementIdentity", report.axisDistributions.elementIdentity));
621
- lines.push(...counterLines("value", report.axisDistributions.value));
622
- lines.push(...counterLines("opClass", report.axisDistributions.opClass));
623
- lines.push(...counterLines("sideEffect", report.axisDistributions.sideEffect));
624
- lines.push(...counterLines("loop", report.axisDistributions.loop));
625
- const recovery = report.recovery;
626
- lines.push("## Recovery", "");
627
- lines.push(`- selfHealPairCount: ${recovery.selfHealPairCount}`);
628
- lines.push(`- recoveredPositiveCount: ${recovery.recoveredPositiveCount}`);
629
- lines.push(`- eligiblePositiveCount: ${recovery.eligiblePositiveCount}`);
630
- lines.push(
631
- `- recoveryAccuracy: ${recovery.recoveryAccuracy === void 0 ? "n/a (no eligible positives)" : recovery.recoveryAccuracy}`
632
- );
633
- lines.push(`- negativeDecisionCount: ${recovery.negativeDecisionCount}`);
634
- lines.push(`- negativeFullMatchCount: ${recovery.negativeFullMatchCount}`);
635
- lines.push("");
636
- return lines.join("\n");
637
- }
638
- var notSupplied;
639
- var init_eval_report_render = __esm({
640
- "src/eval-report-render.ts"() {
641
- "use strict";
642
- notSupplied = "unavailable (not supplied)";
643
- }
644
- });
645
-
646
- // src/model-defaults.ts
647
- var DEFAULT_ANTHROPIC_MODEL;
648
- var init_model_defaults = __esm({
649
- "src/model-defaults.ts"() {
650
- "use strict";
651
- DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-5-5";
652
- }
653
- });
654
-
655
- // src/llm-utils.ts
656
- var llm_utils_exports = {};
657
- __export(llm_utils_exports, {
658
- applyResolvedAuthToModelConfig: () => applyResolvedAuthToModelConfig,
659
- buildAnalysisTokenBudget: () => buildAnalysisTokenBudget,
660
- buildJobModelCapability: () => buildJobModelCapability,
661
- loadAuthPluginsForRawConfig: () => loadAuthPluginsForRawConfig,
662
- loadProviderPluginsForRawConfig: () => loadProviderPluginsForRawConfig,
663
- resolveCliAnalysisTokenBudget: () => resolveCliAnalysisTokenBudget,
664
- resolveCliChatCapabilities: () => resolveCliChatCapabilities,
665
- resolveCliJobModel: () => resolveCliJobModel,
666
- resolveCredentials: () => resolveCredentials,
667
- resolveLLMModels: () => resolveLLMModels,
668
- resolveModelAuth: () => resolveModelAuth,
669
- resolveNamedConfig: () => resolveNamedConfig,
670
- toModelProfileAndAuth: () => toModelProfileAndAuth
671
- });
672
- import { dirname as dirname8, resolve as resolvePath3 } from "path";
673
- import { createModel as createModel2, createTokenAccounting as createTokenAccounting2, getProviderOptions, resolveCapabilities as resolveCapabilities2, resolveVisionGate } from "@vostride/agent-qa-model";
674
- import { generateText as generateText2 } from "ai";
675
- function normalizeLLMConfig(raw) {
676
- return {
677
- name: raw.name,
678
- provider: raw.provider,
679
- model: raw.model,
680
- ...raw.baseURL !== void 0 ? { baseURL: raw.baseURL } : {},
681
- ...raw.providerHeaders !== void 0 ? { providerHeaders: raw.providerHeaders } : {},
682
- ...raw.vision !== void 0 ? { vision: raw.vision } : {},
683
- ...raw.contextWindow !== void 0 ? { contextWindow: raw.contextWindow } : {}
684
- };
685
- }
686
- async function loadAuthPluginsForRawConfig(raw, configPath = "agent-qa.config.yaml") {
687
- const declarations = raw?.plugins?.auth;
688
- if (!Array.isArray(declarations) || declarations.length === 0) return;
689
- const { loadLLMAuthPlugins: loadLLMAuthPlugins2 } = await import("@vostride/agent-qa-core");
690
- await loadLLMAuthPlugins2(declarations, {
691
- baseDir: dirname8(resolvePath3(configPath))
692
- });
693
- }
694
- async function resolveNamedConfig(configName, configPath = "agent-qa.config.yaml", options = {}) {
695
- const { loadConfigFile: loadConfigFile2 } = await Promise.resolve().then(() => (init_config(), config_exports));
696
- const raw = await loadConfigFile2(configPath);
697
- if (options.keyOnly !== true) {
698
- await loadAuthPluginsForRawConfig(raw, configPath);
699
- await loadProviderPluginsForRawConfig(raw, configPath);
700
- }
701
- const llms = raw?.registry?.llms?.map(normalizeLLMConfig);
702
- const defaultLLM = raw?.use?.llm;
703
- if (!llms?.length) {
704
- throw new Error("No LLM configs found. Run `agent-qa init` to set up your config.");
705
- }
706
- const targetName = configName ?? defaultLLM;
707
- if (!targetName) {
708
- throw new Error("No config name specified and no use.llm set in config.");
709
- }
710
- const found = llms.find((c) => c.name === targetName);
711
- if (!found) {
712
- throw new Error(`Config "${targetName}" not found. Available: ${llms.map((c) => c.name).join(", ")}`);
713
- }
714
- return { config: found, allConfigs: llms, defaultName: defaultLLM ?? "" };
715
- }
716
- function resolveLLMModels(config) {
717
- const defaults = { provider: "anthropic-subscription", model: DEFAULT_ANTHROPIC_MODEL };
718
- if (!config.registry?.llms || !config.use?.llm) {
719
- return { planner: defaults, verifier: defaults, configName: "" };
720
- }
721
- const found = config.registry.llms.find((c) => c.name === config.use.llm);
722
- if (!found) {
723
- return { planner: defaults, verifier: defaults, configName: "" };
724
- }
725
- const { name, ...modelFields } = normalizeLLMConfig(found);
726
- return { planner: modelFields, verifier: modelFields, configName: name };
727
- }
728
- async function resolveModelAuth(configName, llmConfig) {
729
- const { resolveLLMAuth } = await import("@vostride/agent-qa-core");
730
- return resolveLLMAuth(configName, llmConfig);
731
- }
732
- function applyResolvedAuthToModelConfig(llmConfig, auth) {
733
- switch (auth.kind) {
734
- case "api-key":
735
- return { ...llmConfig, apiKey: auth.apiKey };
736
- case "bearer-token":
737
- return { ...llmConfig, authToken: auth.token };
738
- case "auth-fetch":
739
- return { ...llmConfig, fetch: auth.fetch, modelAdapter: auth.modelAdapter };
740
- case "unauthenticated":
741
- return { ...llmConfig };
742
- case "missing":
743
- throw new Error(auth.message);
744
- }
745
- }
746
- function toModelProfileAndAuth(llmConfig, resolved) {
747
- const providerOptions = getProviderOptions({
748
- provider: llmConfig.provider,
749
- modelAdapter: resolved.kind === "auth-fetch" ? resolved.modelAdapter : void 0,
750
- fetch: resolved.kind === "auth-fetch" ? resolved.fetch : void 0
751
- });
752
- const profile = {
753
- provider: llmConfig.provider,
754
- model: llmConfig.model,
755
- ...llmConfig.baseURL !== void 0 ? { baseURL: llmConfig.baseURL } : {},
756
- ...llmConfig.providerHeaders !== void 0 ? { headers: llmConfig.providerHeaders } : {},
757
- ...providerOptions ? { providerOptions } : {},
758
- auth: toAuthDeclaration(resolved)
759
- };
760
- switch (resolved.kind) {
761
- case "api-key":
762
- return { profile, auth: { kind: "api-key", apiKey: resolved.apiKey } };
763
- case "bearer-token":
764
- return { profile, auth: { kind: "bearer-token", token: resolved.token } };
765
- case "auth-fetch":
766
- return { profile, auth: { kind: "auth-fetch", modelAdapter: resolved.modelAdapter, fetch: resolved.fetch } };
767
- case "unauthenticated":
768
- return { profile, auth: { kind: "none" } };
769
- case "missing":
770
- throw new Error(resolved.message);
771
- }
772
- }
773
- function toAuthDeclaration(resolved) {
774
- switch (resolved.kind) {
775
- case "api-key":
776
- return { mode: "api-key" };
777
- case "bearer-token":
778
- return { mode: "bearer-token" };
779
- case "auth-fetch":
780
- return { mode: "subscription", plugin: resolved.provider };
781
- case "unauthenticated":
782
- case "missing":
783
- return { mode: "none" };
784
- }
785
- }
786
- function buildJobModelCapability(profile, auth, mask, deps = {}) {
787
- const model = (deps.createModelImpl ?? createModel2)(profile, auth, mask);
788
- const accounting = (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
789
- getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
790
- });
791
- const generate = deps.generateTextImpl ?? generateText2;
792
- const vision = (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile).vision;
793
- return {
794
- modelId: `${profile.provider}:${profile.model}`,
795
- vision,
796
- async generate(request2) {
797
- const base = {
798
- model,
799
- maxRetries: 0,
800
- temperature: 0,
801
- maxOutputTokens: accounting.getCeilings().maxOutput,
802
- ...request2.system === void 0 ? {} : { system: request2.system },
803
- ...request2.signal === void 0 ? {} : { abortSignal: request2.signal }
804
- };
805
- if (request2.images !== void 0 && request2.images.length > 0 && resolveVisionGate({ vision }).attachImage) {
806
- const result2 = await generate({
807
- ...base,
808
- messages: [
809
- {
810
- role: "user",
811
- content: [
812
- { type: "text", text: request2.prompt },
813
- ...request2.images.map((image) => ({ type: "image", image }))
814
- ]
815
- }
816
- ]
817
- });
818
- return { text: result2.text };
819
- }
820
- const result = await generate({
821
- ...base,
822
- prompt: request2.prompt
823
- });
824
- return { text: result.text };
825
- }
826
- };
827
- }
828
- function buildAnalysisTokenBudget(profile, deps = {}) {
829
- const accounting = (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
830
- getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
831
- });
832
- const ceilings = accounting.getCeilings();
833
- return {
834
- estimate: (text) => accounting.estimate([{ role: "user", content: text }]),
835
- inputBudget: Math.max(0, ceilings.contextWindow - ceilings.maxOutput)
836
- };
837
- }
838
- async function resolveCliAnalysisTokenBudget(configPath, deps = {}) {
839
- try {
840
- const { config } = await resolveNamedConfig(void 0, configPath);
841
- const resolved = await resolveModelAuth(config.name, config);
842
- const { profile } = toModelProfileAndAuth(config, resolved);
843
- return buildAnalysisTokenBudget(profile, deps);
844
- } catch {
845
- return void 0;
846
- }
847
- }
848
- async function resolveCliJobModel(configPath, deps = {}) {
849
- const { config } = await resolveNamedConfig(void 0, configPath);
850
- const resolved = await resolveModelAuth(config.name, config);
851
- const { profile, auth } = toModelProfileAndAuth(config, resolved);
852
- return buildJobModelCapability(profile, auth, createDeclaredSecretMask(configPath).mask, deps);
853
- }
854
- async function resolveCliChatCapabilities(configPath, deps = {}) {
855
- const { mask } = createDeclaredSecretMask(configPath);
856
- const build = async (configName) => {
857
- const { config } = await resolveNamedConfig(configName, configPath);
858
- const resolved = await resolveModelAuth(config.name, config);
859
- const { profile, auth } = toModelProfileAndAuth(config, resolved);
860
- return {
861
- model: (deps.createModelImpl ?? createModel2)(profile, auth, mask),
862
- accounting: (deps.createTokenAccountingImpl ?? createTokenAccounting2)({
863
- getCapabilities: () => (deps.resolveCapabilitiesImpl ?? resolveCapabilities2)(profile)
864
- })
865
- };
866
- };
867
- try {
868
- const opened = await build();
869
- return {
870
- model: opened.model,
871
- accounting: opened.accounting,
872
- // A THROW rather than a degrade on this arm: `/model nonsense` is a person naming something,
873
- // and the honest answer is the refusal that says which name failed. The turn engine's one
874
- // refusal path carries it, leaving the same countable row any other refusal leaves.
875
- modelFor: async (name) => (await build(name)).model
876
- };
877
- } catch {
878
- return void 0;
879
- }
880
- }
881
- async function resolveCredentials(configName, llmConfig) {
882
- try {
883
- const auth = await resolveModelAuth(configName, llmConfig);
884
- if (auth.kind === "api-key") return { apiKey: auth.apiKey };
885
- if (auth.kind === "bearer-token") return { authToken: auth.token };
886
- if (auth.kind === "auth-fetch") return { authFetch: auth.fetch };
887
- return {};
888
- } catch {
889
- return {};
890
- }
891
- }
892
- var init_llm_utils = __esm({
893
- "src/llm-utils.ts"() {
894
- "use strict";
895
- init_declared_secrets();
896
- init_model_defaults();
897
- init_provider_plugins();
898
- }
899
- });
900
-
901
- // src/judge-panel-preflight.ts
902
- async function preflightJudgePanel(session, options = {}, deps = {}) {
903
- const minN = resolveJudgeCalibrationMinimum(options.minN);
904
- const names = selectedEvalJudgeNames(session.evalConfig);
905
- if (names.length < 2) return {
906
- kind: "capability-gap",
907
- status: "capability-gap",
908
- reason: "panel-not-configured",
909
- minN,
910
- entries: []
911
- };
912
- const named = deps.resolveNamedConfig ?? resolveNamedConfig;
913
- const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
914
- const rows = [];
915
- for (const [index, name] of names.entries()) {
916
- const diagnostic = {
917
- index,
918
- name,
919
- families: [],
920
- credential: "resolution-failed",
921
- failures: []
922
- };
923
- const row = { diagnostic };
924
- try {
925
- const { config } = await named(name, session.configPath, { keyOnly: true });
926
- row.config = config;
927
- diagnostic.modelId = config.model;
928
- diagnostic.families = [...resolveJudgeFamilies(config.model)].sort();
929
- } catch {
930
- diagnostic.failures = ["config-unresolved"];
931
- }
932
- if (row.config) {
933
- try {
934
- const auth = await resolveAuth(row.config.name, row.config);
935
- row.auth = auth;
936
- diagnostic.credential = auth.kind === "missing" || auth.kind === "unauthenticated" ? auth.kind : "present";
937
- } catch {
938
- }
939
- }
940
- if (diagnostic.credential !== "present") diagnostic.failures = [...diagnostic.failures, "no-credential"];
941
- rows.push(row);
942
- }
943
- let replayProfile;
944
- try {
945
- replayProfile = await resolveReplayProfileDeclaration(session, named);
946
- } catch {
947
- throw new Error("Cannot resolve the replay profile for judge panel preflight. Check use.llm and its registry.llms entry.");
948
- }
949
- const replayFamilies = resolveJudgeFamilies(replayProfile.model);
950
- const sets = rows.map((row) => resolveJudgeFamilies(row.config?.model ?? ""));
951
- for (const [index, row] of rows.entries()) {
952
- const failures = [...row.diagnostic.failures];
953
- if (row.config && (sets[index].size === 0 || replayFamilies.size === 0)) failures.push("unknown-family");
954
- if (judgeFamiliesIntersect(sets[index], replayFamilies)) failures.push("replay-family-collision");
955
- if (sets.some((set, other) => other !== index && judgeFamiliesIntersect(sets[index], set))) failures.push("member-family-collision");
956
- row.diagnostic.failures = failures;
957
- }
958
- const entries = rows.map(({ diagnostic }) => ({
959
- ...diagnostic,
960
- name: session.mask(diagnostic.name),
961
- ...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
962
- }));
963
- const modelIds = rows.flatMap((row) => row.config ? [row.config.model] : []);
964
- const complete = modelIds.length === names.length;
965
- const identity = complete ? createPanelIdentity(modelIds) : void 0;
966
- const safeIdentity = [replayProfile.model, ...modelIds].every((id) => session.mask(id) === id) ? identity : void 0;
967
- const gap = (reason) => ({
968
- kind: "capability-gap",
969
- status: "capability-gap",
970
- reason,
971
- minN,
972
- entries,
973
- ...safeIdentity === void 0 ? {} : { identity: safeIdentity }
974
- });
975
- if (rows.some((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
976
- const eligibility = assessPanelEligibility({ memberModelIds: modelIds, replayedModelId: replayProfile.model });
977
- if (!eligibility.eligible) return gap("single-family-host");
978
- return {
979
- kind: "eligible",
980
- status: "uncalibrated",
981
- n: 0,
982
- minN,
983
- entries,
984
- identity,
985
- replayProfile,
986
- members: rows.map((row) => toModelProfileAndAuth(row.config, row.auth))
987
- };
988
- }
989
- var init_judge_panel_preflight = __esm({
990
- "src/judge-panel-preflight.ts"() {
991
- "use strict";
992
- init_eval_session_composition();
993
- init_llm_utils();
994
- }
995
- });
996
-
997
- // src/adversarial-probe-preflight.ts
998
- var adversarial_probe_preflight_exports = {};
999
- __export(adversarial_probe_preflight_exports, {
1000
- resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability
1001
- });
1002
- function authoringFamily(modelId) {
1003
- return [...resolveJudgeFamilies(modelId)].sort().join(",");
1004
- }
1005
- async function resolveAdversarialProbeCapability(session, deps = {}) {
1006
- const names = selectedEvalJudgeNames(session.evalConfig);
1007
- if (names.length === 0) {
1008
- return { kind: "no-probe-model", reason: "panel-not-configured", entries: [] };
1009
- }
1010
- const named = deps.resolveNamedConfig ?? resolveNamedConfig;
1011
- const resolveAuth = deps.resolveModelAuth ?? resolveModelAuth;
1012
- const rows = [];
1013
- for (const [index, name] of names.entries()) {
1014
- const diagnostic = {
1015
- index,
1016
- name,
1017
- families: [],
1018
- credential: "resolution-failed",
1019
- failures: []
1020
- };
1021
- const row = { diagnostic };
1022
- try {
1023
- const { config } = await named(name, session.configPath, { keyOnly: true });
1024
- row.config = config;
1025
- diagnostic.modelId = config.model;
1026
- diagnostic.families = [...resolveJudgeFamilies(config.model)].sort();
1027
- } catch {
1028
- diagnostic.failures = ["config-unresolved"];
1029
- }
1030
- if (row.config) {
1031
- try {
1032
- const auth2 = await resolveAuth(row.config.name, row.config);
1033
- row.auth = auth2;
1034
- diagnostic.credential = auth2.kind === "missing" || auth2.kind === "unauthenticated" ? auth2.kind : "present";
1035
- } catch {
1036
- }
1037
- }
1038
- if (diagnostic.credential !== "present") {
1039
- diagnostic.failures = [...diagnostic.failures, "no-credential"];
1040
- }
1041
- rows.push(row);
1042
- }
1043
- let replayProfile;
1044
- try {
1045
- replayProfile = await resolveReplayProfileDeclaration(session, named);
1046
- } catch {
1047
- throw new Error("Cannot resolve the replay profile for adversarial probe preflight. Check use.llm and its registry.llms entry.");
1048
- }
1049
- const replayFamilies = resolveJudgeFamilies(replayProfile.model);
1050
- for (const row of rows) {
1051
- if (!row.config) continue;
1052
- const families = resolveJudgeFamilies(row.config.model);
1053
- row.eligible = assessPanelEligibility({
1054
- memberModelIds: [row.config.model],
1055
- replayedModelId: replayProfile.model
1056
- }).eligible;
1057
- const failures = [...row.diagnostic.failures];
1058
- if (families.size === 0 || replayFamilies.size === 0) failures.push("unknown-family");
1059
- else if (!row.eligible) failures.push("replay-family-collision");
1060
- row.diagnostic.failures = failures;
1061
- }
1062
- const entries = rows.map(({ diagnostic }) => ({
1063
- ...diagnostic,
1064
- name: session.mask(diagnostic.name),
1065
- ...diagnostic.modelId === void 0 ? {} : { modelId: session.mask(diagnostic.modelId) }
1066
- }));
1067
- const gap = (reason) => ({ kind: "no-probe-model", reason, entries });
1068
- if (rows.every((row) => row.diagnostic.credential !== "present")) return gap("no-credential");
1069
- const selected = rows.find((row) => row.diagnostic.credential === "present" && row.config !== void 0 && row.eligible === true);
1070
- if (!selected) return gap("single-family-host");
1071
- const { profile, auth } = toModelProfileAndAuth(selected.config, selected.auth);
1072
- return {
1073
- kind: "probe-author",
1074
- entries,
1075
- author: {
1076
- index: selected.diagnostic.index,
1077
- name: selected.config.name,
1078
- authoringModel: { id: profile.model, family: authoringFamily(profile.model) },
1079
- profile,
1080
- auth
1081
- }
1082
- };
1083
- }
1084
- var init_adversarial_probe_preflight = __esm({
1085
- "src/adversarial-probe-preflight.ts"() {
1086
- "use strict";
1087
- init_eval_session_composition();
1088
- init_llm_utils();
1089
- }
1090
- });
1091
-
1092
- // src/adversarial-probe-author.ts
1093
- var adversarial_probe_author_exports = {};
1094
- __export(adversarial_probe_author_exports, {
1095
- PROBE_AUTHOR_MAX_OUTPUT_TOKENS: () => PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
1096
- PROBE_AUTHOR_TIMEOUT_MS: () => PROBE_AUTHOR_TIMEOUT_MS,
1097
- createAdversarialProbeAuthor: () => createAdversarialProbeAuthor
1098
- });
1099
- import { generateText as generateText3 } from "ai";
1100
- import { z as z7 } from "zod";
1101
- function createAdversarialProbeAuthor(input) {
1102
- const { author } = input;
1103
- const create = input.createModelImpl ?? createModel3;
1104
- const generate = input.generateTextImpl ?? generateText3;
1105
- return {
1106
- async author(request2) {
1107
- if (request2.authoringModel.id !== author.authoringModel.id) {
1108
- throw new Error("This probe author is bound to a different authoring model than the request names.");
1109
- }
1110
- await input.onCallAttempted?.({
1111
- seedCassetteId: request2.seedCassetteId,
1112
- kind: request2.kind,
1113
- promptVersion: request2.promptVersion,
1114
- authoringModelId: author.authoringModel.id
1115
- });
1116
- const profile = {
1117
- ...author.profile,
1118
- ...author.profile.capabilities === void 0 ? {} : { capabilities: { ...author.profile.capabilities } }
1119
- };
1120
- return await runSafetyScope({ executionContext: input.executionContext }, async () => {
1121
- const model = create(profile, author.auth, input.mask);
1122
- const prompt = renderAdversarialProbePrompt(request2);
1123
- const result = await generate({
1124
- model,
1125
- maxRetries: 0,
1126
- temperature: 0,
1127
- maxOutputTokens: PROBE_AUTHOR_MAX_OUTPUT_TOKENS,
1128
- abortSignal: AbortSignal.timeout(PROBE_AUTHOR_TIMEOUT_MS),
1129
- system: prompt.system,
1130
- prompt: prompt.user
1131
- });
1132
- const parsed = AUTHORED_NAME.safeParse(parseJson2(result.text));
1133
- if (!parsed.success) {
1134
- throw new Error("The probe author returned a response that is not the single-field name object the frozen prompt requires.");
1135
- }
1136
- return {
1137
- text: parsed.data.name,
1138
- // Reported spend, carried as reported -- and ABSENT when the provider reported
1139
- // nothing, never zero. A zero here would be a measured claim about a call whose
1140
- // cost we do not know, and the disclosure that reads this cannot tell the two
1141
- // apart once they are the same number. The probe budget is denominated in CALLS
1142
- // for exactly this reason, so an unreported token still never buys an extra call.
1143
- usage: {
1144
- ...reportedCount(result.usage?.inputTokens) === void 0 ? {} : { promptTokens: reportedCount(result.usage?.inputTokens) },
1145
- ...reportedCount(result.usage?.outputTokens) === void 0 ? {} : { completionTokens: reportedCount(result.usage?.outputTokens) }
1146
- }
1147
- };
1148
- });
1149
- }
1150
- };
1151
- }
1152
- function parseJson2(text) {
1153
- try {
1154
- return JSON.parse(text);
1155
- } catch {
1156
- return void 0;
1157
- }
1158
- }
1159
- function reportedCount(value) {
1160
- return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : void 0;
1161
- }
1162
- var PROBE_AUTHOR_MAX_OUTPUT_TOKENS, PROBE_AUTHOR_TIMEOUT_MS, AUTHORED_NAME;
1163
- var init_adversarial_probe_author = __esm({
1164
- "src/adversarial-probe-author.ts"() {
1165
- "use strict";
1166
- init_eval_session_composition();
1167
- PROBE_AUTHOR_MAX_OUTPUT_TOKENS = 200;
1168
- PROBE_AUTHOR_TIMEOUT_MS = 2e4;
1169
- AUTHORED_NAME = z7.object({ name: z7.string() }).strict();
1170
- }
1171
- });
1172
-
1173
- // src/curation/trust-priors.ts
1174
- import { dirname as dirname9, resolve as resolve4 } from "path";
1175
- function buildTrustPriorsResolver(input) {
1176
- const configDir = dirname9(resolve4(input.configPath));
1177
- const committedRoot = resolve4(configDir, "agent-qa-eval");
1178
- const read = readLabelCalibration(committedRoot);
1179
- if (read.kind === "refused") return Object.freeze({ kind: "refused", reason: REFUSAL });
1180
- const table = {};
1181
- const nByOrigin = { "agent-verified": 0, "agent-observed": 0 };
1182
- if (read.kind === "loaded") {
1183
- for (const [origin, channel] of [["agent-verified", "recorded-verdict"], ["agent-observed", "self-heal-pair"]]) {
1184
- nByOrigin[origin] = read.artifact.channels[channel].n;
1185
- const prior = read.artifact.priors[origin];
1186
- if (prior !== void 0) table[origin] = Object.freeze({ alpha: prior.alpha, beta: prior.beta });
1187
- }
1188
- }
1189
- const priors = Object.keys(table).length === 0 ? void 0 : Object.freeze(table);
1190
- const disclosure = Object.freeze({
1191
- status: priors === void 0 ? "provisional" : "calibrated",
1192
- ...read.kind === "loaded" ? { digest: read.artifact.generationId } : {},
1193
- nByOrigin: Object.freeze(nByOrigin)
1194
- });
1195
- let calibration;
1196
- if (input.inspectCalibration) {
1197
- if (read.kind === "absent") calibration = "label calibration: absent";
1198
- else {
1199
- const identity = readCalibrationGoldIdentity({ committedRoot, agentQaRoot: resolve4(configDir, ".agent-qa"), calibrationRead: read });
1200
- const freshness = identity.kind === "unavailable" ? "unavailable" : identity.goldSetDigest === read.artifact.goldSetDigest ? "current" : "stale";
1201
- const channels = ["recorded-verdict", "self-heal-pair"].map((channel) => {
1202
- const row = read.artifact.channels[channel];
1203
- return `${channel} ${row.status} n=${row.n}`;
1204
- }).join(", ");
1205
- calibration = `label calibration: ${channels}; goldDigest ${freshness}`;
1206
- }
1207
- }
1208
- return Object.freeze({
1209
- kind: "ready",
1210
- resolve: () => priors,
1211
- disclosure,
1212
- ...calibration === void 0 ? {} : { calibration }
1213
- });
1214
- }
1215
- function trustPriorsForHost(input) {
1216
- const result = buildTrustPriorsResolver(input);
1217
- if (result.kind === "refused") throw new TrustPriorsRefusedError();
1218
- return result;
1219
- }
1220
- var REFUSAL, TrustPriorsRefusedError;
1221
- var init_trust_priors = __esm({
1222
- "src/curation/trust-priors.ts"() {
1223
- "use strict";
1224
- init_eval_session_composition();
1225
- REFUSAL = "Trust prior calibration is invalid or unavailable; inspect the selected config's committed calibration marker and referenced revisions, following https://vostride.com/docs/agent-qa/self-improvement/evals#if-calibration-is-refused before composing this host.";
1226
- TrustPriorsRefusedError = class extends Error {
1227
- kind = "trust-priors-refused";
1228
- constructor() {
1229
- super(REFUSAL);
1230
- this.name = "TrustPriorsRefusedError";
1231
- }
1232
- };
1233
- }
1234
- });
1235
-
1236
- // src/gates/replay-gate-host.ts
1237
- var replay_gate_host_exports = {};
1238
- __export(replay_gate_host_exports, {
1239
- createReplayGateHost: () => createReplayGateHost,
1240
- createReplayGatePortForHost: () => createReplayGatePortForHost
1241
- });
1242
- import { dirname as dirname10, resolve as resolve5 } from "path";
1243
- import { Gate1ReceiptNotPersistedError } from "@vostride/agent-qa-agent";
1244
- import {
1245
- createKnowledgeReader,
1246
- LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES2
1247
- } from "@vostride/agent-qa-agent-local";
1248
- import {
1249
- GateReceiptSchema,
1250
- receiptDigestOf
1251
- } from "@vostride/agent-qa-schema";
1252
- function createReplayGatePortForHost(input) {
1253
- const resolveTrustPriors = input.resolveTrustPriors ?? trustPriorsForHost(input).resolve;
1254
- return {
1255
- async runReplayGate(candidateId) {
1256
- const { resolveEvalSession: resolveEvalSession2, runReplayGateVerb: runReplayGateVerb2 } = await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports));
1257
- const session = await resolveEvalSession2({ configPath: input.configPath, keyOnly: true });
1258
- const borrowed = input.resolveClaimAuthority?.();
1259
- const result = await runReplayGateVerb2(session, candidateId, {
1260
- resolveTrustPriors,
1261
- agentQaRoot: input.agentQaRoot,
1262
- onProgress: () => void 0,
1263
- ...borrowed === void 0 ? {} : { claimAuthority: borrowed }
1264
- });
1265
- if (result.kind === "refused") throw new Error(result.refusal.reason);
1266
- return checkedReceipt(acknowledged(result, candidateId), candidateId);
1267
- }
1268
- };
1269
- }
1270
- function acknowledged(result, candidateId) {
1271
- if (!ACKNOWLEDGED_RECORDINGS.includes(result.recorded)) {
1272
- throw new Gate1ReceiptNotPersistedError(candidateId, result.recorded);
1273
- }
1274
- return result.receipt;
1275
- }
1276
- function checkedReceipt(value, candidateId) {
1277
- const receipt = GateReceiptSchema.parse(value);
1278
- if (receipt.gate !== 1 || receipt.candidateId !== candidateId || receipt.receiptDigest !== receiptDigestOf(receipt))
1279
- throw new Error("Gate 1 returned a foreign candidate, wrong gate or invalid receipt digest");
1280
- return receipt;
1281
- }
1282
- function createReplayGateHost(deps) {
1283
- const resolveTrustPriors = deps.resolveTrustPriors ?? trustPriorsForHost(deps).resolve;
1284
- const configDir = dirname10(resolve5(deps.configPath));
1285
- const knowledgeRoot = resolveConfiguredKnowledgeRoot(deps.session.config, configDir);
1286
- const reader = deps.reader ?? createKnowledgeReader(() => knowledgeRoot, void 0, void 0, resolveTrustPriors);
1287
- const curation = deps.curation ?? LOCAL_CURATION_CAPABILITIES2;
1288
- async function run(candidateId) {
1289
- const read = reader.candidateInjection;
1290
- if (read === void 0) throw new Error("knowledge reader binds no candidateInjection");
1291
- const found = await read.call(reader, { candidateId });
1292
- if (found.kind === "not-found")
1293
- return {
1294
- kind: "refused",
1295
- refusal: {
1296
- path: knowledgeRoot,
1297
- reason: `candidate ${candidateId} not found`
1298
- },
1299
- notices: [],
1300
- exit: 2
1301
- };
1302
- if (found.injection.candidateId !== candidateId)
1303
- throw new Error("Gate 1 reader returned a foreign candidate");
1304
- const runGate = deps.runReplayGate ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).runReplayGate;
1305
- const result = await runGate({
1306
- ...deps.composition,
1307
- injection: found.injection,
1308
- committedRoot: resolve5(configDir, "agent-qa-eval"),
1309
- agentQaRoot: deps.agentQaRoot,
1310
- mask: deps.session.mask,
1311
- producedBy: { opId: "eval.replay-gate" }
1312
- });
1313
- if (result.kind === "refused") return { ...result, exit: 2 };
1314
- const receipt = checkedReceipt(result.receipt, candidateId);
1315
- const answer = {
1316
- kind: "receipt",
1317
- receipt,
1318
- lines: result.lines,
1319
- notices: [...result.notices],
1320
- recorded: "dry-run",
1321
- exit: receipt.verdict === "pass" ? 0 : receipt.verdict === "fail" ? 1 : 2
1322
- };
1323
- if (!deps.record) return answer;
1324
- const borrowed = deps.resolveClaimAuthority?.();
1325
- if (borrowed !== void 0) {
1326
- const outcome = await curation.recordGateReceipt({
1327
- resolveTrustPriors,
1328
- receipt,
1329
- lease: { leaseId: borrowed.leaseId, epoch: borrowed.epoch },
1330
- knowledgeRoot,
1331
- executionContext: deps.session.executionContext,
1332
- sourceBuildOrdinal: receipt.producedAtOrdinal,
1333
- ...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
1334
- });
1335
- answer.recorded = outcome.kind;
1336
- return answer;
1337
- }
1338
- const lease = curation.maintenance.acquireWriterLease(knowledgeRoot);
1339
- if (lease.kind !== "acquired") {
1340
- answer.recorded = lease.kind === "held" ? "lease-held" : "lease-refused";
1341
- answer.exit = 2;
1342
- answer.notices.push(
1343
- lease.kind === "held" ? "writer lease held by another process; receipt printed, not recorded \u2014 re-run when the daemon releases it" : `writer lease refused: ${lease.reason}; receipt printed, not recorded`
1344
- );
1345
- return answer;
1346
- }
1347
- try {
1348
- const outcome = await curation.recordGateReceipt({
1349
- resolveTrustPriors,
1350
- receipt,
1351
- lease: { leaseId: lease.leaseId, epoch: lease.epoch },
1352
- knowledgeRoot,
1353
- executionContext: deps.session.executionContext,
1354
- sourceBuildOrdinal: receipt.producedAtOrdinal,
1355
- ...deps.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: deps.resolvePromotionPosture }
1356
- });
1357
- answer.recorded = outcome.kind;
1358
- return answer;
1359
- } finally {
1360
- curation.maintenance.releaseLease({ knowledgeRoot, leaseId: lease.leaseId });
1361
- }
1362
- }
1363
- return {
1364
- run,
1365
- port: {
1366
- async runReplayGate(id) {
1367
- const result = await run(id);
1368
- if (result.kind === "refused") throw new Error(result.refusal.reason);
1369
- return acknowledged(result, id);
1370
- }
1371
- }
1372
- };
1373
- }
1374
- var ACKNOWLEDGED_RECORDINGS;
1375
- var init_replay_gate_host = __esm({
1376
- "src/gates/replay-gate-host.ts"() {
1377
- "use strict";
1378
- init_trust_priors();
1379
- init_config();
1380
- ACKNOWLEDGED_RECORDINGS = ["appended", "duplicate"];
1381
- }
1382
- });
1383
-
1384
- // src/eval-session-composition.ts
1385
- var eval_session_composition_exports = {};
1386
- __export(eval_session_composition_exports, {
1387
- ADVERSARIAL_STAGING_SUBDIR: () => ADVERSARIAL_STAGING_SUBDIR,
1388
- ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS: () => ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS,
1389
- EvalJudgeDeclarationError: () => EvalJudgeDeclarationError,
1390
- EvalNamedInputError: () => EvalNamedInputError,
1391
- EvalPartitionFlagsRequiredError: () => EvalPartitionFlagsRequiredError,
1392
- EvalPartitionManifestConflictError: () => EvalPartitionManifestConflictError,
1393
- FLOOR_DEFAULT_BOOTSTRAP_SEED: () => FLOOR_DEFAULT_BOOTSTRAP_SEED,
1394
- FLOOR_DEFAULT_REPEATS: () => FLOOR_DEFAULT_REPEATS,
1395
- FLOOR_DEFAULT_SEEDS: () => FLOOR_DEFAULT_SEEDS,
1396
- FsHandoffSource: () => FsHandoffSource,
1397
- FsMarkLedger: () => FsMarkLedger,
1398
- GATE1_DEFAULT_K: () => GATE1_DEFAULT_K,
1399
- GATE1_DEFAULT_MAX_SCREENS: () => GATE1_DEFAULT_MAX_SCREENS,
1400
- GATE2_DEFAULT_CANARY_HORIZON_BUILDS: () => GATE2_DEFAULT_CANARY_HORIZON_BUILDS,
1401
- GATE2_DEFAULT_MAX_ABLATIONS: () => GATE2_DEFAULT_MAX_ABLATIONS,
1402
- HANDOFF_FILE: () => HANDOFF_FILE2,
1403
- LABEL_CALIBRATION_MIN_GOLD: () => LABEL_CALIBRATION_MIN_GOLD,
1404
- MARK_INTENT_FILE: () => MARK_INTENT_FILE,
1405
- MarkLedgerCorruptError: () => MarkLedgerCorruptError,
1406
- NO_PROBE_MODEL_REASONS: () => NO_PROBE_MODEL_REASONS,
1407
- RUNS_SUBDIR: () => RUNS_SUBDIR2,
1408
- SEED_POLICIES: () => SEED_POLICIES2,
1409
- adversarialHostOptions: () => adversarialHostOptions,
1410
- assessPanelEligibility: () => assessPanelEligibility,
1411
- canonicalJson: () => canonicalJson6,
1412
- computeCorpusDigest: () => computeCorpusDigest2,
1413
- computeReplayProfileFingerprintFor: () => computeReplayProfileFingerprintFor,
1414
- createAdversarialProbeAuthor: () => createAdversarialProbeAuthor,
1415
- createModel: () => createModel3,
1416
- createPanelIdentity: () => createPanelIdentity,
1417
- feedbackSeedHostOptions: () => feedbackSeedHostOptions,
1418
- freezeGoldenTrajectory: () => freezeGoldenTrajectory,
1419
- isFeedbackStateNovel: () => isFeedbackStateNovel,
1420
- judgeCapabilityOf: () => judgeCapabilityOf,
1421
- judgeFamiliesIntersect: () => judgeFamiliesIntersect,
1422
- latestMarkReceiptFor: () => latestMarkReceiptFor,
1423
- listAdversarialBatches: () => listAdversarialBatches,
1424
- listStagedAdversarial: () => listStagedAdversarial2,
1425
- locateMisdirectionDecision: () => locateMisdirectionDecision,
1426
- markedDecisionCount: () => markedDecisionCount,
1427
- preflightJudgePanel: () => preflightJudgePanel,
1428
- previewMarkLabels: () => previewMarkLabels,
1429
- projectMatcherConfigContract: () => projectMatcherConfigContract,
1430
- readActiveCommittedCorpus: () => readActiveCommittedCorpus2,
1431
- readAdversarialRefreshRecords: () => readAdversarialRefreshRecords,
1432
- readCalibrationGoldIdentity: () => readCalibrationGoldIdentity,
1433
- readCanaryCorpusDigest: () => readCanaryCorpusDigest,
1434
- readCommittedCorpus: () => readCommittedCorpus2,
1435
- readGatePostureFor: () => readGatePostureFor,
1436
- readJudgeCalibration: () => readJudgeCalibration,
1437
- readLabelCalibration: () => readLabelCalibration,
1438
- readMisdirectionRootPosture: () => readMisdirectionRootPosture,
1439
- renderAdversarialProbePrompt: () => renderAdversarialProbePrompt,
1440
- renderBaselineComparatorRefusal: () => renderBaselineComparatorRefusal2,
1441
- renderCensusMarkdown: () => renderCensusMarkdown,
1442
- renderJudgeTargetedCalibration: () => renderJudgeTargetedCalibration,
1443
- resolveAdversarialProbeCapability: () => resolveAdversarialProbeCapability,
1444
- resolveEvalJudge: () => resolveEvalJudge,
1445
- resolveEvalSession: () => resolveEvalSession,
1446
- resolveGate2Caps: () => resolveGate2Caps,
1447
- resolveGateProfileIdentity: () => resolveGateProfileIdentity,
1448
- resolveJudgeCalibrationMinimum: () => resolveJudgeCalibrationMinimum,
1449
- resolveJudgeFamilies: () => resolveJudgeFamilies,
1450
- resolveMarkLabelTargets: () => resolveMarkLabelTargets,
1451
- resolveMarkedDecisions: () => resolveMarkedDecisions,
1452
- resolveMinimumGateSet: () => resolveMinimumGateSet,
1453
- resolveMisdirectionCaps: () => resolveMisdirectionCaps,
1454
- resolveProductPartition: () => resolveProductPartition2,
1455
- resolveReplayPlanner: () => resolveReplayPlanner,
1456
- resolveReplayProfileDeclaration: () => resolveReplayProfileDeclaration,
1457
- resolveReplayProfileFingerprint: () => resolveReplayProfileFingerprint,
1458
- resolveTargetEnvironmentForProduct: () => resolveTargetEnvironmentForProduct,
1459
- runAdversarialVerb: () => runAdversarialVerb,
1460
- runCalibrateVerb: () => runCalibrateVerb,
1461
- runCanaryAblation: () => runCanaryAblation,
1462
- runCensusVerb: () => runCensusVerb,
1463
- runExtractVerb: () => runExtractVerb,
1464
- runFeedbackStaging: () => runFeedbackStaging,
1465
- runFloorVerb: () => runFloorVerb,
1466
- runGateVerb: () => runGateVerb,
1467
- runJudgeCalibrateVerb: () => runJudgeCalibrateVerb,
1468
- runMisdirectionCheck: () => runMisdirectionCheck,
1469
- runReplayGate: () => runReplayGate,
1470
- runReplayGateVerb: () => runReplayGateVerb,
1471
- runReplayVerb: () => runReplayVerb,
1472
- runReportVerb: () => runReportVerb,
1473
- runRetireAdversarialVerb: () => runRetireAdversarialVerb,
1474
- runSafetyScope: () => runSafetyScope,
1475
- runSeedFromFeedbackVerb: () => runSeedFromFeedbackVerb,
1476
- runSeedVerb: () => runSeedVerb,
1477
- selectGateSet: () => selectGateSet,
1478
- selectedEvalJudgeNames: () => selectedEvalJudgeNames,
1479
- serializeReplayGateReceipt: () => serializeReplayGateReceipt,
1480
- slugBranch: () => slugBranch2
1481
- });
1482
- import { GATE2_DEFAULT_MAX_LOWER_TRUST_PER_SCOPE } from "@vostride/agent-qa-schema";
1483
- import { resolveKnowledgeRoot as resolveKnowledgeRoot2 } from "@vostride/agent-qa-core";
1484
- import { createHash as createHash2, randomBytes } from "crypto";
1485
- import { existsSync as existsSync4, lstatSync as lstatSync2, readFileSync as readFileSync5, readdirSync as readdirSync4, realpathSync as realpathSync2 } from "fs";
1486
- import { readFile as readFile2 } from "fs/promises";
1487
- import { dirname as dirname11, isAbsolute as isAbsolute2, join as join9, relative, resolve as resolve6 } from "path";
1488
- import {
1489
- HANDOFF_FILE,
1490
- RUNS_SUBDIR,
1491
- loadCompletedRunHandoff,
1492
- slugBranch
1493
- } from "@vostride/agent-qa-runner-local";
1494
- import {
1495
- assertBaselineComparator,
1496
- buildEvalReport,
1497
- buildSeedingPlan,
1498
- admitSeedMember,
1499
- canonicalJson as canonicalJson5,
1500
- composeBaselineProfileKey,
1501
- computeBatchKey,
1502
- computeCorpusDigest,
1503
- createPanelIdentity,
1504
- decisionCorrectnessVectors,
1505
- deriveCensus,
1506
- renderCensusMarkdown,
1507
- EXCLUSION_CLASSES,
1508
- extractRunCassettes,
1509
- FLOOR_PROTOCOL_MINIMUM,
1510
- guardClassifierRepeatability,
1511
- guardCampaignManifest,
1512
- JUDGE_CONFIG,
1513
- judgeIdentitySchema,
1514
- maskCorpusText,
1515
- partitionParamsOf,
1516
- projectBaselinePointer,
1517
- projectDecisionVectorsDocument,
1518
- projectPartitionManifest,
1519
- projectRunExclusion,
1520
- renderBaselineComparatorRefusal,
1521
- renderFloorRefusal,
1522
- resolveJudgeFamily,
1523
- resolveProductPartition,
1524
- SEED_POLICIES
1525
- } from "@vostride/agent-qa-eval";
1526
- import {
1527
- COMMITTED_CASSETTES_DIR,
1528
- DEFAULT_EVAL_CASSETTES_SUBDIR,
1529
- DEFAULT_EVAL_REPLAYS_SUBDIR,
1530
- DEFAULT_EVAL_REPORTS_SUBDIR,
1531
- FloorProtocolError,
1532
- FsCassetteStore,
1533
- FsReplayRecordStore,
1534
- FsReportStore,
1535
- readReportProfileProof,
1536
- FsRunArtifactSource,
1537
- FsScoreStore,
1538
- assembleScoredBatch,
1539
- copyJudgeCalibrationToReport,
1540
- readJudgeCalibration,
1541
- ablateCanaryStep,
1542
- locateTargetCassettes,
1543
- GATE2_AUDIT_SUBDIR,
1544
- isNovelState,
1545
- stageFeedbackEntry,
1546
- admitStagedFeedback,
1547
- ADVERSARIAL_STAGING_SUBDIR,
1548
- listStagedAdversarial,
1549
- refreshAdversarialSplit,
1550
- retireStagedAdversarialBatch,
1551
- resolveAdversarialProbeCallCap,
1552
- resolveAdversarialSeedCap,
1553
- stageAdversarialProposal,
1554
- FEEDBACK_STAGING_SUBDIR,
1555
- readCommittedCorpusDigest,
1556
- readProbeAttempt,
1557
- cassetteProductPartition,
1558
- commitCensus,
1559
- commitEvalReport,
1560
- committedExclusionPath,
1561
- committedNoiseFloorPath,
1562
- committedPartitionManifestPath,
1563
- computeReplayProfileFingerprintFor,
1564
- listBaselinePointers,
1565
- listCommittedReports,
1566
- listNoiseFloorArtefacts,
1567
- loadCassetteBundles,
1568
- loadLabelEvidence,
1569
- loadReplayRecord,
1570
- loadScoreRecord,
1571
- readCommittedCorpus,
1572
- readActiveCommittedCorpus,
1573
- readPartitionManifests,
1574
- readRunExclusions,
1575
- replayJudgeRequest,
1576
- runNoiseFloorMeasurement,
1577
- runReleaseGate,
1578
- runLabelCalibration,
1579
- runJudgeCalibration,
1580
- maskJudgeCalibrationText,
1581
- seedCommittedCorpus,
1582
- sweepCassetteReplays,
1583
- writeBaselinePointer,
1584
- writePartitionManifest,
1585
- writeRunExclusion
1586
- } from "@vostride/agent-qa-eval-local";
1587
- import { runSafetyScope as runSafetyScope2 } from "@vostride/agent-qa-safety-local";
1588
- import {
1589
- ANALYSIS_FILE,
1590
- admitAnalysisArtifactForRun,
1591
- guardRunAnalysisArtifact,
1592
- resolveRunVerdicts,
1593
- sha256hex as sha256hex3
1594
- } from "@vostride/agent-qa-schema";
1595
- import {
1596
- LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES3,
1597
- resolveRunExecutionContext
1598
- } from "@vostride/agent-qa-agent-local";
1599
- import { renderBaselineComparatorRefusal as renderBaselineComparatorRefusal2, SEED_POLICIES as SEED_POLICIES2 } from "@vostride/agent-qa-eval";
1600
- import { computeCorpusDigest as computeCorpusDigest2, selectGateSet } from "@vostride/agent-qa-eval";
1601
- import { judgeCapabilityOf } from "@vostride/agent-qa-eval";
1602
- import { readActiveCommittedCorpus as readActiveCommittedCorpus2, readCommittedCorpus as readCommittedCorpus2, readGatePostureFor, resolveMinimumGateSet } from "@vostride/agent-qa-eval-local";
1603
- import {
1604
- assessPanelEligibility,
1605
- judgeFamiliesIntersect,
1606
- resolveJudgeFamilies,
1607
- resolveJudgeCalibrationMinimum
1608
- } from "@vostride/agent-qa-eval";
1609
- import { projectMatcherConfigContract } from "@vostride/agent-qa-eval";
1610
- import { runReplayGate, serializeReplayGateReceipt, readLabelCalibration, readCalibrationGoldIdentity } from "@vostride/agent-qa-eval-local";
1611
- import { listStagedAdversarial as listStagedAdversarial2, readAdversarialRefreshRecords } from "@vostride/agent-qa-eval-local";
1612
- import { canonicalJson as canonicalJson6, markedDecisionCount, previewMarkLabels, resolveMarkLabelTargets, resolveMarkedDecisions, resolveProductPartition as resolveProductPartition2 } from "@vostride/agent-qa-eval";
1613
- import { FsHandoffSource, FsMarkLedger, MARK_INTENT_FILE, MarkLedgerCorruptError, freezeGoldenTrajectory, latestMarkReceiptFor } from "@vostride/agent-qa-eval-local";
1614
- import { HANDOFF_FILE as HANDOFF_FILE2, RUNS_SUBDIR as RUNS_SUBDIR2, slugBranch as slugBranch2 } from "@vostride/agent-qa-runner-local";
1615
- import { runSafetyScope } from "@vostride/agent-qa-safety-local";
1616
- import { createModel as createModel3 } from "@vostride/agent-qa-model";
1617
- import { NO_PROBE_MODEL_REASONS, renderAdversarialProbePrompt } from "@vostride/agent-qa-eval";
1618
- function selectedEvalJudgeNames(config) {
1619
- if (config.judge !== void 0 && config.judges !== void 0) {
1620
- throw new EvalJudgeDeclarationError("Set either eval.judge or eval.judges, never both; remove one declaration.");
1621
- }
1622
- if (config.judge !== void 0 && typeof config.judge !== "string") {
1623
- throw new EvalJudgeDeclarationError("eval.judge must be one nonempty registry name; use eval.judges for a list.");
1624
- }
1625
- const names = config.judge !== void 0 ? [config.judge] : config.judges === void 0 ? [] : config.judges;
1626
- if (!Array.isArray(names) || Array.from(names).some((name) => typeof name !== "string" || name.trim().length === 0)) {
1627
- throw new EvalJudgeDeclarationError("eval.judges must be an array of nonempty registry names.");
1628
- }
1629
- if (names.length > JUDGE_CONFIG.maxPanelMembers) {
1630
- throw new EvalJudgeDeclarationError(`Choose at most ${JUDGE_CONFIG.maxPanelMembers} explicit judge names.`);
1631
- }
1632
- return [...names];
1633
- }
1634
- function withJudgeOverrides(judge, modelOverrides) {
1635
- if (judge === void 0 || modelOverrides === void 0) return judge;
1636
- return "members" in judge ? { ...judge, members: judge.members.map((member) => ({ ...member, modelOverrides })) } : { ...judge, modelOverrides };
1637
- }
1638
- async function resolveEvalSession(input = {}) {
1639
- const configPath = input.configPath ?? "agent-qa.config.yaml";
1640
- const configDir = dirname11(resolve6(configPath));
1641
- const raw = await loadConfigFile(configPath);
1642
- if (raw?.eval && typeof raw.eval === "object") selectedEvalJudgeNames(raw.eval);
1643
- const config = await resolveConfig({ configPath, loadAuthPlugins: input.keyOnly !== true });
1644
- const evalConfig = mergeEvalConfig(config.eval, input.flags);
1645
- if (input.keyOnly === true) {
1646
- return {
1647
- config,
1648
- configPath,
1649
- configDir,
1650
- evalConfig,
1651
- mask: () => {
1652
- throw new Error("A key-only eval session has no declared secrets to mask with");
1653
- },
1654
- executionContext: resolveRunExecutionContext()
1655
- };
1656
- }
1657
- const secretStore = await loadEvalSecretStore(config, configDir);
1658
- const { SecretRedactor: SecretRedactor2 } = await import("@vostride/agent-qa-runner");
1659
- const redactor = new SecretRedactor2(secretStore);
1660
- return {
1661
- config,
1662
- configPath,
1663
- configDir,
1664
- evalConfig,
1665
- mask: (text) => redactor.redactString(text),
1666
- executionContext: resolveRunExecutionContext()
1667
- };
1668
- }
1669
- function committedCorpusMask(session) {
1670
- const declared = createDeclaredSecretMask(session.configPath);
1671
- declared.redactor();
1672
- return declared.mask;
1673
- }
1674
- async function runExtractVerb(session, filters = {}) {
1675
- return await runSafetyScope2(
1676
- { executionContext: session.executionContext },
1677
- async () => sweepAndExtract(resolve6(session.configDir, ".agent-qa"), filters)
1678
- );
1679
- }
1680
- function sweepAndExtract(agentQaRoot, filters) {
1681
- const runsRoot = resolve6(agentQaRoot, RUNS_SUBDIR);
1682
- let runSlugs = listRunSlugsWithHandoff(runsRoot);
1683
- if (filters.run !== void 0) {
1684
- const named = filters.run;
1685
- runSlugs = runSlugs.filter((slug) => slug === named || slug === slugBranch(named));
1686
- if (runSlugs.length === 0) {
1687
- throw new EvalNamedInputError("run", named, runsRoot);
1688
- }
1689
- }
1690
- const store = new FsCassetteStore({ root: agentQaRoot });
1691
- const runsSkipped = [];
1692
- const completeness = {};
1693
- let runsFiltered = 0;
1694
- let cassettesWritten = 0;
1695
- for (const runSlug of runSlugs) {
1696
- let extraction;
1697
- try {
1698
- const handoff = loadCompletedRunHandoff(resolve6(runsRoot, runSlug, HANDOFF_FILE));
1699
- if (filters.product !== void 0 && resolveProductPartition(handoff.scope) !== filters.product) {
1700
- runsFiltered++;
1701
- continue;
1702
- }
1703
- extraction = extractRunCassettes({
1704
- handoff,
1705
- artifacts: new FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
1706
- });
1707
- } catch (err) {
1708
- runsSkipped.push({ runSlug, reason: err instanceof Error ? err.message : String(err) });
1709
- continue;
1710
- }
1711
- for (const extracted of extraction.cassettes) {
1712
- store.write(extracted);
1713
- cassettesWritten++;
1714
- }
1715
- for (const emitted of extraction.report.emitted) {
1716
- bumpCounter(completeness, emitted.completenessStatus);
1717
- }
1718
- for (const skip of extraction.report.skips) {
1719
- bumpCounter(completeness, skip.reason);
1720
- }
1721
- }
1722
- return {
1723
- runsSwept: runSlugs.length,
1724
- runsSkipped,
1725
- runsFiltered,
1726
- cassettesWritten,
1727
- completeness,
1728
- cassettesRoot: resolve6(agentQaRoot, DEFAULT_EVAL_CASSETTES_SUBDIR)
1729
- };
1730
- }
1731
- function listRunSlugsWithHandoff(runsRoot) {
1732
- let entries;
1733
- try {
1734
- entries = readdirSync4(runsRoot, { withFileTypes: true });
1735
- } catch (err) {
1736
- if (err.code === "ENOENT") return [];
1737
- throw err;
1738
- }
1739
- return entries.filter((entry) => entry.isDirectory() && existsSync4(resolve6(runsRoot, entry.name, HANDOFF_FILE))).map((entry) => entry.name).sort();
1740
- }
1741
- function bumpCounter(counters, key) {
1742
- counters[key] = (counters[key] ?? 0) + 1;
1743
- }
1744
- function resolveCassettesRoot(session, source) {
1745
- if (source === "committed") {
1746
- const cassettesRoot = join9(
1747
- resolve6(session.configDir, "agent-qa-eval"),
1748
- COMMITTED_CASSETTES_DIR
1749
- );
1750
- return { cassettesRoot, notice: `cassette source: committed root ${cassettesRoot}` };
1751
- }
1752
- return {
1753
- cassettesRoot: join9(resolve6(session.configDir, ".agent-qa"), DEFAULT_EVAL_CASSETTES_SUBDIR)
1754
- };
1755
- }
1756
- function replayInvocationSpend(entries, seed, modelCalls, started, calibration) {
1757
- const report = buildEvalReport(entries, void 0, void 0, { declaredSeed: seed, calibration });
1758
- const cost = report.totalCost;
1759
- const judgeCost = report.honesty.judge.cost;
1760
- return {
1761
- seed: seed ?? null,
1762
- seedHonored: cost.sampleCount === 0 ? null : report.seedHonored ?? null,
1763
- decisions: report.headline.perDecisionCost.decisionCount,
1764
- modelCalls,
1765
- promptTokens: cost.promptTokens + judgeCost.inputTokens,
1766
- completionTokens: cost.completionTokens + judgeCost.outputTokens,
1767
- wallClockMs: Math.max(0, Math.round(performance.now() - started)),
1768
- sampleCount: cost.sampleCount,
1769
- samplesMissingTokenUsage: cost.samplesMissingTokenUsage,
1770
- judgeCost
1771
- };
1772
- }
1773
- async function runReplayVerb(session, filters = {}, opts = {}) {
1774
- const started = performance.now();
1775
- const agentQaRoot = resolve6(session.configDir, ".agent-qa");
1776
- const source = opts.source ?? "staged";
1777
- const sourceRoot = resolveCassettesRoot(session, source);
1778
- const cassettesRoot = sourceRoot.cassettesRoot;
1779
- const replaysRoot = join9(agentQaRoot, DEFAULT_EVAL_REPLAYS_SUBDIR);
1780
- const { swept, loaded, loadRefusals } = loadCassetteBundles(cassettesRoot);
1781
- const activeCorpus = readActiveCommittedCorpus(resolve6(session.configDir, "agent-qa-eval"), agentQaRoot);
1782
- const activeIds = source !== "committed" ? void 0 : new Set(activeCorpus.snapshot.entries.map((entry) => entry.cassetteId));
1783
- let selected = activeIds === void 0 ? loaded : loaded.filter((entry) => activeIds.has(entry.cassette.id));
1784
- if (filters.cassette !== void 0) {
1785
- const named = filters.cassette;
1786
- selected = selected.filter((entry) => entry.cassette.id === named);
1787
- if (selected.length === 0) {
1788
- throw new EvalNamedInputError("cassette", named, cassettesRoot, "replay");
1789
- }
1790
- }
1791
- if (filters.run !== void 0) {
1792
- const named = filters.run;
1793
- selected = selected.filter((entry) => {
1794
- const head = entry.cassette.id.split("/")[0];
1795
- return head === named || head === slugBranch(named);
1796
- });
1797
- if (selected.length === 0) {
1798
- throw new EvalNamedInputError("run", named, cassettesRoot, "replay");
1799
- }
1800
- }
1801
- if (filters.product !== void 0) {
1802
- const named = filters.product;
1803
- selected = selected.filter((entry) => cassetteProductPartition(entry.cassette) === named);
1804
- }
1805
- selected.sort((a, b) => a.cassette.id < b.cassette.id ? -1 : a.cassette.id > b.cassette.id ? 1 : 0);
1806
- const notices = [];
1807
- if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
1808
- notices.push(...activeCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
1809
- const zeroJudge = (configured) => ({
1810
- configured,
1811
- invoked: 0,
1812
- settled: 0,
1813
- abstained: 0,
1814
- errors: 0,
1815
- notInvoked: 0
1816
- });
1817
- if (selected.length === 0) {
1818
- notices.push(`empty sweep \u2014 no ${source} cassette bundles matched`);
1819
- return {
1820
- batchKey: "empty",
1821
- swept,
1822
- replayed: 0,
1823
- loadRefusals,
1824
- replayRefusals: {},
1825
- scored: 0,
1826
- judge: zeroJudge(selectedEvalJudgeNames(session.evalConfig).length > 0),
1827
- replaysRoot,
1828
- source,
1829
- notices,
1830
- spend: replayInvocationSpend([], session.evalConfig.seed, 0, started, activeCorpus?.calibration)
1831
- };
1832
- }
1833
- const batchKey = computeBatchKey(selected.map((entry) => entry.cassette.id));
1834
- const planner = await resolveReplayPlanner(session);
1835
- const judgeResolution = await resolveEvalJudge(session);
1836
- if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
1837
- const recordStore = new FsReplayRecordStore({ root: agentQaRoot });
1838
- const scoreStore = new FsScoreStore({ root: agentQaRoot });
1839
- let modelCalls = 0;
1840
- const { replayed, replayRefusals, replayedDirs, notices: replayNotices } = await sweepCassetteReplays({
1841
- selected,
1842
- labelRecords: activeCorpus?.snapshot.labelsById,
1843
- profile: planner.profile,
1844
- auth: planner.auth,
1845
- modelCallObservation: { onProviderCall: () => {
1846
- modelCalls++;
1847
- } },
1848
- ...opts.plannerOverrides !== void 0 ? { modelOverrides: opts.plannerOverrides } : {},
1849
- ...session.evalConfig.k !== void 0 ? { k: session.evalConfig.k } : {},
1850
- ...session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed } : {},
1851
- ...judgeResolution.judge !== void 0 ? {
1852
- judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides)
1853
- } : {},
1854
- mask: session.mask,
1855
- recordStore,
1856
- scoreStore,
1857
- replaysRoot,
1858
- // Progress ALWAYS rides stderr — stdout stays reserved for the rendered DTO.
1859
- onProgress: (id) => process.stderr.write(`replay: ${id}
1860
- `)
1861
- });
1862
- notices.push(...replayNotices);
1863
- const judge = zeroJudge(judgeResolution.judge !== void 0);
1864
- let scored = 0;
1865
- for (const recordDir of replayedDirs) {
1866
- const replay = loadReplayRecord(recordDir);
1867
- const load = loadScoreRecord(recordDir);
1868
- if (replay.kind !== "loaded") {
1869
- notices.push(`spend replay read refused under ${recordDir}: ${replay.refusal.detail}`);
1870
- }
1871
- if (load.kind === "absent") continue;
1872
- if (load.kind === "refused") {
1873
- notices.push(`score read refused under ${recordDir}: ${load.refusal.detail}`);
1874
- continue;
1875
- }
1876
- scored++;
1877
- foldJudgeDispositions(load.record.score, judge);
1878
- }
1879
- const spendBatch = assembleScoredBatch({
1880
- replaysRoot,
1881
- cassettesRoot,
1882
- source,
1883
- recordDirs: replayedDirs,
1884
- labelRecords: activeCorpus?.snapshot.labelsById,
1885
- corpusEntriesById: source === "committed" ? activeCorpus.corpusEntriesById : void 0,
1886
- excludedLabelIds: activeCorpus?.snapshot.excludedLabelIds,
1887
- calibration: activeCorpus?.calibration
1888
- });
1889
- notices.push(...spendBatch.notices);
1890
- return {
1891
- batchKey,
1892
- swept,
1893
- replayed,
1894
- loadRefusals,
1895
- replayRefusals,
1896
- scored,
1897
- judge,
1898
- replaysRoot,
1899
- source,
1900
- notices,
1901
- spend: replayInvocationSpend(spendBatch.entries, session.evalConfig.seed, modelCalls, started, spendBatch.calibration)
1902
- };
1903
- }
1904
- function foldJudgeDispositions(score, judge) {
1905
- const entries = score.kind === "leaf" ? [score.entry] : score.kind === "procedure" ? score.entries : [];
1906
- for (const entry of entries) {
1907
- if (entry.kind !== "decision") continue;
1908
- const decision = entry.score;
1909
- if (decision.kind === "unresolved-tie") {
1910
- const outcome = decision.judgeOutcome;
1911
- if (outcome === void 0) continue;
1912
- if (outcome.kind === "abstained") {
1913
- judge.abstained++;
1914
- judge.invoked++;
1915
- } else if (outcome.kind === "judge-error") {
1916
- judge.errors++;
1917
- judge.invoked++;
1918
- } else {
1919
- judge.notInvoked++;
1920
- }
1921
- } else if (decision.kind === "scored" && decision.judgeSettled !== void 0) {
1922
- judge.settled++;
1923
- judge.invoked++;
1924
- }
1925
- }
1926
- }
1927
- async function resolveEvalJudge(session) {
1928
- const names = selectedEvalJudgeNames(session.evalConfig);
1929
- if (names.length === 0) {
1930
- return {
1931
- notice: "no judge configured \u2014 unresolved ties will record not-invoked{no-judge-configured}"
1932
- };
1933
- }
1934
- if (names.length > 1) {
1935
- const members = [];
1936
- for (const name of names) members.push(await resolveNamedProfile(name, session.configPath));
1937
- return { judge: { members }, ...session.evalConfig.judgeAllowSameFamily === true ? { notice: "eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint." } : {} };
1938
- }
1939
- const resolved = await resolveNamedProfile(names[0], session.configPath);
1940
- const planner = await resolveReplayPlanner(session);
1941
- const judgeFamily = resolveJudgeFamily(resolved.profile.model);
1942
- const plannerFamily = resolveJudgeFamily(planner.profile.model);
1943
- const overrideActive = session.evalConfig.judgeAllowSameFamily === true;
1944
- const judge = overrideActive ? { ...resolved, allowSameFamily: true } : resolved;
1945
- if (judgeFamily !== void 0 && plannerFamily !== void 0 && judgeFamily === plannerFamily && overrideActive) {
1946
- return {
1947
- judge,
1948
- notice: `D-10 override active (eval.judgeAllowSameFamily, owner decision 2026-07-12): judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" share the judge family '${judgeFamily}' \u2014 same-family judging runs and every scored entry discloses familyOverride: same-family-allowed`
1949
- };
1950
- }
1951
- if (judgeFamily === void 0 || plannerFamily === void 0 || judgeFamily === plannerFamily) {
1952
- return {
1953
- judge,
1954
- notice: `judge model "${resolved.profile.model}" and planner model "${planner.profile.model}" resolve to the same or unknown family \u2014 per-cassette judge eligibility will refuse self-judging and ties may record not-invoked`
1955
- };
1956
- }
1957
- return { judge };
1958
- }
1959
- async function resolveReplayPlanner(session) {
1960
- const plannerName = session.config.use?.llm;
1961
- if (!plannerName) {
1962
- throw new Error(
1963
- "use.llm is not set in the config \u2014 the replay planner is the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
1964
- );
1965
- }
1966
- return resolveNamedProfile(plannerName, session.configPath);
1967
- }
1968
- async function runJudgeCalibrateVerb(session, opts = {}) {
1969
- if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
1970
- const preflight = await preflightJudgePanel(session, { minN: opts.minN });
1971
- if (preflight.kind === "capability-gap" && !preflight.identity) return {
1972
- ...await runJudgeCalibration({
1973
- kind: "capability-gap",
1974
- reason: preflight.reason,
1975
- minN: preflight.minN,
1976
- commit: opts.commit,
1977
- replace: opts.replace
1978
- }),
1979
- entries: preflight.entries
1980
- };
1981
- const replayProfile = preflight.kind === "eligible" ? preflight.replayProfile : await resolveReplayProfileDeclaration(session);
1982
- const fingerprint = computeReplayProfileFingerprintFor({
1983
- profile: replayProfile,
1984
- seedPolicy: "declared"
1985
- });
1986
- const context = {
1987
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
1988
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
1989
- // Saved evidence must prove this actual planner/profile; judging does not rerun a planner.
1990
- profile: {
1991
- plannerModelId: replayProfile.model,
1992
- fingerprint,
1993
- matcherVersion: (await import("@vostride/agent-qa-eval")).projectMatcherConfigContract().version,
1994
- judgeModelId: preflight.identity.judgeModelId,
1995
- judgeVersion: preflight.identity.judgeVersion
1996
- },
1997
- identity: preflight.identity,
1998
- minN: preflight.minN
1999
- };
2000
- const result = await runJudgeCalibration(preflight.kind === "eligible" ? { kind: "probe", commit: opts.commit, replace: opts.replace, probe: {
2001
- ...context,
2002
- replayProfile,
2003
- members: preflight.members.map((member) => ({ ...member, modelOverrides: opts.judgeOverrides })),
2004
- mask: session.mask
2005
- } } : {
2006
- kind: "capability-gap",
2007
- reason: preflight.reason,
2008
- minN: preflight.minN,
2009
- publication: context,
2010
- commit: opts.commit,
2011
- replace: opts.replace
2012
- });
2013
- return maskJudgeCalibrationText({ ...result, entries: preflight.entries }, session.mask);
2014
- }
2015
- function renderJudgeTargetedCalibration(disclosure) {
2016
- if (disclosure === void 0) {
2017
- return ["judge-targeted adversarial bait: not recorded (this artifact predates the disclosure)"];
2018
- }
2019
- if (disclosure.source === "unavailable") {
2020
- return [`judge-targeted adversarial bait: not determined (${disclosure.reason})`];
2021
- }
2022
- const lines = [
2023
- `judge-targeted adversarial bait: judge-targeted=${disclosure.judgeTargeted} self-authored-by-this-panel=${disclosure.selfAuthored} observed-in-measured-population=${disclosure.observedInCalibration}`,
2024
- `adversarial provenance: model-authored=${disclosure.byProvenance.probe} table-derived=${disclosure.byProvenance.table} legacy-unattributed=${disclosure.byProvenance.absent} malformed=${disclosure.byProvenance.malformed} unresolved=${disclosure.metadataUnavailable} of ${disclosure.total} committed members (arms are counted apart, never summed)`
2025
- ];
2026
- for (const row of disclosure.byAuthoringModel) {
2027
- lines.push(`author ${row.authoringModel} (${row.authoringFamily})${row.panelMember ? " [panel member \u2014 self-authored]" : ""}: judge-targeted=${row.judgeTargeted} authored=${row.probeAuthored}`);
2028
- }
2029
- lines.push("Bait counts above are RAW: the agreement, interval, per-member rows and minN are not adjusted for them.");
2030
- return lines;
2031
- }
2032
- async function runCalibrateVerb(session, opts = {}) {
2033
- const minGold = opts.minGold ?? LABEL_CALIBRATION_MIN_GOLD;
2034
- if (!Number.isSafeInteger(minGold) || minGold < 2) {
2035
- return { kind: "refused", reason: "--min-gold must be a safe integer of at least 2." };
2036
- }
2037
- if (opts.replace && !opts.commit) return { kind: "refused", reason: "--replace requires --commit." };
2038
- const result = await runLabelCalibration({
2039
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
2040
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
2041
- commit: opts.commit,
2042
- replace: opts.replace,
2043
- minGold: { value: minGold, source: opts.minGold === void 0 ? "default" : "cli" }
2044
- });
2045
- if ("artifact" in result) return result;
2046
- const reasons = {
2047
- refused: "Calibration inputs unavailable or invalid; inspect committed labels and run evidence.",
2048
- busy: "Another calibration publisher holds the lock; retry after it finishes.",
2049
- "input-drift": "Calibration inputs changed; retry from the current snapshot.",
2050
- "publication-failed": "Calibration publication interrupted; retry the identical command.",
2051
- "write-conflict": "Calibration bytes conflict; review the dry-run, then use --commit --replace to replace the marker. Immutable revision conflicts require inspection."
2052
- };
2053
- return { kind: result.kind, reason: reasons[result.kind] };
2054
- }
2055
- function resolveGate2Caps(session, opts = {}) {
2056
- const maxAblations = opts.maxAblations ?? session.evalConfig.gate2?.maxAblations ?? GATE2_DEFAULT_MAX_ABLATIONS;
2057
- if (!Number.isSafeInteger(maxAblations) || maxAblations < 1) {
2058
- throw new TypeError("--max-ablations must be a positive integer");
2059
- }
2060
- const maxAblationsSource = opts.maxAblations !== void 0 ? "flag" : session.evalConfig.gate2?.maxAblations !== void 0 ? "config" : "default";
2061
- const canaryHorizonBuilds = session.evalConfig.gate2?.canaryHorizonBuilds ?? GATE2_DEFAULT_CANARY_HORIZON_BUILDS;
2062
- const maxLowerTrustPerScope = session.evalConfig.gate2?.maxLowerTrustPerScope ?? GATE2_DEFAULT_MAX_LOWER_TRUST_PER_SCOPE;
2063
- for (const [name, value] of Object.entries({ canaryHorizonBuilds, maxLowerTrustPerScope })) {
2064
- if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`eval.gate2.${name} must be a positive integer`);
2065
- }
2066
- return {
2067
- maxAblations,
2068
- maxAblationsSource,
2069
- canaryHorizonBuilds,
2070
- maxLowerTrustPerScope,
2071
- canaryHorizonBuildsSource: session.evalConfig.gate2?.canaryHorizonBuilds === void 0 ? "default" : "config",
2072
- maxLowerTrustPerScopeSource: session.evalConfig.gate2?.maxLowerTrustPerScope === void 0 ? "default" : "config"
2073
- };
2074
- }
2075
- function readCanaryCorpusDigest(session) {
2076
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2077
- try {
2078
- lstatSync2(committedRoot);
2079
- } catch (error) {
2080
- return error.code === "ENOENT" ? { kind: "no-eval-root" } : { kind: "unavailable", reason: error instanceof Error ? error.message : String(error) };
2081
- }
2082
- return readCommittedCorpusDigest({ committedRoot });
2083
- }
2084
- async function runCanaryAblation(session, input) {
2085
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2086
- const digest2 = readCanaryCorpusDigest(session);
2087
- if (digest2.kind !== "digest") return digest2;
2088
- const k = input.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
2089
- if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
2090
- throw new TypeError("--k must be an odd integer of at least 3");
2091
- }
2092
- const identity = await resolveGateProfileIdentity(session, input.plannerOverrides);
2093
- const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
2094
- (entry) => entry.artefact.profileKey === identity.profileKey
2095
- )?.artefact;
2096
- const baseSeed = input.seed ?? session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
2097
- const execution = await resolveEvalSession({ configPath: session.configPath });
2098
- const planner = await resolveReplayPlanner(execution);
2099
- return ablateCanaryStep({
2100
- agentQaRoot: input.agentQaRoot,
2101
- committedRoot,
2102
- recordId: input.recordId,
2103
- target: input.target,
2104
- attributedObservationId: input.attributedObservationId,
2105
- k,
2106
- baseSeed,
2107
- replay: {
2108
- profile: planner.profile,
2109
- auth: planner.auth,
2110
- modelOverrides: input.plannerOverrides,
2111
- mask: execution.mask
2112
- }
2113
- });
2114
- }
2115
- async function readMisdirectionRootPosture(session) {
2116
- const digest2 = readCanaryCorpusDigest(session);
2117
- const floorless = !hasNoiseFloorArtefacts(resolve6(session.configDir, "agent-qa-eval"));
2118
- if (digest2.kind === "no-eval-root" || floorless) return { kind: "no-eval-root" };
2119
- if (digest2.kind !== "digest") return { kind: "broken-eval-root", reason: "eval-root-unreadable" };
2120
- const identity = await resolveGateProfileIdentity(session);
2121
- return {
2122
- kind: "eval-root",
2123
- corpusDigest: digest2.corpusDigest,
2124
- profileDigest: sha256hex3(identity.profileKey)
2125
- };
2126
- }
2127
- function resolveMisdirectionCaps(_session, opts = {}) {
2128
- const maxChecks = opts.maxChecks ?? ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS;
2129
- if (!Number.isSafeInteger(maxChecks) || maxChecks < 0) {
2130
- throw new TypeError("--max-misdirection-checks must be a non-negative integer");
2131
- }
2132
- return { maxChecks, source: opts.maxChecks === void 0 ? "default" : "flag" };
2133
- }
2134
- async function locateMisdirectionDecision(session, target) {
2135
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2136
- const corpus = readCommittedCorpus(committedRoot);
2137
- const located = await locateTargetCassettes({
2138
- committedRoot,
2139
- agentQaRoot: target.agentQaRoot,
2140
- gate1Dir: misdirectionAuditDir(target),
2141
- corpusEntries: corpus.snapshot.entries,
2142
- target: { runId: target.runId, testId: target.testId, stepIndex: target.stepIndex },
2143
- source: "handoff"
2144
- });
2145
- if (located.kind === "unavailable") {
2146
- return { kind: "unavailable", reason: located.detail.reason ?? "target-unavailable" };
2147
- }
2148
- if (located.cassettes.length !== 1) {
2149
- return { kind: "unavailable", reason: "ambiguous live step: multiple recorded decisions" };
2150
- }
2151
- return { kind: "located", decisionOrdinal: located.cassettes[0].cassette.decisionOrdinal };
2152
- }
2153
- async function runMisdirectionCheck(session, input) {
2154
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2155
- const digest2 = readCanaryCorpusDigest(session);
2156
- if (digest2.kind !== "digest") {
2157
- return { kind: "unavailable", reason: digest2.kind === "no-eval-root" ? "no-eval-root" : digest2.reason };
2158
- }
2159
- const k = session.evalConfig.k ?? GATE1_DEFAULT_K;
2160
- if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
2161
- throw new TypeError("eval.k must be an odd integer of at least 3");
2162
- }
2163
- const identity = await resolveGateProfileIdentity(session);
2164
- const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
2165
- (entry) => entry.artefact.profileKey === identity.profileKey
2166
- )?.artefact;
2167
- const baseSeed = session.evalConfig.seed ?? floor?.seeds[0] ?? 11;
2168
- const execution = await resolveEvalSession({ configPath: session.configPath });
2169
- const planner = await resolveReplayPlanner(execution);
2170
- const result = await ablateCanaryStep({
2171
- agentQaRoot: input.agentQaRoot,
2172
- committedRoot,
2173
- recordId: input.recordId,
2174
- target: { runId: input.runId, testId: input.testId, stepIndex: input.stepIndex },
2175
- attributedObservationId: misdirectionAuditKey(input),
2176
- k,
2177
- baseSeed,
2178
- admission: { failingPathCandidate: input.authorization },
2179
- replay: {
2180
- profile: planner.profile,
2181
- auth: planner.auth,
2182
- mask: execution.mask
2183
- }
2184
- });
2185
- if (result.kind !== "ablated") return { kind: "unavailable", reason: result.reason };
2186
- return {
2187
- kind: "checked",
2188
- crossCheck: result.crossCheck,
2189
- seedHonored: result.seedHonored,
2190
- samples: { k: result.k, seeds: [...result.seeds], modelCalls: result.spend.modelCalls }
2191
- };
2192
- }
2193
- function misdirectionAuditKey(target) {
2194
- return `misdirection/${slugBranch(target.runId)}/${slugBranch(target.testId)}/${String(target.stepIndex)}${target.recordId === void 0 ? "" : `/${slugBranch(target.recordId)}`}`;
2195
- }
2196
- function misdirectionAuditDir(target) {
2197
- return join9(target.agentQaRoot, GATE2_AUDIT_SUBDIR, slugBranch(misdirectionAuditKey(target)));
2198
- }
2199
- function isFeedbackStateNovel(session, input) {
2200
- const digest2 = readCanaryCorpusDigest(session);
2201
- if (digest2.kind === "no-eval-root") return false;
2202
- if (digest2.kind !== "digest") throw new TypeError("feedback corpus unavailable");
2203
- return isNovelState({ ...input, committedRoot: resolve6(session.configDir, "agent-qa-eval") });
2204
- }
2205
- async function runFeedbackStaging(session, input) {
2206
- const digest2 = readCanaryCorpusDigest(session);
2207
- if (digest2.kind !== "digest") return digest2;
2208
- return stageFeedbackEntry({
2209
- ...input,
2210
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
2211
- mask: committedCorpusMask(session)
2212
- });
2213
- }
2214
- async function runSeedFromFeedbackVerb(session, opts) {
2215
- const stagingDir = join9(opts.agentQaRoot, FEEDBACK_STAGING_SUBDIR);
2216
- const configArg = `'${resolve6(session.configPath).replaceAll("'", `'"'"'`)}'`;
2217
- const floorCommand = `agent-qa --config ${configArg} eval floor`;
2218
- const base = { entries: [], refusals: [], staleReceipts: [], writesStarted: false, failures: [], stagingDir, floorCommand };
2219
- const digest2 = readCanaryCorpusDigest(session);
2220
- if (digest2.kind !== "digest") {
2221
- const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
2222
- return { ...base, kind: "refused", reason, failures: [{ phase: "before-write", reason }] };
2223
- }
2224
- try {
2225
- const receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
2226
- const result = await admitStagedFeedback({
2227
- ...opts,
2228
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
2229
- mask: committedCorpusMask(session)
2230
- });
2231
- const selected = opts.dryRun ? result.entries : result.admitted;
2232
- const entries = selected.map((entry) => ({
2233
- cassetteId: entry.extracted.cassette.id,
2234
- product: entry.product,
2235
- split: "natural",
2236
- source: "canary-feedback",
2237
- benefit: entry.benefit,
2238
- attributedObservationId: entry.attributedObservationId,
2239
- runId: entry.runId,
2240
- stepIndex: entry.stepIndex,
2241
- captured_at_ordinal: entry.extracted.cassette.receipt.buildOrdinal
2242
- }));
2243
- const moves = opts.dryRun ? entries.length > 0 : result.nextCorpusDigest === void 0 ? void 0 : result.nextCorpusDigest !== result.previousCorpusDigest;
2244
- const refused = result.refusals.length > 0 || result.failures.length > 0;
2245
- return {
2246
- kind: refused ? "refused" : result.entries.length === 0 ? "nothing-staged" : opts.dryRun ? "dry-run" : "admitted",
2247
- ...refused ? { reason: "admission-refused" } : {},
2248
- entries,
2249
- refusals: result.refusals,
2250
- writesStarted: result.writesStarted,
2251
- failures: result.failures,
2252
- staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === result.previousCorpusDigest) : [],
2253
- previousCorpusDigest: result.previousCorpusDigest,
2254
- ...result.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: result.nextCorpusDigest },
2255
- stagingDir,
2256
- floorCommand
2257
- };
2258
- } catch {
2259
- return {
2260
- ...base,
2261
- kind: "refused",
2262
- reason: "feedback-unavailable",
2263
- failures: [{ phase: "before-write", reason: "feedback-unavailable" }],
2264
- previousCorpusDigest: digest2.corpusDigest
2265
- };
2266
- }
2267
- }
2268
- function authoringModelDisclosureOf(model) {
2269
- return {
2270
- idDigest: createHash2("sha256").update(model.id, "utf8").digest("hex"),
2271
- family: model.family
2272
- };
2273
- }
2274
- function adversarialHostOptions(session) {
2275
- return {
2276
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
2277
- knowledgeRoot: resolveKnowledgeRoot2(session.config.knowledge, session.configDir),
2278
- readStandingGateReceipts: LOCAL_CURATION_CAPABILITIES3.readStandingGateReceipts
2279
- };
2280
- }
2281
- function runRetireAdversarialVerb(session, opts) {
2282
- const result = retireStagedAdversarialBatch({
2283
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
2284
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
2285
- product: opts.product,
2286
- batchDigest: opts.batch
2287
- });
2288
- return {
2289
- ...result,
2290
- modelCalls: 0,
2291
- admitsNothing: true,
2292
- settledWorkPreserved: true,
2293
- newMeasurementStaleness: false
2294
- };
2295
- }
2296
- async function runAdversarialVerb(session, opts) {
2297
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2298
- const product = opts.product ?? "default";
2299
- const stagingDir = join9(opts.agentQaRoot, ADVERSARIAL_STAGING_SUBDIR, slugBranch(product));
2300
- const configArg = `'${resolve6(session.configPath).replaceAll("'", `'"'"'`)}'`;
2301
- const floorCommand = `agent-qa --config ${configArg} eval floor`;
2302
- const cap = resolveAdversarialSeedCap(
2303
- opts.maxSeeds !== void 0 ? { value: opts.maxSeeds, source: "flag" } : session.evalConfig.adversarial?.maxSeedsPerRefresh !== void 0 ? { value: session.evalConfig.adversarial.maxSeedsPerRefresh, source: "config" } : {}
2304
- );
2305
- const probeCap = resolveAdversarialProbeCallCap(
2306
- opts.maxProbeCalls !== void 0 ? { value: opts.maxProbeCalls, source: "flag" } : session.evalConfig.adversarial?.maxProbeCalls !== void 0 ? { value: session.evalConfig.adversarial.maxProbeCalls, source: "config" } : {}
2307
- );
2308
- const base = {
2309
- kind: "refused",
2310
- product,
2311
- stagingDir,
2312
- staged: [],
2313
- admitted: [],
2314
- unchanged: [],
2315
- remaining: [],
2316
- refusals: [],
2317
- settledRefusals: [],
2318
- stagingSkips: [],
2319
- skips: [],
2320
- seedsConsidered: 0,
2321
- seedsDeferred: 0,
2322
- eligibleSeeds: 0,
2323
- cap,
2324
- writesStarted: false,
2325
- staleReceipts: [],
2326
- floorCommand,
2327
- probeCap,
2328
- spend: { modelCalls: 0, promptTokens: 0, completionTokens: 0, usageKnown: true },
2329
- probeSkips: [],
2330
- probeRefusals: []
2331
- };
2332
- const digest2 = readCanaryCorpusDigest(session);
2333
- if (digest2.kind !== "digest") {
2334
- const reason = digest2.kind === "no-eval-root" ? "no-eval-root" : "corpus-unavailable";
2335
- return { ...base, kind: "refused", reason };
2336
- }
2337
- if (opts.admit === true) {
2338
- if (opts.batch === void 0) {
2339
- return { ...base, kind: "refused", reason: "admit-requires-batch" };
2340
- }
2341
- let receipts = [];
2342
- const refreshed = refreshAdversarialSplit({
2343
- agentQaRoot: opts.agentQaRoot,
2344
- committedRoot,
2345
- product,
2346
- batchDigest: opts.batch,
2347
- beforeWrites: () => {
2348
- receipts = opts.knowledgeRoot !== void 0 && opts.readStandingGateReceipts !== void 0 ? opts.readStandingGateReceipts({ knowledgeRoot: opts.knowledgeRoot }) : [];
2349
- },
2350
- ...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun },
2351
- mask: committedCorpusMask(session)
2352
- });
2353
- const moves = refreshed.nextCorpusDigest === void 0 ? refreshed.writesStarted ? void 0 : false : refreshed.nextCorpusDigest !== refreshed.previousCorpusDigest;
2354
- return {
2355
- ...base,
2356
- kind: refreshed.kind,
2357
- ...refreshed.reason === void 0 ? {} : { reason: refreshed.reason },
2358
- batchDigest: refreshed.batchDigest,
2359
- admitted: refreshed.admitted,
2360
- unchanged: refreshed.unchanged,
2361
- remaining: refreshed.remaining,
2362
- refusals: refreshed.refusals,
2363
- stagingSkips: refreshed.stagingSkips,
2364
- writesStarted: refreshed.writesStarted,
2365
- ...refreshed.retirement === void 0 ? {} : { retirement: refreshed.retirement },
2366
- ...refreshed.lifecycleBusy === void 0 ? {} : { lifecycleBusy: refreshed.lifecycleBusy },
2367
- staleReceipts: moves === void 0 ? null : moves ? receipts.filter((receipt) => receipt.corpusDigest === refreshed.previousCorpusDigest) : [],
2368
- ...refreshed.previousCorpusDigest === void 0 ? {} : { previousCorpusDigest: refreshed.previousCorpusDigest },
2369
- ...refreshed.nextCorpusDigest === void 0 ? {} : { nextCorpusDigest: refreshed.nextCorpusDigest }
2370
- };
2371
- }
2372
- const probe = await resolveProbeBinding(session, opts, product);
2373
- const result = await stageAdversarialProposal({
2374
- agentQaRoot: opts.agentQaRoot,
2375
- committedRoot,
2376
- product,
2377
- ...opts.mutators === void 0 ? {} : { mutators: opts.mutators },
2378
- // Only a NON-default cap is pushed down: the composition owns the recorded default, and
2379
- // handing it back an unattributed value would have it record an operator choice nobody made.
2380
- ...cap.source === "default" ? {} : { maxSeeds: cap.value, maxSeedsSource: cap.source },
2381
- ...probeCap.source === "default" ? {} : { maxProbeCalls: probeCap.value, maxProbeCallsSource: probeCap.source },
2382
- ...probe.noProbeModel === void 0 ? {} : { noProbeModel: probe.noProbeModel },
2383
- // A DRY RUN authors nothing. A preview that spends money is not a preview, and there is no
2384
- // way to un-spend it once the batch it previewed is discarded.
2385
- ...probe.author === void 0 || opts.dryRun === true ? {} : { probeAuthor: probe.author, authoringModel: probe.authoringModel },
2386
- ...opts.dryRun === void 0 ? {} : { dryRun: opts.dryRun }
2387
- });
2388
- return {
2389
- ...base,
2390
- kind: result.kind,
2391
- ...result.batchDigest === void 0 ? {} : { batchDigest: result.batchDigest },
2392
- staged: result.staged,
2393
- refusals: result.refusals,
2394
- settledRefusals: result.settledRefusals,
2395
- stagingSkips: result.stagingSkips,
2396
- skips: result.skips,
2397
- seedsConsidered: result.seedsConsidered,
2398
- seedsDeferred: result.seedsDeferred,
2399
- eligibleSeeds: result.eligibleSeeds,
2400
- cap: result.cap,
2401
- tableVersion: result.tableVersion,
2402
- tableHash: result.tableHash,
2403
- previousCorpusDigest: digest2.corpusDigest,
2404
- // Staging commits nothing, so nothing it does can stale a standing receipt.
2405
- staleReceipts: [],
2406
- ...result.noProbeModel === void 0 ? {} : { noProbeModel: result.noProbeModel },
2407
- // The raw id stays on the binding, where derivation and the committed stamp need it. What
2408
- // leaves on the result is the family and a digest -- see AdversarialVerbResult.
2409
- ...result.authoringModel === void 0 ? {} : { authoringModel: authoringModelDisclosureOf(result.authoringModel) },
2410
- probeCap: result.probeCap,
2411
- spend: result.spend,
2412
- probeSkips: result.probeSkips,
2413
- probeRefusals: result.probeRefusals
2414
- };
2415
- }
2416
- async function resolveProbeBinding(session, opts, product) {
2417
- const capability = opts.probeCapability ?? await (await Promise.resolve().then(() => (init_adversarial_probe_preflight(), adversarial_probe_preflight_exports))).resolveAdversarialProbeCapability(session);
2418
- if (capability.kind !== "probe-author") {
2419
- return { noProbeModel: { kind: "no-probe-model", reason: capability.reason } };
2420
- }
2421
- const bind = opts.bindProbeAuthor ?? await defaultProbeAuthorBinder(session);
2422
- const binding = {
2423
- onCallAttempted: (attempt) => {
2424
- const state = readProbeAttempt({
2425
- agentQaRoot: opts.agentQaRoot,
2426
- key: {
2427
- product,
2428
- seedCassetteId: attempt.seedCassetteId,
2429
- kind: attempt.kind,
2430
- promptVersion: attempt.promptVersion
2431
- }
2432
- });
2433
- if (state.kind === "unattempted") {
2434
- throw new Error("This probe call holds no durable reservation. Nothing reaches a provider before the spend is recorded.");
2435
- }
2436
- }
2437
- };
2438
- return { author: bind(capability.author, binding), authoringModel: capability.author.authoringModel };
2439
- }
2440
- async function defaultProbeAuthorBinder(session) {
2441
- const { createAdversarialProbeAuthor: createAdversarialProbeAuthor2 } = await Promise.resolve().then(() => (init_adversarial_probe_author(), adversarial_probe_author_exports));
2442
- return (author, binding) => createAdversarialProbeAuthor2({
2443
- author,
2444
- executionContext: session.executionContext,
2445
- mask: session.mask,
2446
- onCallAttempted: binding.onCallAttempted
2447
- });
2448
- }
2449
- function listAdversarialBatches(session, opts) {
2450
- void session;
2451
- return listStagedAdversarial(opts).map((batch) => ({
2452
- dir: batch.dir,
2453
- batchDigest: batch.manifest.batchDigest,
2454
- product: batch.manifest.product,
2455
- tableVersion: batch.manifest.tableVersion,
2456
- tableHash: batch.manifest.tableHash,
2457
- stagedAt: batch.manifest.stagedAt,
2458
- cases: batch.manifest.cases.map((row) => ({
2459
- cassetteId: row.cassetteId,
2460
- seedCassetteId: row.seedCassetteId,
2461
- mutator: row.mutator,
2462
- kind: row.kind,
2463
- tableEntryId: row.tableEntryId
2464
- }))
2465
- }));
2466
- }
2467
- function feedbackSeedHostOptions(session) {
2468
- return {
2469
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
2470
- knowledgeRoot: resolveKnowledgeRoot2(session.config.knowledge, session.configDir),
2471
- readStandingGateReceipts: LOCAL_CURATION_CAPABILITIES3.readStandingGateReceipts
2472
- };
2473
- }
2474
- async function runReplayGateVerb(session, candidateId, opts = {}) {
2475
- const k = opts.k ?? session.evalConfig.k ?? GATE1_DEFAULT_K;
2476
- if (!Number.isSafeInteger(k) || k < 3 || k % 2 === 0) {
2477
- throw new TypeError("--k must be an odd integer of at least 3");
2478
- }
2479
- const kSource = opts.k !== void 0 ? "flag" : session.evalConfig.k !== void 0 ? "config" : "default";
2480
- const maxScreens = opts.maxScreens ?? session.evalConfig.gate1?.maxScreens ?? GATE1_DEFAULT_MAX_SCREENS;
2481
- const maxScreensSource = opts.maxScreens !== void 0 ? "flag" : session.evalConfig.gate1?.maxScreens !== void 0 ? "config" : "default";
2482
- if (!Number.isSafeInteger(maxScreens) || maxScreens < 1) {
2483
- throw new TypeError("--max-screens must be a positive integer");
2484
- }
2485
- const identity = await resolveGateProfileIdentity(session, opts.plannerOverrides);
2486
- const profile = await resolveReplayProfileDeclaration(session);
2487
- let execution;
2488
- let mask = session.mask;
2489
- const names = selectedEvalJudgeNames(session.evalConfig);
2490
- const prepare = () => execution ??= resolveEvalSession({
2491
- configPath: session.configPath,
2492
- flags: { judge: names }
2493
- }).then(async (full) => {
2494
- if ((await resolveGateProfileIdentity(full, opts.plannerOverrides)).profileKey !== identity.profileKey) {
2495
- throw new EvalJudgeDeclarationError("Gate profile identity changed before execution; re-run eval replay-gate.");
2496
- }
2497
- mask = full.mask;
2498
- return full;
2499
- });
2500
- const declarations = await resolveEvalJudgeDeclarations(session);
2501
- const lazyJudges = declarations.map(({ name, profile: profile2 }) => ({
2502
- profile: profile2,
2503
- modelOverrides: opts.judgeOverrides,
2504
- resolveAuth: async () => {
2505
- const full = await prepare();
2506
- return (await resolveNamedProfile(name, full.configPath)).auth;
2507
- }
2508
- }));
2509
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2510
- const floor = listNoiseFloorArtefacts(committedRoot).artefacts.find(
2511
- (entry) => entry.path === join9(committedRoot, committedNoiseFloorPath(identity.profileKey)) && entry.artefact.profileKey === identity.profileKey
2512
- )?.artefact;
2513
- const { createReplayGateHost: createReplayGateHost2 } = await Promise.resolve().then(() => (init_replay_gate_host(), replay_gate_host_exports));
2514
- const result = await createReplayGateHost2({
2515
- ...opts.resolveTrustPriors === void 0 ? {} : { resolveTrustPriors: opts.resolveTrustPriors },
2516
- // 422 D-14: the lease the caller lent this verb, re-presented to the bridge as a resolver
2517
- // over the value this call already holds. The bridge calls it once, at its own claim.
2518
- ...opts.claimAuthority === void 0 ? {} : { resolveClaimAuthority: () => opts.claimAuthority },
2519
- configPath: session.configPath,
2520
- agentQaRoot: opts.agentQaRoot ?? resolve6(session.configDir, ".agent-qa"),
2521
- record: opts.dryRun !== true,
2522
- session: {
2523
- ...session,
2524
- mask: (text) => mask(text)
2525
- },
2526
- composition: {
2527
- identity,
2528
- baselineWarmup: {
2529
- configPath: session.configPath,
2530
- ...names.length > 1 ? { judges: names } : { judge: names[0] }
2531
- },
2532
- k,
2533
- kSource,
2534
- maxScreens,
2535
- planner: {
2536
- profile,
2537
- modelOverrides: opts.plannerOverrides,
2538
- resolveAuth: async () => (await resolveReplayPlanner(await prepare())).auth
2539
- },
2540
- ...lazyJudges.length > 1 ? { judgeMembers: lazyJudges } : { judge: lazyJudges[0] },
2541
- judgeOverrideConfigured: names.length <= 1 && session.evalConfig.judgeAllowSameFamily === true,
2542
- baselineSeed: session.evalConfig.seed !== void 0 ? { seed: session.evalConfig.seed, source: "config" } : { floorSeeds: floor?.seeds ?? [] },
2543
- clock: opts.clock,
2544
- onProgress: opts.onProgress
2545
- }
2546
- }).run(candidateId);
2547
- const capNotice = `cap: maxScreens=${maxScreens} (${maxScreensSource})`;
2548
- return {
2549
- ...result,
2550
- maxScreens,
2551
- maxScreensSource,
2552
- k,
2553
- kSource,
2554
- notices: [...result.notices, ...names.length > 1 && session.evalConfig.judgeAllowSameFamily === true ? ["eval.judgeAllowSameFamily is ignored for a composite judge panel; every member must be disjoint."] : [], capNotice],
2555
- ...result.kind === "receipt" ? { lines: [...result.lines, capNotice] } : {}
2556
- };
2557
- }
2558
- async function runFloorVerb(session, opts = {}) {
2559
- const repeats = opts.repeats ?? FLOOR_DEFAULT_REPEATS;
2560
- const seeds = [...new Set(opts.seeds ?? FLOOR_DEFAULT_SEEDS)];
2561
- const bootstrapSeed = opts.bootstrapSeed ?? FLOOR_DEFAULT_BOOTSTRAP_SEED;
2562
- if (!Number.isSafeInteger(repeats) || repeats < FLOOR_PROTOCOL_MINIMUM.repeats || seeds.length < FLOOR_PROTOCOL_MINIMUM.seeds || !seeds.every(Number.isSafeInteger)) {
2563
- throw new FloorProtocolError();
2564
- }
2565
- if (!Number.isSafeInteger(bootstrapSeed)) throw new TypeError("--bootstrap-seed must be a safe integer");
2566
- let classifierRepeatability;
2567
- if (opts.classifierRepeatability !== void 0) {
2568
- const path = resolve6(session.configDir, opts.classifierRepeatability);
2569
- let value;
2570
- try {
2571
- value = JSON.parse(await readFile2(path, "utf8"));
2572
- } catch (error) {
2573
- throw new TypeError(`classifier repeatability unreadable at ${path}: ${error.message}`);
2574
- }
2575
- const guarded = guardClassifierRepeatability(value);
2576
- if (guarded.kind === "rejected") {
2577
- throw new TypeError(`classifier repeatability rejected at ${path}: ${guarded.reason}`);
2578
- }
2579
- classifierRepeatability = guarded.repeatability;
2580
- }
2581
- const planner = opts.fromMeasured === void 0 ? await resolveReplayPlanner(session) : { profile: await resolveReplayProfileDeclaration(session) };
2582
- const judgeResolution = opts.fromMeasured === void 0 ? await resolveEvalJudge(session) : {};
2583
- const judge = withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides);
2584
- const { plannerModelId, matcherVersion, judgeModelId, judgeVersion } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
2585
- if (opts.fromMeasured === void 0 && replayJudgeRequest(judge)?.judgeModelId !== judgeModelId) {
2586
- throw new EvalJudgeDeclarationError("Floor judge identity changed before execution; re-run eval floor.");
2587
- }
2588
- const profileKeyInputs = {
2589
- plannerModelId,
2590
- matcherVersion,
2591
- ...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
2592
- };
2593
- const costLines = [];
2594
- const result = await runNoiseFloorMeasurement({
2595
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
2596
- cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
2597
- agentQaRoot: resolve6(session.configDir, ".agent-qa"),
2598
- ...planner,
2599
- ...opts.fromMeasured === void 0 ? {} : { fromMeasured: resolve6(session.configDir, opts.fromMeasured) },
2600
- modelOverrides: opts.plannerOverrides,
2601
- judge,
2602
- mask: session.mask,
2603
- profileKeyInputs,
2604
- seeds,
2605
- repeats,
2606
- bootstrapSeed,
2607
- classifierRepeatability,
2608
- explanation: opts.explanation,
2609
- commit: opts.commit === true,
2610
- onProgress: (line) => {
2611
- if (/^floor r\d+ seed=/.test(line)) costLines.push(line);
2612
- process.stderr.write(line + "\n");
2613
- }
2614
- });
2615
- const notices = [...result.notices];
2616
- if (judgeResolution.notice !== void 0) notices.unshift(judgeResolution.notice);
2617
- if (result.committed?.kind === "refused") notices.push(renderFloorRefusal(result.committed.refusal));
2618
- return {
2619
- ...result,
2620
- notices,
2621
- repeats,
2622
- seeds,
2623
- bootstrapSeed,
2624
- costLines,
2625
- recordedDefaults: opts.repeats === void 0 && opts.seeds === void 0 && opts.bootstrapSeed === void 0,
2626
- corpusDigest: result.artefact.corpusDigest
2627
- };
2628
- }
2629
- async function runGateVerb(session, opts = {}) {
2630
- if (opts.stagedDir !== void 0 && opts.report === void 0) {
2631
- throw new TypeError("--staged-dir requires --report <batchKey>");
2632
- }
2633
- const declaredSeed = opts.seed ?? session.evalConfig.seed;
2634
- if (declaredSeed !== void 0 && !Number.isSafeInteger(declaredSeed)) {
2635
- throw new TypeError("--seed must be a safe integer");
2636
- }
2637
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2638
- const agentQaRoot = resolve6(session.configDir, ".agent-qa");
2639
- const { plannerModelId, judgeModelId, judgeVersion, fingerprint, matcherVersion, profileKey } = await resolveGateProfileIdentity(session, opts.plannerOverrides);
2640
- const profileKeyInputs = {
2641
- plannerModelId,
2642
- matcherVersion,
2643
- ...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion }
2644
- };
2645
- const floorPath = join9(committedRoot, committedNoiseFloorPath(profileKey));
2646
- const floorSeed = declaredSeed === void 0 ? listNoiseFloorArtefacts(committedRoot).artefacts.find((entry) => entry.path === floorPath && entry.profileKey === profileKey)?.artefact.seeds[0] : void 0;
2647
- const seed = declaredSeed ?? floorSeed ?? FLOOR_DEFAULT_SEEDS[0];
2648
- const seedSource = opts.seed !== void 0 ? "flag" : session.evalConfig.seed !== void 0 ? "config" : floorSeed !== void 0 ? "floor" : "default";
2649
- const probedDirs = [];
2650
- const notices = [];
2651
- let mode;
2652
- if (opts.report !== void 0) {
2653
- const choices = opts.stagedDir !== void 0 ? [{ dir: resolve6(session.configDir, opts.stagedDir), label: "--staged-dir" }] : [
2654
- { dir: join9(agentQaRoot, "eval", "gate", slugBranch(profileKey), "reports"), label: "gate root" },
2655
- { dir: join9(agentQaRoot, DEFAULT_EVAL_REPORTS_SUBDIR), label: "default staging root" }
2656
- ];
2657
- const files = [];
2658
- let stagedOutputDir;
2659
- for (const choice of choices) {
2660
- probedDirs.push(choice.dir);
2661
- const file = join9(choice.dir, `${slugBranch(opts.report)}.json`);
2662
- files.push(file);
2663
- try {
2664
- lstatSync2(file);
2665
- } catch (error) {
2666
- if (error.code === "ENOENT") continue;
2667
- }
2668
- stagedOutputDir = choice.dir;
2669
- notices.push(`gate --report: reading ${file} (${choice.label})`);
2670
- break;
2671
- }
2672
- if (stagedOutputDir === void 0) {
2673
- return {
2674
- kind: "refused",
2675
- refusal: {
2676
- path: files[files.length - 1],
2677
- reason: `staged report absent; probed ${files.join(", ")}`
2678
- },
2679
- notices,
2680
- seed,
2681
- seedSource,
2682
- probedDirs
2683
- };
2684
- }
2685
- mode = {
2686
- kind: "report",
2687
- batchKey: opts.report,
2688
- stagedOutputDir,
2689
- profile: await resolveReplayProfileDeclaration(session),
2690
- modelOverrides: opts.plannerOverrides
2691
- };
2692
- } else {
2693
- const planner = await resolveReplayPlanner(session);
2694
- const judgeResolution = await resolveEvalJudge(session);
2695
- if (judgeResolution.notice !== void 0) notices.push(judgeResolution.notice);
2696
- mode = {
2697
- kind: "fresh",
2698
- ...planner,
2699
- modelOverrides: opts.plannerOverrides,
2700
- judge: withJudgeOverrides(judgeResolution.judge, opts.judgeOverrides),
2701
- mask: session.mask,
2702
- seed,
2703
- cassettesRoot: resolveCassettesRoot(session, "committed").cassettesRoot,
2704
- onProgress: (line) => process.stderr.write(line + "\n")
2705
- };
2706
- }
2707
- const result = await runReleaseGate({ committedRoot, agentQaRoot, profileKeyInputs, mode });
2708
- return { ...result, notices: [...notices, ...result.notices], seed, seedSource, probedDirs };
2709
- }
2710
- function isExclusionVerdict(verdict) {
2711
- return EXCLUDED_VERDICT_CLASSES.has(verdict.class);
2712
- }
2713
- async function runSeedVerb(session, filters = {}, params = {}) {
2714
- const agentQaRoot = resolve6(session.configDir, ".agent-qa");
2715
- const runsRoot = resolve6(agentQaRoot, RUNS_SUBDIR);
2716
- let runSlugs = listRunSlugsWithHandoff(runsRoot);
2717
- if (filters.run !== void 0) {
2718
- const named = filters.run;
2719
- runSlugs = runSlugs.filter((slug) => slug === named || slug === slugBranch(named));
2720
- if (runSlugs.length === 0) {
2721
- throw new EvalNamedInputError("run", named, runsRoot, "seed");
2722
- }
2723
- }
2724
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
2725
- const silverPerTestCap = params.silverPerTestCap ?? session.evalConfig.seeding?.silverPerTestCap;
2726
- const result = {
2727
- membersExcluded: [],
2728
- admissionConflicts: [],
2729
- silverExclusionRows: [],
2730
- supersededOriginalCount: 0,
2731
- retirementCounts: { missing: 0, conflict: 0 },
2732
- runsSwept: runSlugs.length,
2733
- runsSkipped: [],
2734
- runsFiltered: 0,
2735
- runsExcluded: [],
2736
- runsAdmitted: [],
2737
- notices: [],
2738
- handoffsSeeded: 0,
2739
- seededCount: 0,
2740
- unchangedCount: 0,
2741
- refusals: [],
2742
- pairSkips: 0,
2743
- silverExclusions: 0,
2744
- pairCounts: { total: 0, sameState: 0, differingState: 0 },
2745
- contracts: {},
2746
- scaffold: {},
2747
- partitions: [],
2748
- exclusionsWritten: [],
2749
- committedRoot
2750
- };
2751
- const admitted = [];
2752
- const initialCommitted = readCommittedCorpus(committedRoot);
2753
- for (const runSlug of runSlugs) {
2754
- try {
2755
- const handoffPath = resolve6(runsRoot, runSlug, HANDOFF_FILE);
2756
- const located = relative(realpathSync2(agentQaRoot), realpathSync2(handoffPath));
2757
- if (located === ".." || located.startsWith("../") || located.startsWith("..\\") || isAbsolute2(located)) {
2758
- throw new Error("seed handoff path is outside the operator run root");
2759
- }
2760
- const handoff = loadCompletedRunHandoff(handoffPath);
2761
- if (slugBranch(handoff.runId) !== runSlug) throw new Error("seed handoff run identity does not match its directory");
2762
- const product = resolveProductPartition(handoff.scope);
2763
- if (filters.product !== void 0 && product !== filters.product) {
2764
- result.runsFiltered++;
2765
- continue;
2766
- }
2767
- const verified = loadLabelEvidence({ snapshot: initialCommitted.snapshot, agentQaRoot, incoming: handoff });
2768
- if (!verified.handoffs.some((value) => value.runId === handoff.runId)) {
2769
- throw new Error("seed incoming handoff evidence could not be positively verified before admission");
2770
- }
2771
- const analysis = readRunAnalysisArtifact(runsRoot, runSlug, handoff.runId, result.notices);
2772
- const row = {
2773
- runSlug,
2774
- product,
2775
- verdicts: resolveSeedVerdictRows(analysis.artifact, handoff)
2776
- };
2777
- const excluding = row.verdicts.filter(isExclusionVerdict);
2778
- const memberVerdicts = resolveSeedMemberVerdicts(analysis.artifact, handoff);
2779
- const eligible = handoff.tests.filter((member) => admitSeedMember(member, memberVerdicts.get(member.testIndex)));
2780
- const hasEligibleRerun = eligible.some((member) => member.isRerun);
2781
- if (excluding.length > 0 && !hasEligibleRerun) {
2782
- result.runsExcluded.push(row);
2783
- const record = projectRunExclusion({
2784
- runId: handoff.runId,
2785
- runSlug,
2786
- product,
2787
- verdicts: excluding,
2788
- analysisDigest: analysis.text === void 0 ? sha256hex3(canonicalJson5(row.verdicts)) : sha256hex3(analysis.text)
2789
- });
2790
- const outcome = writeRunExclusion({
2791
- record,
2792
- committedRoot
2793
- });
2794
- result.exclusionsWritten.push({
2795
- runSlug,
2796
- product,
2797
- path: committedExclusionPath(product, runSlug),
2798
- outcome: outcome.kind
2799
- });
2800
- continue;
2801
- }
2802
- const exclusionPath = committedExclusionPath(product, runSlug);
2803
- if (existsSync4(join9(committedRoot, exclusionPath))) {
2804
- result.admissionConflicts.push({ runSlug, product, path: exclusionPath, reason: "existing-run-exclusion" });
2805
- result.notices.push(`seed ${runSlug}: existing run exclusion requires the reviewed exclusion path before admission`);
2806
- continue;
2807
- }
2808
- const testIndexes = new Set(eligible.map((member) => member.testIndex));
2809
- for (const member of handoff.tests) {
2810
- const verdict = memberVerdicts.get(member.testIndex);
2811
- if (!testIndexes.has(member.testIndex) && verdict) {
2812
- result.membersExcluded.push({ runSlug, product, testIndex: member.testIndex, verdict });
2813
- }
2814
- }
2815
- result.runsAdmitted.push(row);
2816
- admitted.push({ runSlug, handoff, product, testIndexes });
2817
- } catch (err) {
2818
- result.runsSkipped.push({
2819
- runSlug,
2820
- reason: err instanceof Error ? err.message : String(err)
2821
- });
2822
- continue;
2823
- }
2824
- }
2825
- const manifests = ensurePartitionManifests(session, admitted, committedRoot, result, params);
2826
- for (const { runSlug, handoff, product, testIndexes } of admitted) {
2827
- let report;
2828
- try {
2829
- const committed = readCommittedCorpus(committedRoot);
2830
- const evidence = loadLabelEvidence({ snapshot: committed.snapshot, agentQaRoot, incoming: handoff });
2831
- for (const refusal of evidence.refusals) result.notices.push(`SOFT evidence ${refusal.runId}: ${refusal.reason}`);
2832
- if (!evidence.handoffs.some((value) => value.runId === handoff.runId)) {
2833
- throw new Error("seed incoming handoff evidence could not be positively verified");
2834
- }
2835
- const mask = committedCorpusMask(session);
2836
- const admittedHandoff = maskCorpusText(
2837
- { ...handoff, tests: handoff.tests.filter((member) => testIndexes.has(member.testIndex)) },
2838
- mask
2839
- );
2840
- const populationHandoffs = evidence.handoffs.map((source) => {
2841
- if (source.runId === handoff.runId) return admittedHandoff;
2842
- const sourceAnalysis = readRunAnalysisArtifact(runsRoot, slugBranch(source.runId), source.runId, result.notices);
2843
- const verdicts = resolveSeedMemberVerdicts(sourceAnalysis.artifact, source);
2844
- return maskCorpusText(
2845
- { ...source, tests: source.tests.filter((member) => admitSeedMember(member, verdicts.get(member.testIndex))) },
2846
- mask
2847
- );
2848
- });
2849
- const manifest = manifests.get(product);
2850
- if (manifest === void 0) {
2851
- throw new Error(`no committed partition manifest for product ${product}`);
2852
- }
2853
- const extraction = extractRunCassettes({
2854
- handoff: admittedHandoff,
2855
- artifacts: new FsRunArtifactSource({ root: agentQaRoot, runId: handoff.runId })
2856
- });
2857
- const targetEnvironment = resolveTargetEnvironmentForProduct(session.config, product);
2858
- const plan = buildSeedingPlan({
2859
- handoff: admittedHandoff,
2860
- populationContext: { snapshot: committed.snapshot, handoffs: populationHandoffs, marks: evidence.marks },
2861
- cassettes: extraction.cassettes,
2862
- // 414 D-07: the split comes from the product's COMMITTED manifest params — the cli adds
2863
- // no selection or relabel logic of its own.
2864
- partition: partitionParamsOf(manifest),
2865
- capturedAt: new Date(handoff.endedAt).toISOString(),
2866
- ...targetEnvironment === void 0 ? {} : { targetEnvironment },
2867
- params: {
2868
- ...silverPerTestCap !== void 0 ? { silverPerTestCap } : {}
2869
- },
2870
- mask
2871
- });
2872
- report = seedCommittedCorpus({
2873
- plan,
2874
- cassettes: extraction.cassettes,
2875
- committedRoot,
2876
- mask
2877
- });
2878
- for (const skip of report.softEvidence?.skips ?? []) {
2879
- result.notices.push(`SOFT evidence ${skip.runId}/${skip.testIndex}/${skip.stepIndex}/${skip.attempt}: ${skip.reason}`);
2880
- }
2881
- } catch (err) {
2882
- result.runsSkipped.push({
2883
- runSlug,
2884
- reason: err instanceof Error ? err.message : String(err)
2885
- });
2886
- continue;
2887
- }
2888
- result.handoffsSeeded++;
2889
- result.seededCount += report.seededCount;
2890
- result.unchangedCount += report.unchangedCount;
2891
- result.supersededOriginalCount += report.supersededOriginalCount;
2892
- result.retirementCounts.missing += report.retirementCounts.missing;
2893
- result.retirementCounts.conflict += report.retirementCounts.conflict;
2894
- result.silverExclusionRows.push(...report.silverExclusions.map((row) => ({ ...row, runSlug, product })));
2895
- result.refusals.push(...report.refusals);
2896
- result.pairSkips += report.pairSkips.length;
2897
- result.silverExclusions += report.silverExclusions.length;
2898
- result.pairCounts.total += report.pairCounts.total;
2899
- result.pairCounts.sameState += report.pairCounts.sameState;
2900
- result.pairCounts.differingState += report.pairCounts.differingState;
2901
- for (const contract of report.contracts) {
2902
- bumpCounter(result.contracts, contract.outcome);
2903
- }
2904
- bumpCounter(result.scaffold, report.scaffold.gitignore);
2905
- bumpCounter(result.scaffold, report.scaffold.marker);
2906
- }
2907
- return result;
2908
- }
2909
- function ensurePartitionManifests(session, admitted, committedRoot, result, params) {
2910
- const byProduct = /* @__PURE__ */ new Map();
2911
- const read = readPartitionManifests(committedRoot);
2912
- for (const manifest of read.manifests) byProduct.set(manifest.product, manifest);
2913
- result.notices.push(...read.notices);
2914
- for (const refusal of read.refusals) {
2915
- result.notices.push(`partition manifest refused at ${refusal.path}: ${refusal.reason}`);
2916
- }
2917
- const products = [...new Set(admitted.map((entry) => entry.product))].sort();
2918
- const { heldOutFraction, privateFraction } = params;
2919
- const unpartitioned = products.filter((product) => !byProduct.has(product));
2920
- if (unpartitioned.length > 0 && (heldOutFraction === void 0 || privateFraction === void 0)) {
2921
- throw new EvalPartitionFlagsRequiredError(unpartitioned);
2922
- }
2923
- for (const product of products) {
2924
- const committed = byProduct.get(product);
2925
- const relPath = committedPartitionManifestPath(product);
2926
- if (committed === void 0) {
2927
- const saltHex = randomBytes(32).toString("hex");
2928
- const manifest = projectPartitionManifest({
2929
- product,
2930
- saltHex,
2931
- heldOutFraction,
2932
- privateFraction
2933
- });
2934
- const outcome2 = writePartitionManifest({
2935
- manifest,
2936
- committedRoot
2937
- });
2938
- byProduct.set(product, manifest);
2939
- result.partitions.push(partitionRow(manifest, relPath, requireManifestWritten(outcome2, product, relPath)));
2940
- continue;
2941
- }
2942
- if (heldOutFraction === void 0 || privateFraction === void 0) {
2943
- result.partitions.push(partitionRow(committed, relPath, "unchanged"));
2944
- continue;
2945
- }
2946
- const reprojected = projectPartitionManifest({
2947
- product,
2948
- saltHex: committed.salt,
2949
- heldOutFraction,
2950
- privateFraction
2951
- });
2952
- const outcome = writePartitionManifest({
2953
- manifest: reprojected,
2954
- committedRoot
2955
- });
2956
- result.partitions.push(
2957
- partitionRow(committed, relPath, requireManifestWritten(outcome, product, relPath))
2958
- );
2959
- }
2960
- return byProduct;
2961
- }
2962
- function partitionRow(manifest, path, outcome) {
2963
- return {
2964
- product: manifest.product,
2965
- path,
2966
- outcome,
2967
- saltDigest: sha256hex3(manifest.salt),
2968
- heldOutFraction: manifest.heldOutFraction,
2969
- privateFraction: manifest.privateFraction,
2970
- partitionVersion: manifest.partitionVersion
2971
- };
2972
- }
2973
- function resolveTargetEnvironmentForProduct(config, product) {
2974
- const declared = /* @__PURE__ */ new Set();
2975
- for (const summary of listTargets(config)) {
2976
- const target = resolveTarget(config, summary.name);
2977
- if (target.product !== product) continue;
2978
- if (target.environment === void 0) continue;
2979
- declared.add(target.environment);
2980
- }
2981
- return declared.size === 1 ? [...declared][0] : void 0;
2982
- }
2983
- function requireManifestWritten(outcome, product, path) {
2984
- if (outcome.kind === "write-conflict") {
2985
- throw new EvalPartitionManifestConflictError(product, path);
2986
- }
2987
- return outcome.kind;
2988
- }
2989
- function readRunAnalysisArtifact(runsRoot, runSlug, runId, notices) {
2990
- const path = join9(runsRoot, runSlug, ANALYSIS_FILE);
2991
- if (!existsSync4(path)) return {};
2992
- try {
2993
- const text = readFileSync5(path, "utf8");
2994
- const admission = admitAnalysisArtifactForRun(
2995
- guardRunAnalysisArtifact(JSON.parse(text)),
2996
- runId
2997
- );
2998
- if (admission.kind === "admitted") return { artifact: admission.artifact, text };
2999
- notices.push(
3000
- `${ANALYSIS_FILE} for ${runSlug} records run ${admission.recordedRunId}, not ${admission.resolvingRunId} \u2014 deterministic floor used`
3001
- );
3002
- return {};
3003
- } catch (err) {
3004
- notices.push(
3005
- `${ANALYSIS_FILE} unreadable for ${runSlug}: ${err instanceof Error ? err.message : String(err)} \u2014 deterministic floor used`
3006
- );
3007
- return {};
3008
- }
3009
- }
3010
- function resolveSeedVerdictRows(artifact, handoff) {
3011
- return [...resolveSeedMemberVerdicts(artifact, handoff).entries()].sort(([first], [second]) => first - second).map(([, verdict]) => verdict);
3012
- }
3013
- function resolveSeedMemberVerdicts(artifact, handoff) {
3014
- const byPerTest = /* @__PURE__ */ new Map();
3015
- for (const member of handoff.tests) {
3016
- if (member.rootCause === void 0) continue;
3017
- byPerTest.set(member.testIndex, {
3018
- class: member.rootCause.class,
3019
- producer: member.rootCause.producer
3020
- });
3021
- }
3022
- for (const verdict of resolveRunVerdicts(artifact, handoff)) {
3023
- byPerTest.set(verdict.perTest, { class: verdict.class, producer: verdict.producer });
3024
- }
3025
- return byPerTest;
3026
- }
3027
- async function runReportVerb(session, filters = {}, opts = {}) {
3028
- if (opts.fromCommitted !== void 0) {
3029
- if (opts.baseline !== true) throw new TypeError("--from-committed requires --baseline");
3030
- if (opts.seedPolicy === void 0) {
3031
- throw new TypeError("--from-committed requires --seed-policy <none|declared>");
3032
- }
3033
- if (opts.commit === true) throw new TypeError("--from-committed cannot be combined with --commit");
3034
- if (!/^[0-9a-f]{64}$/.test(opts.fromCommitted)) {
3035
- throw new TypeError("--from-committed requires a batchKey of 64 lowercase hexadecimal characters");
3036
- }
3037
- if (!SEED_POLICIES.includes(opts.seedPolicy)) {
3038
- throw new TypeError('--seed-policy must be "none" or "declared"');
3039
- }
3040
- const committedRoot2 = resolve6(session.configDir, "agent-qa-eval");
3041
- const reportList = listCommittedReports(committedRoot2);
3042
- const requestedFile = join9(committedRoot2, "reports", opts.fromCommitted, "report.json");
3043
- const entry = reportList.reports.find((candidate) => candidate.batchKey === opts.fromCommitted);
3044
- if (entry === void 0) {
3045
- const refusal = reportList.refusals.find((candidate) => candidate.path === requestedFile);
3046
- throw new EvalNamedInputError(
3047
- "batchKey",
3048
- opts.fromCommitted,
3049
- refusal?.path ?? join9(committedRoot2, "reports"),
3050
- "designate",
3051
- refusal?.reason
3052
- );
3053
- }
3054
- if (entry.path !== requestedFile || entry.report.batchKey !== opts.fromCommitted) {
3055
- throw new EvalNamedInputError(
3056
- "batchKey",
3057
- opts.fromCommitted,
3058
- entry.path,
3059
- "designate",
3060
- `report batchKey "${entry.report.batchKey}" does not match the requested directory`
3061
- );
3062
- }
3063
- const notices2 = reportList.refusals.map(
3064
- (refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`
3065
- );
3066
- const designation = await designateReportBaseline(
3067
- session,
3068
- entry.report,
3069
- opts.seedPolicy,
3070
- opts.plannerOverrides,
3071
- notices2
3072
- );
3073
- return {
3074
- batchKey: entry.report.batchKey,
3075
- entries: 0,
3076
- report: entry.report,
3077
- committedFile: entry.path,
3078
- judgeIdentity: entry.report.judgeIdentity === "mixed" ? "mixed" : entry.report.judge === void 0 ? "absent" : "present",
3079
- source: "committed",
3080
- ...designation,
3081
- notices: notices2
3082
- };
3083
- }
3084
- if (opts.baseline === true && opts.commit !== true) {
3085
- throw new Error(
3086
- "--baseline requires --commit: the pointer designates a COMMITTED report (413 D-19)"
3087
- );
3088
- }
3089
- const agentQaRoot = resolve6(session.configDir, ".agent-qa");
3090
- const replaysRoot = join9(agentQaRoot, DEFAULT_EVAL_REPLAYS_SUBDIR);
3091
- const source = opts.source ?? "staged";
3092
- const sourceRoot = resolveCassettesRoot(session, source);
3093
- const cassettesRoot = sourceRoot.cassettesRoot;
3094
- const notices = [];
3095
- if (sourceRoot.notice !== void 0) notices.push(sourceRoot.notice);
3096
- const labelCorpus = readActiveCommittedCorpus(resolve6(session.configDir, "agent-qa-eval"), agentQaRoot);
3097
- const committedCorpus = source === "committed" ? labelCorpus : void 0;
3098
- notices.push(...labelCorpus.refusals.map((refusal) => `committed read refused ${refusal.path}: ${refusal.reason}`));
3099
- const {
3100
- entries: assembled,
3101
- judgeBlocks,
3102
- partitionsById,
3103
- missingBundleCount,
3104
- notices: assemblyNotices
3105
- } = assembleScoredBatch({
3106
- replaysRoot,
3107
- cassettesRoot,
3108
- source,
3109
- labelRecords: labelCorpus.snapshot.labelsById,
3110
- excludedLabelIds: labelCorpus.snapshot.excludedLabelIds,
3111
- ...committedCorpus === void 0 ? {} : {
3112
- productByCassetteId: new Map(committedCorpus.snapshot.entries.map((entry) => [entry.cassetteId, entry.product])),
3113
- eligibleCassetteIds: new Set(committedCorpus.snapshot.entries.map((entry) => entry.cassetteId)),
3114
- corpusEntriesById: committedCorpus.corpusEntriesById
3115
- }
3116
- });
3117
- notices.push(...assemblyNotices);
3118
- if (assembled.some((entry) => entry.split === void 0)) {
3119
- notices.push("corpus split ownership unknown; entries without corpus metadata are unassigned");
3120
- }
3121
- if (missingBundleCount > 0) {
3122
- notices.push(
3123
- `${missingBundleCount} replay record(s) have no ${source} cassette bundle \u2014 label legs absent`
3124
- );
3125
- }
3126
- let entries = assembled;
3127
- if (filters.cassette !== void 0) {
3128
- const named = filters.cassette;
3129
- entries = entries.filter((entry) => entry.cassetteId === named);
3130
- if (entries.length === 0) {
3131
- throw new EvalNamedInputError("cassette", named, replaysRoot, "report");
3132
- }
3133
- }
3134
- if (filters.run !== void 0) {
3135
- const named = filters.run;
3136
- entries = entries.filter((entry) => {
3137
- const head = entry.cassetteId.split("/")[0];
3138
- return head === named || head === slugBranch(named);
3139
- });
3140
- if (entries.length === 0) {
3141
- throw new EvalNamedInputError("run", named, replaysRoot, "report");
3142
- }
3143
- }
3144
- if (filters.product !== void 0) {
3145
- const named = filters.product;
3146
- entries = entries.filter((entry) => partitionsById.get(entry.cassetteId) === named);
3147
- }
3148
- entries.sort((a, b) => a.cassetteId < b.cassetteId ? -1 : a.cassetteId > b.cassetteId ? 1 : 0);
3149
- let judgeIdentity;
3150
- let judgeForReport;
3151
- let mixedIdentity;
3152
- if (judgeBlocks.length === 0) {
3153
- judgeIdentity = "absent";
3154
- notices.push("no judge identity on any persisted score record \u2014 the report carries no judge block");
3155
- } else {
3156
- const identities = judgeBlocks.map((block) => judgeIdentitySchema.strip().parse(block));
3157
- const first = canonicalJson5(identities[0]);
3158
- if (identities.every((block) => canonicalJson5(block) === first)) {
3159
- judgeIdentity = "present";
3160
- judgeForReport = identities[0];
3161
- } else {
3162
- judgeIdentity = "mixed";
3163
- mixedIdentity = {
3164
- judgeIdentity: "mixed",
3165
- judgeModelIds: [...new Set(judgeBlocks.map((block) => block.judgeModelId))].sort()
3166
- };
3167
- notices.push(
3168
- "mixed judge identities across persisted score records \u2014 the report carries no judge block (never fabricated)"
3169
- );
3170
- }
3171
- }
3172
- const report = buildEvalReport(entries, judgeForReport, mixedIdentity, { calibration: labelCorpus.calibration });
3173
- const reportSeedPolicy = session.evalConfig.seed !== void 0 ? "declared" : "none";
3174
- let reportProfileProof;
3175
- if (report.judge?.panelSize !== void 0) {
3176
- let calibrationRead;
3177
- try {
3178
- const replayProfile = await resolveReplayProfileDeclaration(session);
3179
- const actualModels = entries.flatMap((entry) => {
3180
- if ("kind" in entry.outcome) return [];
3181
- const results = entry.outcome.granularity === "leaf" ? [entry.outcome] : entry.outcome.steps.flatMap((step) => step.kind === "decision" ? [step.result] : []);
3182
- return results.map((result) => result.replayedModel.modelId);
3183
- });
3184
- if (actualModels.length === 0 || actualModels.some((model) => model !== replayProfile.model)) {
3185
- throw new Error("report replay profile unavailable");
3186
- }
3187
- const fingerprint = computeReplayProfileFingerprintFor({
3188
- profile: replayProfile,
3189
- seedPolicy: reportSeedPolicy,
3190
- modelOverrides: opts.plannerOverrides
3191
- });
3192
- reportProfileProof = readReportProfileProof({
3193
- agentQaRoot,
3194
- replaysRoot,
3195
- report,
3196
- entries,
3197
- fingerprint,
3198
- corpusDigest: computeCorpusDigest(labelCorpus.snapshot.entries),
3199
- seed: session.evalConfig.seed
3200
- });
3201
- if (!reportProfileProof) throw new Error("report full profile proof unavailable");
3202
- calibrationRead = readJudgeCalibration({
3203
- committedRoot: resolve6(session.configDir, "agent-qa-eval"),
3204
- agentQaRoot,
3205
- identity: report.judge,
3206
- profile: {
3207
- plannerModelId: replayProfile.model,
3208
- fingerprint,
3209
- matcherVersion: report.matcherVersion,
3210
- judgeModelId: report.judge.judgeModelId,
3211
- judgeVersion: report.judge.judgeVersion
3212
- }
3213
- });
3214
- } catch {
3215
- }
3216
- copyJudgeCalibrationToReport(report, calibrationRead);
3217
- if (calibrationRead?.kind !== "loaded") notices.push("judge calibration unavailable: no matching guarded artifact");
3218
- }
3219
- const vectors = decisionCorrectnessVectors(entries, partitionsById);
3220
- const decisions = projectDecisionVectorsDocument({
3221
- batchKey: report.batchKey,
3222
- matcherVersion: report.matcherVersion,
3223
- vectors
3224
- });
3225
- if (reportProfileProof) report.runIdentity = reportProfileProof.runIdentity;
3226
- const staged = new FsReportStore({ root: agentQaRoot }).save(report, decisions, reportProfileProof?.replayGeneration);
3227
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
3228
- let committed;
3229
- if (opts.commit === true) {
3230
- committed = commitEvalReport({
3231
- report,
3232
- decisions,
3233
- markdown: renderEvalReportMarkdown(report),
3234
- committedRoot
3235
- });
3236
- }
3237
- let baseline;
3238
- let profileKey;
3239
- if (opts.baseline === true) {
3240
- if (committed?.kind !== "committed") {
3241
- notices.push(
3242
- `baseline pointer skipped: report commit outcome was ${committed?.kind ?? "absent"}`
3243
- );
3244
- } else {
3245
- const designation = await designateReportBaseline(
3246
- session,
3247
- report,
3248
- reportSeedPolicy,
3249
- opts.plannerOverrides,
3250
- notices
3251
- );
3252
- profileKey = designation.profileKey;
3253
- baseline = designation.baseline;
3254
- }
3255
- }
3256
- return {
3257
- batchKey: report.batchKey,
3258
- entries: entries.length,
3259
- report,
3260
- stagedFile: staged.file,
3261
- decisionVectorCount: decisions.vectors.length,
3262
- ...committed !== void 0 ? { committed } : {},
3263
- judgeIdentity,
3264
- source,
3265
- ...baseline !== void 0 ? { baseline } : {},
3266
- ...profileKey !== void 0 ? { profileKey } : {},
3267
- notices
3268
- };
3269
- }
3270
- async function designateReportBaseline(session, report, seedPolicy, plannerOverrides, notices) {
3271
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
3272
- const read = readActiveCommittedCorpus(committedRoot, resolve6(session.configDir, ".agent-qa"));
3273
- notices.push(...read.notices);
3274
- for (const refusal of read.refusals) {
3275
- notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
3276
- }
3277
- const pointer = projectBaselinePointer({
3278
- plannerModelId: await resolvePlannerModelId(session),
3279
- fingerprint: await resolveReplayProfileFingerprint(session, seedPolicy, plannerOverrides),
3280
- corpusDigest: computeCorpusDigest(read.snapshot.entries),
3281
- matcherVersion: report.matcherVersion,
3282
- ...report.judge === void 0 ? {} : {
3283
- judgeModelId: report.judge.judgeModelId,
3284
- ...report.judge.judgeVersion === void 0 ? {} : { judgeVersion: report.judge.judgeVersion }
3285
- },
3286
- batchKey: report.batchKey
3287
- });
3288
- const baseline = writeBaselinePointer({ report, pointer, committedRoot });
3289
- if (baseline.kind === "refused") notices.push(renderBaselineComparatorRefusal(baseline.refusal));
3290
- if (seedPolicy === "none" && (baseline.kind === "written" || baseline.kind === "unchanged")) {
3291
- notices.push(
3292
- "baseline designated under seed policy none \u2014 eval gate compares against a declared-policy pointer; re-designate with eval report --baseline --from-committed <batchKey> --seed-policy declared"
3293
- );
3294
- }
3295
- return { profileKey: pointer.profileKey, baseline };
3296
- }
3297
- async function resolvePlannerModelId(session) {
3298
- return (await resolveBaselinePlannerConfig(session)).model;
3299
- }
3300
- async function resolveGateProfileIdentity(session, modelOverrides) {
3301
- const plannerModelId = await resolvePlannerModelId(session);
3302
- const fingerprint = await resolveReplayProfileFingerprint(session, "declared", modelOverrides);
3303
- const judgeProfiles = await resolveEvalJudgeDeclarations(session);
3304
- const judgeMemberModelIds = judgeProfiles.map((member) => member.profile.model);
3305
- const judgeIdentity = judgeMemberModelIds.length === 0 ? void 0 : createPanelIdentity(judgeMemberModelIds);
3306
- const judgeModelId = judgeIdentity?.judgeModelId;
3307
- const identity = {
3308
- plannerModelId,
3309
- fingerprint,
3310
- matcherVersion: buildEvalReport([]).matcherVersion,
3311
- ...judgeModelId === void 0 ? {} : { judgeModelId, judgeVersion: JUDGE_CONFIG.judgeVersion }
3312
- };
3313
- return {
3314
- ...identity,
3315
- profileKey: composeBaselineProfileKey(identity),
3316
- ...judgeMemberModelIds.length > 1 ? { judgeMemberModelIds } : {}
3317
- };
3318
- }
3319
- async function resolveEvalJudgeDeclarations(session) {
3320
- const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
3321
- const members = [];
3322
- for (const name of selectedEvalJudgeNames(session.evalConfig)) {
3323
- const { config } = await resolveNamedConfig2(name, session.configPath, { keyOnly: true });
3324
- members.push({ name, profile: {
3325
- provider: config.provider,
3326
- model: config.model,
3327
- auth: { mode: "none" },
3328
- ...config.baseURL === void 0 ? {} : { baseURL: config.baseURL },
3329
- ...config.providerHeaders === void 0 ? {} : { headers: config.providerHeaders }
3330
- } });
3331
- }
3332
- return members;
3333
- }
3334
- async function resolveReplayProfileFingerprint(session, seedPolicy, modelOverrides) {
3335
- const profile = await resolveReplayProfileDeclaration(session);
3336
- return computeReplayProfileFingerprintFor({ profile, seedPolicy, modelOverrides });
3337
- }
3338
- async function resolveReplayProfileDeclaration(session, namedResolver) {
3339
- const config = await resolveBaselinePlannerConfig(session, namedResolver);
3340
- return {
3341
- provider: config.provider,
3342
- model: config.model,
3343
- ...config.baseURL !== void 0 ? { baseURL: config.baseURL } : {},
3344
- ...config.providerHeaders !== void 0 ? { headers: config.providerHeaders } : {},
3345
- auth: { mode: "none" }
3346
- };
3347
- }
3348
- async function resolveBaselinePlannerConfig(session, namedResolver) {
3349
- const plannerName = session.config.use?.llm;
3350
- if (!plannerName) {
3351
- throw new Error(
3352
- "use.llm is not set in the config \u2014 the baseline profile key is keyed on the replay planner, the named use.llm registry entry. Set use.llm in agent-qa.config.yaml."
3353
- );
3354
- }
3355
- const { resolveNamedConfig: resolveNamedConfig2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
3356
- const { config } = await (namedResolver ?? resolveNamedConfig2)(plannerName, session.configPath, { keyOnly: true });
3357
- return config;
3358
- }
3359
- async function runCensusVerb(session, opts = {}) {
3360
- const committedRoot = resolve6(session.configDir, "agent-qa-eval");
3361
- const notices = [];
3362
- const read = readActiveCommittedCorpus(committedRoot, resolve6(session.configDir, ".agent-qa"));
3363
- const manifestRead = readPartitionManifests(committedRoot);
3364
- const exclusionRead = readRunExclusions(committedRoot);
3365
- notices.push(...read.notices, ...manifestRead.notices);
3366
- for (const refusal of [...read.refusals, ...manifestRead.refusals, ...exclusionRead.refusals]) {
3367
- notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
3368
- }
3369
- let campaign;
3370
- let campaignManifest;
3371
- if (opts.campaign !== void 0) {
3372
- campaignManifest = resolve6(process.cwd(), opts.campaign);
3373
- const manifest = guardCampaignManifest(JSON.parse(readFileSync5(campaignManifest, "utf8")));
3374
- campaign = { legs: manifest.legs, manifestDigest: sha256hex3(canonicalJson5(manifest)) };
3375
- }
3376
- const reportList = listCommittedReports(committedRoot);
3377
- const pointerList = listBaselinePointers(committedRoot);
3378
- for (const refusal of [...reportList.refusals, ...pointerList.refusals]) {
3379
- notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
3380
- }
3381
- const reports = [];
3382
- for (const entry of reportList.reports) {
3383
- const comparator = assertBaselineComparator(entry.report);
3384
- const designatedBy = pointerList.pointers.filter((pointer) => pointer.batchKey === entry.batchKey).map((pointer) => pointer.profileKey).sort();
3385
- notices.push(
3386
- `cross-check ${entry.batchKey}: report decisions ${entry.report.headline.perDecisionCost.decisionCount} vs corpus cassettes ${read.snapshot.entries.length} \u2014 batch membership differs (batchKey only)`
3387
- );
3388
- if (!comparator.ok) notices.push(renderBaselineComparatorRefusal(comparator.refusal));
3389
- reports.push({ report: entry.report, comparator, designatedBy });
3390
- }
3391
- const floorList = listNoiseFloorArtefacts(committedRoot);
3392
- for (const refusal of floorList.refusals) {
3393
- notices.push(`committed read refused ${refusal.path}: ${refusal.reason}`);
3394
- }
3395
- const noiseFloor = floorList.artefacts.map(({ profileKey, path, artefact }) => {
3396
- const repeatability = artefact.classifierRepeatability;
3397
- return {
3398
- profileKey,
3399
- path,
3400
- ...artefact.minimumGateSet.value === void 0 ? {} : { minimumGateSet: artefact.minimumGateSet.value },
3401
- minimumGateSetDerivation: artefact.minimumGateSet.derivation,
3402
- ...repeatability === void 0 ? {} : {
3403
- classifierRepeatability: {
3404
- n: repeatability.n,
3405
- k: repeatability.k,
3406
- ...repeatability.classFlipRate === void 0 ? {} : { classFlipRate: repeatability.classFlipRate },
3407
- ...repeatability.admitExcludeFlipRate === void 0 ? {} : { admitExcludeFlipRate: repeatability.admitExcludeFlipRate }
3408
- }
3409
- }
3410
- };
3411
- });
3412
- const census = deriveCensus({
3413
- snapshot: read.snapshot,
3414
- ...read.corpusEntriesById === void 0 ? {} : { corpusEntriesById: read.corpusEntriesById },
3415
- ...read.retiredMarkedCassetteIds === void 0 ? {} : { retiredMarkedCassetteIds: read.retiredMarkedCassetteIds },
3416
- ...campaign === void 0 ? {} : { campaign },
3417
- reports,
3418
- manifests: manifestRead.manifests,
3419
- exclusions: exclusionRead.exclusions,
3420
- schemaVersionsSeen: read.schemaVersionsSeen
3421
- });
3422
- let committed;
3423
- if (opts.commit === true) {
3424
- committed = commitCensus({
3425
- census,
3426
- markdown: renderCensusMarkdown(census),
3427
- committedRoot
3428
- });
3429
- }
3430
- return {
3431
- corpusDigest: census.corpusDigest,
3432
- entryCount: census.entryCount,
3433
- productsSeen: read.productsSeen,
3434
- census,
3435
- ...campaignManifest !== void 0 ? { campaignManifest } : {},
3436
- ...committed !== void 0 ? { committed } : {},
3437
- ...noiseFloor.length === 0 ? {} : { noiseFloor },
3438
- notices
3439
- };
3440
- }
3441
- async function resolveNamedProfile(name, configPath) {
3442
- const { resolveNamedConfig: resolveNamedConfig2, resolveModelAuth: resolveModelAuth2, toModelProfileAndAuth: toModelProfileAndAuth2 } = await Promise.resolve().then(() => (init_llm_utils(), llm_utils_exports));
3443
- const { config } = await resolveNamedConfig2(name, configPath);
3444
- const auth = await resolveModelAuth2(config.name, config);
3445
- if (auth.kind === "missing") {
3446
- throw new Error(
3447
- `No credentials for LLM config "${config.name}": ${auth.message} Run \`agent-qa auth login\` to authenticate.`
3448
- );
3449
- }
3450
- return toModelProfileAndAuth2(config, auth);
3451
- }
3452
- async function loadEvalSecretStore(config, configDir) {
3453
- const secretsPath = resolve6(configDir, config.workspace.secretsFile);
3454
- let content;
3455
- try {
3456
- content = await readFile2(secretsPath, "utf-8");
3457
- } catch {
3458
- throw new Error(
3459
- `Secrets file not found or unreadable: ${secretsPath}. Create the file before running eval verbs (it may be empty, but it must exist).`
3460
- );
3461
- }
3462
- try {
3463
- const { SecretStore: SecretStore2 } = await import("@vostride/agent-qa-runner");
3464
- return SecretStore2.fromEnvContent(content);
3465
- } catch (err) {
3466
- const message = err instanceof Error ? err.message : String(err);
3467
- throw new Error(`Secrets file is invalid: ${secretsPath}: ${message}`);
3468
- }
3469
- }
3470
- function mergeEvalConfig(fromConfig, flags) {
3471
- const configured = selectedEvalJudgeNames(fromConfig ?? {});
3472
- const names = flags?.judge === void 0 ? configured : selectedEvalJudgeNames(typeof flags.judge === "string" ? { judge: flags.judge } : { judges: flags.judge });
3473
- const judgeAllowSameFamily = fromConfig?.judgeAllowSameFamily;
3474
- const k = flags?.k ?? fromConfig?.k;
3475
- const seed = flags?.seed ?? fromConfig?.seed;
3476
- const silverPerTestCap = flags?.silverCap ?? fromConfig?.seeding?.silverPerTestCap;
3477
- return {
3478
- ...names.length === 1 ? { judge: names[0] } : names.length > 1 ? { judges: names } : {},
3479
- ...judgeAllowSameFamily !== void 0 ? { judgeAllowSameFamily } : {},
3480
- ...k !== void 0 ? { k } : {},
3481
- ...seed !== void 0 ? { seed } : {},
3482
- ...fromConfig?.gate2 === void 0 ? {} : { gate2: fromConfig.gate2 },
3483
- ...fromConfig?.adversarial === void 0 ? {} : { adversarial: fromConfig.adversarial },
3484
- ...silverPerTestCap !== void 0 ? { seeding: { silverPerTestCap } } : {},
3485
- // 422-09: forwarded WHOLE, exactly as `gate2` and `adversarial` above are. The field-by-field
3486
- // rebuild this replaces silently dropped every member the schema gained after it was written.
3487
- ...fromConfig?.gate1 === void 0 ? {} : { gate1: fromConfig.gate1 }
3488
- };
3489
- }
3490
- var EvalJudgeDeclarationError, EvalNamedInputError, FLOOR_DEFAULT_REPEATS, FLOOR_DEFAULT_SEEDS, FLOOR_DEFAULT_BOOTSTRAP_SEED, LABEL_CALIBRATION_MIN_GOLD, GATE1_DEFAULT_MAX_SCREENS, GATE1_DEFAULT_K, GATE2_DEFAULT_MAX_ABLATIONS, GATE2_DEFAULT_CANARY_HORIZON_BUILDS, ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS, EXCLUDED_VERDICT_CLASSES, EvalPartitionFlagsRequiredError, EvalPartitionManifestConflictError;
3491
- var init_eval_session_composition = __esm({
3492
- "src/eval-session-composition.ts"() {
3493
- "use strict";
3494
- init_config();
3495
- init_declared_secrets();
3496
- init_targets();
3497
- init_promotion_posture();
3498
- init_eval_report_render();
3499
- init_judge_panel_preflight();
3500
- init_judge_panel_preflight();
3501
- init_adversarial_probe_preflight();
3502
- init_adversarial_probe_author();
3503
- EvalJudgeDeclarationError = class extends TypeError {
3504
- constructor(message) {
3505
- super(message);
3506
- this.name = "EvalJudgeDeclarationError";
3507
- }
3508
- };
3509
- EvalNamedInputError = class extends Error {
3510
- inputKind;
3511
- inputName;
3512
- searchRoot;
3513
- constructor(inputKind, inputName, searchRoot, verb = "extract", detail) {
3514
- super(
3515
- detail === void 0 ? `No recorded ${inputKind} named "${inputName}" under ${searchRoot} \u2014 nothing to ${verb}.` : `Cannot ${verb} ${inputKind} "${inputName}" at ${searchRoot}: ${detail}`
3516
- );
3517
- this.name = "EvalNamedInputError";
3518
- this.inputKind = inputKind;
3519
- this.inputName = inputName;
3520
- this.searchRoot = searchRoot;
3521
- }
3522
- };
3523
- FLOOR_DEFAULT_REPEATS = 3;
3524
- FLOOR_DEFAULT_SEEDS = [11, 29];
3525
- FLOOR_DEFAULT_BOOTSTRAP_SEED = 7;
3526
- LABEL_CALIBRATION_MIN_GOLD = 20;
3527
- GATE1_DEFAULT_MAX_SCREENS = 64;
3528
- GATE1_DEFAULT_K = 3;
3529
- GATE2_DEFAULT_MAX_ABLATIONS = 4;
3530
- GATE2_DEFAULT_CANARY_HORIZON_BUILDS = 5;
3531
- ATTR_DEFAULT_MAX_MISDIRECTION_CHECKS = 4;
3532
- EXCLUDED_VERDICT_CLASSES = new Set(
3533
- // 414 D-02: the local literal triplet is gone. `EXCLUSION_CLASSES` is the SAME closed
3534
- // vocabulary, and it now also decides what the committed exclusion record admits — a second
3535
- // spelling here could let the seeder drop a run the record schema would refuse to describe.
3536
- EXCLUSION_CLASSES
3537
- );
3538
- EvalPartitionFlagsRequiredError = class extends Error {
3539
- products;
3540
- constructor(products) {
3541
- super(
3542
- `partition of ${products.join(", ")} requires --held-out-fraction and --private-fraction on its first seed (414 D-07); no default exists`
3543
- );
3544
- this.name = "EvalPartitionFlagsRequiredError";
3545
- this.products = products;
3546
- }
3547
- };
3548
- EvalPartitionManifestConflictError = class extends Error {
3549
- product;
3550
- path;
3551
- constructor(product, path) {
3552
- super(
3553
- `partition manifest for ${product} at ${path}: write-conflict \u2014 the committed manifest disagrees with the supplied fractions; changing a manifest is a reviewed git edit and a named migration (414 D-05)`
3554
- );
3555
- this.name = "EvalPartitionManifestConflictError";
3556
- this.product = product;
3557
- this.path = path;
3558
- }
356
+ platform: typedEntry.platform
3559
357
  };
3560
- }
3561
- });
3562
-
3563
- // src/index.ts
3564
- init_config();
3565
- init_targets();
3566
- import { getAgentQaVersion } from "@vostride/agent-qa-core";
3567
- export * from "@vostride/agent-qa-core";
358
+ });
359
+ }
3568
360
 
3569
361
  // src/commands/cache.ts
3570
- init_config();
3571
362
  import { Command } from "commander";
3572
363
  import pc2 from "picocolors";
3573
- import { readdir, rm, stat } from "fs/promises";
3574
- import { join, resolve, basename } from "path";
364
+ import { dirname as dirname3, resolve } from "path";
365
+ import { purgeActionCache } from "@vostride/agent-qa-agent-local";
3575
366
  import { DEFAULT_AGENT_QA_CACHE_DIR } from "@vostride/agent-qa-core";
3576
- async function removeCacheEntries(cacheRoot) {
3577
- let branchDirs;
3578
- try {
3579
- branchDirs = await readdir(cacheRoot);
3580
- } catch (err) {
3581
- if (err.code === "ENOENT") return 0;
3582
- throw err;
3583
- }
3584
- let purged = 0;
3585
- for (const branch of branchDirs) {
3586
- const branchPath = join(cacheRoot, branch);
3587
- let entryStat;
3588
- try {
3589
- entryStat = await stat(branchPath);
3590
- } catch {
3591
- continue;
3592
- }
3593
- if (!entryStat.isDirectory()) {
3594
- await rm(branchPath, { force: true });
3595
- continue;
3596
- }
3597
- let files;
3598
- try {
3599
- files = await readdir(branchPath);
3600
- } catch {
3601
- continue;
3602
- }
3603
- for (const file of files) {
3604
- if (!file.endsWith(".json")) continue;
3605
- await rm(join(branchPath, file), { force: true });
3606
- purged++;
3607
- }
3608
- await rm(branchPath, { recursive: true, force: true }).catch(() => {
3609
- });
3610
- }
3611
- return purged;
3612
- }
3613
- async function countCacheEntries(cacheRoot) {
3614
- let branchDirs;
3615
- try {
3616
- branchDirs = await readdir(cacheRoot);
3617
- } catch (err) {
3618
- if (err.code === "ENOENT") return 0;
3619
- throw err;
3620
- }
3621
- let count2 = 0;
3622
- for (const branch of branchDirs) {
3623
- try {
3624
- const files = await readdir(join(cacheRoot, branch));
3625
- count2 += files.filter((f) => f.endsWith(".json")).length;
3626
- } catch {
3627
- }
3628
- }
3629
- return count2;
3630
- }
3631
- async function purgeTest(_testPath, cacheDir) {
3632
- return removeCacheEntries(resolve(cacheDir));
3633
- }
3634
367
  async function purgeAll(cacheDir) {
3635
- const resolvedDir = resolve(cacheDir);
3636
- const total = await countCacheEntries(resolvedDir);
3637
- if (total === 0) return 0;
3638
- return removeCacheEntries(resolvedDir);
368
+ return purgeActionCache(resolve(cacheDir));
3639
369
  }
3640
370
 
3641
371
  // src/curation/curator-author.ts
@@ -3654,7 +384,7 @@ import {
3654
384
  } from "@vostride/agent-qa-schema";
3655
385
 
3656
386
  // src/curation/emit-candidates.ts
3657
- import { isAbsolute, join as join2 } from "path";
387
+ import { isAbsolute, join } from "path";
3658
388
  import { z } from "zod";
3659
389
  import {
3660
390
  LOCAL_CURATION_CAPABILITIES
@@ -3965,9 +695,7 @@ var trajectoryCompositeValueSchema = z.union([
3965
695
  z.record(z.string(), procedureJsonValueSchema)
3966
696
  ]);
3967
697
  var trajectoryValueSlotSchema = z.object({
3968
- // WR-04: boolean and numeric action fields are replay semantics, not decoration. Dropping
3969
- // fill.submit, scroll.maxSwipes/retryOpposite, durations, or scale here would author a
3970
- // procedure whose behavior differs from the verified trajectory.
698
+ // Boolean and numeric action fields are replay semantics; dropping one changes the replay.
3971
699
  observed: z.union([
3972
700
  z.string(),
3973
701
  z.number(),
@@ -3975,11 +703,7 @@ var trajectoryValueSlotSchema = z.object({
3975
703
  trajectoryCompositeValueSchema
3976
704
  ]),
3977
705
  source: z.enum(["literal", "test-variable", "masked-secret"]),
3978
- // G-381-3: the live binding name behind this slot, when the evidence names it —
3979
- // a test variable matched against Plan 381-41's interpolation provenance, or a secret
3980
- // recovered from its recorded masked form. `curator-author` forces the covering
3981
- // parameter to exactly this name so consult-time fillability (D-08, which matches on
3982
- // binding NAMES) succeeds. Absent when the source is an anonymous literal.
706
+ // The binding name behind this slot; `curator-author` names the covering parameter after it.
3983
707
  sourceName: z.string().min(1).optional()
3984
708
  }).strict();
3985
709
  var trajectoryStepSchema = z.object({
@@ -4249,7 +973,7 @@ function projectCurationCandidates(handoff, mask) {
4249
973
  function bindArtifactSources(projection, runDir) {
4250
974
  return projection.artifactDescriptors.map((descriptor) => ({
4251
975
  artifactId: descriptor.ref.id,
4252
- sourcePath: join2(runDir, RUN_EVIDENCE_BLOBS_SUBDIR, descriptor.ref.sha)
976
+ sourcePath: join(runDir, RUN_EVIDENCE_BLOBS_SUBDIR, descriptor.ref.sha)
4253
977
  }));
4254
978
  }
4255
979
  function incrementReport(report, outcome) {
@@ -4449,9 +1173,9 @@ function buildPrompt(job, neighbors, accounting) {
4449
1173
  const { contextWindow, maxOutput } = accounting.getCeilings();
4450
1174
  const inputBudget = contextWindow - maxOutput;
4451
1175
  if (inputBudget <= 0) return void 0;
4452
- for (let count2 = limited.length; count2 >= 0; count2 -= 1) {
4453
- const presented = limited.slice(0, count2);
4454
- const messages = renderPrompt(job, presented, neighbors.length - count2);
1176
+ for (let count = limited.length; count >= 0; count -= 1) {
1177
+ const presented = limited.slice(0, count);
1178
+ const messages = renderPrompt(job, presented, neighbors.length - count);
4455
1179
  if (accounting.estimate(messages) <= inputBudget) {
4456
1180
  return {
4457
1181
  messages,
@@ -5258,38 +1982,38 @@ function buildCuratorAuthor(deps) {
5258
1982
  }
5259
1983
 
5260
1984
  // src/curation/curation-tail.ts
5261
- import { randomUUID as randomUUID5 } from "crypto";
1985
+ import { randomUUID as randomUUID3 } from "crypto";
5262
1986
  import {
5263
- closeSync as closeSync5,
5264
- existsSync as existsSync5,
5265
- fsyncSync as fsyncSync5,
5266
- openSync as openSync5,
5267
- readFileSync as readFileSync8,
5268
- readdirSync as readdirSync5,
5269
- renameSync as renameSync3,
5270
- rmSync as rmSync6,
5271
- writeFileSync as writeFileSync5
1987
+ closeSync as closeSync3,
1988
+ existsSync as existsSync3,
1989
+ fsyncSync as fsyncSync3,
1990
+ openSync as openSync3,
1991
+ readFileSync as readFileSync5,
1992
+ readdirSync,
1993
+ renameSync as renameSync2,
1994
+ rmSync as rmSync4,
1995
+ writeFileSync as writeFileSync3
5272
1996
  } from "fs";
5273
- import { join as join12 } from "path";
5274
- import { resolveKnowledgeRoot as resolveKnowledgeRoot3 } from "@vostride/agent-qa-core";
1997
+ import { join as join7 } from "path";
1998
+ import { resolveKnowledgeRoot as resolveKnowledgeRoot2 } from "@vostride/agent-qa-core";
5275
1999
  import {
5276
- LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES5,
2000
+ LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES3,
5277
2001
  QUARANTINE_CLASS_DISPOSITIONS,
5278
2002
  QUARANTINE_FLOOR_ELIGIBILITY
5279
2003
  } from "@vostride/agent-qa-agent-local";
5280
2004
  import {
5281
2005
  EVENTS_FILE,
5282
- HANDOFF_FILE as HANDOFF_FILE3,
5283
- RUNS_SUBDIR as RUNS_SUBDIR3,
5284
- loadCompletedRunHandoff as loadCompletedRunHandoff2,
5285
- slugBranch as slugBranch3
2006
+ HANDOFF_FILE,
2007
+ RUNS_SUBDIR,
2008
+ loadCompletedRunHandoff,
2009
+ slugBranch
5286
2010
  } from "@vostride/agent-qa-runner-local";
5287
- import { runSafetyScope as runSafetyScope3 } from "@vostride/agent-qa-safety-local";
5288
- import { canonicalJson as canonicalJson8, sha256hex as sha256hex4, CurationSnapshotSchema, CURATION_BREAKDOWN_BUCKETS } from "@vostride/agent-qa-schema";
2011
+ import { runSafetyScope } from "@vostride/agent-qa-safety-local";
2012
+ import { canonicalJson as canonicalJson3, sha256hex, CurationSnapshotSchema, CURATION_BREAKDOWN_BUCKETS } from "@vostride/agent-qa-schema";
5289
2013
 
5290
2014
  // src/daemon/client.ts
5291
2015
  import { request } from "http";
5292
- import { dirname as dirname5, join as join5 } from "path";
2016
+ import { dirname as dirname6, join as join4 } from "path";
5293
2017
  import {
5294
2018
  FunctionInputError,
5295
2019
  FunctionNotBoundError,
@@ -5312,7 +2036,7 @@ import {
5312
2036
  unlinkSync as unlinkSync2,
5313
2037
  writeFileSync
5314
2038
  } from "fs";
5315
- import { dirname as dirname4, join as join4 } from "path";
2039
+ import { dirname as dirname5, join as join3 } from "path";
5316
2040
  import { LOCAL_DAEMON_CAPABILITIES } from "@vostride/agent-qa-agent-local";
5317
2041
  import { z as z4 } from "zod";
5318
2042
 
@@ -5326,7 +2050,7 @@ import {
5326
2050
  } from "http";
5327
2051
  import { connect } from "net";
5328
2052
  import { tmpdir } from "os";
5329
- import { basename as basename2, dirname as dirname3, join as join3, resolve as resolve2, sep } from "path";
2053
+ import { basename, dirname as dirname4, join as join2, resolve as resolve2, sep } from "path";
5330
2054
  import { invokeFunction } from "@vostride/agent-qa-agent";
5331
2055
  import { ANALYTICS_AGENT_PRODUCTS } from "@vostride/agent-qa-analytics";
5332
2056
  import { z as z3 } from "zod";
@@ -5450,7 +2174,7 @@ var daemonRegistrationSchema = z4.object({
5450
2174
  startedAtMs: z4.number().int().nonnegative()
5451
2175
  }).strict();
5452
2176
  function daemonRegistrationPath(agentQaRoot) {
5453
- return join4(agentQaRoot, DAEMON_REGISTRATION_FILE);
2177
+ return join3(agentQaRoot, DAEMON_REGISTRATION_FILE);
5454
2178
  }
5455
2179
  function readDaemonRegistrationFile(file) {
5456
2180
  let raw;
@@ -5551,15 +2275,7 @@ async function classifyDaemonRegistration(input) {
5551
2275
  );
5552
2276
  switch (verdict.kind) {
5553
2277
  case "fresh":
5554
- /**
5555
- * CR-01: `unknown` answers held, with the same holder payload as `fresh`.
5556
- *
5557
- * This verdict selects registration deletion and socket unlink, and the two preconditions
5558
- * are correlated rather than independent — fd exhaustion produces both the failed identity
5559
- * handshake and the unreadable epoch at once. Treating "I could not read the authority
5560
- * state" as "nobody holds it" therefore de-registers a live, healthy daemon precisely when
5561
- * the host is under stress. The refusal is the only recoverable answer.
5562
- */
2278
+ // `unknown` answers held: fd exhaustion fails both checks at once, so unknown is not free.
5563
2279
  case "unknown":
5564
2280
  return {
5565
2281
  kind: "unreachable-lease-fresh",
@@ -5578,7 +2294,7 @@ async function classifyDaemonRegistration(input) {
5578
2294
  // src/daemon/client.ts
5579
2295
  var DAEMON_NUDGE_TIMEOUT_MS = 5e3;
5580
2296
  function defaultKnowledgeRoot(agentQaRoot) {
5581
- return join5(dirname5(agentQaRoot), "agent-qa-knowledge");
2297
+ return join4(dirname6(agentQaRoot), "agent-qa-knowledge");
5582
2298
  }
5583
2299
  function parseJson(text, context) {
5584
2300
  try {
@@ -5673,401 +2389,36 @@ async function nudgeDaemon(agentQaRoot, options = {}) {
5673
2389
  }
5674
2390
 
5675
2391
  // src/curation/emit-canary-outcomes.ts
5676
- import { readFileSync as readFileSync6 } from "fs";
5677
- import { isAbsolute as isAbsolute3, join as join10 } from "path";
5678
- import { LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES4 } from "@vostride/agent-qa-agent-local";
5679
- import {
5680
- admitAnalysisArtifactForRun as admitAnalysisArtifactForRun2,
5681
- ANALYSIS_FILE as ANALYSIS_FILE2,
5682
- canonicalJson as canonicalJson7,
5683
- deriveMemoryOnPathSites,
5684
- guardRunAnalysisArtifact as guardRunAnalysisArtifact2,
5685
- misdirectionSourceDigest,
5686
- normalizeAnchor as normalizeAnchor2,
5687
- resolveRunVerdicts as resolveRunVerdicts2
5688
- } from "@vostride/agent-qa-schema";
5689
-
5690
- // src/curation/canary-run-state.ts
5691
- import { closeSync as closeSync2, fsyncSync as fsyncSync2, linkSync, mkdirSync as mkdirSync3, openSync as openSync2, readdirSync, readFileSync as readFileSync2, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
5692
- import { randomUUID as randomUUID2 } from "crypto";
5693
- import { join as join6 } from "path";
5694
- import { z as z5 } from "zod";
5695
- import { CROSS_CHECKS, canonicalJson as canonicalJson2, sha256hex } from "@vostride/agent-qa-schema";
5696
- var POLICY_FILE = "canary-policy.json";
5697
- function readRunCanaryPolicy(runDir, runId) {
5698
- let text;
5699
- try {
5700
- text = readFileSync2(join6(runDir, POLICY_FILE), "utf8");
5701
- } catch (error) {
5702
- if (error.code === "ENOENT") return void 0;
5703
- throw new Error("canary policy unavailable");
5704
- }
5705
- const policy = JSON.parse(text);
5706
- if (typeof policy !== "object" || policy === null || Array.isArray(policy) || Object.keys(policy).sort().join(",") !== "maxAblations,runId,schemaVersion,source" || !("schemaVersion" in policy) || policy.schemaVersion !== 2 || !("runId" in policy) || policy.runId !== runId || !("maxAblations" in policy) || typeof policy.maxAblations !== "number" || !Number.isSafeInteger(policy.maxAblations) || policy.maxAblations < 1 || !("source" in policy) || typeof policy.source !== "string" || !["flag", "config", "default"].includes(policy.source)) {
5707
- throw new Error("canary policy invalid");
5708
- }
5709
- return policy;
5710
- }
5711
- var count = z5.number().int().nonnegative();
5712
- var digest = z5.string().regex(/^[a-f0-9]{64}$/);
5713
- var completionSchema = z5.object({
5714
- schemaVersion: z5.literal(2),
5715
- runId: z5.string().min(1),
5716
- candidateId: z5.string().min(1),
5717
- attributedObservationId: z5.string().min(1),
5718
- observedStateDigest: digest,
5719
- result: z5.union([
5720
- z5.object({
5721
- kind: z5.literal("ablated"),
5722
- crossCheck: z5.enum(CROSS_CHECKS),
5723
- corpusDigest: digest,
5724
- k: z5.number().int().min(3).refine((k) => k % 2 === 1),
5725
- baseSeed: z5.number().int(),
5726
- seeds: z5.array(z5.number().int()),
5727
- seedHonored: z5.boolean(),
5728
- spend: z5.object({ modelCalls: count, screens: count, promptTokens: count, completionTokens: count }).strict(),
5729
- gate2Dir: z5.string().min(1)
5730
- }).strict().refine((result) => result.seeds.length === result.k && result.seeds[0] === result.baseSeed),
5731
- z5.object({ kind: z5.literal("error"), message: z5.literal("ablation-error") }).strict(),
5732
- z5.object({ kind: z5.literal("unavailable"), reason: z5.literal("ablation-unavailable") }).strict(),
5733
- z5.object({ kind: z5.literal("no-eval-root") }).strict()
5734
- ])
5735
- }).strict();
5736
- var COMPLETIONS_DIR = "canary-completions";
5737
- function readCompletedCanaryAblations(runDir, runId) {
5738
- const dir = join6(runDir, COMPLETIONS_DIR);
5739
- let names;
5740
- try {
5741
- names = readdirSync(dir).sort();
5742
- } catch (error) {
5743
- if (error.code === "ENOENT") return [];
5744
- throw new Error("completed canary evidence unavailable");
5745
- }
5746
- return names.filter((name) => !name.startsWith(".")).map((name) => {
5747
- let value;
5748
- try {
5749
- value = completionSchema.parse(JSON.parse(readFileSync2(join6(dir, name), "utf8")));
5750
- } catch {
5751
- throw new Error("completed canary evidence invalid");
5752
- }
5753
- if (value.runId !== runId || name !== `${sha256hex(value.candidateId)}.json`) {
5754
- throw new Error("completed canary evidence identity mismatch");
5755
- }
5756
- return value;
5757
- });
5758
- }
5759
- function saveCompletedCanaryAblation(runDir, input) {
5760
- const value = completionSchema.parse(input);
5761
- const dir = join6(runDir, COMPLETIONS_DIR);
5762
- mkdirSync3(dir, { recursive: true });
5763
- const file = join6(dir, `${sha256hex(value.candidateId)}.json`);
5764
- const temporary = join6(dir, `.${randomUUID2()}.tmp`);
5765
- const text = canonicalJson2(value) + "\n";
5766
- try {
5767
- writeFileSync2(temporary, text, { flag: "wx", mode: 384 });
5768
- const fd = openSync2(temporary, "r");
5769
- try {
5770
- fsyncSync2(fd);
5771
- } finally {
5772
- closeSync2(fd);
5773
- }
5774
- try {
5775
- linkSync(temporary, file);
5776
- } catch (error) {
5777
- if (error.code !== "EEXIST" || readFileSync2(file, "utf8") !== text) {
5778
- throw new Error("completed canary evidence conflict");
5779
- }
5780
- }
5781
- const directory = openSync2(dir, "r");
5782
- try {
5783
- fsyncSync2(directory);
5784
- } finally {
5785
- closeSync2(directory);
5786
- }
5787
- } finally {
5788
- rmSync3(temporary, { force: true });
5789
- }
5790
- }
5791
-
5792
- // src/curation/misdirection-run-state.ts
5793
- import {
5794
- chmodSync as chmodSync2,
5795
- closeSync as closeSync3,
5796
- fsyncSync as fsyncSync3,
5797
- linkSync as linkSync2,
5798
- mkdirSync as mkdirSync4,
5799
- openSync as openSync3,
5800
- readFileSync as readFileSync3,
5801
- readdirSync as readdirSync2,
5802
- renameSync as renameSync2,
5803
- rmSync as rmSync4,
5804
- statSync,
5805
- writeFileSync as writeFileSync3
5806
- } from "fs";
5807
- import { randomUUID as randomUUID3 } from "crypto";
5808
- import { dirname as dirname6, join as join7 } from "path";
5809
- import { z as z6 } from "zod";
2392
+ import { readFileSync as readFileSync2 } from "fs";
2393
+ import { isAbsolute as isAbsolute2, join as join5 } from "path";
2394
+ import { LOCAL_CURATION_CAPABILITIES as LOCAL_CURATION_CAPABILITIES2 } from "@vostride/agent-qa-agent-local";
5810
2395
  import {
5811
- MisdirectionRecordResultSchema,
5812
- canonicalJson as canonicalJson3,
5813
- sha256hex as sha256hex2
2396
+ admitAnalysisArtifactForRun,
2397
+ ANALYSIS_FILE,
2398
+ canonicalJson as canonicalJson2,
2399
+ guardRunAnalysisArtifact,
2400
+ normalizeAnchor,
2401
+ resolveRunVerdicts
5814
2402
  } from "@vostride/agent-qa-schema";
5815
- var MISDIRECTION_COMPLETIONS_DIR = "misdirection-completions";
5816
- function misdirectionCheckKeyDigest(key) {
5817
- return sha256hex2(
5818
- canonicalJson3({
5819
- runId: key.runId,
5820
- perTest: key.perTest,
5821
- stepIndex: key.stepIndex,
5822
- decisionOrdinal: key.decisionOrdinal,
5823
- recordId: key.recordId
5824
- })
5825
- );
5826
- }
5827
- var policySchema = z6.object({
5828
- schemaVersion: z6.literal(2),
5829
- runId: z6.string().min(1),
5830
- maxChecks: z6.number().int().nonnegative(),
5831
- source: z6.enum(["flag", "default"])
5832
- }).strict();
5833
- var reservationSchema = z6.object({
5834
- schemaVersion: z6.literal(2),
5835
- runId: z6.string().min(1),
5836
- perTest: z6.number().int().nonnegative(),
5837
- stepIndex: z6.number().int().nonnegative(),
5838
- decisionOrdinal: z6.number().int().nonnegative(),
5839
- recordId: z6.string().min(1),
5840
- handoffBaseDigest: z6.string().min(1),
5841
- briefDigest: z6.string().min(1),
5842
- corpusDigest: z6.string().min(1),
5843
- profileDigest: z6.string().min(1)
5844
- }).strict();
5845
- function readDocuments(dir, unavailable) {
5846
- let names;
5847
- try {
5848
- names = readdirSync2(dir).sort();
5849
- } catch (error) {
5850
- if (error.code === "ENOENT") return [];
5851
- throw new Error(unavailable);
5852
- }
5853
- return names.filter((name) => !name.startsWith(".")).map((name) => ({ name, text: readFileSync3(join7(dir, name), "utf8") }));
5854
- }
5855
- function readCompletedMisdirectionChecks(runDir, runId) {
5856
- return readDocuments(
5857
- join7(runDir, MISDIRECTION_COMPLETIONS_DIR),
5858
- "completed misdirection evidence unavailable"
5859
- ).map(({ name, text }) => {
5860
- let value;
5861
- try {
5862
- value = MisdirectionRecordResultSchema.parse(JSON.parse(text));
5863
- } catch {
5864
- throw new Error("completed misdirection evidence invalid");
5865
- }
5866
- if (value.runId !== runId || name !== `${misdirectionCheckKeyDigest(value)}.json`) {
5867
- throw new Error("completed misdirection evidence identity mismatch");
5868
- }
5869
- return value;
5870
- });
5871
- }
5872
- var postureSchema = z6.object({
5873
- schemaVersion: z6.literal(2),
5874
- runId: z6.string().min(1),
5875
- kind: z6.enum(["eval-root", "no-eval-root", "broken-eval-root"]),
5876
- corpusDigest: z6.string().min(1).optional(),
5877
- profileDigest: z6.string().min(1).optional(),
5878
- reason: z6.string().min(1).optional()
5879
- }).strict();
5880
2403
 
5881
2404
  // src/curation/canary-credit.ts
5882
- import {
5883
- actedPair as actedPair3,
5884
- canonicalJson as canonicalJson4,
5885
- distilledScreenStateDigest,
5886
- exercisedAnchors as exercisedAnchors2,
5887
- normalizeAnchor
5888
- } from "@vostride/agent-qa-schema";
5889
2405
  function emptyCanaryCreditCounters() {
5890
- return {
5891
- credited: 0,
5892
- rideAlongUncredited: 0,
5893
- ablationCapExceeded: 0,
5894
- ablationHeld: 0,
5895
- ablationInconclusive: 0,
5896
- ablationError: 0,
5897
- rerunExcluded: 0,
5898
- nonSignal: { environment: 0, timeout: 0, flake: 0 },
5899
- staged: 0,
5900
- stagingRefused: 0
5901
- };
2406
+ return { rideAlongUncredited: 0, rerunExcluded: 0, nonSignal: { environment: 0, timeout: 0, flake: 0 } };
5902
2407
  }
5903
- async function creditCanaryAttributions(input) {
5904
- if (!Number.isSafeInteger(input.maxAblations) || input.maxAblations < 1) {
5905
- throw new TypeError("--max-ablations must be a positive integer");
5906
- }
2408
+ function creditCanaryAttributions(input) {
5907
2409
  const counters = emptyCanaryCreditCounters();
5908
2410
  const attributions = [];
5909
- let digestRead = false;
5910
- let corpusDigest;
5911
- const ablatedCandidates = /* @__PURE__ */ new Set();
5912
- const completedByCandidate = new Map(input.completedAblations.map((entry) => [entry.candidateId, entry]));
5913
- let ablationsRun = completedByCandidate.size;
5914
2411
  for (const row of input.attributions) {
5915
2412
  if (row.kind !== "outcome") {
5916
2413
  attributions.push(row);
5917
2414
  continue;
5918
2415
  }
5919
- const served = row.source === "served-brief";
5920
- if (served && input.ablation !== void 0 && !digestRead) {
5921
- digestRead = true;
5922
- try {
5923
- const digest2 = await input.ablation.corpusDigest();
5924
- if (digest2.kind === "digest") corpusDigest = digest2.corpusDigest;
5925
- } catch {
5926
- }
5927
- }
5928
2416
  const outcome = {
5929
2417
  ...row.outcome,
5930
- stateValidity: "unknown",
5931
- benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along",
5932
- ...served && corpusDigest !== void 0 ? { corpusDigest } : {}
2418
+ benefit: row.outcome.runOutcome === "negative" ? "exercised-harmed" : "rode-along"
5933
2419
  };
5934
- if (!served || corpusDigest === void 0) delete outcome.corpusDigest;
5935
- delete outcome.observedStateDigest;
5936
- delete outcome.ablation;
5937
- if (served) {
5938
- const step = input.handoff.tests.find((test) => test.testIndex === outcome.testIndex && test.testId === outcome.testId)?.steps.find((candidate) => candidate.stepIndex === outcome.stepIndex);
5939
- const live = step?.subActions?.filter((action2) => action2.exchanges !== void 0) ?? [];
5940
- const action = live.length === 1 ? live[0] : void 0;
5941
- const acted = action === void 0 ? void 0 : actedSubAction(action);
5942
- const observed = step !== void 0 && acted !== void 0 && exercisedAnchors2({ ...step, subActions: [acted] }).some((anchor) => canonicalJson4(normalizeAnchor(anchor)) === canonicalJson4(normalizeAnchor(outcome.exercisedAnchor))) ? actedPair3(acted).stateBefore : void 0;
5943
- if (observed !== void 0) outcome.observedStateDigest = distilledScreenStateDigest(observed);
5944
- const bundle = input.readBundleStateDigests({
5945
- knowledgeRoot: input.knowledgeRoot,
5946
- candidateId: outcome.candidateId
5947
- });
5948
- if (outcome.observedStateDigest !== void 0 && bundle.kind === "found" && bundle.digests.length > 0) {
5949
- outcome.stateValidity = bundle.digests.includes(outcome.observedStateDigest) ? "valid" : "stale";
5950
- }
5951
- }
5952
- let crossCheck = "not-run";
5953
- let corpusDigestMoved;
5954
- const completed = completedByCandidate.get(outcome.candidateId);
5955
- if (outcome.runOutcome === "positive") {
5956
- if (!served || input.ablation === void 0) {
5957
- counters.rideAlongUncredited += 1;
5958
- } else if (outcome.observedStateDigest === void 0) {
5959
- counters.ablationInconclusive += 1;
5960
- } else if (ablatedCandidates.has(outcome.candidateId)) {
5961
- counters.rideAlongUncredited += 1;
5962
- } else if (completed !== void 0 && (completed.attributedObservationId !== input.observationIdOf(outcome) || completed.observedStateDigest !== outcome.observedStateDigest)) {
5963
- counters.rideAlongUncredited += 1;
5964
- } else if (completed === void 0 && ablationsRun >= input.maxAblations) {
5965
- counters.ablationCapExceeded += 1;
5966
- } else {
5967
- ablatedCandidates.add(outcome.candidateId);
5968
- let result;
5969
- if (completed !== void 0) result = completed.result;
5970
- else {
5971
- ablationsRun += 1;
5972
- try {
5973
- result = await input.ablation.ablate({
5974
- handoff: input.handoff,
5975
- runDir: input.runDir,
5976
- candidateId: outcome.candidateId,
5977
- runId: outcome.runId,
5978
- testId: outcome.testId,
5979
- testIndex: outcome.testIndex,
5980
- stepIndex: outcome.stepIndex,
5981
- attributedObservationId: input.observationIdOf(outcome)
5982
- });
5983
- } catch (error) {
5984
- result = { kind: "error", message: error instanceof Error ? error.message : String(error) };
5985
- }
5986
- input.saveCompletedAblation({
5987
- schemaVersion: 2,
5988
- runId: outcome.runId,
5989
- candidateId: outcome.candidateId,
5990
- attributedObservationId: input.observationIdOf(outcome),
5991
- observedStateDigest: outcome.observedStateDigest,
5992
- result: result.kind === "ablated" ? {
5993
- kind: "ablated",
5994
- crossCheck: result.crossCheck,
5995
- corpusDigest: result.corpusDigest,
5996
- k: result.k,
5997
- baseSeed: result.baseSeed,
5998
- seeds: result.seeds,
5999
- seedHonored: result.seedHonored,
6000
- spend: result.spend,
6001
- gate2Dir: result.gate2Dir
6002
- } : result.kind === "error" ? { kind: "error", message: "ablation-error" } : result.kind === "unavailable" ? { kind: "unavailable", reason: "ablation-unavailable" } : { kind: "no-eval-root" }
6003
- });
6004
- }
6005
- switch (result.kind) {
6006
- case "ablated":
6007
- crossCheck = result.crossCheck;
6008
- if (corpusDigest !== void 0 && corpusDigest !== result.corpusDigest) corpusDigestMoved = true;
6009
- outcome.corpusDigest = result.corpusDigest;
6010
- outcome.ablation = {
6011
- crossCheck,
6012
- k: result.k,
6013
- baseSeed: result.baseSeed,
6014
- seeds: result.seeds,
6015
- seedHonored: result.seedHonored,
6016
- spend: result.spend
6017
- };
6018
- if (crossCheck === "flipped") {
6019
- outcome.benefit = "exercised-benefited";
6020
- counters.credited += 1;
6021
- } else if (crossCheck === "held") counters.ablationHeld += 1;
6022
- else counters.ablationInconclusive += 1;
6023
- break;
6024
- case "no-eval-root":
6025
- counters.rideAlongUncredited += 1;
6026
- break;
6027
- case "unavailable":
6028
- counters.ablationInconclusive += 1;
6029
- break;
6030
- case "error":
6031
- counters.ablationError += 1;
6032
- break;
6033
- }
6034
- }
6035
- }
6036
- let staged;
6037
- if (served && input.ablation !== void 0 && outcome.observedStateDigest !== void 0) {
6038
- try {
6039
- const product = input.handoff.scope.tier === "product" && input.handoff.scope.key !== "" && input.handoff.scope.key !== "unknown" ? input.handoff.scope.key : "default";
6040
- const stateDigest = outcome.observedStateDigest;
6041
- if (await input.ablation.isNovel({ product, stateDigest })) {
6042
- const stage = await input.ablation.stage({
6043
- handoff: input.handoff,
6044
- runDir: input.runDir,
6045
- candidateId: outcome.candidateId,
6046
- runId: outcome.runId,
6047
- testId: outcome.testId,
6048
- testIndex: outcome.testIndex,
6049
- stepIndex: outcome.stepIndex,
6050
- attributedObservationId: input.observationIdOf(outcome),
6051
- product,
6052
- stateDigest,
6053
- benefit: outcome.benefit
6054
- });
6055
- if (stage.kind === "staged") {
6056
- counters.staged += 1;
6057
- staged = true;
6058
- } else if (!["already-staged", "not-novel", "no-eval-root"].includes(stage.kind)) counters.stagingRefused += 1;
6059
- }
6060
- } catch {
6061
- counters.stagingRefused += 1;
6062
- }
6063
- }
6064
- attributions.push({
6065
- ...row,
6066
- outcome,
6067
- crossCheck,
6068
- ...staged === void 0 ? {} : { staged },
6069
- ...corpusDigestMoved === void 0 ? {} : { corpusDigestMoved }
6070
- });
2420
+ if (outcome.runOutcome === "positive") counters.rideAlongUncredited += 1;
2421
+ attributions.push({ ...row, outcome });
6071
2422
  }
6072
2423
  return { attributions, counters };
6073
2424
  }
@@ -6083,7 +2434,7 @@ function resolveAttributionClasses(artifact, handoff) {
6083
2434
  for (const member of handoff.tests) {
6084
2435
  if (member.rootCause !== void 0) classes.set(member.testIndex, member.rootCause.class);
6085
2436
  }
6086
- for (const verdict of resolveRunVerdicts2(artifact, handoff)) {
2437
+ for (const verdict of resolveRunVerdicts(artifact, handoff)) {
6087
2438
  classes.set(verdict.perTest, verdict.class);
6088
2439
  }
6089
2440
  return classes;
@@ -6091,13 +2442,13 @@ function resolveAttributionClasses(artifact, handoff) {
6091
2442
  function loadRunAnalysisArtifact(runDir, runId) {
6092
2443
  let bytes;
6093
2444
  try {
6094
- bytes = readFileSync6(join10(runDir, ANALYSIS_FILE2), "utf8");
2445
+ bytes = readFileSync2(join5(runDir, ANALYSIS_FILE), "utf8");
6095
2446
  } catch (error) {
6096
2447
  if (error.code === "ENOENT") return void 0;
6097
2448
  throw error;
6098
2449
  }
6099
- const admission = admitAnalysisArtifactForRun2(
6100
- guardRunAnalysisArtifact2(JSON.parse(bytes)),
2450
+ const admission = admitAnalysisArtifactForRun(
2451
+ guardRunAnalysisArtifact(JSON.parse(bytes)),
6101
2452
  runId
6102
2453
  );
6103
2454
  return admission.kind === "admitted" ? admission.artifact : void 0;
@@ -6108,7 +2459,7 @@ var STALENESS_MISS_REASONS = /* @__PURE__ */ new Set([
6108
2459
  "page-anchor-mismatch"
6109
2460
  ]);
6110
2461
  function normalizedKey(anchor) {
6111
- return canonicalJson7(normalizeAnchor2(anchor));
2462
+ return canonicalJson2(normalizeAnchor(anchor));
6112
2463
  }
6113
2464
  function nonAttributableFailure(test, resolved) {
6114
2465
  const resolvedClass = resolved.get(test.testIndex);
@@ -6119,45 +2470,6 @@ function attributedOutcome(test, step, resolved) {
6119
2470
  if (nonAttributableFailure(test, resolved)) return void 0;
6120
2471
  return "negative";
6121
2472
  }
6122
- var EXPOSURE_SEPARATOR = "";
6123
- function exposureKey(perTest, stepIndex, recordId) {
6124
- return [perTest, stepIndex, recordId].join(EXPOSURE_SEPARATOR);
6125
- }
6126
- function flippedExposures(handoff, evidence) {
6127
- if (evidence === void 0 || evidence.length === 0) return /* @__PURE__ */ new Set();
6128
- const runId = String(handoff.runId);
6129
- const baseDigest = misdirectionSourceDigest(handoff);
6130
- const exposed = /* @__PURE__ */ new Set();
6131
- for (const test of handoff.tests) {
6132
- for (const site of deriveMemoryOnPathSites(test.steps)) {
6133
- exposed.add(exposureKey(test.testIndex, site.stepIndex, site.recordId));
6134
- }
6135
- }
6136
- const flipped = /* @__PURE__ */ new Set();
6137
- const answered = /* @__PURE__ */ new Set();
6138
- for (const row of evidence) {
6139
- if (row.runId !== runId) continue;
6140
- if (row.handoffBaseDigest !== baseDigest) continue;
6141
- const key = exposureKey(row.perTest, row.stepIndex, row.recordId);
6142
- if (!exposed.has(key)) continue;
6143
- if (answered.has(key)) {
6144
- flipped.delete(key);
6145
- continue;
6146
- }
6147
- answered.add(key);
6148
- if (row.crossCheck === "flipped") flipped.add(key);
6149
- }
6150
- return flipped;
6151
- }
6152
- function causalRecordsOn(test, step, flipped) {
6153
- if (flipped.size === 0 || step.verifierVerdict.pass) return [];
6154
- const ids = [];
6155
- for (const site of deriveMemoryOnPathSites([step])) {
6156
- if (!flipped.has(exposureKey(test.testIndex, site.stepIndex, site.recordId))) continue;
6157
- if (!ids.includes(site.recordId)) ids.push(site.recordId);
6158
- }
6159
- return ids;
6160
- }
6161
2473
  function procedureReplayOutcome(test, step, resolved) {
6162
2474
  const source = step.planSource;
6163
2475
  if (source === void 0 || source.kind !== "procedure-replay") return void 0;
@@ -6193,7 +2505,7 @@ function servedCanaryIds(step) {
6193
2505
  }
6194
2506
  function firstMatchingAnchor(candidateAnchors, stepAnchors) {
6195
2507
  const exercisedByKey = new Map(
6196
- stepAnchors.map((anchor) => [normalizedKey(anchor), normalizeAnchor2(anchor)])
2508
+ stepAnchors.map((anchor) => [normalizedKey(anchor), normalizeAnchor(anchor)])
6197
2509
  );
6198
2510
  for (const candidateAnchor of candidateAnchors) {
6199
2511
  const match = exercisedByKey.get(normalizedKey(candidateAnchor));
@@ -6218,10 +2530,6 @@ function projectCanaryAttributions(input) {
6218
2530
  return resolutionByCandidate.get(id);
6219
2531
  };
6220
2532
  const resolved = resolveAttributionClasses(input.artifact, input.handoff);
6221
- const flipped = flippedExposures(
6222
- input.handoff,
6223
- input.misdirectionEvidence ?? input.handoff.misdirectionEvidence
6224
- );
6225
2533
  const projected = [];
6226
2534
  for (const test of input.handoff.tests) {
6227
2535
  if (test.isRerun === true) {
@@ -6229,45 +2537,33 @@ function projectCanaryAttributions(input) {
6229
2537
  continue;
6230
2538
  }
6231
2539
  for (const step of test.steps) {
6232
- projectBriefAttributions(input, test, step, resolveAnchors, resolved, flipped, projected);
2540
+ projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected);
6233
2541
  projectProcedureAttribution(input, test, step, resolveAnchors, resolved, projected);
6234
2542
  }
6235
2543
  }
6236
2544
  return projected;
6237
2545
  }
6238
- function projectBriefAttributions(input, test, step, resolveAnchors, resolved, flipped, projected) {
2546
+ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, projected) {
6239
2547
  const runOutcome = attributedOutcome(test, step, resolved);
6240
2548
  const resolvedClass = resolved.get(test.testIndex);
6241
- const causal = causalRecordsOn(test, step, flipped);
6242
- const causalSet = new Set(causal);
6243
- const nonSignal = (candidateId) => {
2549
+ if (runOutcome === void 0) {
6244
2550
  if (resolvedClass === void 0 || !isNonSignalClass(resolvedClass)) return;
6245
- projected.push({
6246
- kind: "non-signal",
6247
- source: "served-brief",
6248
- class: resolvedClass,
6249
- candidateId,
6250
- testId: test.testId,
6251
- testIndex: test.testIndex,
6252
- stepIndex: step.stepIndex
6253
- });
6254
- };
6255
- if (runOutcome === void 0 && causal.length === 0) {
6256
- for (const candidateId of servedCanaryIds(step)) nonSignal(candidateId);
2551
+ for (const candidateId of servedCanaryIds(step)) {
2552
+ projected.push({
2553
+ kind: "non-signal",
2554
+ source: "served-brief",
2555
+ class: resolvedClass,
2556
+ candidateId,
2557
+ testId: test.testId,
2558
+ testIndex: test.testIndex,
2559
+ stepIndex: step.stepIndex
2560
+ });
2561
+ }
6257
2562
  return;
6258
2563
  }
6259
2564
  const stepAnchors = exercisedAnchors(step);
6260
- if (stepAnchors.length === 0 && causal.length === 0) return;
6261
- const served = servedCanaryIds(step);
6262
- const candidateIds = [...served, ...causal.filter((id) => !served.includes(id))];
6263
- for (const candidateId of candidateIds) {
6264
- const causalRow = causalSet.has(candidateId);
6265
- const rowOutcome = causalRow ? "negative" : runOutcome;
6266
- if (rowOutcome === void 0) {
6267
- nonSignal(candidateId);
6268
- continue;
6269
- }
6270
- if (!causalRow && stepAnchors.length === 0) continue;
2565
+ if (stepAnchors.length === 0) return;
2566
+ for (const candidateId of servedCanaryIds(step)) {
6271
2567
  const resolution = resolveAnchors(candidateId);
6272
2568
  if (resolution.kind === "unresolved") {
6273
2569
  projected.push({
@@ -6282,20 +2578,16 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
6282
2578
  });
6283
2579
  continue;
6284
2580
  }
6285
- const contacted = firstMatchingAnchor(resolution.anchors, stepAnchors);
6286
- const fallback = resolution.anchors[0];
6287
- const exercisedAnchor = contacted ?? (causalRow && fallback !== void 0 ? normalizeAnchor2(fallback) : void 0);
2581
+ const exercisedAnchor = firstMatchingAnchor(resolution.anchors, stepAnchors);
6288
2582
  if (exercisedAnchor === void 0) continue;
6289
2583
  projected.push({
6290
2584
  kind: "outcome",
6291
2585
  source: "served-brief",
6292
- ...causalRow ? { misdirectionFlipped: true } : {},
6293
2586
  outcome: {
6294
2587
  candidateId,
6295
- runOutcome: rowOutcome,
6296
- // 2026-09-07, 418 D-01/D-08: pure projection stays uncredited until ablated.
2588
+ runOutcome,
2589
+ // The credit pass sets the benefit from the run outcome.
6297
2590
  benefit: "rode-along",
6298
- stateValidity: "unknown",
6299
2591
  runId: String(input.handoff.runId),
6300
2592
  testId: test.testId,
6301
2593
  testIndex: test.testIndex,
@@ -6304,10 +2596,9 @@ function projectBriefAttributions(input, test, step, resolveAnchors, resolved, f
6304
2596
  briefDigest: step.servedBrief.briefDigest,
6305
2597
  attribution: [
6306
2598
  `Candidate ${candidateId}`,
6307
- `matched ${canonicalJson7(exercisedAnchor)}`,
2599
+ `matched ${canonicalJson2(exercisedAnchor)}`,
6308
2600
  `on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
6309
- `with ${rowOutcome} verdict`,
6310
- ...causalRow ? ["after the misdirection cross-check flipped"] : []
2601
+ `with ${runOutcome} verdict`
6311
2602
  ].join(" "),
6312
2603
  exercisedAnchor
6313
2604
  }
@@ -6360,9 +2651,8 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
6360
2651
  outcome: {
6361
2652
  candidateId: source.procedureId,
6362
2653
  runOutcome,
6363
- // 2026-09-07, 418 D-01/D-08: pure projection stays uncredited until ablated.
2654
+ // The credit pass sets the benefit from the run outcome.
6364
2655
  benefit: "rode-along",
6365
- stateValidity: "unknown",
6366
2656
  runId: String(input.handoff.runId),
6367
2657
  testId: test.testId,
6368
2658
  testIndex: test.testIndex,
@@ -6374,7 +2664,7 @@ function projectProcedureAttribution(input, test, step, resolveAnchors, resolved
6374
2664
  `on test ${test.testIndex} (${test.testId}) step ${step.stepIndex}`,
6375
2665
  attributionDetail
6376
2666
  ].join(" "),
6377
- exercisedAnchor: normalizeAnchor2(anchor)
2667
+ exercisedAnchor: normalizeAnchor(anchor)
6378
2668
  }
6379
2669
  });
6380
2670
  }
@@ -6384,45 +2674,27 @@ function projectCanaryOutcomes(input) {
6384
2674
  );
6385
2675
  }
6386
2676
  async function emitCanaryOutcomes(opts) {
6387
- if (!isAbsolute3(opts.knowledgeRoot)) throw new TypeError("knowledgeRoot must be absolute");
6388
- if (!isAbsolute3(opts.runDir)) throw new TypeError("runDir must be absolute");
2677
+ if (!isAbsolute2(opts.knowledgeRoot)) throw new TypeError("knowledgeRoot must be absolute");
2678
+ if (!isAbsolute2(opts.runDir)) throw new TypeError("runDir must be absolute");
6389
2679
  const artifact = loadRunAnalysisArtifact(opts.runDir, String(opts.handoff.runId));
6390
- const recovered = opts.handoff.misdirectionEvidence === void 0 ? readCompletedMisdirectionChecks(opts.runDir, String(opts.handoff.runId)) : void 0;
6391
2680
  const projected = projectCanaryAttributions({
6392
2681
  handoff: opts.handoff,
6393
2682
  ...artifact === void 0 ? {} : { artifact },
6394
- ...recovered === void 0 || recovered.length === 0 ? {} : { misdirectionEvidence: recovered },
6395
- resolveCandidateAnchors: (id) => LOCAL_CURATION_CAPABILITIES4.readCanonicalBundleAnchors({
2683
+ resolveCandidateAnchors: (id) => LOCAL_CURATION_CAPABILITIES2.readCanonicalBundleAnchors({
6396
2684
  knowledgeRoot: opts.knowledgeRoot,
6397
2685
  bundleId: id
6398
2686
  })
6399
2687
  });
6400
- const credit = opts.credit === void 0 ? void 0 : await creditCanaryAttributions({
6401
- completedAblations: readCompletedCanaryAblations(opts.runDir, String(opts.handoff.runId)),
6402
- saveCompletedAblation: (entry) => saveCompletedCanaryAblation(opts.runDir, entry),
6403
- attributions: projected,
6404
- handoff: opts.handoff,
6405
- runDir: opts.runDir,
6406
- ...opts.credit,
6407
- knowledgeRoot: opts.knowledgeRoot,
6408
- observationIdOf: LOCAL_CURATION_CAPABILITIES4.canaryOutcomeObservationId,
6409
- maxAblations: opts.credit.maxAblations ?? (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
6410
- readBundleStateDigests: opts.credit.readBundleStateDigests ?? LOCAL_CURATION_CAPABILITIES4.readCanonicalBundleStateDigests
6411
- });
6412
- const canary = credit?.counters ?? emptyCanaryCreditCounters();
6413
- if (credit === void 0) {
6414
- canary.rideAlongUncredited = projected.filter((row) => row.kind === "outcome" && row.outcome.runOutcome === "positive").length;
6415
- }
2688
+ const credit = creditCanaryAttributions({ attributions: projected });
6416
2689
  const report = {
6417
- canary,
2690
+ canary: credit.counters,
6418
2691
  emitted: 0,
6419
2692
  refused: 0,
6420
2693
  unresolved: 0,
6421
2694
  procedure: { emitted: 0, refused: 0, unresolved: 0 },
6422
2695
  details: []
6423
2696
  };
6424
- const attributions = credit?.attributions ?? projected;
6425
- for (const attribution of attributions) {
2697
+ for (const attribution of credit.attributions) {
6426
2698
  if (attribution.kind === "rerun-excluded") {
6427
2699
  report.canary.rerunExcluded += 1;
6428
2700
  report.details.push({
@@ -6462,7 +2734,6 @@ async function emitCanaryOutcomes(opts) {
6462
2734
  });
6463
2735
  continue;
6464
2736
  }
6465
- const causal = attribution.misdirectionFlipped === true;
6466
2737
  let outcome;
6467
2738
  try {
6468
2739
  opts.emit(attribution.outcome);
@@ -6477,10 +2748,6 @@ async function emitCanaryOutcomes(opts) {
6477
2748
  }
6478
2749
  report.details.push({
6479
2750
  benefit: attribution.outcome.benefit,
6480
- crossCheck: attribution.crossCheck ?? "not-run",
6481
- ...attribution.staged === void 0 ? {} : { staged: attribution.staged },
6482
- ...attribution.corpusDigestMoved === void 0 ? {} : { corpusDigestMoved: attribution.corpusDigestMoved },
6483
- ...causal ? { misdirectionFlipped: true } : {},
6484
2751
  candidateId: attribution.outcome.candidateId,
6485
2752
  testId: attribution.outcome.testId,
6486
2753
  testIndex: attribution.outcome.testIndex,
@@ -6491,42 +2758,126 @@ async function emitCanaryOutcomes(opts) {
6491
2758
  return report;
6492
2759
  }
6493
2760
 
6494
- // src/curation/curation-tail.ts
6495
- init_promotion_posture();
2761
+ // src/curation/promotion-line.ts
2762
+ function renderPromotionLine(entry) {
2763
+ return entry.promotionPath === void 0 ? `curation: ${entry.candidateId} held: ${entry.promotionHoldReason}` : `curation: ${entry.candidateId} promoted via ${entry.promotionPath}`;
2764
+ }
6496
2765
 
6497
2766
  // src/curation/run-curation-context.ts
6498
- init_declared_secrets();
6499
- init_config();
6500
- import { randomUUID as randomUUID4 } from "crypto";
2767
+ import { randomUUID as randomUUID2 } from "crypto";
6501
2768
  import {
6502
- closeSync as closeSync4,
6503
- fsyncSync as fsyncSync4,
6504
- linkSync as linkSync3,
6505
- lstatSync as lstatSync3,
6506
- openSync as openSync4,
6507
- readFileSync as readFileSync7,
6508
- rmSync as rmSync5,
6509
- writeFileSync as writeFileSync4
2769
+ closeSync as closeSync2,
2770
+ fsyncSync as fsyncSync2,
2771
+ linkSync,
2772
+ lstatSync as lstatSync2,
2773
+ openSync as openSync2,
2774
+ readFileSync as readFileSync4,
2775
+ rmSync as rmSync3,
2776
+ writeFileSync as writeFileSync2
6510
2777
  } from "fs";
6511
- import { dirname as dirname12, isAbsolute as isAbsolute4, join as join11, resolve as resolve7 } from "path";
6512
- import { z as z8 } from "zod";
2778
+ import { dirname as dirname7, isAbsolute as isAbsolute3, join as join6, resolve as resolve3 } from "path";
2779
+ import { z as z5 } from "zod";
2780
+
2781
+ // src/composition/declared-secrets.ts
2782
+ import { existsSync as existsSync2, readFileSync as readFileSync3, statSync } from "fs";
2783
+ import { parse as parseYaml2 } from "yaml";
2784
+ import { resolveWorkspacePaths } from "@vostride/agent-qa-core";
2785
+ import { SecretRedactor, SecretStore } from "@vostride/agent-qa-runner";
2786
+ var DeclaredSecretsUnavailableError = class _DeclaredSecretsUnavailableError extends Error {
2787
+ lines;
2788
+ constructor(lines) {
2789
+ super(lines.join(" "));
2790
+ this.name = "DeclaredSecretsUnavailableError";
2791
+ this.lines = lines;
2792
+ }
2793
+ static required() {
2794
+ return new _DeclaredSecretsUnavailableError([
2795
+ "Error: workspace.secretsFile is required. Run `agent-qa init` or set it in agent-qa.config.yaml."
2796
+ ]);
2797
+ }
2798
+ };
2799
+ function readDeclaredSecrets(secretsPath) {
2800
+ if (!existsSync2(secretsPath)) {
2801
+ throw new DeclaredSecretsUnavailableError([
2802
+ `Error: Secrets file not found: ${secretsPath}`,
2803
+ "Create the file before running tests. It may be empty, but it must exist."
2804
+ ]);
2805
+ }
2806
+ let content;
2807
+ try {
2808
+ content = readFileSync3(secretsPath, "utf-8");
2809
+ } catch {
2810
+ throw new DeclaredSecretsUnavailableError([
2811
+ `Error: Secrets file could not be read: ${secretsPath}`,
2812
+ "Check the path and file permissions before running tests."
2813
+ ]);
2814
+ }
2815
+ try {
2816
+ return SecretStore.fromEnvContent(content);
2817
+ } catch (err) {
2818
+ const message = err instanceof Error ? err.message : String(err);
2819
+ throw new DeclaredSecretsUnavailableError([`Error: Secrets file is invalid: ${message}`]);
2820
+ }
2821
+ }
2822
+ function resolveSecretsPath(configPath) {
2823
+ const file = parseYaml2(readFileSync3(configPath, "utf-8")) ?? {};
2824
+ const config = mergeConfigs(file, loadEnvOverrides(), {});
2825
+ const configured = config?.workspace?.secretsFile;
2826
+ if (typeof configured !== "string" || configured.trim().length === 0) {
2827
+ throw DeclaredSecretsUnavailableError.required();
2828
+ }
2829
+ const workspace = resolveWorkspacePaths({ config, configPath });
2830
+ return workspace.secretsFile.absolutePath;
2831
+ }
2832
+ function statKey(path) {
2833
+ const stat = statSync(path);
2834
+ return `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
2835
+ }
2836
+ function createDeclaredSecretMask(configPath) {
2837
+ let resolved;
2838
+ let cached;
2839
+ const currentSecretsPath = () => {
2840
+ const configKey = statKey(configPath);
2841
+ if (resolved?.configKey !== configKey) {
2842
+ resolved = void 0;
2843
+ resolved = { configKey, secretsPath: resolveSecretsPath(configPath) };
2844
+ }
2845
+ return resolved.secretsPath;
2846
+ };
2847
+ const redactor = () => {
2848
+ const secretsPath = currentSecretsPath();
2849
+ let key;
2850
+ try {
2851
+ key = `${secretsPath}\0${statKey(secretsPath)}`;
2852
+ } catch {
2853
+ key = void 0;
2854
+ }
2855
+ if (key !== void 0 && cached?.key === key) return cached.redactor;
2856
+ const current = new SecretRedactor(readDeclaredSecrets(secretsPath));
2857
+ cached = key === void 0 ? void 0 : { key, redactor: current };
2858
+ return current;
2859
+ };
2860
+ return { mask: (text) => redactor().redactString(text), redactor };
2861
+ }
2862
+
2863
+ // src/curation/run-curation-context.ts
6513
2864
  var RUN_CONFIG_RECORD_FILE = "run-config.json";
6514
2865
  var RUN_CONFIG_RECORD_SCHEMA_VERSION = 2;
6515
- var runConfigRecordSchema = z8.object({
6516
- schemaVersion: z8.literal(RUN_CONFIG_RECORD_SCHEMA_VERSION),
6517
- runId: z8.string().min(1),
6518
- configPath: z8.string().min(1)
2866
+ var runConfigRecordSchema = z5.object({
2867
+ schemaVersion: z5.literal(RUN_CONFIG_RECORD_SCHEMA_VERSION),
2868
+ runId: z5.string().min(1),
2869
+ configPath: z5.string().min(1)
6519
2870
  }).strict();
6520
2871
  function runConfigRecordPath(agentQaRoot, runId) {
6521
- return join11(completedRunDir(agentQaRoot, runId), RUN_CONFIG_RECORD_FILE);
2872
+ return join6(completedRunDir(agentQaRoot, runId), RUN_CONFIG_RECORD_FILE);
6522
2873
  }
6523
2874
  function readRunConfigRecord(agentQaRoot, runId) {
6524
2875
  const path = runConfigRecordPath(agentQaRoot, runId);
6525
2876
  try {
6526
- if (!lstatSync3(path).isFile()) return void 0;
6527
- const parsed = runConfigRecordSchema.safeParse(JSON.parse(readFileSync7(path, "utf8")));
2877
+ if (!lstatSync2(path).isFile()) return void 0;
2878
+ const parsed = runConfigRecordSchema.safeParse(JSON.parse(readFileSync4(path, "utf8")));
6528
2879
  if (!parsed.success) return void 0;
6529
- if (parsed.data.runId !== runId || !isAbsolute4(parsed.data.configPath)) return void 0;
2880
+ if (parsed.data.runId !== runId || !isAbsolute3(parsed.data.configPath)) return void 0;
6530
2881
  return parsed.data;
6531
2882
  } catch {
6532
2883
  return void 0;
@@ -6554,13 +2905,13 @@ async function resolveRunCurationContext(input) {
6554
2905
  let knowledgeRoot;
6555
2906
  try {
6556
2907
  const config = await resolveConfig({ configPath, loadAuthPlugins: false });
6557
- knowledgeRoot = resolveConfiguredKnowledgeRoot(config, dirname12(configPath));
2908
+ knowledgeRoot = resolveConfiguredKnowledgeRoot(config, dirname7(configPath));
6558
2909
  } catch {
6559
2910
  return {
6560
2911
  handBack: `Run ${runId} was recorded under ${configPath}, which cannot be read as an agent-qa config, so it is left for a run that can read it.`
6561
2912
  };
6562
2913
  }
6563
- if (resolve7(knowledgeRoot) !== resolve7(input.drainerKnowledgeRoot)) {
2914
+ if (resolve3(knowledgeRoot) !== resolve3(input.drainerKnowledgeRoot)) {
6564
2915
  return {
6565
2916
  handBack: `Run ${runId} keeps its knowledge in ${knowledgeRoot}; it is left for a run of its own project.`
6566
2917
  };
@@ -6588,18 +2939,18 @@ var CURATION_TAIL_RECEIPT_FILE = "curation-tail-receipt.json";
6588
2939
  var CURATION_TAIL_RECEIPT_SCHEMA_VERSION = 2;
6589
2940
  function defaultOperations() {
6590
2941
  return {
6591
- loadCompletedRunHandoff: loadCompletedRunHandoff2,
6592
- resolveKnowledgeRoot: resolveKnowledgeRoot3,
6593
- runSafetyScope: runSafetyScope3,
2942
+ loadCompletedRunHandoff,
2943
+ resolveKnowledgeRoot: resolveKnowledgeRoot2,
2944
+ runSafetyScope,
6594
2945
  emitCurationCandidates,
6595
- openKnowledgeReader: LOCAL_CURATION_CAPABILITIES5.openKnowledgeReader,
2946
+ openKnowledgeReader: LOCAL_CURATION_CAPABILITIES3.openKnowledgeReader,
6596
2947
  emitCanaryOutcomes,
6597
- foldRunOutcomes: LOCAL_CURATION_CAPABILITIES5.foldRunOutcomes,
6598
- reconcileRunOutcomesProjection: LOCAL_CURATION_CAPABILITIES5.reconcileRunOutcomesProjection,
6599
- acquireWriterLease: LOCAL_CURATION_CAPABILITIES5.maintenance.acquireWriterLease,
6600
- releaseLease: LOCAL_CURATION_CAPABILITIES5.maintenance.releaseLease,
2948
+ foldRunOutcomes: LOCAL_CURATION_CAPABILITIES3.foldRunOutcomes,
2949
+ reconcileRunOutcomesProjection: LOCAL_CURATION_CAPABILITIES3.reconcileRunOutcomesProjection,
2950
+ acquireWriterLease: LOCAL_CURATION_CAPABILITIES3.maintenance.acquireWriterLease,
2951
+ releaseLease: LOCAL_CURATION_CAPABILITIES3.maintenance.releaseLease,
6601
2952
  buildCuratorAuthor,
6602
- drainQueue: LOCAL_CURATION_CAPABILITIES5.drainQueue,
2953
+ drainQueue: LOCAL_CURATION_CAPABILITIES3.drainQueue,
6603
2954
  listCompletedRunDirs,
6604
2955
  readCurationTailReceipt,
6605
2956
  writeCurationTailReceipt,
@@ -6608,15 +2959,15 @@ function defaultOperations() {
6608
2959
  };
6609
2960
  }
6610
2961
  function completedRunDir(agentQaRoot, runId) {
6611
- return join12(agentQaRoot, RUNS_SUBDIR3, slugBranch3(runId));
2962
+ return join7(agentQaRoot, RUNS_SUBDIR, slugBranch(runId));
6612
2963
  }
6613
2964
  function completedRunsRoot(agentQaRoot) {
6614
- return join12(agentQaRoot, RUNS_SUBDIR3);
2965
+ return join7(agentQaRoot, RUNS_SUBDIR);
6615
2966
  }
6616
2967
  function listCompletedRunDirs(agentQaRoot) {
6617
2968
  const runsRoot = completedRunsRoot(agentQaRoot);
6618
- if (!existsSync5(runsRoot)) return [];
6619
- return readdirSync5(runsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join12(runsRoot, entry.name)).filter((runDir) => existsSync5(join12(runDir, HANDOFF_FILE3))).sort();
2969
+ if (!existsSync3(runsRoot)) return [];
2970
+ return readdirSync(runsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join7(runsRoot, entry.name)).filter((runDir) => existsSync3(join7(runDir, HANDOFF_FILE))).sort();
6620
2971
  }
6621
2972
  function isSortedCandidateIds(value) {
6622
2973
  if (!Array.isArray(value)) return false;
@@ -6628,12 +2979,12 @@ function isSortedCandidateIds(value) {
6628
2979
  function isCanaryCreditCounters(value) {
6629
2980
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
6630
2981
  const record = value;
6631
- if (Object.keys(record).sort().join(",") !== "ablationCapExceeded,ablationError,ablationHeld,ablationInconclusive,credited,nonSignal,rerunExcluded,rideAlongUncredited,staged,stagingRefused") return false;
6632
- const count2 = (entry) => typeof entry === "number" && Number.isInteger(entry) && entry >= 0;
6633
- if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" || count2(entry))) return false;
2982
+ if (Object.keys(record).sort().join(",") !== "nonSignal,rerunExcluded,rideAlongUncredited") return false;
2983
+ const count = (entry) => typeof entry === "number" && Number.isInteger(entry) && entry >= 0;
2984
+ if (!Object.entries(record).every(([key, entry]) => key === "nonSignal" || count(entry))) return false;
6634
2985
  const nonSignal = record.nonSignal;
6635
2986
  if (typeof nonSignal !== "object" || nonSignal === null || Array.isArray(nonSignal)) return false;
6636
- return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(count2);
2987
+ return Object.keys(nonSignal).sort().join(",") === [...NON_SIGNAL_CLASSES].sort().join(",") && Object.values(nonSignal).every(count);
6637
2988
  }
6638
2989
  function isCurationTailReceipt(value) {
6639
2990
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
@@ -6644,7 +2995,7 @@ function isCurationTailReceipt(value) {
6644
2995
  function readCurationTailReceipt(runDir) {
6645
2996
  try {
6646
2997
  const parsed = JSON.parse(
6647
- readFileSync8(join12(runDir, CURATION_TAIL_RECEIPT_FILE), "utf8")
2998
+ readFileSync5(join7(runDir, CURATION_TAIL_RECEIPT_FILE), "utf8")
6648
2999
  );
6649
3000
  return isCurationTailReceipt(parsed) ? parsed : void 0;
6650
3001
  } catch {
@@ -6652,36 +3003,36 @@ function readCurationTailReceipt(runDir) {
6652
3003
  }
6653
3004
  }
6654
3005
  function fsyncFile(file) {
6655
- const fd = openSync5(file, "r");
3006
+ const fd = openSync3(file, "r");
6656
3007
  try {
6657
- fsyncSync5(fd);
3008
+ fsyncSync3(fd);
6658
3009
  } finally {
6659
- closeSync5(fd);
3010
+ closeSync3(fd);
6660
3011
  }
6661
3012
  }
6662
3013
  function writeCurationTailReceipt(runDir, receipt) {
6663
3014
  if (!isCurationTailReceipt(receipt)) throw new TypeError("invalid curation tail receipt");
6664
- const file = join12(runDir, CURATION_TAIL_RECEIPT_FILE);
6665
- const staged = join12(runDir, `.${CURATION_TAIL_RECEIPT_FILE}.${randomUUID5()}.tmp`);
3015
+ const file = join7(runDir, CURATION_TAIL_RECEIPT_FILE);
3016
+ const staged = join7(runDir, `.${CURATION_TAIL_RECEIPT_FILE}.${randomUUID3()}.tmp`);
6666
3017
  try {
6667
- writeFileSync5(staged, `${canonicalJson8(receipt)}
3018
+ writeFileSync3(staged, `${canonicalJson3(receipt)}
6668
3019
  `, { encoding: "utf8", mode: 384 });
6669
3020
  fsyncFile(staged);
6670
- renameSync3(staged, file);
3021
+ renameSync2(staged, file);
6671
3022
  fsyncFile(runDir);
6672
3023
  } finally {
6673
- rmSync6(staged, { force: true });
3024
+ rmSync4(staged, { force: true });
6674
3025
  }
6675
3026
  }
6676
3027
  function curationTailHandoffDigest(handoff) {
6677
- return sha256hex4(canonicalJson8(handoff));
3028
+ return sha256hex(canonicalJson3(handoff));
6678
3029
  }
6679
3030
  function receiptIdentityForHandoff(handoff) {
6680
3031
  return {
6681
3032
  schemaVersion: CURATION_TAIL_RECEIPT_SCHEMA_VERSION,
6682
3033
  runId: String(handoff.runId),
6683
3034
  handoffDigest: curationTailHandoffDigest(handoff),
6684
- projectionVersion: LOCAL_CURATION_CAPABILITIES5.runOutcomesProjectionVersion
3035
+ projectionVersion: LOCAL_CURATION_CAPABILITIES3.runOutcomesProjectionVersion
6685
3036
  };
6686
3037
  }
6687
3038
  function receiptMatches(receipt, expected) {
@@ -6706,12 +3057,6 @@ function candidateSummaryLine(runId, report) {
6706
3057
  });
6707
3058
  return `Knowledge curation candidates for ${runId}: ${segments.join("; ")}.`;
6708
3059
  }
6709
- function misdirectionHarmLine(runId, rows) {
6710
- const landed = rows.filter((row) => row.outcome.kind === "emitted");
6711
- const refused = rows.length - landed.length;
6712
- const named = rows.map((row) => row.candidateId).join(", ");
6713
- return `Misdirection harm for ${runId}: ${landed.length} harmed outcome(s) emitted from flipped cross-check evidence regardless of verdict class (${named})${refused === 0 ? "" : `; ${refused} refused by the queue`}.`;
6714
- }
6715
3060
  function curationOutcomeLine(input) {
6716
3061
  if (input.committed <= 0) return void 0;
6717
3062
  const proposed = Math.min(Math.max(input.downgraded, 0), input.committed);
@@ -6779,7 +3124,6 @@ async function curateCompletedRun(input) {
6779
3124
  deprecated: previous.deprecated ?? 0,
6780
3125
  complete: true,
6781
3126
  drainKind: previous.drainKind,
6782
- candidateIds: previous.candidateIds,
6783
3127
  snapshot: previous.snapshot
6784
3128
  };
6785
3129
  }
@@ -6794,7 +3138,7 @@ async function curateCompletedRun(input) {
6794
3138
  const budgetMs = input.deadlineMs === void 0 ? void 0 : Math.max(0, input.deadlineMs - tailStartedAtMs);
6795
3139
  await operations.runSafetyScope(
6796
3140
  {
6797
- // SAFE-01: posture is persisted run truth, never an ambient re-detection.
3141
+ // Posture is persisted run truth, never an ambient re-detection.
6798
3142
  executionContext: input.handoff.executionContext
6799
3143
  },
6800
3144
  async () => {
@@ -6805,29 +3149,13 @@ async function curateCompletedRun(input) {
6805
3149
  ...input.mask === void 0 ? {} : { mask: input.mask }
6806
3150
  });
6807
3151
  input.log.info?.(candidateSummaryLine(receiptIdentity.runId, candidates));
6808
- const reader = operations.openKnowledgeReader({
6809
- knowledgeRoot: input.knowledgeRoot,
6810
- priors: input.resolveTrustPriors?.()
6811
- });
6812
- const ablation = operations.canaryAblation;
6813
- const recordedPolicy = readRunCanaryPolicy(input.runDir, String(input.handoff.runId));
6814
- const caps = ablation === void 0 ? void 0 : recordedPolicy ?? (input.maxAblations !== void 0 ? { maxAblations: input.maxAblations, source: "flag" } : await ablation.resolveCaps?.() ?? {
6815
- maxAblations: (await Promise.resolve().then(() => (init_eval_session_composition(), eval_session_composition_exports))).GATE2_DEFAULT_MAX_ABLATIONS,
6816
- source: "default"
6817
- });
6818
- if (ablation !== void 0 && recordedPolicy === void 0) input.log.info?.("canary cap: legacy run without recorded policy; using explicit run override or current config/default");
6819
- if (caps !== void 0) input.log.info?.(`canary cap: maxAblations=${caps.maxAblations} (${caps.source})`);
3152
+ const reader = operations.openKnowledgeReader({ knowledgeRoot: input.knowledgeRoot });
6820
3153
  const canaries = await operations.emitCanaryOutcomes({
6821
3154
  handoff: input.handoff,
6822
3155
  knowledgeRoot: input.knowledgeRoot,
6823
3156
  runDir: input.runDir,
6824
- emit: reader.emitCanaryOutcome.bind(reader),
6825
- ...operations.canaryAblation === void 0 ? {} : { credit: { ablation: operations.canaryAblation, maxAblations: caps.maxAblations, readBundleStateDigests: LOCAL_CURATION_CAPABILITIES5.readCanonicalBundleStateDigests } }
3157
+ emit: reader.emitCanaryOutcome.bind(reader)
6826
3158
  });
6827
- const causalHarm = (canaries.details ?? []).filter((row) => row.misdirectionFlipped === true);
6828
- if (causalHarm.length > 0) {
6829
- input.log.info?.(misdirectionHarmLine(receiptIdentity.runId, causalHarm));
6830
- }
6831
3159
  try {
6832
3160
  const fold = operations.foldRunOutcomes({
6833
3161
  knowledgeRoot: input.knowledgeRoot,
@@ -6852,15 +3180,13 @@ async function curateCompletedRun(input) {
6852
3180
  }));
6853
3181
  const drain = await operations.drainQueue({
6854
3182
  sourceRunId: String(input.handoff.runId),
6855
- observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [LOCAL_CURATION_CAPABILITIES5.canaryOutcomeObservationId({
3183
+ observationIds: (canaries.details ?? []).flatMap((row) => row.candidateId === void 0 || row.outcome.kind !== "emitted" ? [] : [LOCAL_CURATION_CAPABILITIES3.canaryOutcomeObservationId({
6856
3184
  candidateId: row.candidateId,
6857
3185
  runId: input.handoff.runId,
6858
3186
  testId: row.testId,
6859
3187
  testIndex: row.testIndex,
6860
3188
  stepIndex: row.stepIndex
6861
3189
  })]),
6862
- priors: input.resolveTrustPriors?.(),
6863
- ...input.resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture: input.resolvePromotionPosture },
6864
3190
  knowledgeRoot: input.knowledgeRoot,
6865
3191
  author,
6866
3192
  executionContext: input.handoff.executionContext,
@@ -6902,8 +3228,7 @@ async function curateCompletedRun(input) {
6902
3228
  drainKind: drain.kind,
6903
3229
  handoffHolderPid: drain.kind === "handed-off" ? drain.holder.pid : null,
6904
3230
  candidateIds: sortedCandidateIds(candidates),
6905
- // WR-08: the counts travel beside the snapshot, which is exactly when the rerun short-circuit
6906
- // answers from this receipt; a handed-off receipt has no counts of its own to record.
3231
+ // Counts travel with the snapshot, when the rerun short-circuit answers from the receipt.
6907
3232
  ...validated?.success === true ? { snapshot: validated.data, curated, deprecated } : {}
6908
3233
  };
6909
3234
  }
@@ -6948,7 +3273,6 @@ async function curateCompletedRun(input) {
6948
3273
  deprecated,
6949
3274
  complete: true,
6950
3275
  drainKind,
6951
- candidateIds: completedReceipt.candidateIds,
6952
3276
  ...completedReceipt.snapshot === void 0 ? {} : { snapshot: completedReceipt.snapshot }
6953
3277
  };
6954
3278
  }
@@ -6961,7 +3285,7 @@ async function runCurationTail(opts) {
6961
3285
  let currentHandoff;
6962
3286
  try {
6963
3287
  currentHandoff = operations.loadCompletedRunHandoff(
6964
- join12(currentRunDir, HANDOFF_FILE3)
3288
+ join7(currentRunDir, HANDOFF_FILE)
6965
3289
  );
6966
3290
  } catch {
6967
3291
  opts.log.warn(
@@ -7003,7 +3327,7 @@ async function runCurationTail(opts) {
7003
3327
  opts.log.warn("curation: run-outcome projection not reconciled (no writer authority)");
7004
3328
  } else if (reconcile.kind === "rebuilt") {
7005
3329
  opts.log.info?.(
7006
- `curation: run-outcome projection rebuilt at version ${LOCAL_CURATION_CAPABILITIES5.runOutcomesProjectionVersion} (folded ${reconcile.report.folded}, skipped ${reconcile.report.skippedHandoffs} handoff, ${reconcile.report.skippedArtifacts} artifact)`
3330
+ `curation: run-outcome projection rebuilt at version ${LOCAL_CURATION_CAPABILITIES3.runOutcomesProjectionVersion} (folded ${reconcile.report.folded}, skipped ${reconcile.report.skippedHandoffs} handoff, ${reconcile.report.skippedArtifacts} artifact)`
7007
3331
  );
7008
3332
  }
7009
3333
  } catch (error) {
@@ -7025,7 +3349,7 @@ async function runCurationTail(opts) {
7025
3349
  for (const runDir of runDirs) {
7026
3350
  let handoff;
7027
3351
  try {
7028
- handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff(join12(runDir, HANDOFF_FILE3));
3352
+ handoff = runDir === currentRunDir ? currentHandoff : operations.loadCompletedRunHandoff(join7(runDir, HANDOFF_FILE));
7029
3353
  } catch {
7030
3354
  opts.log.warn(`Skipping post-verdict knowledge curation for ${runDir}: handoff is malformed.`);
7031
3355
  continue;
@@ -7047,11 +3371,7 @@ async function runCurationTail(opts) {
7047
3371
  }
7048
3372
  runMask = context.mask;
7049
3373
  }
7050
- const resolvePromotionPosture = opts.resolvePromotionPostureForRun === void 0 ? opts.resolvePromotionPosture : await opts.resolvePromotionPostureForRun(handoff.buildOrdinal, handoff.scope);
7051
3374
  const folded = await curateCompletedRun({
7052
- resolveTrustPriors: opts.resolveTrustPriors,
7053
- ...runDir !== currentRunDir || opts.maxAblations === void 0 ? {} : { maxAblations: opts.maxAblations },
7054
- ...resolvePromotionPosture === void 0 ? {} : { resolvePromotionPosture },
7055
3375
  runDir,
7056
3376
  handoff,
7057
3377
  knowledgeRoot,
@@ -7133,7 +3453,6 @@ export {
7133
3453
  normalizeConfigForValidation,
7134
3454
  projectCanaryOutcomes,
7135
3455
  purgeAll,
7136
- purgeTest,
7137
3456
  readAnalyticsPrivacyDeclaration,
7138
3457
  resolveConfig,
7139
3458
  resolveConfiguredKnowledgeRoot,