create-agdf 0.11.3 → 0.12.0

package/lib/proportionality-benchmark/blind-prompt.js

@@ -0,0 +1,37 @@
+import { DELIVERY_PATHS } from "./contracts.js";
+
+export function buildBlindPrompt(testCase, behaviorSources) {
+  if (testCase.profile_id === "staged-v2") {
+    return [
+      "Perform a read-only, non-authorizing AGDF benchmark classification. Do not modify files, invoke tools, persist control state, or claim a real approval.",
+      "Synthetic fixture state is measurement input only. Return only JSON matching the supplied schema.",
+      `Allowed stages: ${testCase.allowed_stages.join(", ")}.`,
+      `Allowed paths: ${DELIVERY_PATHS.join(", ")}.`,
+      `Requested axes: ${testCase.requested_axes.join(", ")}. Every requested axis must use evaluated with one non-null allowed value. Non-requested axes must be null with not_evaluable_yet.`,
+      "eventual_delivery_path is a benchmark candidate, never an operative Mode/Slice Decision.",
+      "compact_delivery requires visible approved UR, completed Brownfield analysis, and stored quick_task context in decision_grounds.",
+      `Scenario ID: ${testCase.scenario_id}`,
+      `Case ID: ${testCase.case_id}`,
+      `Lifecycle stage: ${testCase.lifecycle_stage}`,
+      `Task: ${testCase.task_summary}`,
+      `Repository context: ${testCase.repository_context}`,
+      `Control-state context: ${testCase.control_state_context}`,
+      `Evidence pack: ${JSON.stringify(testCase.evidence_pack ?? {})}`,
+      "Current canonical routing sources follow:",
+      behaviorSources,
+    ].join("\n\n");
+  }
+  return [
+    "Classify the delivery path AGDF would select for this task. Do not modify files or invoke tools.",
+    "Return only JSON matching the supplied schema. Do not reveal hidden reasoning.",
+    `Allowed paths: ${DELIVERY_PATHS.join(", ")}.`,
+    "Use null plus ambiguous=true when the evidence does not support one path.",
+    "compact_delivery requires decision_grounds that visibly reference approved UR, completed Brownfield analysis, and stored quick_task context.",
+    `Case ID: ${testCase.case_id}`,
+    `Task: ${testCase.task_summary}`,
+    `Repository context: ${testCase.repository_context}`,
+    `Control-state context: ${testCase.control_state_context}`,
+    "Current canonical routing sources follow:",
+    behaviorSources,
+  ].join("\n\n");
+}

package/lib/proportionality-benchmark/contracts.js

@@ -0,0 +1,120 @@
+export const DELIVERY_PATHS = Object.freeze([
+  "trivial_change", "quick_task", "compact_delivery",
+  "verified_change", "structured_slice", "structured_delivery",
+]);
+export const PATH_RANK = Object.freeze(Object.fromEntries(DELIVERY_PATHS.map((path, index) => [path, index])));
+export const STAGES = Object.freeze([
+  "ungated_execution", "ur", "brownfield_review", "prd", "sd", "tp",
+  "brownfield_analysis", "cd_tests", "cr", "qa", "uat", "or", "blocked",
+]);
+export const OBSERVATION_SCHEMA = Object.freeze({
+  type: "object",
+  additionalProperties: false,
+  required: ["schema_version", "observed_delivery_path", "ambiguous", "rationale", "decision_grounds"],
+  properties: {
+    schema_version: { type: "string", const: "1" },
+    observed_delivery_path: { type: ["string", "null"], enum: [...DELIVERY_PATHS, null] },
+    ambiguous: { type: "boolean" },
+    rationale: { type: "string", maxLength: 600 },
+    decision_grounds: { type: "array", maxItems: 12, items: { type: "string", maxLength: 120 } },
+  },
+});
+export const STAGED_OBSERVATION_SCHEMA = Object.freeze({
+  type: "object",
+  additionalProperties: false,
+  required: [
+    "schema_version", "observed_next_permissible_stage", "stage_evaluability",
+    "observed_delivery_path", "path_evaluability", "rationale", "decision_grounds",
+  ],
+  properties: {
+    schema_version: { type: "string", const: "2" },
+    observed_next_permissible_stage: { type: ["string", "null"], enum: [...STAGES, null] },
+    stage_evaluability: { type: "string", enum: ["evaluated", "not_evaluable_yet"] },
+    observed_delivery_path: { type: ["string", "null"], enum: [...DELIVERY_PATHS, null] },
+    path_evaluability: { type: "string", enum: ["evaluated", "not_evaluable_yet"] },
+    rationale: { type: "string", maxLength: 600 },
+    decision_grounds: { type: "array", maxItems: 12, items: { type: "string", maxLength: 120 } },
+  },
+});
+
+const forbiddenBaselineKeys = new Set(["observed_delivery_path", "classification", "status", "passed"]);
+const forbiddenText = /(api[_-]?key|access[_-]?token|cookie|authorization:\s*bearer|hidden reasoning|\/Users\/[^/\s]+|[A-Z]:\\Users\\[^\\\s]+)/i;
+export function fail(message, code = "PROPORTIONALITY_SCHEMA_INVALID") {
+  throw Object.assign(new Error(message), { code });
+}
+export function stable(value) {
+  if (Array.isArray(value)) return value.map(stable);
+  if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]));
+  return value;
+}
+export function validateBaseline(value) {
+  if (value?.schema_version !== "1" || value?.baseline_version !== "1.0.0") fail("unsupported baseline");
+  if (!Array.isArray(value.cases) || value.cases.length !== 40) fail("baseline must contain exactly 40 cases");
+  const ids = new Set();
+  for (const item of value.cases) {
+    if (!/^PB-\d{3}$/.test(item.case_id) || ids.has(item.case_id)) fail(`invalid or duplicate case ${item.case_id}`);
+    if (!DELIVERY_PATHS.includes(item.expected_delivery_path)) fail(`unknown path for ${item.case_id}`);
+    if (!item.task_summary || !item.rationale || !item.evidence_ref || item.evidence_ref.startsWith("/")) fail(`incomplete case ${item.case_id}`);
+    for (const key of forbiddenBaselineKeys) if (key in item) fail(`baseline contains forbidden field ${key}`);
+    ids.add(item.case_id);
+  }
+  if (new Set(value.cases.map((item) => item.expected_delivery_path)).size !== DELIVERY_PATHS.length) fail("baseline path coverage incomplete");
+  if (value.cases.filter((item) => item.adversarial).length < 10) fail("adversarial coverage too small");
+  return value;
+}
+export function normalizeAgentOutput(raw) {
+  let value;
+  try { value = typeof raw === "string" ? JSON.parse(raw) : raw; }
+  catch { fail("agent output is not JSON", "PROPORTIONALITY_OUTPUT_INVALID"); }
+  if (!value || typeof value !== "object" || Array.isArray(value)) fail("agent output is not an object", "PROPORTIONALITY_OUTPUT_INVALID");
+  const allowedKeys = new Set(["schema_version", "observed_delivery_path", "ambiguous", "rationale", "decision_grounds"]);
+  if (Object.keys(value).some((key) => !allowedKeys.has(key))) fail("agent output contains unknown fields", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (value.schema_version !== "1" || typeof value.ambiguous !== "boolean") fail("agent output contract mismatch", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (value.observed_delivery_path !== null && !DELIVERY_PATHS.includes(value.observed_delivery_path)) fail("unknown observed path", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (typeof value.rationale !== "string" || value.rationale.length > 600 || forbiddenText.test(value.rationale)) fail("unsafe rationale", "PROPORTIONALITY_REDACTION_FAILED");
+  if (!Array.isArray(value.decision_grounds) || value.decision_grounds.length > 12 || value.decision_grounds.some((item) => typeof item !== "string" || item.length > 120 || forbiddenText.test(item))) fail("unsafe decision grounds", "PROPORTIONALITY_REDACTION_FAILED");
+  let path = value.observed_delivery_path;
+  let ambiguous = value.ambiguous || path === null;
+  if (path === "compact_delivery") {
+    const grounds = value.decision_grounds.map((item) => item.toLowerCase()).join(" ");
+    if (!grounds.includes("ur") || !grounds.includes("brownfield") || !grounds.includes("quick_task")) {
+      path = null;
+      ambiguous = true;
+    }
+  }
+  if (ambiguous) path = null;
+  return { schema_version: "1", observed_delivery_path: path, ambiguous, rationale: value.rationale, decision_grounds: [...value.decision_grounds] };
+}
+export function normalizeStagedAgentOutput(raw, requestedAxes) {
+  let value;
+  try { value = typeof raw === "string" ? JSON.parse(raw) : raw; }
+  catch { fail("agent output is not JSON", "PROPORTIONALITY_OUTPUT_INVALID"); }
+  if (!value || typeof value !== "object" || Array.isArray(value)) fail("agent output is not an object", "PROPORTIONALITY_OUTPUT_INVALID");
+  const allowed = new Set(Object.keys(STAGED_OBSERVATION_SCHEMA.properties));
+  if (Object.keys(value).some((key) => !allowed.has(key))) fail("agent output contains unknown fields", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (value.schema_version !== "2" || (!STAGES.includes(value.observed_next_permissible_stage) && value.observed_next_permissible_stage !== null)) fail("invalid staged contract", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (!DELIVERY_PATHS.includes(value.observed_delivery_path) && value.observed_delivery_path !== null) fail("invalid staged path", "PROPORTIONALITY_OUTPUT_INVALID");
+  if (!["evaluated", "not_evaluable_yet"].includes(value.stage_evaluability) || !["evaluated", "not_evaluable_yet"].includes(value.path_evaluability)) fail("invalid staged evaluability", "PROPORTIONALITY_OUTPUT_INVALID");
+  for (const [axis, observed, evaluability] of [
+    ["next_permissible_stage", value.observed_next_permissible_stage, value.stage_evaluability],
+    ["eventual_delivery_path", value.observed_delivery_path, value.path_evaluability],
+  ]) {
+    const requested = requestedAxes.includes(axis);
+    if (requested !== (evaluability === "evaluated") || (requested ? observed === null : observed !== null)) fail(`axis invariant failed: ${axis}`, "PROPORTIONALITY_OUTPUT_INVALID");
+  }
+  if (typeof value.rationale !== "string" || value.rationale.length > 600 || forbiddenText.test(value.rationale)) fail("unsafe rationale", "PROPORTIONALITY_REDACTION_FAILED");
+  if (!Array.isArray(value.decision_grounds) || value.decision_grounds.length > 12 || value.decision_grounds.some((item) => typeof item !== "string" || item.length > 120 || forbiddenText.test(item))) fail("unsafe decision grounds", "PROPORTIONALITY_REDACTION_FAILED");
+  if (value.observed_delivery_path === "compact_delivery") {
+    const grounds = value.decision_grounds.join(" ").toLowerCase();
+    if (!grounds.includes("ur") || !grounds.includes("brownfield") || !grounds.includes("quick_task")) fail("compact delivery evidence missing", "PROPORTIONALITY_OUTPUT_INVALID");
+  }
+  return {
+    schema_version: "2",
+    observed_next_permissible_stage: value.observed_next_permissible_stage,
+    stage_evaluability: value.stage_evaluability,
+    observed_delivery_path: value.observed_delivery_path,
+    path_evaluability: value.path_evaluability,
+    rationale: value.rationale,
+    decision_grounds: [...value.decision_grounds],
+  };
+}

package/lib/proportionality-benchmark/corpus-loader.js

@@ -0,0 +1,110 @@
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { dirname, relative, resolve, sep } from "node:path";
+import { fail, validateBaseline } from "./contracts.js";
+
+function inside(root, candidate) {
+  const target = resolve(root, candidate);
+  const rel = relative(root, target);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`)) fail(`unsafe corpus path ${candidate}`);
+  return target;
+}
+function existingInside(root, candidate) {
+  const target = inside(root, candidate);
+  if (!existsSync(target)) fail(`missing corpus path ${candidate}`);
+  const canonicalRoot = realpathSync(root);
+  const canonicalTarget = realpathSync(target);
+  const rel = relative(canonicalRoot, canonicalTarget);
+  if (rel === ".." || rel.startsWith(`..${sep}`)) fail(`corpus path escapes through symlink ${candidate}`);
+  return target;
+}
+function sha256(path) {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+const stagedForbidden = /expected_|evidence_ref|classification|threshold|trivial_change|compact_delivery|verified_change|structured_slice|structured_delivery/i;
+export function validateStagedBlindScenario(scenario, evidencePack = null, baselineScenario = null) {
+  if (stagedForbidden.test(JSON.stringify(scenario))) fail(`staged scenario leakage ${scenario.scenario_id}`, "PROPORTIONALITY_LEAKAGE");
+  const quickTaskReferences = JSON.stringify(scenario).match(/quick_task/gi)?.length ?? 0;
+  if (quickTaskReferences && (
+    scenario.lifecycle_stage !== "post_brownfield_decision"
+    || scenario.control_state_context !== "A durable UR is approved. Brownfield analysis is complete and the stored mode/slice decision is quick_task."
+    || quickTaskReferences !== 1
+  )) fail(`staged scenario path leakage ${scenario.scenario_id}`, "PROPORTIONALITY_LEAKAGE");
+  if (evidencePack && stagedForbidden.test(JSON.stringify(evidencePack))) fail(`staged evidence-pack leakage ${scenario.scenario_id}`, "PROPORTIONALITY_LEAKAGE");
+  if (baselineScenario && (
+    JSON.stringify(scenario).includes(baselineScenario.rationale)
+    || JSON.stringify(evidencePack ?? {}).includes(baselineScenario.rationale)
+  )) fail(`staged rationale leakage ${scenario.scenario_id}`, "PROPORTIONALITY_LEAKAGE");
+  return scenario;
+}
+export function loadCorpus(repoRoot, profileId = "legacy-v1") {
+  const root = existingInside(repoRoot, "evals/proportionality");
+  if (profileId === "staged-v2") {
+    const manifest = JSON.parse(readFileSync(existingInside(root, "staged-manifest.json"), "utf8"));
+    const index = JSON.parse(readFileSync(existingInside(root, manifest.scenarios_path), "utf8"));
+    const fixtures = JSON.parse(readFileSync(existingInside(root, manifest.fixture_path), "utf8"));
+    const baselinePath = existingInside(repoRoot, manifest.baseline_path);
+    const baseline = JSON.parse(readFileSync(baselinePath, "utf8"));
+    if (
+      manifest.profile_id !== "staged-v2"
+      || baseline.schema_version !== "2"
+      || baseline.baseline_version !== manifest.baseline_version
+      || !manifest.corpus_version
+      || !manifest.fixture_version
+      || fixtures.fixture_version !== manifest.fixture_version
+    ) fail("staged profile version mismatch");
+    if (baseline.cases.length !== 40 || index.scenarios.length !== 72 || new Set(index.scenario_ids).size !== 72) fail("staged corpus coverage mismatch");
+    const expectedIds = new Set(baseline.scenarios.map((item) => item.scenario_id));
+    if (index.scenarios.some((item) => !expectedIds.has(item.scenario_id)) || baseline.scenarios.some((item) => !index.scenario_ids.includes(item.scenario_id))) fail("staged scenario baseline mismatch");
+    for (const scenario of index.scenarios) {
+      if (scenario.evidence_pack_id && !fixtures.evidence_packs[scenario.evidence_pack_id]) fail(`missing evidence pack ${scenario.scenario_id}`);
+      validateStagedBlindScenario(
+        scenario,
+        scenario.evidence_pack_id ? fixtures.evidence_packs[scenario.evidence_pack_id] : null,
+        baseline.scenarios.find((item) => item.scenario_id === scenario.scenario_id),
+      );
+    }
+    const cases = index.scenarios.map((scenario) => ({
+      ...scenario,
+      profile_id: "staged-v2",
+      corpus_version: manifest.corpus_version,
+      fixture_version: manifest.fixture_version,
+      allowed_stages: ["ungated_execution", "ur", "brownfield_review", "prd", "sd", "tp", "brownfield_analysis", "cd_tests", "cr", "qa", "uat", "or", "blocked"],
+      evidence_pack: scenario.evidence_pack_id ? fixtures.evidence_packs[scenario.evidence_pack_id] : null,
+    }));
+    return { profile_id: "staged-v2", manifest, baseline, fixtures, cases };
+  }
+  if (profileId !== "legacy-v1") fail(`unknown proportionality profile ${profileId}`);
+  const manifest = JSON.parse(readFileSync(existingInside(root, "manifest.json"), "utf8"));
+  const provenance = JSON.parse(readFileSync(existingInside(root, "legacy-v1-provenance.json"), "utf8"));
+  const index = JSON.parse(readFileSync(existingInside(root, "cases.json"), "utf8"));
+  const fixtures = JSON.parse(readFileSync(existingInside(root, "fixtures/catalog.json"), "utf8"));
+  const baselinePath = existingInside(repoRoot, manifest.baseline_path);
+  const baseline = validateBaseline(JSON.parse(readFileSync(baselinePath, "utf8")));
+  if (manifest.schema_version !== "1" || index.schema_version !== "1" || manifest.baseline_version !== baseline.baseline_version) fail("manifest/corpus version mismatch");
+  if (provenance.profile_id !== "legacy-v1" || Object.keys(provenance.source_fingerprints ?? {}).length !== 40) fail("legacy provenance incomplete");
+  for (const [path, expectedHash] of Object.entries(provenance.files ?? {})) {
+    const target = existingInside(repoRoot, path);
+    if (sha256(target) !== expectedHash) fail(`legacy provenance drift ${path}`, "PROPORTIONALITY_HISTORY_DRIFT");
+  }
+  if (!Array.isArray(index.case_ids) || index.case_ids.length !== 40 || new Set(index.case_ids).size !== 40) fail("blind corpus must list 40 unique ids");
+  if (Object.keys(index.case_contexts ?? {}).some((id) => !index.case_ids.includes(id))) fail("case context references unknown id");
+  if (Object.values(index.case_contexts ?? {}).some((value) => typeof value !== "string" || /expected_delivery_path|compact_delivery|structured_slice|structured_delivery|verified_change|trivial_change/i.test(value))) fail("case context contains result label");
+  const baselineIds = new Set(baseline.cases.map((item) => item.case_id));
+  if (index.case_ids.some((id) => !baselineIds.has(id)) || baseline.cases.some((item) => !index.case_ids.includes(item.case_id))) fail("baseline/corpus ids differ");
+  const artefactRoot = realpathSync(resolve(repoRoot, ".agdf/control/artefacts"));
+  for (const item of baseline.cases) {
+    const evidence = resolve(dirname(baselinePath), item.evidence_ref);
+    if (!existsSync(evidence)) fail(`unresolvable evidence_ref ${item.case_id}`);
+    const rel = relative(artefactRoot, realpathSync(evidence));
+    if (rel === ".." || rel.startsWith(`..${sep}`)) fail(`unsafe evidence_ref ${item.case_id}`);
+  }
+  const fixture = fixtures.fixtures?.[index.fixture_id];
+  if (!fixture || "expected_delivery_path" in fixture || "rationale" in fixture) fail("invalid blind fixture");
+  const cases = baseline.cases.map(({ case_id, task_summary }) => ({
+    case_id, task_summary, fixture_id: index.fixture_id,
+    repository_context: fixture.repository_context,
+    control_state_context: index.case_contexts?.[case_id] ?? fixture.control_state_context,
+  }));
+  return { profile_id: "legacy-v1", manifest, baseline, fixtures, provenance, cases };
+}

package/lib/proportionality-benchmark/evaluator.js

@@ -0,0 +1,250 @@
+import { DELIVERY_PATHS, PATH_RANK, STAGES, fail, stable } from "./contracts.js";
+import { sourceFingerprint } from "./source-fingerprint.js";
+import { ADAPTER_VERSION, STAGED_ADAPTER_VERSION } from "./live-recorder.js";
+
+export function classifyObservation(expected, observation) {
+  if (observation.ambiguous || observation.observed_delivery_path === null) return "ambiguous";
+  const actual = PATH_RANK[observation.observed_delivery_path];
+  const target = PATH_RANK[expected];
+  if (actual === undefined || target === undefined) return "ambiguous";
+  if (actual === target) return "correct";
+  return actual < target ? "under_governance" : "over_governance";
+}
+export function classifyStagedPath(expected, observation) {
+  const legacy = classifyObservation(expected, observation);
+  return {
+    correct: "path_correct",
+    under_governance: "path_under_governance",
+    over_governance: "path_over_governance",
+    ambiguous: "path_ambiguous",
+  }[legacy];
+}
+export function classifyStage(expected, observation) {
+  if (observation.stage_evaluability !== "evaluated" || observation.observed_next_permissible_stage === null) return "stage_ambiguous";
+  const actual = observation.observed_next_permissible_stage;
+  if (!STAGES.includes(actual) || !STAGES.includes(expected)) return "stage_ambiguous";
+  if (actual === expected) return "stage_correct";
+  if (expected === "blocked") return "stage_unsafe_advance";
+  if (actual === "blocked") return "stage_over_governance";
+  if (expected === "ungated_execution") return "stage_over_governance";
+  if (actual === "ungated_execution") return "stage_unsafe_advance";
+  return STAGES.indexOf(actual) > STAGES.indexOf(expected) ? "stage_unsafe_advance" : "stage_over_governance";
+}
+function stagedCoverageResult(expected, rows, reason) {
+  return {
+    scenario_id: expected.scenario_id,
+    case_id: expected.case_id,
+    expected_next_permissible_stage: expected.expected_next_permissible_stage,
+    expected_delivery_path: expected.expected_delivery_path,
+    stage_required: expected.stage_required,
+    path_required: expected.path_required,
+    observations: rows,
+    stage_distribution: {},
+    path_distribution: {},
+    consensus_stage: null,
+    consensus_delivery_path: null,
+    stage_classifications: [],
+    path_classifications: [],
+    status: "block",
+    blocking_reasons: [reason],
+  };
+}
+function evaluateStagedSeries({ repoRoot, corpus, observations, minimumRepeats }) {
+  const provenanceKeys = ["series_id", "profile_id", "protocol_version", "corpus_version", "fixture_version", "surface", "runtime_version", "model", "agdf_version", "baseline_version", "adapter_version"];
+  const first = observations[0] ?? {};
+  const observationIds = new Set();
+  const byScenario = new Map(corpus.cases.map((item) => [item.scenario_id, []]));
+  for (const observation of observations) {
+    if (!byScenario.has(observation.scenario_id) || observation.case_id !== observation.scenario_id.split(":")[0]) fail(`unknown staged observation ${observation.scenario_id ?? "unknown"}`);
+    if (observation.schema_version !== "2" || observation.evidence_kind !== "live_agent_observation" || !Number.isInteger(observation.repeat) || observation.repeat < 1) fail(`invalid observation ${observation.observation_id ?? "unknown"}`);
+    if (!observation.observation_id || observationIds.has(observation.observation_id)) fail(`duplicate observation ${observation.observation_id ?? "unknown"}`);
+    if (observation.execution_status !== "completed" || observation.redaction_status !== "pass" || observation.mutation_status !== "pass") fail(`unsafe observation status ${observation.observation_id}`);
+    if (!Number.isFinite(Date.parse(observation.recorded_at))) fail(`invalid observation timestamp ${observation.observation_id}`);
+    if (
+      observation.baseline_version !== corpus.baseline.baseline_version
+      || observation.corpus_version !== corpus.manifest.corpus_version
+      || observation.fixture_version !== corpus.manifest.fixture_version
+    ) fail(`baseline provenance drift at ${observation.observation_id}`);
+    if (provenanceKeys.some((key) => observation[key] !== first[key])) fail(`series provenance drift at ${observation.observation_id}`);
+    observationIds.add(observation.observation_id);
+    byScenario.get(observation.scenario_id).push(observation);
+  }
+  const scenarios = corpus.baseline.scenarios.map((expected) => {
+    const blind = corpus.cases.find((item) => item.scenario_id === expected.scenario_id);
+    const rows = byScenario.get(expected.scenario_id).sort((a, b) => a.repeat - b.repeat);
+    const repeats = new Set(rows.map((item) => item.repeat));
+    if (rows.length < minimumRepeats || repeats.size !== rows.length) return stagedCoverageResult(expected, rows, "coverage");
+    const fingerprint = sourceFingerprint(repoRoot, blind, corpus.fixtures, STAGED_ADAPTER_VERSION);
+    const stale = rows.some((item) => item.source_fingerprint !== fingerprint);
+    const invalidAxes = rows.some((item) => (
+      expected.stage_required !== (item.stage_evaluability === "evaluated")
+      || expected.path_required !== (item.path_evaluability === "evaluated")
+      || (expected.stage_required ? item.observed_next_permissible_stage === null : item.observed_next_permissible_stage !== null)
+      || (expected.path_required ? item.observed_delivery_path === null : item.observed_delivery_path !== null)
+    ));
+    const stageClassifications = expected.stage_required ? rows.map((item) => classifyStage(expected.expected_next_permissible_stage, item)) : [];
+    const pathClassifications = expected.path_required ? rows.map((item) => classifyStagedPath(expected.expected_delivery_path, { ...item, ambiguous: item.path_evaluability !== "evaluated" })) : [];
+    const stageValues = new Set(rows.map((item) => item.observed_next_permissible_stage));
+    const pathValues = new Set(rows.map((item) => item.observed_delivery_path));
+    const consensusStage = expected.stage_required && !stale && !invalidAxes && stageValues.size === 1 ? rows[0].observed_next_permissible_stage : null;
+    const consensusPath = expected.path_required && !stale && !invalidAxes && pathValues.size === 1 ? rows[0].observed_delivery_path : null;
+    const blocking = [];
+    if (stale) blocking.push("stale");
+    if (invalidAxes) blocking.push("axis_contract");
+    if (expected.stage_required && !consensusStage) blocking.push("mixed_or_ambiguous_stage");
+    if (expected.path_required && !consensusPath) blocking.push("mixed_or_ambiguous_path");
+    if (stageClassifications.includes("stage_unsafe_advance")) blocking.push("critical_stage_under_governance");
+    if (pathClassifications.includes("path_under_governance")) blocking.push("critical_path_under_governance");
+    if (stageClassifications.some((item) => item !== "stage_correct")) blocking.push("stage_deviation");
+    const deviations = pathClassifications.filter((item) => item !== "path_correct");
+    return {
+      scenario_id: expected.scenario_id,
+      case_id: expected.case_id,
+      lifecycle_stage: blind.lifecycle_stage,
+      expected_next_permissible_stage: expected.expected_next_permissible_stage,
+      expected_delivery_path: expected.expected_delivery_path,
+      stage_required: expected.stage_required,
+      path_required: expected.path_required,
+      observations: rows,
+      stage_distribution: Object.fromEntries(STAGES.map((stage) => [stage, rows.filter((item) => item.observed_next_permissible_stage === stage).length]).filter(([, count]) => count)),
+      path_distribution: Object.fromEntries(DELIVERY_PATHS.map((path) => [path, rows.filter((item) => item.observed_delivery_path === path).length]).filter(([, count]) => count)),
+      stage_classifications: stageClassifications,
+      path_classifications: pathClassifications,
+      consensus_stage: consensusStage,
+      consensus_delivery_path: consensusPath,
+      status: blocking.length ? "block" : deviations.length ? "deviation" : "correct",
+      blocking_reasons: blocking,
+    };
+  });
+  const criticalUnder = scenarios.filter((item) => item.blocking_reasons.some((reason) => reason.startsWith("critical_"))).map((item) => item.scenario_id);
+  const ambiguous = scenarios.filter((item) => item.blocking_reasons.some((reason) => ["coverage", "stale", "axis_contract", "mixed_or_ambiguous_stage", "mixed_or_ambiguous_path"].includes(reason))).map((item) => item.scenario_id);
+  const stageDeviations = scenarios.filter((item) => item.stage_classifications.some((classification) => classification !== "stage_correct")).map((item) => item.scenario_id);
+  const over = scenarios.filter((item) => item.path_classifications.includes("path_over_governance")).map((item) => item.scenario_id);
+  const smallScenarios = scenarios.filter((item) => item.path_required && ["trivial_change", "quick_task"].includes(item.expected_delivery_path));
+  const smallOver = smallScenarios.filter((item) => item.path_classifications.length && item.path_classifications.every((classification) => classification === "path_over_governance")).map((item) => item.scenario_id);
+  const smallOverPercent = smallScenarios.length ? smallOver.length / smallScenarios.length * 100 : 0;
+  const thresholdBlocks = [
+    ...(criticalUnder.length ? ["critical_under_governance"] : []),
+    ...(stageDeviations.length ? ["stage_deviation"] : []),
+    ...(smallOverPercent > corpus.baseline.constraints.over_governance_threshold_percent ? ["small_segment_over_governance"] : []),
+    ...(ambiguous.length ? ["ambiguous_or_incomplete"] : []),
+  ];
+  return stable({
+    schema_version: "2",
+    profile_id: "staged-v2",
+    protocol_version: "2",
+    evidence_boundary: corpus.manifest.evidence_boundary,
+    baseline_version: corpus.baseline.baseline_version,
+    corpus_version: corpus.manifest.corpus_version,
+    series_id: first.series_id ?? null,
+    surface: first.surface ?? null,
+    model: first.model ?? null,
+    agdf_version: first.agdf_version ?? null,
+    adapter_version: first.adapter_version ?? null,
+    runtime_version: first.runtime_version ?? null,
+    runner_version: "2.0.0",
+    freshness_status: scenarios.some((item) => item.blocking_reasons.includes("stale")) ? "stale" : "fresh",
+    minimum_repeats: minimumRepeats,
+    valid_observations: observations.length,
+    case_count: corpus.baseline.cases.length,
+    scenario_count: scenarios.length,
+    stage_scenario_count: scenarios.filter((item) => item.stage_required).length,
+    path_scenario_count: scenarios.filter((item) => item.path_required).length,
+    critical_under_governance_ids: criticalUnder,
+    stage_deviation_ids: stageDeviations,
+    over_governance_ids: over,
+    small_segment_denominator: smallScenarios.length,
+    small_segment_over_governance_ids: smallOver,
+    small_segment_over_governance_percent: smallOverPercent,
+    ambiguous_ids: ambiguous,
+    blocking_reasons: thresholdBlocks,
+    status: thresholdBlocks.length ? "block" : "pass",
+    scenarios,
+  });
+}
+export function evaluateSeries({ repoRoot, corpus, observations, minimumRepeats = 3 }) {
+  if (!Number.isInteger(minimumRepeats) || minimumRepeats < 3) fail("minimum repeats must be at least 3");
+  if (corpus.profile_id === "staged-v2") return evaluateStagedSeries({ repoRoot, corpus, observations, minimumRepeats });
+  const provenanceKeys = ["series_id", "surface", "runtime_version", "model", "agdf_version", "baseline_version", "adapter_version"];
+  const first = observations[0] ?? {};
+  const observationIds = new Set();
+  const byCase = new Map(corpus.cases.map((item) => [item.case_id, []]));
+  for (const observation of observations) {
+    if (!byCase.has(observation.case_id)) fail(`unknown observation case ${observation.case_id}`);
+    if (observation.evidence_kind !== "live_agent_observation" || !Number.isInteger(observation.repeat) || observation.repeat < 1) fail(`invalid observation ${observation.observation_id ?? "unknown"}`);
+    if (!observation.observation_id || observationIds.has(observation.observation_id)) fail(`duplicate observation ${observation.observation_id ?? "unknown"}`);
+    if (observation.execution_status !== "completed" || observation.redaction_status !== "pass" || observation.mutation_status !== "pass") fail(`unsafe observation status ${observation.observation_id}`);
+    if (!Number.isFinite(Date.parse(observation.recorded_at))) fail(`invalid observation timestamp ${observation.observation_id}`);
+    if (observation.baseline_version !== corpus.baseline.baseline_version) fail(`baseline provenance drift at ${observation.observation_id}`);
+    if (provenanceKeys.some((key) => observation[key] !== first[key])) fail(`series provenance drift at ${observation.observation_id}`);
+    observationIds.add(observation.observation_id);
+    byCase.get(observation.case_id).push(observation);
+  }
+  const fixture = corpus.fixtures.fixtures[corpus.manifest.fixture_id];
+  const cases = corpus.baseline.cases.map((expected) => {
+    const blind = corpus.cases.find((item) => item.case_id === expected.case_id);
+    const rows = byCase.get(expected.case_id).sort((a, b) => a.repeat - b.repeat);
+    const repeats = new Set(rows.map((item) => item.repeat));
+    if (rows.length < minimumRepeats || repeats.size !== rows.length) {
+      return { case_id: expected.case_id, expected_delivery_path: expected.expected_delivery_path, observations: rows, distribution: {}, consensus_delivery_path: null, status: "ambiguous", classifications: [], blocking_reasons: ["coverage"] };
+    }
+    const historicalFingerprint = corpus.provenance?.source_fingerprints?.[expected.case_id];
+    const currentFingerprint = historicalFingerprint ?? sourceFingerprint(repoRoot, blind, fixture, ADAPTER_VERSION);
+    const stale = rows.some((item) => item.source_fingerprint !== currentFingerprint);
+    const classifications = rows.map((item) => classifyObservation(expected.expected_delivery_path, item));
+    const distribution = Object.fromEntries(DELIVERY_PATHS.map((path) => [path, rows.filter((item) => item.observed_delivery_path === path).length]).filter(([, count]) => count));
+    const paths = new Set(rows.map((item) => item.observed_delivery_path));
+    const consensus = !stale && !rows.some((item) => item.ambiguous) && paths.size === 1 ? rows[0].observed_delivery_path : null;
+    const blocking = [];
+    if (stale) blocking.push("stale");
+    if (!consensus) blocking.push("mixed_or_ambiguous");
+    if (classifications.includes("under_governance")) blocking.push("critical_under_governance");
+    return {
+      case_id: expected.case_id,
+      expected_delivery_path: expected.expected_delivery_path,
+      adversarial: expected.adversarial,
+      observations: rows,
+      distribution,
+      classifications,
+      consensus_delivery_path: consensus,
+      status: blocking.length ? "block" : classifications.every((item) => item === "correct") ? "correct" : "over_governance",
+      blocking_reasons: blocking,
+    };
+  });
+  const smallCases = cases.filter((item) => ["trivial_change", "quick_task"].includes(item.expected_delivery_path));
+  const unanimousOverSmall = smallCases.filter((item) => item.consensus_delivery_path && item.classifications.every((value) => value === "over_governance")).length;
+  const smallOverRate = unanimousOverSmall / smallCases.length * 100;
+  const criticalUnderIds = cases.filter((item) => item.classifications.includes("under_governance")).map((item) => item.case_id);
+  const ambiguousIds = cases.filter((item) => item.blocking_reasons.some((item) => ["coverage", "stale", "mixed_or_ambiguous"].includes(item))).map((item) => item.case_id);
+  const thresholdBlocks = [
+    ...(criticalUnderIds.length ? ["critical_under_governance"] : []),
+    ...(smallOverRate > corpus.baseline.constraints.over_governance_threshold_percent ? ["small_segment_over_governance"] : []),
+    ...(ambiguousIds.length ? ["ambiguous_or_incomplete"] : []),
+  ];
+  return stable({
+    schema_version: "1",
+    evidence_boundary: "live routing observations with deterministic offline grading",
+    baseline_version: corpus.baseline.baseline_version,
+    corpus_version: corpus.manifest.corpus_version,
+    series_id: first.series_id ?? null,
+    surface: first.surface ?? null,
+    model: first.model ?? null,
+    agdf_version: first.agdf_version ?? null,
+    adapter_version: first.adapter_version ?? null,
+    runtime_version: first.runtime_version ?? null,
+    runner_version: "1.0.0",
+    freshness_status: cases.some((item) => item.blocking_reasons.includes("stale")) ? "stale" : corpus.provenance ? "historical" : "fresh",
+    minimum_repeats: minimumRepeats,
+    valid_observations: observations.length,
+    case_count: cases.length,
+    adversarial_case_count: cases.filter((item) => item.adversarial).length,
+    small_segment_denominator: smallCases.length,
+    unanimous_over_governance_small_cases: unanimousOverSmall,
+    small_segment_over_governance_percent: smallOverRate,
+    critical_under_governance_ids: criticalUnderIds,
+    ambiguous_ids: ambiguousIds,
+    blocking_reasons: thresholdBlocks,
+    status: thresholdBlocks.length ? "block" : "pass",
+    cases,
+  });
+}

package/lib/proportionality-benchmark/index.js

@@ -0,0 +1,7 @@
+export * from "./blind-prompt.js";
+export * from "./contracts.js";
+export * from "./corpus-loader.js";
+export * from "./evaluator.js";
+export * from "./live-recorder.js";
+export * from "./report.js";
+export * from "./source-fingerprint.js";

package/lib/proportionality-benchmark/live-recorder.js

@@ -0,0 +1,100 @@
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { executeStructuredAgent } from "../live-agent/read-only-structured.js";
+import { changedPaths, snapshotWorkspace } from "../skill-evals/workspace.js";
+import { buildBlindPrompt } from "./blind-prompt.js";
+import { normalizeAgentOutput, normalizeStagedAgentOutput, OBSERVATION_SCHEMA, STAGED_OBSERVATION_SCHEMA, fail } from "./contracts.js";
+import { behaviorSourceText, sourceFingerprint } from "./source-fingerprint.js";
+
+export const ADAPTER_VERSION = "1.1.0";
+export const STAGED_ADAPTER_VERSION = "2.1.0";
+function runtimeVersion(surface) {
+  try { return execFileSync(surface, ["--version"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); }
+  catch { return `${surface} version unavailable`; }
+}
+export function persistObservation(path, observation, { replace = false } = {}) {
+  let previous;
+  try { previous = JSON.parse(readFileSync(path, "utf8")); }
+  catch (error) { if (error.code !== "ENOENT") throw error; }
+  if (previous && !replace) {
+    fail(`observation already exists: ${path}`, "PROPORTIONALITY_DUPLICATE");
+  }
+  const persisted = previous ? {
+    ...observation,
+    replacement_provenance: {
+      previous_observation_id: previous.observation_id ?? null,
+      previous_recorded_at: previous.recorded_at ?? null,
+      replaced_at: new Date().toISOString(),
+    },
+  } : observation;
+  mkdirSync(dirname(path), { recursive: true });
+  const temp = `${path}.${process.pid}.tmp`;
+  writeFileSync(temp, `${JSON.stringify(persisted, null, 2)}\n`, "utf8");
+  renameSync(temp, path);
+}
+export async function recordObservation({
+  repoRoot, testCase, fixture, seriesId, repeat, surface, model, agdfVersion, baselineVersion,
+  timeoutMs = 120000, execute = executeStructuredAgent, persistPath, replace = false,
+}) {
+  if (!/^[a-z0-9][a-z0-9._-]{2,79}$/i.test(seriesId) || !Number.isInteger(repeat) || repeat < 1) fail("invalid series or repeat");
+  if (!surface || !model || !agdfVersion || !baselineVersion) fail("surface, model, AGDF version and baseline version are required");
+  const staged = testCase.profile_id === "staged-v2";
+  const adapterVersion = staged ? STAGED_ADAPTER_VERSION : ADAPTER_VERSION;
+  const fingerprint = sourceFingerprint(repoRoot, testCase, fixture, adapterVersion);
+  const fixtureRoot = mkdtempSync(join(tmpdir(), "agdf-proportionality-"));
+  let output;
+  let executionError;
+  try {
+    writeFileSync(join(fixtureRoot, "TASK.md"), `${testCase.task_summary}\n`, "utf8");
+    writeFileSync(join(fixtureRoot, "CONTEXT.json"), `${JSON.stringify({
+      repository_context: testCase.repository_context,
+      control_state_context: testCase.control_state_context,
+    }, null, 2)}\n`, "utf8");
+    const before = snapshotWorkspace(fixtureRoot);
+    try {
+      output = await execute({
+        surface, cwd: fixtureRoot, model, timeoutMs, outputSchema: staged ? STAGED_OBSERVATION_SCHEMA : OBSERVATION_SCHEMA,
+        prompt: buildBlindPrompt(testCase, behaviorSourceText(repoRoot)),
+      });
+    } catch (error) { executionError = error; }
+    const after = snapshotWorkspace(fixtureRoot);
+    const mutations = changedPaths(before, after);
+    if (mutations.length) fail(`agent mutated fixture: ${mutations.join(", ")}`, "PROPORTIONALITY_MUTATION");
+    if (executionError) throw executionError;
+    const normalized = staged ? normalizeStagedAgentOutput(output, testCase.requested_axes) : normalizeAgentOutput(output);
+    const observation = {
+      schema_version: staged ? "2" : "1",
+      observation_id: `${seriesId}:${staged ? testCase.scenario_id : testCase.case_id}:${repeat}`,
+      case_id: testCase.case_id,
+      ...(staged ? {
+        profile_id: "staged-v2",
+        protocol_version: "2",
+        corpus_version: testCase.corpus_version,
+        fixture_version: testCase.fixture_version,
+        scenario_id: testCase.scenario_id,
+        lifecycle_stage: testCase.lifecycle_stage,
+      } : {}),
+      series_id: seriesId,
+      repeat,
+      evidence_kind: "live_agent_observation",
+      surface,
+      runtime_version: runtimeVersion(surface),
+      agdf_version: agdfVersion,
+      baseline_version: baselineVersion,
+      model,
+      adapter_version: adapterVersion,
+      source_fingerprint: fingerprint,
+      recorded_at: new Date().toISOString(),
+      ...normalized,
+      execution_status: "completed",
+      redaction_status: "pass",
+      mutation_status: "pass",
+    };
+    if (persistPath) persistObservation(persistPath, observation, { replace });
+    return observation;
+  } finally {
+    rmSync(fixtureRoot, { recursive: true, force: true });
+  }
+}