create-agdf 0.14.4 → 0.20.0

package/lib/proportionality-benchmark/evaluator.js

@@ -1,260 +0,0 @@
-import { DELIVERY_PATHS, PATH_RANK, STAGES, fail, stable } from "./contracts.js";
-import { sourceFingerprint } from "./source-fingerprint.js";
-import { ADAPTER_VERSION } from "./live-recorder.js";
-import { getProfileDefinition, isStagedProfile } from "./profiles.js";
-
-export function classifyObservation(expected, observation) {
-  if (observation.ambiguous || observation.observed_delivery_path === null) return "ambiguous";
-  const actual = PATH_RANK[observation.observed_delivery_path];
-  const target = PATH_RANK[expected];
-  if (actual === undefined || target === undefined) return "ambiguous";
-  if (actual === target) return "correct";
-  return actual < target ? "under_governance" : "over_governance";
-}
-export function classifyStagedPath(expected, observation) {
-  const legacy = classifyObservation(expected, observation);
-  return {
-    correct: "path_correct",
-    under_governance: "path_under_governance",
-    over_governance: "path_over_governance",
-    ambiguous: "path_ambiguous",
-  }[legacy];
-}
-export function classifyStage(expected, observation) {
-  if (observation.stage_evaluability !== "evaluated" || observation.observed_next_permissible_stage === null) return "stage_ambiguous";
-  const actual = observation.observed_next_permissible_stage;
-  if (!STAGES.includes(actual) || !STAGES.includes(expected)) return "stage_ambiguous";
-  if (actual === expected) return "stage_correct";
-  if (expected === "blocked") return "stage_unsafe_advance";
-  if (actual === "blocked") return "stage_over_governance";
-  if (expected === "ungated_execution") return "stage_over_governance";
-  if (actual === "ungated_execution") return "stage_unsafe_advance";
-  return STAGES.indexOf(actual) > STAGES.indexOf(expected) ? "stage_unsafe_advance" : "stage_over_governance";
-}
-function stagedCoverageResult(expected, rows, reason) {
-  return {
-    scenario_id: expected.scenario_id,
-    case_id: expected.case_id,
-    expected_next_permissible_stage: expected.expected_next_permissible_stage,
-    expected_delivery_path: expected.expected_delivery_path,
-    stage_required: expected.stage_required,
-    path_required: expected.path_required,
-    observations: rows,
-    stage_distribution: {},
-    path_distribution: {},
-    consensus_stage: null,
-    consensus_delivery_path: null,
-    stage_classifications: [],
-    path_classifications: [],
-    status: "block",
-    blocking_reasons: [reason],
-  };
-}
-function evaluateStagedSeries({ repoRoot, corpus, observations, minimumRepeats }) {
-  const profile = getProfileDefinition(corpus.profile_id);
-  const provenanceKeys = ["series_id", "profile_id", "protocol_version", "corpus_version", "fixture_version", "surface", "runtime_version", "model", "agdf_version", "baseline_version", "adapter_version", ...(profile.evidence_kind_provenance ? ["evidence_kind"] : []), ...(profile.series_profile_metadata ? ["runner_version"] : [])];
-  const first = observations[0] ?? {};
-  const observationIds = new Set();
-  const byScenario = new Map(corpus.cases.map((item) => [item.scenario_id, []]));
-  for (const observation of observations) {
-    if (!byScenario.has(observation.scenario_id) || observation.case_id !== observation.scenario_id.split(":")[0]) fail(`unknown staged observation ${observation.scenario_id ?? "unknown"}`);
-    if (observation.schema_version !== profile.schema_version || !profile.evidence_kinds.includes(observation.evidence_kind) || !Number.isInteger(observation.repeat) || observation.repeat < 1) fail(`invalid observation ${observation.observation_id ?? "unknown"}`);
-    if (!observation.observation_id || observationIds.has(observation.observation_id)) fail(`duplicate observation ${observation.observation_id ?? "unknown"}`);
-    if (observation.execution_status !== "completed" || observation.redaction_status !== "pass" || observation.mutation_status !== "pass") fail(`unsafe observation status ${observation.observation_id}`);
-    if (!Number.isFinite(Date.parse(observation.recorded_at))) fail(`invalid observation timestamp ${observation.observation_id}`);
-    const versionDrift = [
-      ["baseline_version", corpus.baseline.baseline_version],
-      ["corpus_version", corpus.manifest.corpus_version],
-      ["fixture_version", corpus.manifest.fixture_version],
-    ].find(([key, expected]) => observation[key] !== expected);
-    if (versionDrift) fail(`baseline provenance drift (${versionDrift[0]}) at ${observation.observation_id}`);
-    const provenanceDrift = provenanceKeys.find((key) => observation[key] !== first[key]);
-    if (provenanceDrift) fail(`series provenance drift (${provenanceDrift}) at ${observation.observation_id}`);
-    observationIds.add(observation.observation_id);
-    byScenario.get(observation.scenario_id).push(observation);
-  }
-  const scenarios = corpus.baseline.scenarios.map((expected) => {
-    const blind = corpus.cases.find((item) => item.scenario_id === expected.scenario_id);
-    const rows = byScenario.get(expected.scenario_id).sort((a, b) => a.repeat - b.repeat);
-    const repeats = new Set(rows.map((item) => item.repeat));
-    if (rows.length < minimumRepeats || repeats.size !== rows.length) return stagedCoverageResult(expected, rows, "coverage");
-    const fingerprint = sourceFingerprint(repoRoot, blind, corpus.fixtures, profile.adapter_version);
-    const stale = rows.some((item) => item.source_fingerprint !== fingerprint);
-    const invalidAxes = rows.some((item) => (
-      expected.stage_required !== (item.stage_evaluability === "evaluated")
-      || expected.path_required !== (item.path_evaluability === "evaluated")
-      || (expected.stage_required ? item.observed_next_permissible_stage === null : item.observed_next_permissible_stage !== null)
-      || (expected.path_required ? item.observed_delivery_path === null : item.observed_delivery_path !== null)
-    ));
-    const stageClassifications = expected.stage_required ? rows.map((item) => classifyStage(expected.expected_next_permissible_stage, item)) : [];
-    const pathClassifications = expected.path_required ? rows.map((item) => classifyStagedPath(expected.expected_delivery_path, { ...item, ambiguous: item.path_evaluability !== "evaluated" })) : [];
-    const stageValues = new Set(rows.map((item) => item.observed_next_permissible_stage));
-    const pathValues = new Set(rows.map((item) => item.observed_delivery_path));
-    const consensusStage = expected.stage_required && !stale && !invalidAxes && stageValues.size === 1 ? rows[0].observed_next_permissible_stage : null;
-    const consensusPath = expected.path_required && !stale && !invalidAxes && pathValues.size === 1 ? rows[0].observed_delivery_path : null;
-    const blocking = [];
-    if (stale) blocking.push("stale");
-    if (invalidAxes) blocking.push("axis_contract");
-    if (expected.stage_required && !consensusStage) blocking.push("mixed_or_ambiguous_stage");
-    if (expected.path_required && !consensusPath) blocking.push("mixed_or_ambiguous_path");
-    if (stageClassifications.includes("stage_unsafe_advance")) blocking.push("critical_stage_under_governance");
-    if (pathClassifications.includes("path_under_governance")) blocking.push("critical_path_under_governance");
-    if (stageClassifications.some((item) => item !== "stage_correct")) blocking.push("stage_deviation");
-    const deviations = pathClassifications.filter((item) => item !== "path_correct");
-    return {
-      scenario_id: expected.scenario_id,
-      case_id: expected.case_id,
-      lifecycle_stage: blind.lifecycle_stage,
-      expected_next_permissible_stage: expected.expected_next_permissible_stage,
-      expected_delivery_path: expected.expected_delivery_path,
-      stage_required: expected.stage_required,
-      path_required: expected.path_required,
-      observations: rows,
-      stage_distribution: Object.fromEntries(STAGES.map((stage) => [stage, rows.filter((item) => item.observed_next_permissible_stage === stage).length]).filter(([, count]) => count)),
-      path_distribution: Object.fromEntries(DELIVERY_PATHS.map((path) => [path, rows.filter((item) => item.observed_delivery_path === path).length]).filter(([, count]) => count)),
-      stage_classifications: stageClassifications,
-      path_classifications: pathClassifications,
-      consensus_stage: consensusStage,
-      consensus_delivery_path: consensusPath,
-      status: blocking.length ? "block" : deviations.length ? "deviation" : "correct",
-      blocking_reasons: blocking,
-    };
-  });
-  const criticalUnder = scenarios.filter((item) => item.blocking_reasons.some((reason) => reason.startsWith("critical_"))).map((item) => item.scenario_id);
-  const ambiguous = scenarios.filter((item) => item.blocking_reasons.some((reason) => ["coverage", "stale", "axis_contract", "mixed_or_ambiguous_stage", "mixed_or_ambiguous_path"].includes(reason))).map((item) => item.scenario_id);
-  const stageDeviations = scenarios.filter((item) => item.stage_classifications.some((classification) => classification !== "stage_correct")).map((item) => item.scenario_id);
-  const over = scenarios.filter((item) => item.path_classifications.includes("path_over_governance")).map((item) => item.scenario_id);
-  const smallScenarios = scenarios.filter((item) => item.path_required && ["trivial_change", "quick_task"].includes(item.expected_delivery_path));
-  const smallOver = smallScenarios.filter((item) => item.path_classifications.length && item.path_classifications.every((classification) => classification === "path_over_governance")).map((item) => item.scenario_id);
-  const smallOverPercent = smallScenarios.length ? smallOver.length / smallScenarios.length * 100 : 0;
-  const thresholdBlocks = [
-    ...(criticalUnder.length ? ["critical_under_governance"] : []),
-    ...(stageDeviations.length ? ["stage_deviation"] : []),
-    ...(smallOverPercent > corpus.baseline.constraints.over_governance_threshold_percent ? ["small_segment_over_governance"] : []),
-    ...(ambiguous.length ? ["ambiguous_or_incomplete"] : []),
-  ];
-  return stable({
-    schema_version: profile.schema_version,
-    profile_id: profile.profile_id,
-    protocol_version: profile.protocol_version,
-    evidence_boundary: corpus.manifest.evidence_boundary,
-    baseline_version: corpus.baseline.baseline_version,
-    corpus_version: corpus.manifest.corpus_version,
-    series_id: first.series_id ?? null,
-    surface: first.surface ?? null,
-    model: first.model ?? null,
-    agdf_version: first.agdf_version ?? null,
-    adapter_version: first.adapter_version ?? null,
-    runtime_version: first.runtime_version ?? null,
-    runner_version: profile.runner_version,
-    ...(profile.repository_replay_non_claim ? {
-      evidence_class: first.evidence_kind ?? "none",
-      report_version: profile.report_version,
-      authenticated_live_host_evidence: false,
-      live_host_non_claim: "No authenticated staged-v3 live-host series was executed or accepted by this repository replay.",
-    } : {}),
-    freshness_status: scenarios.some((item) => item.blocking_reasons.includes("stale")) ? "stale" : "fresh",
-    minimum_repeats: minimumRepeats,
-    valid_observations: observations.length,
-    case_count: corpus.baseline.cases.length,
-    scenario_count: scenarios.length,
-    stage_scenario_count: scenarios.filter((item) => item.stage_required).length,
-    path_scenario_count: scenarios.filter((item) => item.path_required).length,
-    critical_under_governance_ids: criticalUnder,
-    stage_deviation_ids: stageDeviations,
-    over_governance_ids: over,
-    small_segment_denominator: smallScenarios.length,
-    small_segment_over_governance_ids: smallOver,
-    small_segment_over_governance_percent: smallOverPercent,
-    ambiguous_ids: ambiguous,
-    blocking_reasons: thresholdBlocks,
-    status: thresholdBlocks.length ? "block" : "pass",
-    scenarios,
-  });
-}
-export function evaluateSeries({ repoRoot, corpus, observations, minimumRepeats = 3 }) {
-  if (!Number.isInteger(minimumRepeats) || minimumRepeats < 3) fail("minimum repeats must be at least 3");
-  if (isStagedProfile(getProfileDefinition(corpus.profile_id))) return evaluateStagedSeries({ repoRoot, corpus, observations, minimumRepeats });
-  const provenanceKeys = ["series_id", "surface", "runtime_version", "model", "agdf_version", "baseline_version", "adapter_version"];
-  const first = observations[0] ?? {};
-  const observationIds = new Set();
-  const byCase = new Map(corpus.cases.map((item) => [item.case_id, []]));
-  for (const observation of observations) {
-    if (!byCase.has(observation.case_id)) fail(`unknown observation case ${observation.case_id}`);
-    if (observation.evidence_kind !== "live_agent_observation" || !Number.isInteger(observation.repeat) || observation.repeat < 1) fail(`invalid observation ${observation.observation_id ?? "unknown"}`);
-    if (!observation.observation_id || observationIds.has(observation.observation_id)) fail(`duplicate observation ${observation.observation_id ?? "unknown"}`);
-    if (observation.execution_status !== "completed" || observation.redaction_status !== "pass" || observation.mutation_status !== "pass") fail(`unsafe observation status ${observation.observation_id}`);
-    if (!Number.isFinite(Date.parse(observation.recorded_at))) fail(`invalid observation timestamp ${observation.observation_id}`);
-    if (observation.baseline_version !== corpus.baseline.baseline_version) fail(`baseline provenance drift at ${observation.observation_id}`);
-    if (provenanceKeys.some((key) => observation[key] !== first[key])) fail(`series provenance drift at ${observation.observation_id}`);
-    observationIds.add(observation.observation_id);
-    byCase.get(observation.case_id).push(observation);
-  }
-  const fixture = corpus.fixtures.fixtures[corpus.manifest.fixture_id];
-  const cases = corpus.baseline.cases.map((expected) => {
-    const blind = corpus.cases.find((item) => item.case_id === expected.case_id);
-    const rows = byCase.get(expected.case_id).sort((a, b) => a.repeat - b.repeat);
-    const repeats = new Set(rows.map((item) => item.repeat));
-    if (rows.length < minimumRepeats || repeats.size !== rows.length) {
-      return { case_id: expected.case_id, expected_delivery_path: expected.expected_delivery_path, observations: rows, distribution: {}, consensus_delivery_path: null, status: "ambiguous", classifications: [], blocking_reasons: ["coverage"] };
-    }
-    const historicalFingerprint = corpus.provenance?.source_fingerprints?.[expected.case_id];
-    const currentFingerprint = historicalFingerprint ?? sourceFingerprint(repoRoot, blind, fixture, ADAPTER_VERSION);
-    const stale = rows.some((item) => item.source_fingerprint !== currentFingerprint);
-    const classifications = rows.map((item) => classifyObservation(expected.expected_delivery_path, item));
-    const distribution = Object.fromEntries(DELIVERY_PATHS.map((path) => [path, rows.filter((item) => item.observed_delivery_path === path).length]).filter(([, count]) => count));
-    const paths = new Set(rows.map((item) => item.observed_delivery_path));
-    const consensus = !stale && !rows.some((item) => item.ambiguous) && paths.size === 1 ? rows[0].observed_delivery_path : null;
-    const blocking = [];
-    if (stale) blocking.push("stale");
-    if (!consensus) blocking.push("mixed_or_ambiguous");
-    if (classifications.includes("under_governance")) blocking.push("critical_under_governance");
-    return {
-      case_id: expected.case_id,
-      expected_delivery_path: expected.expected_delivery_path,
-      adversarial: expected.adversarial,
-      observations: rows,
-      distribution,
-      classifications,
-      consensus_delivery_path: consensus,
-      status: blocking.length ? "block" : classifications.every((item) => item === "correct") ? "correct" : "over_governance",
-      blocking_reasons: blocking,
-    };
-  });
-  const smallCases = cases.filter((item) => ["trivial_change", "quick_task"].includes(item.expected_delivery_path));
-  const unanimousOverSmall = smallCases.filter((item) => item.consensus_delivery_path && item.classifications.every((value) => value === "over_governance")).length;
-  const smallOverRate = unanimousOverSmall / smallCases.length * 100;
-  const criticalUnderIds = cases.filter((item) => item.classifications.includes("under_governance")).map((item) => item.case_id);
-  const ambiguousIds = cases.filter((item) => item.blocking_reasons.some((item) => ["coverage", "stale", "mixed_or_ambiguous"].includes(item))).map((item) => item.case_id);
-  const thresholdBlocks = [
-    ...(criticalUnderIds.length ? ["critical_under_governance"] : []),
-    ...(smallOverRate > corpus.baseline.constraints.over_governance_threshold_percent ? ["small_segment_over_governance"] : []),
-    ...(ambiguousIds.length ? ["ambiguous_or_incomplete"] : []),
-  ];
-  return stable({
-    schema_version: "1",
-    evidence_boundary: "live routing observations with deterministic offline grading",
-    baseline_version: corpus.baseline.baseline_version,
-    corpus_version: corpus.manifest.corpus_version,
-    series_id: first.series_id ?? null,
-    surface: first.surface ?? null,
-    model: first.model ?? null,
-    agdf_version: first.agdf_version ?? null,
-    adapter_version: first.adapter_version ?? null,
-    runtime_version: first.runtime_version ?? null,
-    runner_version: "1.0.0",
-    freshness_status: cases.some((item) => item.blocking_reasons.includes("stale")) ? "stale" : corpus.provenance ? "historical" : "fresh",
-    minimum_repeats: minimumRepeats,
-    valid_observations: observations.length,
-    case_count: cases.length,
-    adversarial_case_count: cases.filter((item) => item.adversarial).length,
-    small_segment_denominator: smallCases.length,
-    unanimous_over_governance_small_cases: unanimousOverSmall,
-    small_segment_over_governance_percent: smallOverRate,
-    critical_under_governance_ids: criticalUnderIds,
-    ambiguous_ids: ambiguousIds,
-    blocking_reasons: thresholdBlocks,
-    status: thresholdBlocks.length ? "block" : "pass",
-    cases,
-  });
-}

package/lib/proportionality-benchmark/index.js

@@ -1,8 +0,0 @@
-export * from "./blind-prompt.js";
-export * from "./contracts.js";
-export * from "./corpus-loader.js";
-export * from "./evaluator.js";
-export * from "./live-recorder.js";
-export * from "./profiles.js";
-export * from "./report.js";
-export * from "./source-fingerprint.js";

package/lib/proportionality-benchmark/live-recorder.js

@@ -1,122 +0,0 @@
-import { execFileSync } from "node:child_process";
-import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
-import { executeStructuredAgent } from "../live-agent/read-only-structured.js";
-import { changedPaths, snapshotWorkspace } from "../skill-evals/workspace.js";
-import { buildBlindPrompt } from "./blind-prompt.js";
-import { normalizeAgentOutput, normalizeStagedAgentOutput, OBSERVATION_SCHEMA, fail, stagedSchemaForVersion } from "./contracts.js";
-import { getProfileDefinition, isStagedProfile } from "./profiles.js";
-import { behaviorSourceText, sourceFingerprint } from "./source-fingerprint.js";
-
-export const ADAPTER_VERSION = getProfileDefinition("legacy-v1").adapter_version;
-export const STAGED_ADAPTER_VERSION = getProfileDefinition("staged-v2").adapter_version;
-export const STAGED_V3_ADAPTER_VERSION = getProfileDefinition("staged-v3").adapter_version;
-export function isRetryableObservationError(error) {
-  return error?.code === "GENERATOR_TIMEOUT";
-}
-export function observationAttemptFailure(error, attemptCount, attemptLimit) {
-  const messages = {
-    GENERATOR_TIMEOUT: "read-only process timed out",
-    PROPORTIONALITY_OUTPUT_INVALID: "agent output failed the structured contract",
-    PROPORTIONALITY_REDACTION_FAILED: "agent output failed redaction",
-    PROPORTIONALITY_MUTATION: "agent mutated the disposable fixture",
-  };
-  return {
-    status: "invalid",
-    code: error?.code ?? "EXECUTION_ERROR",
-    message: messages[error?.code] ?? "agent execution failed",
-    retryable: isRetryableObservationError(error),
-    remaining_attempt_budget: Math.max(0, attemptLimit - attemptCount),
-  };
-}
-function runtimeVersion(surface) {
-  try { return execFileSync(surface, ["--version"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); }
-  catch { return `${surface} version unavailable`; }
-}
-export function persistObservation(path, observation, { replace = false } = {}) {
-  let previous;
-  try { previous = JSON.parse(readFileSync(path, "utf8")); }
-  catch (error) { if (error.code !== "ENOENT") throw error; }
-  if (previous && !replace) {
-    fail(`observation already exists: ${path}`, "PROPORTIONALITY_DUPLICATE");
-  }
-  const persisted = previous ? {
-    ...observation,
-    replacement_provenance: {
-      previous_observation_id: previous.observation_id ?? null,
-      previous_recorded_at: previous.recorded_at ?? null,
-      replaced_at: new Date().toISOString(),
-    },
-  } : observation;
-  mkdirSync(dirname(path), { recursive: true });
-  const temp = `${path}.${process.pid}.tmp`;
-  writeFileSync(temp, `${JSON.stringify(persisted, null, 2)}\n`, "utf8");
-  renameSync(temp, path);
-}
-export async function recordObservation({
-  repoRoot, testCase, fixture, seriesId, repeat, surface, model, agdfVersion, baselineVersion,
-  timeoutMs = 120000, execute = executeStructuredAgent, persistPath, replace = false,
-}) {
-  if (!/^[a-z0-9][a-z0-9._-]{2,79}$/i.test(seriesId) || !Number.isInteger(repeat) || repeat < 1) fail("invalid series or repeat");
-  if (!surface || !model || !agdfVersion || !baselineVersion) fail("surface, model, AGDF version and baseline version are required");
-  const profile = getProfileDefinition(testCase.profile_id ?? "legacy-v1");
-  const staged = isStagedProfile(profile);
-  const adapterVersion = profile.adapter_version;
-  const fingerprint = sourceFingerprint(repoRoot, testCase, fixture, adapterVersion);
-  const fixtureRoot = mkdtempSync(join(tmpdir(), "agdf-proportionality-"));
-  let output;
-  let executionError;
-  try {
-    writeFileSync(join(fixtureRoot, "TASK.md"), `${testCase.task_summary}\n`, "utf8");
-    writeFileSync(join(fixtureRoot, "CONTEXT.json"), `${JSON.stringify({
-      repository_context: testCase.repository_context,
-      control_state_context: testCase.control_state_context,
-    }, null, 2)}\n`, "utf8");
-    const before = snapshotWorkspace(fixtureRoot);
-    try {
-      output = await execute({
-        surface, cwd: fixtureRoot, model, timeoutMs, outputSchema: staged ? stagedSchemaForVersion(profile.schema_version) : OBSERVATION_SCHEMA,
-        prompt: buildBlindPrompt(testCase, behaviorSourceText(repoRoot)),
-      });
-    } catch (error) { executionError = error; }
-    const after = snapshotWorkspace(fixtureRoot);
-    const mutations = changedPaths(before, after);
-    if (mutations.length) fail(`agent mutated fixture: ${mutations.join(", ")}`, "PROPORTIONALITY_MUTATION");
-    if (executionError) throw executionError;
-    const normalized = staged ? normalizeStagedAgentOutput(output, testCase.requested_axes, profile.schema_version) : normalizeAgentOutput(output);
-    const observation = {
-      schema_version: profile.schema_version,
-      observation_id: `${seriesId}:${staged ? testCase.scenario_id : testCase.case_id}:${repeat}`,
-      case_id: testCase.case_id,
-      ...(staged ? {
-        profile_id: profile.profile_id,
-        protocol_version: profile.protocol_version,
-        corpus_version: testCase.corpus_version,
-        fixture_version: testCase.fixture_version,
-        scenario_id: testCase.scenario_id,
-        lifecycle_stage: testCase.lifecycle_stage,
-        ...(profile.series_profile_metadata ? { runner_version: profile.runner_version } : {}),
-      } : {}),
-      series_id: seriesId,
-      repeat,
-      evidence_kind: "live_agent_observation",
-      surface,
-      runtime_version: runtimeVersion(surface),
-      agdf_version: agdfVersion,
-      baseline_version: baselineVersion,
-      model,
-      adapter_version: adapterVersion,
-      source_fingerprint: fingerprint,
-      recorded_at: new Date().toISOString(),
-      ...normalized,
-      execution_status: "completed",
-      redaction_status: "pass",
-      mutation_status: "pass",
-    };
-    if (persistPath) persistObservation(persistPath, observation, { replace });
-    return observation;
-  } finally {
-    rmSync(fixtureRoot, { recursive: true, force: true });
-  }
-}

package/lib/proportionality-benchmark/profiles.js

@@ -1,102 +0,0 @@
-const definitions = {
-  "legacy-v1": {
-    profile_id: "legacy-v1",
-    family: "legacy",
-    manifest_path: "evals/proportionality/manifest.json",
-    schema_version: "1",
-    protocol_version: "1",
-    adapter_version: "1.1.0",
-    runner_version: "1.0.0",
-    report_version: "1.0.0",
-    observation_key: "case_id",
-    fixture_strategy: "catalog_fixture",
-    evidence_kinds: ["live_agent_observation"],
-    evidence_kind_provenance: false,
-    history_provenance: false,
-    semantic_fact_validation: false,
-    fingerprint_profile_metadata: false,
-    repository_replay_non_claim: false,
-    strict_version_links: false,
-    series_profile_metadata: false,
-  },
-  "staged-v2": {
-    profile_id: "staged-v2",
-    family: "staged",
-    manifest_path: "evals/proportionality/staged-manifest.json",
-    schema_version: "2",
-    protocol_version: "2",
-    adapter_version: "2.1.0",
-    runner_version: "2.0.0",
-    report_version: "2.0.0",
-    observation_key: "scenario_id",
-    fixture_strategy: "staged_catalog",
-    evidence_kinds: ["live_agent_observation"],
-    evidence_kind_provenance: false,
-    history_provenance: false,
-    semantic_fact_validation: false,
-    fingerprint_profile_metadata: false,
-    repository_replay_non_claim: false,
-    strict_version_links: false,
-    series_profile_metadata: false,
-  },
-  "staged-v3": {
-    profile_id: "staged-v3",
-    family: "staged",
-    manifest_path: "evals/proportionality/staged-v3-manifest.json",
-    schema_version: "3",
-    protocol_version: "3",
-    adapter_version: "3.0.0",
-    runner_version: "3.0.0",
-    report_version: "3.0.0",
-    observation_key: "scenario_id",
-    fixture_strategy: "staged_catalog",
-    evidence_kinds: ["synthetic_replay", "live_agent_observation"],
-    evidence_kind_provenance: true,
-    history_provenance: true,
-    semantic_fact_validation: true,
-    fingerprint_profile_metadata: true,
-    repository_replay_non_claim: true,
-    strict_version_links: true,
-    series_profile_metadata: true,
-  },
-};
-
-function deepFreeze(value) {
-  if (value && typeof value === "object" && !Object.isFrozen(value)) {
-    Object.freeze(value);
-    for (const item of Object.values(value)) deepFreeze(item);
-  }
-  return value;
-}
-
-export const PROPORTIONALITY_PROFILES = deepFreeze(definitions);
-export const SUPPORTED_PROFILE_IDS = Object.freeze(Object.keys(PROPORTIONALITY_PROFILES));
-
-export function getProfileDefinition(profileId) {
-  const definition = PROPORTIONALITY_PROFILES[profileId];
-  if (!definition) throw Object.assign(new Error(`unknown proportionality profile ${profileId}`), { code: "PROPORTIONALITY_PROFILE_UNKNOWN" });
-  return definition;
-}
-
-export function isStagedProfile(profileOrId) {
-  const definition = typeof profileOrId === "string" ? getProfileDefinition(profileOrId) : profileOrId;
-  return definition.family === "staged";
-}
-
-export function observationKey(profileOrId, testCase) {
-  const definition = typeof profileOrId === "string" ? getProfileDefinition(profileOrId) : profileOrId;
-  const value = testCase[definition.observation_key];
-  if (!value) throw new Error(`missing ${definition.observation_key} for ${definition.profile_id}`);
-  return definition.observation_key === "scenario_id" ? value.replaceAll(":", "__") : value;
-}
-
-export function fixtureForProfile(profileOrId, corpus) {
-  const definition = typeof profileOrId === "string" ? getProfileDefinition(profileOrId) : profileOrId;
-  return definition.fixture_strategy === "staged_catalog"
-    ? corpus.fixtures
-    : corpus.fixtures.fixtures[corpus.manifest.fixture_id];
-}
-
-export function profileUsage() {
-  return SUPPORTED_PROFILE_IDS.join("|");
-}

package/lib/proportionality-benchmark/report.js

@@ -1,60 +0,0 @@
-import { getProfileDefinition, isStagedProfile } from "./profiles.js";
-
-export function renderMarkdown(report) {
-  if (report.profile_id && isStagedProfile(getProfileDefinition(report.profile_id))) {
-    const lines = [
-      "# AGDF Staged Proportionality Observation",
-      "",
-      `- Status: \`${report.status}\``,
-      `- Evidenzgrenze: ${report.evidence_boundary}`,
-      `- Profil/Protokoll: \`${report.profile_id}\` / \`${report.protocol_version}\``,
-      ...(report.evidence_class ? [
-        `- Corpus/Baseline: \`${report.corpus_version}\` / \`${report.baseline_version}\``,
-        `- Report-Version: \`${report.report_version}\``,
-        `- Evidenzklasse: \`${report.evidence_class}\``,
-        `- Authentifizierte Live-Host-Evidenz: \`${report.authenticated_live_host_evidence}\``,
-        `- Live-Host-Nichtbehauptung: ${report.live_host_non_claim}`,
-      ] : []),
-      `- Serie: \`${report.series_id ?? "none"}\``,
-      `- Surface/Modell: \`${report.surface ?? "none"}\` / \`${report.model ?? "none"}\``,
-      `- Runtime/AGDF/Adapter/Runner: \`${report.runtime_version ?? "none"}\` / \`${report.agdf_version ?? "none"}\` / \`${report.adapter_version ?? "none"}\` / \`${report.runner_version}\``,
-      `- Freshness: \`${report.freshness_status}\``,
-      `- Beobachtungen: ${report.valid_observations}`,
-      `- Coverage: ${report.scenarios.filter((item) => !item.blocking_reasons.includes("coverage")).length}/${report.scenario_count} Szenarien aus ${report.case_count} Fällen`,
-      `- Stage-/Pfad-Szenarien: ${report.stage_scenario_count}/${report.path_scenario_count}`,
-      `- Kritische Under-Governance: ${report.critical_under_governance_ids.length}`,
-      `- Stage-Abweichungen: ${report.stage_deviation_ids.length}`,
-      `- Pfad-Over-Governance: ${report.over_governance_ids.length}`,
-      `- Small-Segment Over-Governance: ${report.small_segment_over_governance_ids.length}/${report.small_segment_denominator} (${report.small_segment_over_governance_percent} %)`,
-      `- Ambiguous/Stale/Incomplete: ${report.ambiguous_ids.length}`,
-      "",
-      "| Szenario | Lifecycle | Soll-Stage | Stage-Konsens | Soll-Pfad | Pfad-Konsens | Status |",
-      "|---|---|---|---|---|---|---|",
-    ];
-    for (const item of report.scenarios) lines.push(`| ${item.scenario_id} | ${item.lifecycle_stage} | ${item.expected_next_permissible_stage ?? "n/a"} | ${item.consensus_stage ?? "n/a"} | ${item.expected_delivery_path ?? "n/a"} | ${item.consensus_delivery_path ?? "n/a"} | ${item.status} |`);
-    return `${lines.join("\n")}\n`;
-  }
-  const lines = [
-    "# AGDF Proportionality Benchmark",
-    "",
-    `- Status: \`${report.status}\``,
-    `- Evidenzgrenze: ${report.evidence_boundary}`,
-    `- Serie: \`${report.series_id ?? "none"}\``,
-    `- Surface/Modell: \`${report.surface ?? "none"}\` / \`${report.model ?? "none"}\``,
-    `- Runtime/AGDF/Adapter/Runner: \`${report.runtime_version ?? "none"}\` / \`${report.agdf_version ?? "none"}\` / \`${report.adapter_version ?? "none"}\` / \`${report.runner_version}\``,
-    `- Freshness: \`${report.freshness_status}\``,
-    `- Beobachtungen: ${report.valid_observations}`,
-    `- Coverage: ${report.cases.filter((item) => !item.blocking_reasons.includes("coverage")).length}/${report.case_count}`,
-    `- Kritische Under-Governance: ${report.critical_under_governance_ids.length}`,
-    `- Small-Segment Over-Governance: ${report.unanimous_over_governance_small_cases}/${report.small_segment_denominator} (${report.small_segment_over_governance_percent} %)`,
-    `- Ambiguous/Stale/Incomplete: ${report.ambiguous_ids.length}`,
-    "",
-    "| Case | Soll | Verteilung | Konsens | Status |",
-    "|---|---|---|---|---|",
-  ];
-  for (const item of report.cases) {
-    const distribution = Object.entries(item.distribution).map(([path, count]) => `${path}:${count}`).join(", ") || "none";
-    lines.push(`| ${item.case_id} | ${item.expected_delivery_path} | ${distribution} | ${item.consensus_delivery_path ?? "none"} | ${item.status} |`);
-  }
-  return `${lines.join("\n")}\n`;
-}

package/lib/proportionality-benchmark/source-fingerprint.js

@@ -1,34 +0,0 @@
-import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { stable } from "./contracts.js";
-import { getProfileDefinition } from "./profiles.js";
-
-export const BEHAVIOR_SOURCES = Object.freeze([
-  "plugin/meta/agdf-agent-router.md",
-  "plugin/meta/contracts/modes.md",
-  "plugin/meta/contracts/gate-transition.md",
-  "plugin/meta/contracts/interaction.md",
-  "plugin/skills/gate-check/SKILL.md",
-]);
-export const IMPLEMENTATION_SOURCES = Object.freeze([
-  "create-agdf/lib/live-agent/read-only-structured.js",
-  "create-agdf/lib/proportionality-benchmark/blind-prompt.js",
-  "create-agdf/lib/proportionality-benchmark/contracts.js",
-  "create-agdf/lib/proportionality-benchmark/corpus-loader.js",
-  "create-agdf/lib/proportionality-benchmark/evaluator.js",
-  "create-agdf/lib/proportionality-benchmark/live-recorder.js",
-  "create-agdf/lib/proportionality-benchmark/source-fingerprint.js",
-]);
-export function behaviorSourceText(repoRoot) {
-  return BEHAVIOR_SOURCES.map((path) => `--- ${path} ---\n${readFileSync(join(repoRoot, path), "utf8")}`).join("\n\n");
-}
-export function sourceFingerprint(repoRoot, testCase, fixture, adapterVersion) {
-  const hash = createHash("sha256");
-  const selectedProfile = testCase.profile_id ? getProfileDefinition(testCase.profile_id) : null;
-  const profile = selectedProfile?.fingerprint_profile_metadata ? selectedProfile : null;
-  hash.update(JSON.stringify(stable(profile ? { testCase, fixture, adapterVersion, profile } : { testCase, fixture, adapterVersion })));
-  const sources = profile ? [...BEHAVIOR_SOURCES, ...IMPLEMENTATION_SOURCES, "create-agdf/lib/proportionality-benchmark/profiles.js"] : [...BEHAVIOR_SOURCES, ...IMPLEMENTATION_SOURCES];
-  for (const path of sources) hash.update(path).update(readFileSync(join(repoRoot, path)));
-  return hash.digest("hex");
-}