chainlesschain 0.166.48 → 0.166.58

package/src/harness/prompt-compressor.js

@@ -315,7 +315,11 @@ export class PromptCompressor {
       !options.maxMessages &&
       !options.maxTokens
     ) {
-      const ctxWindow = getContextWindow(options.model, options.provider);
+      const ctxWindow = getContextWindow(
+        options.model,
+        options.provider,
+        options,
+      );
       const adaptive = adaptiveThresholds(ctxWindow);
       this.maxMessages = adaptive.maxMessages;
       this.maxTokens = adaptive.maxTokens;
@@ -337,8 +341,8 @@ export class PromptCompressor {
     );
   }
 
-  adaptToModel(model, provider) {
-    const ctxWindow = getContextWindow(model, provider);
+  adaptToModel(model, provider, options = {}) {
+    const ctxWindow = getContextWindow(model, provider, options);
     const adaptive = adaptiveThresholds(ctxWindow);
     this.maxMessages = adaptive.maxMessages;
     this.maxTokens = adaptive.maxTokens;
@@ -483,8 +487,12 @@ export class PromptCompressor {
   }
 
   shouldAutoCompact(messages) {
+    // Large adaptive windows use token pressure, not an arbitrary 50-message
+    // trigger. Small local models and explicit fixed limits retain their
+    // count-based policy; maxMessages remains a compaction target in all modes.
     return (
-      messages.length > this.maxMessages ||
+      ((!this._adaptive || this._contextWindow < 128000) &&
+        messages.length > this.maxMessages) ||
       estimateMessagesTokens(messages) > this.maxTokens
     );
   }

package/src/lib/diagnostic-excerpt.js

@@ -0,0 +1,51 @@
+/** Retain source context for several errors; an error line is not a root cause. */
+export function diagnosticExcerpt(value, limit = 1600) {
+  const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
+  if (text.length <= limit) return text;
+  const marker = "\n[... omitted ...]\n";
+  const lines = text.split("\n");
+  const errors = [];
+  const workers = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (
+      /\b\w*Error:|Exception in thread|Caused by:|\bpanic:|panicked at|\bFAIL\b|error TS\d+|##\[error\]/.test(
+        lines[i],
+      )
+    )
+      errors.push(i);
+    if (
+      /\[vitest-pool\]|Worker.*(?:error|exited)|Unhandled (?:Error|Rejection)/i.test(
+        lines[i],
+      )
+    )
+      workers.push(i);
+  }
+  // Keep both an early diagnostic (possibly expected negative-test output)
+  // and later runner failures. Never infer causality from their ordering.
+  const indices = [
+    ...new Set(
+      [workers[0], errors[0], errors.at(-1)].filter((i) => i !== undefined),
+    ),
+  ].sort((a, b) => a - b);
+  if (!indices.length) {
+    const head = Math.floor((limit - marker.length) / 2);
+    return (
+      text.slice(0, head) + marker + text.slice(-(limit - marker.length - head))
+    );
+  }
+  const available = limit - marker.length * (indices.length + 1);
+  const head = Math.floor(available * 0.1);
+  const tail = Math.floor(available * 0.15);
+  const perWindow = Math.floor((available - head - tail) / indices.length);
+  const windows = indices.map((index) => {
+    const before = lines
+      .slice(Math.max(0, index - 2), index)
+      .join("\n")
+      .slice(-Math.floor(perWindow * 0.2));
+    return (before + "\n" + lines.slice(index, index + 10).join("\n")).slice(
+      0,
+      perWindow,
+    );
+  });
+  return [text.slice(0, head), ...windows, text.slice(-tail)].join(marker);
+}

package/src/lib/diagnostic-workflow.js

@@ -0,0 +1,10 @@
+// Shared across ordinary debugging and recovery. Tool observations establish
+// process outcomes; the model still has to validate the causal hypothesis.
+export const DIAGNOSTIC_WORKFLOW_GUIDANCE =
+  "For debugging, retain the original failure signature (error, source location, stack, command, environment and revision when verified). " +
+  "Label each observation by its source and reproduction scope. A passing subset is not a successful full reproduction; a different error is a separate finding until evidence connects it to the original failure. " +
+  "Expected negative-test output, dependency/setup failures, failed assertions and runner/infrastructure failures are different possibilities. Inspect surrounding output and process outcomes before selecting one. Timing alone does not identify the cause, and argument order is not execution order. " +
+  "State one hypothesis and the smallest check that can disprove it; after that check, update the hypothesis from the observed result instead of restarting discovery. " +
+  "Do not change dependencies while a reproduction using them is running. Poll background tasks by task_id and drain has_more_output after exit. " +
+  "Bind reports to verified paths, runs and attempts; do not substitute a local or stale report for the target run. A green report alone does not override a nonzero exit, unhandled errors or incomplete output. " +
+  "A fix requires validation against the original failure and relevant regression checks. Otherwise report what was reproduced, what differs and the specific missing evidence; do not claim the original issue is fixed.";

package/src/lib/eval/evidence.js

@@ -146,6 +146,10 @@ export function createEvalHistoryRecord(
       executionSucceeded: result.executionSucceeded,
       agentOk: result.agentOk,
       executionEvidence: result.executionEvidence,
+      usage: result.usage ?? null,
+      totalCostUsd: Number.isFinite(result.totalCostUsd)
+        ? result.totalCostUsd
+        : null,
       error:
         result.error == null
           ? null
@@ -209,7 +213,19 @@ function validateComparison(value) {
   }
 }
 
-function recordIssues(record, now, maxAgeMs) {
+export function evalExecutionSucceeded(result) {
+  const execution = result?.executionEvidence;
+  return (
+    result?.executionSucceeded === true &&
+    result.error === null &&
+    execution?.protocol === EVAL_EXECUTION_PROTOCOL &&
+    execution.exitCode === 0 &&
+    execution.signal === null &&
+    execution.terminalVerified === true
+  );
+}
+
+export function evalHistoryRecordIssues(record, now, maxAgeMs) {
   if (!record || typeof record !== "object" || Array.isArray(record))
     return ["invalid_history_record"];
   const issues = [];
@@ -247,14 +263,7 @@ function recordIssues(record, now, maxAgeMs) {
       result.pass !== (result.artifactCheckPassed && result.executionSucceeded)
     )
       issues.push("inconsistent_task_verdict");
-    if (
-      !result.executionSucceeded ||
-      result.error !== null ||
-      execution?.protocol !== EVAL_EXECUTION_PROTOCOL ||
-      execution.exitCode !== 0 ||
-      execution.signal !== null ||
-      execution.terminalVerified !== true
-    )
+    if (!evalExecutionSucceeded(result))
       issues.push("execution_not_successful");
     if (execution?.observedFallback !== false)
       issues.push("fallback_or_missing_model_continuity");
@@ -296,7 +305,7 @@ export function evaluateStrictEvalGate(
   if (!Array.isArray(runs) || runs.length < 2) reasons.push("missing_baseline");
   const pair = Array.isArray(runs) ? runs.slice(-2) : [];
   for (const record of pair)
-    reasons.push(...recordIssues(record, now, maxAgeMs));
+    reasons.push(...evalHistoryRecordIssues(record, now, maxAgeMs));
   if (pair.length === 2) {
     const [baseline, candidate] = pair;
     if (

package/src/lib/eval/outcomes.js

@@ -0,0 +1,373 @@
+import {
+  evalDigest,
+  evalHistoryRecordIssues,
+  evalExecutionSucceeded,
+} from "./evidence.js";
+
+export const OUTCOME_PLAN_SCHEMA = "chainlesschain.outcome-plan/v1";
+export const OUTCOME_REPORT_SCHEMA = "chainlesschain.outcome-report/v1";
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const CATEGORIES = new Set([
+  "repair",
+  "support",
+  "compatibility",
+  "release",
+  "test-infrastructure",
+  "construction",
+]);
+const STAGES = ["install", "configure", "authenticate", "tool", "artifact"];
+const FAILURE_CAUSES = new Set([
+  "install",
+  "configuration-identity",
+  "model-protocol",
+  "tools-permissions",
+  "retrieval-context",
+  "recovery",
+  "artifact",
+  "environment",
+  "budget",
+  "manual-repair",
+]);
+const nonempty = (value) =>
+  typeof value === "string" && value.trim().length > 0;
+const nonnegative = (value) => Number.isFinite(value) && value >= 0;
+const count = (value) => Number.isSafeInteger(value) && value >= 0;
+const rate = (numerator, denominator) =>
+  denominator ? numerator / denominator : null;
+function assert(value, message) {
+  if (!value) throw new Error(message);
+}
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (!value || typeof value !== "object") return value;
+  return Object.fromEntries(
+    Object.keys(value)
+      .sort()
+      .map((key) => [key, canonical(value[key])]),
+  );
+}
+export function outcomeDigest(value) {
+  return evalDigest(JSON.stringify(canonical(value)));
+}
+function epoch(value) {
+  const time = Date.parse(value);
+  return typeof value === "string" &&
+    Number.isFinite(time) &&
+    new Date(time).toISOString() === value
+    ? time
+    : NaN;
+}
+function uniqueIndex(rows, key, label) {
+  assert(Array.isArray(rows), `${label} must be an array`);
+  const index = new Map();
+  for (const row of rows) {
+    assert(
+      row && nonempty(row[key]) && !index.has(row[key]),
+      `${label} contains a missing or duplicate ${key}`,
+    );
+    index.set(row[key], row);
+  }
+  return index;
+}
+
+export function validateOutcomePlan(plan) {
+  assert(plan?.schema === OUTCOME_PLAN_SCHEMA, "invalid outcome plan schema");
+  assert(
+    /^[a-f0-9]{40}$/u.test(plan.commitSha || ""),
+    "plan requires an exact commit SHA",
+  );
+  assert(/^[A-Z]{3}$/u.test(plan.currency || ""), "plan requires one currency");
+  assert(
+    epoch(plan.window?.start) < epoch(plan.window?.end),
+    "invalid observation window",
+  );
+  const samples = uniqueIndex(plan.samples, "id", "plan samples");
+  assert(
+    samples.size > 0 && samples.size <= 10000,
+    "plan requires 1–10000 samples",
+  );
+  for (const sample of samples.values()) {
+    assert(["task", "first-run"].includes(sample.kind), "invalid sample kind");
+    assert(
+      nonempty(sample.taskId) && nonempty(sample.stratum),
+      "sample requires taskId and stratum",
+    );
+    assert(
+      DIGEST.test(sample.comparisonDigest || ""),
+      "sample requires an Eval comparison digest",
+    );
+    assert(
+      nonnegative(sample.costBudget) &&
+        Number.isFinite(sample.timeBudgetMs) &&
+        sample.timeBudgetMs > 0,
+      "sample requires finite cost and time budgets",
+    );
+  }
+  return samples;
+}
+
+function summarize(rows) {
+  const succeeded = rows.filter((row) => row.succeeded).length;
+  const observed = rows.filter((row) => row.observed).length;
+  const knownCost = rows.reduce((sum, row) => sum + (row.cost ?? 0), 0);
+  const costsComplete = rows.every((row) => row.observed && row.cost !== null);
+  return {
+    planned: rows.length,
+    observed,
+    missing: rows.length - observed,
+    succeeded,
+    successRate: rate(succeeded, rows.length),
+    knownCost,
+    totalCost: costsComplete ? knownCost : null,
+    costPerSuccess: costsComplete && succeeded ? knownCost / succeeded : null,
+    retries: rows.every((row) => row.retries !== null)
+      ? rows.reduce((sum, row) => sum + row.retries, 0)
+      : null,
+    manualRepairs: rows.every((row) => row.manualRepairs !== null)
+      ? rows.reduce((sum, row) => sum + row.manualRepairs, 0)
+      : null,
+  };
+}
+
+/** Read-only measurement. Declarations and local Eval files are not attestations. */
+export function buildOutcomeReport(
+  { plan, history, observations = [], maintenance = null },
+  { now = Date.now(), expectedPlanDigest } = {},
+) {
+  const samples = validateOutcomePlan(plan);
+  assert(Number.isFinite(now), "invalid report time");
+  const planDigest = outcomeDigest(plan);
+  assert(
+    !expectedPlanDigest || expectedPlanDigest === planDigest,
+    "frozen plan digest mismatch",
+  );
+  const observed = uniqueIndex(observations, "sampleId", "observations");
+  for (const id of observed.keys())
+    assert(samples.has(id), "observation is outside the frozen plan");
+  const runs = uniqueIndex(history?.runs ?? [], "runId", "Eval history");
+  const issues = (history?.issues ?? []).map((issue) => issue.code);
+  if (epoch(plan.window.end) > now)
+    issues.push("observation_window_not_closed");
+  const usedEvidence = new Set();
+  const rows = [];
+  for (const sample of samples.values()) {
+    const observation = observed.get(sample.id);
+    const reasons = [];
+    const row = {
+      sampleId: sample.id,
+      kind: sample.kind,
+      taskId: sample.taskId,
+      stratum: sample.stratum,
+      observed: Boolean(observation),
+      succeeded: false,
+      reasons,
+      cost: null,
+      elapsedMs: null,
+      retries: null,
+      manualRepairs: null,
+      failureCause: null,
+      firstRunCompletedStages: [],
+    };
+    rows.push(row);
+    if (!observation) {
+      reasons.push("missing_observation");
+      issues.push(`${sample.id}:missing_observation`);
+      continue;
+    }
+    for (const key of ["cost", "elapsedMs", "retries", "manualRepairs"]) {
+      const valid = ["retries", "manualRepairs"].includes(key)
+        ? count(observation[key])
+        : nonnegative(observation[key]);
+      if (valid) row[key] = observation[key];
+      else reasons.push(`missing_or_invalid_${key}`);
+    }
+    const observedAt = epoch(observation.observedAt);
+    if (!(
+      observedAt <= Math.min(now, epoch(plan.window.end)) &&
+      observedAt - (row.elapsedMs ?? 0) >= epoch(plan.window.start)
+    ))
+      reasons.push("observation_outside_window");
+    const run = runs.get(observation.runId);
+    const stoppedBeforeTask =
+      sample.kind === "first-run" &&
+      observation.runId === null &&
+      observation.firstRun?.stages?.tool?.passed === false;
+    const evidenceKey = JSON.stringify([observation.runId, sample.taskId]);
+    if (observation.runId !== null) {
+      assert(
+        !usedEvidence.has(evidenceKey),
+        "a task execution cannot count as multiple samples",
+      );
+      usedEvidence.add(evidenceKey);
+    }
+    if (!run && !stoppedBeforeTask) reasons.push("missing_eval_run");
+    if (run) {
+      // Failed executions are valid observations, not successful outcomes.
+      // All other existing Eval integrity rules remain authoritative.
+      const integrity = evalHistoryRecordIssues(
+        run,
+        now,
+        Math.max(1, now - epoch(plan.window.start)),
+      ).filter((reason) => reason !== "execution_not_successful");
+      reasons.push(...integrity);
+      if (run.label !== plan.commitSha) reasons.push("commit_mismatch");
+      if (outcomeDigest(run.comparison) !== sample.comparisonDigest)
+        reasons.push("comparison_mismatch");
+      if (!(
+        epoch(run.ranAt) >= epoch(plan.window.start) &&
+        epoch(run.ranAt) <= epoch(plan.window.end)
+      ))
+        reasons.push("run_outside_window");
+    }
+    const result = run?.results?.find((item) => item?.id === sample.taskId);
+    if (!result && !stoppedBeforeTask) reasons.push("missing_task_result");
+    if (result && (!nonnegative(result.ms) || row.elapsedMs < result.ms))
+      reasons.push("invalid_elapsed_time");
+    if (sample.kind === "first-run") {
+      if (typeof observation.firstRun?.cleanEnvironment !== "boolean")
+        reasons.push("missing_clean_environment_evidence");
+      for (const stage of STAGES) {
+        const evidence = observation.firstRun?.stages?.[stage];
+        if (
+          !evidence ||
+          typeof evidence.passed !== "boolean" ||
+          !nonempty(evidence.receipt)
+        )
+          reasons.push(`missing_stage_evidence:${stage}`);
+        else if (evidence.passed) row.firstRunCompletedStages.push(stage);
+      }
+      // A later completed stage without its prerequisites is not a valid funnel.
+      if (
+        row.firstRunCompletedStages.some(
+          (stage, index) => stage !== STAGES[index],
+        )
+      )
+        reasons.push("inconsistent_first_run_stages");
+    }
+    for (const reason of reasons) issues.push(`${sample.id}:${reason}`);
+    const successfulExecution =
+      result?.pass === true &&
+      result.artifactCheckPassed === true &&
+      evalExecutionSucceeded(result) &&
+      result.executionEvidence.observedFallback === false;
+    if (!successfulExecution) reasons.push("task_not_successful");
+    if (row.elapsedMs > sample.timeBudgetMs || row.cost > sample.costBudget)
+      reasons.push("budget_exceeded");
+    if (row.manualRepairs > 0) reasons.push("manual_repair");
+    if (
+      sample.kind === "first-run" &&
+      (observation.firstRun?.cleanEnvironment !== true ||
+        row.firstRunCompletedStages.length !== STAGES.length)
+    )
+      reasons.push("first_run_not_completed");
+    row.succeeded = reasons.length === 0;
+    if (!row.succeeded) {
+      if (FAILURE_CAUSES.has(observation.failureCause))
+        row.failureCause = observation.failureCause;
+      else issues.push(`${sample.id}:missing_failure_cause`);
+    }
+  }
+  const categories = Object.fromEntries(
+    [...CATEGORIES].map((category) => [category, 0]),
+  );
+  let maintenanceComplete = false;
+  if (maintenance) {
+    const entries = uniqueIndex(
+      maintenance.entries,
+      "id",
+      "maintenance entries",
+    );
+    maintenanceComplete =
+      maintenance.complete === true &&
+      outcomeDigest(maintenance.window) === outcomeDigest(plan.window);
+    for (const entry of entries.values()) {
+      assert(
+        CATEGORIES.has(entry.category) &&
+          nonnegative(entry.hours) &&
+          nonempty(entry.evidenceRef),
+        "invalid maintenance entry",
+      );
+      categories[entry.category] += entry.hours;
+    }
+  }
+  if (!maintenanceComplete) issues.push("incomplete_maintenance_records");
+  const maintenanceHours = [...CATEGORIES]
+    .filter((category) => category !== "construction")
+    .reduce((sum, category) => sum + categories[category], 0);
+  const attempts = rows.filter((row) => row.observed).length;
+  const allObserved = rows.every((row) => row.observed);
+  const firstRuns = rows.filter((row) => row.kind === "first-run");
+  const tasks = rows.filter((row) => row.kind === "task");
+  if (!firstRuns.length || !tasks.length)
+    issues.push("missing_metric_population");
+  return {
+    schema: OUTCOME_REPORT_SCHEMA,
+    generatedAt: new Date(now).toISOString(),
+    scope: "local-outcome-baseline",
+    productionAttested: false,
+    status: issues.length ? "INSUFFICIENT_EVIDENCE" : "BASELINE_RECORDED",
+    // Neither a baseline nor operator declarations constitute an improvement gate.
+    improvementVerdict: "NOT_EVALUATED",
+    planDigest,
+    commitSha: plan.commitSha,
+    currency: plan.currency,
+    window: plan.window,
+    task: summarize(tasks),
+    firstRun: summarize(firstRuns),
+    maintenance: {
+      complete: maintenanceComplete,
+      knownHours: maintenanceHours,
+      totalHours: maintenanceComplete ? maintenanceHours : null,
+      per100Attempts:
+        maintenanceComplete && allObserved && attempts
+          ? (maintenanceHours / attempts) * 100
+          : null,
+      observedAttempts: attempts,
+      constructionHours: maintenanceComplete ? categories.construction : null,
+      categories,
+    },
+    groups: [
+      ...new Set(
+        rows.map((row) => JSON.stringify([row.kind, row.stratum, row.taskId])),
+      ),
+    ].map((key) => {
+      const [kind, stratum, taskId] = JSON.parse(key);
+      return {
+        kind,
+        stratum,
+        taskId,
+        ...summarize(
+          rows.filter(
+            (row) =>
+              row.kind === kind &&
+              row.stratum === stratum &&
+              row.taskId === taskId,
+          ),
+        ),
+      };
+    }),
+    firstRunFunnel: STAGES.map((stage, index) => ({
+      stage,
+      planned: firstRuns.length,
+      passed: firstRuns.filter((row) =>
+        STAGES.slice(0, index + 1).every((required) =>
+          row.firstRunCompletedStages.includes(required),
+        ),
+      ).length,
+    })),
+    failureCauses: [
+      ...new Set(
+        rows
+          .filter((row) => !row.succeeded)
+          .map((row) => row.failureCause || "unknown"),
+      ),
+    ].map((cause) => ({
+      cause,
+      count: rows.filter(
+        (row) => !row.succeeded && (row.failureCause || "unknown") === cause,
+      ).length,
+    })),
+    issues: [...new Set(issues)],
+    rows,
+  };
+}

package/src/lib/eval/runner.js

@@ -109,6 +109,8 @@ export async function runEvalSuite(tasks, opts = {}) {
       executionSucceeded: false,
       agentOk: false,
       executionEvidence: null,
+      usage: null,
+      totalCostUsd: null,
       ms: 0,
       detail: "",
       error: null,
@@ -153,6 +155,10 @@ export async function runEvalSuite(tasks, opts = {}) {
         (agentResult.error == null || agentResult.error === "");
       rec.agentOk = rec.executionSucceeded;
       rec.executionEvidence = agentResult?.executionEvidence || null;
+      rec.usage = agentResult?.usage || null;
+      rec.totalCostUsd = Number.isFinite(agentResult?.totalCostUsd)
+        ? agentResult.totalCostUsd
+        : null;
       if (agentResult?.evaluationMetrics !== undefined) {
         rec.evaluationMetrics = agentResult.evaluationMetrics;
       }

package/src/lib/evolution/evolution-deployment-config.js

@@ -80,7 +80,7 @@ export function assessEvolutionDeploymentReadiness(
         : "no enabled signed Evolution deployment",
       disabled
         ? "review the saved signed deployment, then run cc evolution deployment enable"
-        : "configure a signed descriptor and trust root with cc evolution deployment configure --descriptor <path> --trust-root <path>",
+        : "before production, run cc evolution deployment init-test --module <absolute-path>; otherwise configure a signed descriptor and trust root",
     );
   }
   if (missingCommands.length > 0) {
@@ -126,6 +126,10 @@ export async function getEvolutionDeploymentStatus(options = {}) {
     fromEnvironment || (saved.profile?.enabled ? saved.profile : null);
   const base = {
     source,
+    deploymentMode:
+      source === "environment"
+        ? "external"
+        : saved.profile?.deploymentMode || "none",
     effectiveEnabled: source !== "none",
     profileEnabled: saved.profile?.enabled === true,
     profilePath: saved.filePath,
@@ -183,6 +187,8 @@ export async function configureEvolutionDeployment(
     rootRotationPath = null,
     revocationPath = null,
     enabled = true,
+    deploymentMode,
+    testPrivateKeyPath,
   },
   options = {},
 ) {
@@ -239,6 +245,11 @@ export async function configureEvolutionDeployment(
         verified.descriptor.revision,
       ),
     };
+    const preserveTestMode =
+      deploymentMode === undefined &&
+      current?.deploymentMode === "test" &&
+      current.descriptorPath === verified.descriptorPath &&
+      current.trustRootPath === verified.trustRootPath;
     return {
       enabled: enabled === true,
       descriptorPath: verified.descriptorPath,
@@ -246,6 +257,10 @@ export async function configureEvolutionDeployment(
       revisionFloors,
       activeTrustRootDigest: verified.descriptor.trustRootDigest,
       revokedDescriptorRevisions,
+      deploymentMode: preserveTestMode ? "test" : deploymentMode || "managed",
+      testPrivateKeyPath: preserveTestMode
+        ? current.testPrivateKeyPath
+        : testPrivateKeyPath || null,
     };
   }, options);
   return getEvolutionDeploymentStatus(options);