harnery 0.38.0 → 0.40.0

package/src/core/diagnostics/replay.ts

@@ -6,11 +6,13 @@ import type {
   ObservedHookHealth,
   ObservedServiceHealth,
   SupervisorActivitySnapshot,
+  SupervisorCapability,
   SupervisorFinding,
   SupervisorFindingExplanation,
   SupervisorFindings,
   SupervisorHistory,
   SupervisorLogFeed,
+  SupervisorPressureRecord,
   SupervisorSnapshot,
   SupervisorTimeline,
 } from "../supervisor/contract.ts";
@@ -23,14 +25,24 @@ import { explainSupervisorFinding } from "../supervisor/explanations.ts";
 import { updateSupervisorFindings } from "../supervisor/findings.ts";
 import { projectHookHealth } from "../supervisor/hook-health.ts";
 import { buildSupervisorTimeline } from "../supervisor/timeline.ts";
-import { buildDiagnosticAdvice } from "./advice.ts";
+import {
+  buildDiagnosticAdvice,
+  pressureHistoryFromSupervisor,
+  unknownPressureAssessment,
+} from "./advice.ts";
 import {
   DIAGNOSTIC_EXPECTED_SCHEMA_VERSION,
+  type DiagnosticAdvice,
   type DiagnosticExpected,
   type DiagnosticObservations,
   type DiagnosticSelection,
   type DiagnosticThresholds,
 } from "./contract.ts";
+import { assessPressure } from "./pressure.ts";
+import type { PressureFindingInput } from "./pressure-contract.ts";
+
+const PRESSURE_SOURCE = "supervisor.pressure";
+const FINDINGS_SOURCE = "supervisor.findings";
 
 export function canonicalJson(value: unknown): string {
   return JSON.stringify(sortValue(value));
@@ -60,7 +72,7 @@ export function deriveCapturedExpected(
     findings,
     findings.map((finding) => diagnosticTimeline(finding, relatedSources)),
     findings.map(explainSupervisorFinding),
-    observations,
+    capturedAdvice(observations, findings),
   );
 }
 
@@ -75,8 +87,15 @@ export function replayDiagnosticInputs(
   const supervisor =
     optionalSource<SupervisorSnapshot>(observations, "supervisor.snapshot") ??
     neutralSupervisor(observations.captured_at);
+  // A captured history is untrusted JSON from a bundle that may predate the
+  // current point shape. Reading it unchecked would let an older capture reach
+  // the trend projection with no pressure readings on its points. The live
+  // observer and the dashboard validate the same way.
+  const capturedHistory = optionalSource<SupervisorHistory>(observations, "supervisor.history");
   const history =
-    optionalSource<SupervisorHistory>(observations, "supervisor.history") ?? neutralHistory();
+    capturedHistory?.schema_version === SUPERVISOR_HISTORY_SCHEMA_VERSION
+      ? capturedHistory
+      : neutralHistory();
   const logFeed =
     optionalSource<SupervisorLogFeed>(observations, "supervisor.log-feed") ??
     neutralLogFeed(observations.captured_at);
@@ -115,10 +134,100 @@ export function replayDiagnosticInputs(
     findings,
     findings.map((finding) => diagnosticTimeline(finding, relatedSources)),
     findings.map(explainSupervisorFinding),
-    observations,
+    replayedAdvice(observations, resource, history, evaluated),
   );
 }
 
+/**
+ * The live assessment, exactly as the observer published it. Nothing is
+ * recomputed here, because `expected.json` freezes what the machine actually
+ * reported at capture time.
+ */
+function capturedAdvice(
+  observations: DiagnosticObservations,
+  findings: readonly SupervisorFinding[],
+): DiagnosticAdvice {
+  const record = optionalSource<SupervisorPressureRecord>(observations, PRESSURE_SOURCE);
+  const capability = capturedSourceCapability(observations, PRESSURE_SOURCE);
+  if (!record?.assessment) {
+    return unavailableAdvice(observations.captured_at, capability);
+  }
+  return buildDiagnosticAdvice({
+    assessment: record.assessment,
+    priorHysteresis: record.prior_hysteresis ?? null,
+    sourceCapability: capability,
+    activeFindingCount: openedCount(findings),
+    evaluatedAt: observations.captured_at,
+  });
+}
+
+/**
+ * Recompute the assessment from frozen inputs. `prior_hysteresis` comes from
+ * the captured record, so a transition that depended on the previous sample is
+ * reproduced instead of restarting from a cold state.
+ */
+function replayedAdvice(
+  observations: DiagnosticObservations,
+  resource: ResourceSnapshot,
+  history: SupervisorHistory,
+  evaluated: SupervisorFindings,
+): DiagnosticAdvice {
+  const record = optionalSource<SupervisorPressureRecord>(observations, PRESSURE_SOURCE);
+  const capability = capturedSourceCapability(observations, PRESSURE_SOURCE);
+  if (!record?.assessment) {
+    return unavailableAdvice(observations.captured_at, capability);
+  }
+  const snapshotAvailable = optionalSource<ResourceSnapshot>(observations, "resources.snapshot");
+  const findingsCapability = capturedSourceCapability(observations, FINDINGS_SOURCE);
+  const active = [...evaluated.active, ...evaluated.transitions].filter(
+    (finding) => finding.state === "opened",
+  );
+  const assessment = assessPressure({
+    snapshot: snapshotAvailable ? resource : null,
+    snapshot_reason: snapshotAvailable ? null : "captured_source_unavailable",
+    history: pressureHistoryFromSupervisor(history.points),
+    findings: active as readonly PressureFindingInput[],
+    findings_capability: {
+      source_kind: findingsCapability.source_kind,
+      state: findingsCapability.state,
+      ...(findingsCapability.reason_code ? { reason_code: findingsCapability.reason_code } : {}),
+    },
+    prior: record.prior_hysteresis ?? null,
+    observer_generation: record.observer_generation,
+    // Recover the observer's exact clock. `observed_at` is the sample instant
+    // and `sample_age_ms` is how far past it the observer read, so the sum is
+    // the original `now`, which staleness and the OOM hold both depend on.
+    now_ms: Date.parse(record.assessment.observed_at) + (record.assessment.sample_age_ms ?? 0),
+  });
+  return buildDiagnosticAdvice({
+    assessment,
+    priorHysteresis: record.prior_hysteresis ?? null,
+    sourceCapability: capability,
+    activeFindingCount: openedCount(
+      filterFindings(findingsProjection(evaluated), observations.selection),
+    ),
+    evaluatedAt: observations.captured_at,
+  });
+}
+
+function unavailableAdvice(capturedAt: string, capability: SupervisorCapability): DiagnosticAdvice {
+  return buildDiagnosticAdvice({
+    assessment: unknownPressureAssessment({
+      observedAt: capturedAt,
+      reasonCode: "evidence_unavailable",
+      summary: `Local resource pressure cannot be determined because the published assessment is ${capability.state}${capability.reason_code ? ` (${capability.reason_code})` : ""}.`,
+    }),
+    priorHysteresis: null,
+    sourceCapability: capability,
+    activeFindingCount: 0,
+    evaluatedAt: capturedAt,
+  });
+}
+
+function openedCount(findings: readonly SupervisorFinding[]): number {
+  return findings.filter((finding) => finding.state === "opened").length;
+}
+
 function diagnosticTimeline(
   finding: SupervisorFinding,
   relatedSources: readonly CoordinationHealthSnapshot["recent_events"][number][],
@@ -138,7 +247,7 @@ function expected(
   findings: readonly SupervisorFinding[],
   timelines: readonly SupervisorTimeline[],
   explanations: readonly SupervisorFindingExplanation[],
-  observations: DiagnosticObservations,
+  advice: DiagnosticAdvice,
 ): DiagnosticExpected {
   return {
     schema_version: DIAGNOSTIC_EXPECTED_SCHEMA_VERSION,
@@ -151,18 +260,14 @@ function expected(
     explanations: [...explanations].sort((left, right) =>
       left.finding_id.localeCompare(right.finding_id),
     ),
-    advice: buildDiagnosticAdvice({
-      findings,
-      sourceCapability: capturedSourceCapability(observations, "supervisor.findings"),
-      evaluatedAt: observations.captured_at,
-    }),
+    advice,
   };
 }
 
 function capturedSourceCapability(
   observations: DiagnosticObservations,
   sourceKind: string,
-): import("../supervisor/contract.ts").SupervisorCapability {
+): SupervisorCapability {
   const source = observations.sources.find((candidate) => candidate.source_kind === sourceKind);
   return source
     ? {

package/src/core/events/v3/adapter-id.ts

@@ -0,0 +1,12 @@
+import type { Adapter } from "../../adapter.ts";
+
+/** Harness identities that may produce or describe Event Ledger V3 evidence. */
+export type EventAdapterIdV3 = Adapter | "openclaw";
+
+/** Canonical event-adapter census. Workflow launch adapters remain in core/adapter.ts. */
+export const EVENT_ADAPTER_IDS_V3 = [
+  "claude-code",
+  "codex",
+  "cursor",
+  "openclaw",
+] as const satisfies readonly EventAdapterIdV3[];

package/src/core/events/v3/archive-retention.ts

@@ -10,11 +10,7 @@ import {
   writeFileSync,
 } from "node:fs";
 import { basename, join, relative, resolve } from "node:path";
-import {
-  artifactAutoCleanIntervalHours,
-  type EventLedgerArchivePolicy,
-  eventLedgerArchivePolicy,
-} from "../../config.ts";
+import { type EventLedgerArchivePolicy, eventLedgerArchivePolicy } from "../../config.ts";
 
 export type EventV3ArchiveClassification =
   | "retained"
@@ -151,7 +147,8 @@ export function autoCleanEventV3Archives(
   const policy = eventLedgerArchivePolicy(repoRoot);
   if (!policy.autoClean) return { ran: false, reason: "disabled", deleted: 0, bytes: 0 };
   const stampPath = join(resolve(repoRoot), AUTO_CLEAN_STAMP);
-  const intervalMs = artifactAutoCleanIntervalHours() * 60 * 60 * 1000;
+  // Archive cleanup remains daily, independent of short-lived working files.
+  const intervalMs = 24 * 60 * 60 * 1000;
   try {
     const stamp = JSON.parse(readFileSync(stampPath, "utf8")) as { last_run_at?: string };
     const last = Date.parse(stamp.last_run_at ?? "");

package/src/core/events/v3/bootstrap.ts

@@ -10,15 +10,18 @@ import {
   readFileSync,
   renameSync,
   statSync,
+  unlinkSync,
   writeFileSync,
 } from "node:fs";
 import { hostname } from "node:os";
 import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 import { fsyncParentDirectory } from "../../workflow/durable-record.ts";
 import { acquireNoClobberLease } from "../../workflow/workspaces/leases.ts";
 import { canonicalJsonV3, sha256V3 } from "./canonical.ts";
 import { ADAPTER_CAPABILITY_PROFILES_V3 } from "./capabilities.ts";
 import {
+  type ActivationManifestV3,
   buildActivationManifestV3,
   buildCandidateGenesisManifestV3,
   type CandidateGenesisManifestV3,
@@ -42,6 +45,16 @@ export interface InitializeEventLedgerV3Input {
   configDigest: `sha256:${string}`;
   approvalRecordId: string;
   forceNewEpoch?: boolean;
+  /**
+   * Activate a candidate epoch that was already created for this root.
+   *
+   * Unset, this activates a candidate that carries this initializer's producer
+   * identity and refuses every other one. Creation and activation are one
+   * locked step here, so such a candidate is the residue of a crash or a
+   * failed activation, never work in progress; a candidate from another
+   * producer is a deliberate cutover state. Set it explicitly to force either
+   * answer.
+   */
   resumeCandidate?: boolean;
   now?: () => Date;
 }
@@ -59,6 +72,17 @@ export interface RefreshIncompatibleEventLedgerV3Result {
   control: Extract<EventV3ControlState, { state: "active" }>;
 }
 
+/**
+ * The control producer every epoch created by this initializer carries.
+ *
+ * It separates a stranded epoch from a deliberate one. A candidate minted here
+ * was always meant to be activated in the same locked step, so finishing it is
+ * a repair. A candidate published by any other producer (an operator cutover or
+ * rollback rehearsal) is a state someone chose, and automatic activation would
+ * overrule them.
+ */
+const BOOTSTRAP_CONTROL_PRODUCER_ID = "prd_harnery-init" as const;
+
 const BOOTSTRAP_LEASE_RETRIES = 12;
 const BOOTSTRAP_LEASE_RETRY_MS = 25;
 const BOOTSTRAP_LEASE_STALE_MS = 10_000;
@@ -141,6 +165,74 @@ export function refreshIncompatibleEventLedgerV3(
   });
 }
 
+export interface RepairStrandedEventLedgerV3CandidateResult {
+  state: "repaired" | "not_stranded" | "unavailable";
+  reason?: string;
+  control: EventV3ControlState;
+}
+
+/** The approval record every automatic stranded-candidate repair is bound to. */
+export const EVENT_V3_STRANDED_CANDIDATE_APPROVAL_RECORD_ID =
+  "harnery-runtime-v3-stranded-candidate" as const;
+
+/**
+ * Complete an epoch that was created but never activated.
+ *
+ * This initializer creates a candidate and activates it inside one lease, so a
+ * candidate carrying its producer identity is residue: some process published
+ * the genesis and died or failed before publishing the activation, and every
+ * hook afterwards served that half-built epoch with a candidate-only write
+ * gate. Finishing it is a repair, not a decision, and it is exactly as safe as
+ * the creation it completes: the same lease serializes it, the activation is
+ * derived from the immutable candidate packet, and no ledger row is edited or
+ * synthesized.
+ *
+ * A candidate from any other producer is a deliberate cutover state and is
+ * left untouched. A live bootstrap lease makes this a no-op rather than a
+ * failure, so the next boundary retries.
+ */
+export function repairStrandedEventLedgerV3Candidate(
+  coordRoot: string,
+): RepairStrandedEventLedgerV3CandidateResult {
+  const root = resolve(coordRoot);
+  const observed = readEventV3ControlState(root);
+  const deliberateCandidate = observed.state === "candidate";
+  if (!strandedBootstrapCandidate(observed)) {
+    return {
+      state: "not_stranded",
+      control: observed,
+      ...(deliberateCandidate ? { reason: "candidate_not_bootstrap_created" } : {}),
+    };
+  }
+  try {
+    return withBootstrapLease(root, () => {
+      const current = readEventV3ControlState(root);
+      if (!strandedBootstrapCandidate(current)) return { state: "not_stranded", control: current };
+      const activated = activateCandidateEpoch(
+        root,
+        current.genesis,
+        EVENT_V3_STRANDED_CANDIDATE_APPROVAL_RECORD_ID,
+      );
+      return { state: "repaired", control: activated.control };
+    });
+  } catch (error) {
+    return {
+      state: "unavailable",
+      reason: error instanceof Error ? error.message : String(error),
+      control: observed,
+    };
+  }
+}
+
+function strandedBootstrapCandidate(
+  control: EventV3ControlState,
+): control is Extract<EventV3ControlState, { state: "candidate" }> {
+  return (
+    control.state === "candidate" &&
+    control.genesis.event.producer.producer_id === BOOTSTRAP_CONTROL_PRODUCER_ID
+  );
+}
+
 export interface RotateOversizedEventLedgerV3Result {
   state: "rotated" | "not_oversized" | "not_active" | "disabled";
   active_bytes: number;
@@ -158,10 +250,13 @@ export interface RotateOversizedEventLedgerV3Result {
  * (events, spool, producer states, manifests) moves whole into the V3
  * archive, live sessions re-onboard into the new epoch on their next signal,
  * and the writer's epoch fence keeps in-flight producers of the old epoch
- * from ever committing into the new one. Only a currently valid, active
- * ledger rotates; integrity failures stay closed for the explicit recovery
- * command. The threshold comes from the isolated Event V3 config resolver
- * unless the caller pins one; a non-positive threshold disables rotation.
+ * from ever committing into the new one. A candidate epoch rotates on the same
+ * terms as an active one: its control pair is valid, every reader still
+ * validates all of its history, and refusing to bound it is how a stranded
+ * epoch grew to 100 MB. Integrity failures stay closed for the explicit
+ * recovery command. The threshold comes from the isolated Event V3 config
+ * resolver unless the caller pins one; a non-positive threshold disables
+ * rotation.
  */
 export function rotateOversizedEventLedgerV3(
   coordRoot: string,
@@ -189,7 +284,7 @@ export function rotateOversizedEventLedgerV3(
       return { state: "not_oversized", active_bytes: activeBytes, threshold_bytes: threshold };
     }
     const current = readEventV3ControlState(root);
-    if (current.state !== "active") {
+    if (current.state !== "active" && current.state !== "candidate") {
       return { state: "not_active", active_bytes: activeBytes, threshold_bytes: threshold };
     }
     // Flush durable ready rows into the epoch they were produced for, so the
@@ -235,13 +330,14 @@ function initializeEventLedgerV3Locked(
 ): InitializeEventLedgerV3Result {
   const root = resolve(input.coordRoot);
   let current = readEventV3ControlState(root);
-  if (input.resumeCandidate && current.state === "repairable") {
+  const resumeCandidate = input.resumeCandidate ?? strandedBootstrapCandidate(current);
+  if (resumeCandidate && current.state === "repairable") {
     current = repairEventV3ControlPair(root);
   }
   if (!input.forceNewEpoch && current.state === "active") {
     return { state: "active", initialized: false, control: current };
   }
-  if (!input.forceNewEpoch && input.resumeCandidate && current.state === "candidate") {
+  if (!input.forceNewEpoch && resumeCandidate && current.state === "candidate") {
     return activateCandidateEpoch(root, current.genesis, input.approvalRecordId);
   }
   if (!input.forceNewEpoch && current.state === "candidate") {
@@ -250,11 +346,10 @@ function initializeEventLedgerV3Locked(
 
   const now = input.now ?? (() => new Date());
   const createdAt = now().toISOString();
-  const archivedEpoch = archiveCurrentEpoch(root, createdAt);
   const keys = loadOrCreateFingerprintKeyStoreV3(root, now);
   const harneryBuild = safeBuild(input.harneryBuild);
   const hostBuild = safeBuild(input.hostBuild);
-  const producerId = "prd_harnery-init" as const;
+  const producerId = BOOTSTRAP_CONTROL_PRODUCER_ID;
   const bootId =
     `boot_${createHash("sha256").update(`${root}\0${createdAt}`).digest("hex")}` as const;
   const buildId = liveEventV3BuildId(harneryBuild);
@@ -288,18 +383,42 @@ function initializeEventLedgerV3Locked(
       platform: livePlatformV3(),
     },
   });
+  // Mint the complete control pair before any epoch bytes move. Every approval
+  // and validation failure then happens while the current epoch is still
+  // whole, so it can never leave a replaced epoch stranded in candidate state.
+  const activation = buildActivationManifestV3(activationInput(candidate, input.approvalRecordId));
+
+  const archivedEpoch = archiveCurrentEpoch(root, createdAt);
   publishControlFile(join(root, EVENT_V3_GENESIS_MANIFEST), candidate);
-  const candidateState = repairEventV3ControlPair(root);
-  if (candidateState.state !== "candidate") {
-    throw new Error(`event_v3_candidate_initialization_failed:${candidateState.state}`);
+  let activated: InitializeEventLedgerV3Result;
+  try {
+    activated = completeCandidateActivation(root, activation);
+  } catch (error) {
+    // The epoch has already been replaced, so returning here would strand it.
+    // Retry the completion inside the same lease and, failing that, name the
+    // stranded epoch rather than reporting a successful initialization.
+    activated = completeStrandedCandidateActivation(root, activation, error);
   }
-  const activated = activateCandidateEpoch(root, candidate, input.approvalRecordId);
   return {
     ...activated,
     ...(archivedEpoch ? { archived_epoch: archivedEpoch } : {}),
   };
 }
 
+function completeStrandedCandidateActivation(
+  root: string,
+  activation: ActivationManifestV3,
+  cause: unknown,
+): InitializeEventLedgerV3Result {
+  try {
+    return completeCandidateActivation(root, activation);
+  } catch {
+    throw new Error(
+      `event_v3_epoch_stranded_before_activation:${cause instanceof Error ? cause.message : String(cause)}`,
+    );
+  }
+}
+
 function runtimeCapabilityProfileCurrent(
   control: Extract<EventV3ControlState, { state: "candidate" | "active" }>,
 ): boolean {
@@ -355,8 +474,16 @@ function activateCandidateEpoch(
   candidate: CandidateGenesisManifestV3,
   approvalRecordId: string,
 ): InitializeEventLedgerV3Result {
+  return completeCandidateActivation(
+    root,
+    buildActivationManifestV3(activationInput(candidate, approvalRecordId)),
+  );
+}
+
+/** The activation packet one exact candidate deserves, derived and pure. */
+function activationInput(candidate: CandidateGenesisManifestV3, approvalRecordId: string) {
   const producer = candidate.event.producer;
-  const activation = buildActivationManifestV3({
+  return {
     candidate,
     approval_record_id: approvalRecordId,
     activation_approved_at: candidate.profile.candidate_created_at,
@@ -368,7 +495,23 @@ function activateCandidateEpoch(
       platform: producer.platform,
       ...(producer.bridge ? { bridge: producer.bridge } : {}),
     },
-  });
+  };
+}
+
+/**
+ * Bring a published candidate to active: append the pre-minted genesis event
+ * if a crash left it missing, publish the activation manifest, then append its
+ * pre-minted event. Every step is idempotent, so the caller may retry it, and
+ * anything short of `active` throws with the state that blocked it.
+ */
+function completeCandidateActivation(
+  root: string,
+  activation: ActivationManifestV3,
+): InitializeEventLedgerV3Result {
+  const candidateState = repairEventV3ControlPair(root);
+  if (candidateState.state !== "candidate") {
+    throw new Error(`event_v3_candidate_initialization_failed:${candidateState.state}`);
+  }
   publishControlFile(join(root, EVENT_V3_ACTIVATION_MANIFEST), activation);
   const active = repairEventV3ControlPair(root);
   if (active.state !== "active") {
@@ -394,6 +537,10 @@ function archiveCurrentEpoch(root: string, createdAt: string): string | undefine
 function publishControlFile(path: string, value: unknown): void {
   mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
   const temp = `${path}.tmp.${process.pid}`;
+  // A prior failed publish by this same pid would otherwise make every retry
+  // fail on the exclusive create, which is exactly how one transient error
+  // turns into a permanently stranded epoch.
+  if (existsSync(temp)) unlinkSync(temp);
   const fd = openSync(temp, "wx", 0o600);
   try {
     writeFileSync(fd, `${canonicalJsonV3(value)}\n`, "utf8");
@@ -415,7 +562,7 @@ function safeBuild(value: string): string {
 }
 
 function rootOfHarnery(): string {
-  return resolve(import.meta.dir, "../../../..");
+  return resolve(dirname(fileURLToPath(import.meta.url)), "../../../..");
 }
 
 function repositoryBuild(root: string): string {