harnery 0.35.0 → 0.36.0

package/src/core/events/legacy-storage/reader.ts

@@ -0,0 +1,178 @@
+import { createHash } from "node:crypto";
+import { constants, createReadStream } from "node:fs";
+import { lstat, open, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve } from "node:path";
+import { createGunzip } from "node:zlib";
+import { type LegacyV1SegmentManifest, validateLegacyV1SegmentManifest } from "./manifest.ts";
+
+export interface LegacyV1LogicalRow {
+  row_number: number;
+  bytes: Buffer;
+}
+
+export interface LegacyV1ReaderLimits {
+  max_manifest_bytes?: number;
+  max_payload_bytes?: number;
+  max_decompressed_bytes?: number;
+  max_row_bytes?: number;
+  max_rows?: number;
+  fault?: (boundary: "after_source_digest", path: string) => void;
+}
+
+const DEFAULTS: Required<Omit<LegacyV1ReaderLimits, "fault">> = {
+  max_manifest_bytes: 1024 * 1024,
+  max_payload_bytes: 8 * 1024 * 1024 * 1024,
+  max_decompressed_bytes: 16 * 1024 * 1024 * 1024,
+  max_row_bytes: 64 * 1024 * 1024,
+  max_rows: 20_000_000,
+};
+
+/** Stream rows from one loose sealed segment or one manifest-bound gzip canary. */
+export async function* streamLegacyV1Rows(
+  sourcePath: string,
+  limits: LegacyV1ReaderLimits = {},
+): AsyncGenerator<LegacyV1LogicalRow> {
+  const bounded = { ...DEFAULTS, ...limits };
+  const source = resolve(sourcePath);
+  let input = source;
+  let expectedRows: number | undefined;
+  let expectedSourceBytes: number | undefined;
+  let expectedSourceDigest: `sha256:${string}` | undefined;
+  let expectedPayloadBytes: number | undefined;
+  let expectedPayloadDigest: `sha256:${string}` | undefined;
+  let containmentRoot: string | undefined;
+  if (source.endsWith(".manifest.json")) {
+    const manifestHandle = await openStableRegularFile(source, undefined, "manifest");
+    let manifest: LegacyV1SegmentManifest;
+    try {
+      const stat = await manifestHandle.stat();
+      if (stat.size > bounded.max_manifest_bytes) throw new Error("legacy_v1_manifest_too_large");
+      manifest = validateLegacyV1SegmentManifest(
+        JSON.parse(await manifestHandle.readFile("utf8")) as unknown,
+      );
+    } finally {
+      await manifestHandle.close();
+    }
+    const directory = await realpath(dirname(source));
+    containmentRoot = directory;
+    input = resolve(directory, manifest.payload.path);
+    expectedRows = manifest.source.row_count;
+    expectedSourceBytes = manifest.source.bytes;
+    expectedSourceDigest = manifest.source.digest;
+    expectedPayloadBytes = manifest.payload.bytes;
+    expectedPayloadDigest = manifest.payload.digest;
+  }
+
+  const compressed = source.endsWith(".manifest.json") || source.endsWith(".gz");
+  const kind = source.endsWith(".manifest.json") ? "payload" : "segment";
+  const inputHandle = await openStableRegularFile(input, containmentRoot, kind);
+  try {
+    const inputStat = await inputHandle.stat();
+    if (source.endsWith(".manifest.json")) {
+      if (inputStat.size !== expectedPayloadBytes)
+        throw new Error("legacy_v1_payload_length_mismatch");
+      if (inputStat.size > bounded.max_payload_bytes)
+        throw new Error("legacy_v1_payload_too_large");
+      if ((await hashHandle(input, inputHandle)) !== expectedPayloadDigest) {
+        throw new Error("legacy_v1_payload_digest_mismatch");
+      }
+    }
+    limits.fault?.("after_source_digest", input);
+    const file = createReadStream(input, { fd: inputHandle.fd, autoClose: false, start: 0 });
+    const stream = compressed ? file.pipe(createGunzip()) : file;
+    let pending: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+    let rows = 0;
+    let decompressed = 0;
+    const sourceHash = createHash("sha256");
+    for await (const current of stream) {
+      const chunk = current as Buffer;
+      decompressed += chunk.length;
+      sourceHash.update(chunk);
+      if (decompressed > bounded.max_decompressed_bytes) {
+        throw new Error("legacy_v1_decompression_bound_exceeded");
+      }
+      pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]);
+      if (pending.length > bounded.max_row_bytes && !pending.includes(0x0a)) {
+        throw new Error("legacy_v1_row_bound_exceeded");
+      }
+      let newline = pending.indexOf(0x0a);
+      while (newline >= 0) {
+        const row = pending.subarray(0, newline);
+        pending = pending.subarray(newline + 1);
+        if (row.length === 0) throw new Error("legacy_v1_empty_row");
+        if (row.length > bounded.max_row_bytes) throw new Error("legacy_v1_row_bound_exceeded");
+        rows += 1;
+        if (rows > bounded.max_rows) throw new Error("legacy_v1_row_count_bound_exceeded");
+        yield { row_number: rows, bytes: Buffer.from(row) };
+        newline = pending.indexOf(0x0a);
+      }
+    }
+    if (pending.length !== 0) throw new Error("legacy_v1_segment_unterminated_or_truncated");
+    if (expectedRows !== undefined && rows !== expectedRows)
+      throw new Error("legacy_v1_row_count_mismatch");
+    if (expectedSourceBytes !== undefined && decompressed !== expectedSourceBytes) {
+      throw new Error("legacy_v1_source_length_mismatch");
+    }
+    const digest = `sha256:${sourceHash.digest("hex")}`;
+    if (expectedSourceDigest !== undefined && digest !== expectedSourceDigest) {
+      throw new Error("legacy_v1_source_digest_mismatch");
+    }
+  } finally {
+    await inputHandle.close();
+  }
+}
+
+async function hashHandle(
+  path: string,
+  handle: Awaited<ReturnType<typeof open>>,
+): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  const stream = createReadStream(path, { fd: handle.fd, autoClose: false, start: 0 });
+  for await (const current of stream) hash.update(current as Buffer);
+  return `sha256:${hash.digest("hex")}`;
+}
+
+async function openStableRegularFile(
+  path: string,
+  containmentRoot: string | undefined,
+  kind: "manifest" | "payload" | "segment",
+) {
+  const before = await lstat(path);
+  if (!before.isFile() || before.isSymbolicLink()) throw new Error(`legacy_v1_${kind}_not_regular`);
+  const beforeReal = await realpath(path);
+  if (containmentRoot) assertContained(containmentRoot, beforeReal);
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const opened = await handle.stat();
+    const after = await lstat(path);
+    const afterReal = await realpath(path);
+    if (
+      !opened.isFile() ||
+      !after.isFile() ||
+      after.isSymbolicLink() ||
+      opened.dev !== before.dev ||
+      opened.ino !== before.ino ||
+      opened.dev !== after.dev ||
+      opened.ino !== after.ino ||
+      beforeReal !== afterReal
+    )
+      throw new Error(`legacy_v1_${kind}_changed_during_open`);
+    if (containmentRoot) assertContained(containmentRoot, afterReal);
+    return handle;
+  } catch (error) {
+    await handle.close();
+    throw error;
+  }
+}
+
+function assertContained(root: string, candidate: string): void {
+  const path = relative(root, candidate);
+  if (
+    path === "" ||
+    (path !== ".." &&
+      !path.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) &&
+      !isAbsolute(path))
+  )
+    return;
+  throw new Error("legacy_v1_payload_escape");
+}

package/src/core/events/v3/authority-outbox.ts

@@ -14,9 +14,15 @@ import { join } from "node:path";
 import { fsyncParentDirectory } from "../../workflow/durable-record.ts";
 import { canonicalJsonV3, sha256V3 } from "./canonical.ts";
 import type { EventV3 } from "./contract.ts";
+import { liveGenesisIdV3 } from "./control.ts";
 import { assertEventV3 } from "./validate.ts";
 import { ensureEventV3Layout, eventV3Paths, writeEventV3 } from "./writer.ts";
 
+export interface AuthorityEpochFenceV3 {
+  /** Genesis id of the epoch the transaction's event was produced for. */
+  expectedGenesisId?: `gex_${string}`;
+}
+
 const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/;
 const INSTANCE_PATTERN = /^inst_[a-zA-Z0-9._-]{1,128}$/;
 const TX_PATTERN = /^txn_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
@@ -146,9 +152,13 @@ export function buildAuthorityTransactionV3(
 export function publishAuthorityTransactionV3(
   coordRoot: string,
   transaction: AuthorityTransactionV3,
+  fence: AuthorityEpochFenceV3 = {},
 ): AuthorityTransactionV3 {
   validateTransaction(transaction);
   const paths = ensureEventV3Layout(coordRoot);
+  if (fence.expectedGenesisId && liveGenesisIdV3(coordRoot) !== fence.expectedGenesisId) {
+    throw new Error("authority transaction epoch was replaced before publication");
+  }
   const readyPath = authorityReadyPath(paths.authorityOutbox, transaction.transaction_id);
   const serialized = `${canonicalJsonV3(transaction)}\n`;
   if (existsSync(readyPath)) {
@@ -173,6 +183,7 @@ export function reconcileAuthorityTransactionV3(
   coordRoot: string,
   transactionId: string,
   reconciler: AuthorityReconcilerV3,
+  fence: AuthorityEpochFenceV3 = {},
 ): AuthorityReceiptV3 {
   assertTransactionId(transactionId);
   const paths = ensureEventV3Layout(coordRoot);
@@ -191,7 +202,12 @@ export function reconcileAuthorityTransactionV3(
   }
   const event = JSON.parse(transaction.event_row) as unknown;
   assertEventV3(event);
-  writeEventV3(coordRoot, event);
+  const durability = writeEventV3(coordRoot, event, {
+    ...(fence.expectedGenesisId ? { expectedGenesisId: fence.expectedGenesisId } : {}),
+  });
+  if (durability.state === "epoch_replaced") {
+    throw new Error("authority transaction epoch was replaced before its event committed");
+  }
   const receipt: AuthorityReceiptV3 = {
     format: "harnery-event-v3-authority-receipt",
     format_version: 1,
@@ -222,6 +238,11 @@ export function listPendingAuthorityTransactionsV3(coordRoot: string): Authority
     .map((name) => readAuthorityTransactionV3(join(outbox, name)));
 }
 
+export function authorityRecoveryIntentPathV3(coordRoot: string, transactionId: string): string {
+  assertTransactionId(transactionId);
+  return join(eventV3Paths(coordRoot).root, "authority-recoveries", `${transactionId}.ready.json`);
+}
+
 export function readAuthorityTransactionV3(path: string): AuthorityTransactionV3 {
   const serialized = readFileSync(path, "utf8");
   let parsed: unknown;

package/src/core/events/v3/base-contract.ts

@@ -109,6 +109,8 @@ const RuntimeTelemetryCapabilityValueV3BaseSchema = StrictObject({
   source: SafeToken,
   completeness: Type.Union([
     Type.Literal("exact"),
+    Type.Literal("inferred"),
+    Type.Literal("percentage_only"),
     Type.Literal("lower_bound"),
     Type.Literal("unknown"),
   ]),
@@ -430,13 +432,21 @@ const AuthorityReferenceSchema = StrictObject({
   record_id: Type.Optional(SafeToken),
 });
 
-const MeasurementSchema = StrictObject({
-  used_tokens: Type.Integer({ minimum: 0 }),
-  limit_tokens: Type.Integer({ minimum: 1 }),
-  remaining_tokens: Type.Optional(Type.Integer({ minimum: 0 })),
-  measured_at: Timestamp,
-  method: SafeToken,
-});
+const MeasurementSchema = Type.Union([
+  StrictObject({
+    used_tokens: Type.Integer({ minimum: 0 }),
+    limit_tokens: Type.Integer({ minimum: 1 }),
+    remaining_tokens: Type.Optional(Type.Integer({ minimum: 0 })),
+    measured_at: Timestamp,
+    method: SafeToken,
+  }),
+  StrictObject({
+    used_percent: Type.Number({ minimum: 0, maximum: 100 }),
+    remaining_percent: Type.Optional(Type.Number({ minimum: 0, maximum: 100 })),
+    measured_at: Timestamp,
+    method: SafeToken,
+  }),
+]);
 
 const ArtifactReferenceSchema = StrictObject({
   artifact_id: Type.String({ pattern: "^art_[a-zA-Z0-9._-]{1,128}$" }),

package/src/core/events/v3/bootstrap.ts

@@ -9,10 +9,13 @@ import {
   openSync,
   readFileSync,
   renameSync,
+  statSync,
   writeFileSync,
 } from "node:fs";
+import { hostname } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { fsyncParentDirectory } from "../../workflow/durable-record.ts";
+import { acquireNoClobberLease } from "../../workflow/workspaces/leases.ts";
 import { canonicalJsonV3, sha256V3 } from "./canonical.ts";
 import { ADAPTER_CAPABILITY_PROFILES_V3 } from "./capabilities.ts";
 import {
@@ -23,11 +26,13 @@ import {
   EVENT_V3_GENESIS_MANIFEST,
   type EventV3ControlState,
   readEventV3ControlState,
-  repairEventV3ControlPair,
 } from "./control.ts";
+import { repairEventV3ControlPair } from "./control-writer.ts";
 import { loadOrCreateFingerprintKeyStoreV3 } from "./fingerprint-keys.ts";
 import { EVENT_V3_SCHEMA_DIGEST } from "./generated.ts";
-import { liveEventV3BuildId, livePlatformV3 } from "./live-routing.ts";
+import { resolveEventLedgerRotateActiveBytesV3 } from "./rotation-config.ts";
+import { currentHarneryRuntimeBuild, liveEventV3BuildId, livePlatformV3 } from "./runtime-build.ts";
+import { drainReadyEventsV3 } from "./writer.ts";
 
 export interface InitializeEventLedgerV3Input {
   coordRoot: string;
@@ -47,6 +52,17 @@ export interface InitializeEventLedgerV3Result {
   control: Extract<EventV3ControlState, { state: "active" }>;
 }
 
+export interface RefreshIncompatibleEventLedgerV3Result {
+  state: "current" | "refreshed";
+  archived_epoch?: string;
+  control: Extract<EventV3ControlState, { state: "active" }>;
+}
+
+const BOOTSTRAP_LEASE_RETRIES = 12;
+const BOOTSTRAP_LEASE_RETRY_MS = 25;
+const BOOTSTRAP_LEASE_STALE_MS = 10_000;
+const bootstrapSleepCell = new Int32Array(new SharedArrayBuffer(4));
+
 /**
  * Ensure programmatic Harnery entry points have the same universal V3
  * boundary as `harn init`. Only a genuinely absent control pair is created;
@@ -79,6 +95,133 @@ export function ensureEventLedgerV3(
  */
 export function initializeEventLedgerV3(
   input: InitializeEventLedgerV3Input,
+): InitializeEventLedgerV3Result {
+  return withBootstrapLease(input.coordRoot, () => initializeEventLedgerV3Locked(input));
+}
+
+/**
+ * Replace only a runtime-incompatible epoch that the current code can name
+ * exactly. Corruption and ambiguous control failures remain closed for the
+ * explicit recovery command.
+ */
+export function refreshIncompatibleEventLedgerV3(
+  coordRoot: string,
+): RefreshIncompatibleEventLedgerV3Result {
+  const root = resolve(coordRoot);
+  return withBootstrapLease(root, () => {
+    const current = readEventV3ControlState(root);
+    if (current.state === "active" && runtimeCapabilityProfileCurrent(current)) {
+      return { state: "current", control: current };
+    }
+    const refreshable =
+      (current.state === "invalid" && current.reason === "genesis_schema_digest_incompatible") ||
+      ((current.state === "candidate" || current.state === "active") &&
+        !runtimeCapabilityProfileCurrent(current));
+    if (!refreshable) {
+      const reason = "reason" in current ? `${current.state}:${current.reason}` : current.state;
+      throw new Error(`event_v3_runtime_refresh_refused:${reason}`);
+    }
+    const configPath = join(root, ".harnery", "config.jsonc");
+    const initialized = initializeEventLedgerV3Locked({
+      coordRoot: root,
+      harneryBuild: currentHarneryRuntimeBuild(),
+      hostBuild: repositoryBuild(root),
+      configDigest: sha256V3(
+        existsSync(configPath) ? readFileSync(configPath) : Buffer.from("{}\n"),
+      ),
+      approvalRecordId: "harnery-runtime-v3-auto-refresh",
+      forceNewEpoch: true,
+    });
+    return {
+      state: "refreshed",
+      ...(initialized.archived_epoch ? { archived_epoch: initialized.archived_epoch } : {}),
+      control: initialized.control,
+    };
+  });
+}
+
+export interface RotateOversizedEventLedgerV3Result {
+  state: "rotated" | "not_oversized" | "not_active" | "disabled";
+  active_bytes: number;
+  threshold_bytes: number;
+  archived_epoch?: string;
+  control?: Extract<EventV3ControlState, { state: "active" }>;
+}
+
+/**
+ * Archive a valid oversized epoch intact and start a fresh one.
+ *
+ * Every canonical read validates the complete epoch, and hook producers are
+ * one-shot processes, so an unbounded active segment makes every hook's cold
+ * read scale with all history. Rotation bounds that cost: the replaced epoch
+ * (events, spool, producer states, manifests) moves whole into the V3
+ * archive, live sessions re-onboard into the new epoch on their next signal,
+ * and the writer's epoch fence keeps in-flight producers of the old epoch
+ * from ever committing into the new one. Only a currently valid, active
+ * ledger rotates; integrity failures stay closed for the explicit recovery
+ * command. The threshold comes from the isolated Event V3 config resolver
+ * unless the caller pins one; a non-positive threshold disables rotation.
+ */
+export function rotateOversizedEventLedgerV3(
+  coordRoot: string,
+  options: { thresholdBytes?: number } = {},
+): RotateOversizedEventLedgerV3Result {
+  const root = resolve(coordRoot);
+  const threshold = options.thresholdBytes ?? resolveEventLedgerRotateActiveBytesV3(root);
+  const activePath = join(root, ".harnery", "ledgers", "v3", "active.ndjson");
+  const measure = () => {
+    try {
+      return statSync(activePath).size;
+    } catch {
+      return 0;
+    }
+  };
+  if (threshold <= 0) {
+    return { state: "disabled", active_bytes: measure(), threshold_bytes: threshold };
+  }
+  if (measure() < threshold) {
+    return { state: "not_oversized", active_bytes: measure(), threshold_bytes: threshold };
+  }
+  return withBootstrapLease(root, () => {
+    const activeBytes = measure();
+    if (activeBytes < threshold) {
+      return { state: "not_oversized", active_bytes: activeBytes, threshold_bytes: threshold };
+    }
+    const current = readEventV3ControlState(root);
+    if (current.state !== "active") {
+      return { state: "not_active", active_bytes: activeBytes, threshold_bytes: threshold };
+    }
+    // Flush durable ready rows into the epoch they were produced for, so the
+    // archive carries them committed instead of stranded in its spool.
+    try {
+      drainReadyEventsV3(root);
+    } catch {
+      // A busy append lease only leaves rows in the archived spool; rotation
+      // itself stays safe.
+    }
+    const configPath = join(root, ".harnery", "config.jsonc");
+    const initialized = initializeEventLedgerV3Locked({
+      coordRoot: root,
+      harneryBuild: currentHarneryRuntimeBuild(),
+      hostBuild: repositoryBuild(root),
+      configDigest: sha256V3(
+        existsSync(configPath) ? readFileSync(configPath) : Buffer.from("{}\n"),
+      ),
+      approvalRecordId: "harnery-runtime-v3-size-rotation",
+      forceNewEpoch: true,
+    });
+    return {
+      state: "rotated",
+      active_bytes: activeBytes,
+      threshold_bytes: threshold,
+      ...(initialized.archived_epoch ? { archived_epoch: initialized.archived_epoch } : {}),
+      control: initialized.control,
+    };
+  });
+}
+
+function initializeEventLedgerV3Locked(
+  input: InitializeEventLedgerV3Input,
 ): InitializeEventLedgerV3Result {
   const root = resolve(input.coordRoot);
   let current = readEventV3ControlState(root);
@@ -147,6 +290,56 @@ export function initializeEventLedgerV3(
   };
 }
 
+function runtimeCapabilityProfileCurrent(
+  control: Extract<EventV3ControlState, { state: "candidate" | "active" }>,
+): boolean {
+  const expected = Object.values(ADAPTER_CAPABILITY_PROFILES_V3).map((profile) =>
+    sha256V3(canonicalJsonV3(profile)),
+  );
+  const approved = control.genesis.profile.adapter_capability_profile_digests;
+  const expectedDigests = new Set<string>(expected);
+  return control.state === "candidate"
+    ? approved.some((digest) => expectedDigests.has(digest))
+    : expected.every((digest) => approved.includes(digest));
+}
+
+function withBootstrapLease<T>(coordRoot: string, operation: () => T): T {
+  const root = resolve(coordRoot);
+  const authority = createHash("sha256").update(root).digest("hex");
+  let lease: ReturnType<typeof acquireNoClobberLease> | undefined;
+  for (let attempt = 0; attempt < BOOTSTRAP_LEASE_RETRIES; attempt += 1) {
+    try {
+      lease = acquireNoClobberLease({
+        path: join(root, ".harnery", "private", "event-v3-bootstrap-lease"),
+        scope: "event-v3-bootstrap",
+        authoritySha256: authority,
+        staleAfterMs: BOOTSTRAP_LEASE_STALE_MS,
+        validateStaleOwner: (owner) => owner.host === hostname() && !pidIsAlive(owner.pid),
+      });
+      break;
+    } catch (error) {
+      if (attempt === BOOTSTRAP_LEASE_RETRIES - 1) throw error;
+      Atomics.wait(bootstrapSleepCell, 0, 0, BOOTSTRAP_LEASE_RETRY_MS);
+    }
+  }
+  if (!lease) throw new Error("event_v3_bootstrap_lease_busy");
+  try {
+    return operation();
+  } finally {
+    lease.release();
+  }
+}
+
+function pidIsAlive(pid: number): boolean {
+  if (!Number.isSafeInteger(pid) || pid < 1) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code !== "ESRCH";
+  }
+}
+
 function activateCandidateEpoch(
   root: string,
   candidate: CandidateGenesisManifestV3,

package/src/core/events/v3/capabilities.ts

@@ -84,7 +84,7 @@ const BASE_SIGNAL_SUPPORT: Record<Adapter, Record<BaseAdapterSignalV3, Capabilit
     turn_id: "conditional",
     permission: "native",
     post_compaction: "native",
-    context_usage: "unsupported",
+    context_usage: "conditional",
   },
   codex: {
     ...SHARED_SIGNAL_SUPPORT,
@@ -92,7 +92,7 @@ const BASE_SIGNAL_SUPPORT: Record<Adapter, Record<BaseAdapterSignalV3, Capabilit
     turn_id: "native",
     permission: "native",
     post_compaction: "native",
-    context_usage: "unsupported",
+    context_usage: "derived",
   },
   cursor: {
     ...SHARED_SIGNAL_SUPPORT,
@@ -106,7 +106,7 @@ const BASE_SIGNAL_SUPPORT: Record<Adapter, Record<BaseAdapterSignalV3, Capabilit
     shell: "conditional",
     permission: "conditional",
     post_compaction: "unsupported",
-    context_usage: "unsupported",
+    context_usage: "conditional",
   },
 };