harnery 0.34.1 → 0.36.0

package/src/core/events/legacy-storage/inventory.ts

@@ -0,0 +1,51 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir, realpath } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+
+export interface LegacyV1SegmentInventoryEntry {
+  filename: string;
+  path: string;
+  bytes: number;
+  digest: `sha256:${string}`;
+  compressed: boolean;
+}
+
+const LEGACY_ACTIVE_NAME = ["events", "ndjson"].join(".");
+
+/** Enumerate all root events*.ndjson* frozen-history candidates, including manual variants. */
+export async function inventoryLegacyV1Segments(
+  coordRoot: string,
+): Promise<LegacyV1SegmentInventoryEntry[]> {
+  const harnery = resolve(coordRoot, ".harnery");
+  const stat = await lstat(harnery);
+  if (!stat.isDirectory() || stat.isSymbolicLink())
+    throw new Error("legacy_v1_inventory_root_invalid");
+  const rootReal = await realpath(harnery);
+  const result: LegacyV1SegmentInventoryEntry[] = [];
+  for (const name of (await readdir(harnery)).sort()) {
+    if (!/^events.*\.ndjson(?:\.gz)?$/.test(name) || name === LEGACY_ACTIVE_NAME) continue;
+    const path = join(harnery, name);
+    const entryStat = await lstat(path);
+    if (!entryStat.isFile() || entryStat.isSymbolicLink()) {
+      throw new Error(`legacy_v1_inventory_segment_invalid:${name}`);
+    }
+    const pathReal = await realpath(path);
+    if (relative(rootReal, pathReal).startsWith(".."))
+      throw new Error("legacy_v1_inventory_escape");
+    result.push({
+      filename: name,
+      path,
+      bytes: entryStat.size,
+      digest: await hashFile(path),
+      compressed: name.endsWith(".gz"),
+    });
+  }
+  return result;
+}
+
+async function hashFile(path: string): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  for await (const current of createReadStream(path)) hash.update(current as Buffer);
+  return `sha256:${hash.digest("hex")}`;
+}

package/src/core/events/legacy-storage/manifest.ts

@@ -0,0 +1,136 @@
+export interface LegacyV1SegmentManifest {
+  format: "harnery-legacy-v1-segment";
+  format_version: 1;
+  segment_id: `v1s_${string}`;
+  source: {
+    filename: string;
+    bytes: number;
+    digest: `sha256:${string}`;
+    row_count: number;
+    minimum_recorded_at: string | null;
+    maximum_recorded_at: string | null;
+  };
+  payload: {
+    algorithm: "gzip";
+    path: string;
+    bytes: number;
+    digest: `sha256:${string}`;
+  };
+  minimum_harnery_version: string;
+  created_at: string;
+}
+
+export function validateLegacyV1SegmentManifest(value: unknown): LegacyV1SegmentManifest {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("legacy_v1_manifest_invalid");
+  }
+  const manifest = value as Record<string, unknown>;
+  exact(manifest, [
+    "format",
+    "format_version",
+    "segment_id",
+    "source",
+    "payload",
+    "minimum_harnery_version",
+    "created_at",
+  ]);
+  if (manifest.format !== "harnery-legacy-v1-segment" || manifest.format_version !== 1) {
+    throw new Error("legacy_v1_manifest_version_unsupported");
+  }
+  const segmentId = text(manifest.segment_id);
+  if (!/^v1s_[0-9a-f]{32}$/.test(segmentId)) throw new Error("legacy_v1_segment_id_invalid");
+  const source = object(manifest.source);
+  exact(source, [
+    "filename",
+    "bytes",
+    "digest",
+    "row_count",
+    "minimum_recorded_at",
+    "maximum_recorded_at",
+  ]);
+  const filename = text(source.filename);
+  if (!/^events.+\.ndjson$/.test(filename) || filename.includes("/") || filename.includes("\\")) {
+    throw new Error("legacy_v1_source_filename_invalid");
+  }
+  const minimum = nullableTimestamp(source.minimum_recorded_at);
+  const maximum = nullableTimestamp(source.maximum_recorded_at);
+  if ((minimum === null) !== (maximum === null) || (minimum && maximum && minimum > maximum)) {
+    throw new Error("legacy_v1_manifest_time_range_invalid");
+  }
+  const payload = object(manifest.payload);
+  exact(payload, ["algorithm", "path", "bytes", "digest"]);
+  if (payload.algorithm !== "gzip" || payload.path !== `${segmentId}.ndjson.gz`) {
+    throw new Error("legacy_v1_payload_contract_invalid");
+  }
+  const version = text(manifest.minimum_harnery_version);
+  if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) {
+    throw new Error("legacy_v1_minimum_version_invalid");
+  }
+  return {
+    format: "harnery-legacy-v1-segment",
+    format_version: 1,
+    segment_id: segmentId as `v1s_${string}`,
+    source: {
+      filename,
+      bytes: integer(source.bytes),
+      digest: digest(source.digest),
+      row_count: integer(source.row_count),
+      minimum_recorded_at: minimum,
+      maximum_recorded_at: maximum,
+    },
+    payload: {
+      algorithm: "gzip",
+      path: text(payload.path),
+      bytes: integer(payload.bytes),
+      digest: digest(payload.digest),
+    },
+    minimum_harnery_version: version,
+    created_at: timestamp(manifest.created_at),
+  };
+}
+
+function object(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("legacy_v1_manifest_object_invalid");
+  }
+  return value as Record<string, unknown>;
+}
+
+function exact(value: Record<string, unknown>, keys: string[]): void {
+  if (Object.keys(value).sort().join("\0") !== [...keys].sort().join("\0")) {
+    throw new Error("legacy_v1_manifest_unknown_field");
+  }
+}
+
+function text(value: unknown): string {
+  if (typeof value !== "string") throw new Error("legacy_v1_manifest_string_invalid");
+  return value;
+}
+
+function integer(value: unknown): number {
+  if (!Number.isSafeInteger(value) || (value as number) < 0) {
+    throw new Error("legacy_v1_manifest_integer_invalid");
+  }
+  return value as number;
+}
+
+function digest(value: unknown): `sha256:${string}` {
+  const result = text(value);
+  if (!/^sha256:[0-9a-f]{64}$/.test(result)) throw new Error("legacy_v1_manifest_digest_invalid");
+  return result as `sha256:${string}`;
+}
+
+function timestamp(value: unknown): string {
+  const result = text(value);
+  if (
+    !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(result) ||
+    !Number.isFinite(Date.parse(result))
+  ) {
+    throw new Error("legacy_v1_manifest_timestamp_invalid");
+  }
+  return result;
+}
+
+function nullableTimestamp(value: unknown): string | null {
+  return value === null ? null : timestamp(value);
+}

package/src/core/events/legacy-storage/reader.ts

@@ -0,0 +1,178 @@
+import { createHash } from "node:crypto";
+import { constants, createReadStream } from "node:fs";
+import { lstat, open, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve } from "node:path";
+import { createGunzip } from "node:zlib";
+import { type LegacyV1SegmentManifest, validateLegacyV1SegmentManifest } from "./manifest.ts";
+
+export interface LegacyV1LogicalRow {
+  row_number: number;
+  bytes: Buffer;
+}
+
+export interface LegacyV1ReaderLimits {
+  max_manifest_bytes?: number;
+  max_payload_bytes?: number;
+  max_decompressed_bytes?: number;
+  max_row_bytes?: number;
+  max_rows?: number;
+  fault?: (boundary: "after_source_digest", path: string) => void;
+}
+
+const DEFAULTS: Required<Omit<LegacyV1ReaderLimits, "fault">> = {
+  max_manifest_bytes: 1024 * 1024,
+  max_payload_bytes: 8 * 1024 * 1024 * 1024,
+  max_decompressed_bytes: 16 * 1024 * 1024 * 1024,
+  max_row_bytes: 64 * 1024 * 1024,
+  max_rows: 20_000_000,
+};
+
+/** Stream rows from one loose sealed segment or one manifest-bound gzip canary. */
+export async function* streamLegacyV1Rows(
+  sourcePath: string,
+  limits: LegacyV1ReaderLimits = {},
+): AsyncGenerator<LegacyV1LogicalRow> {
+  const bounded = { ...DEFAULTS, ...limits };
+  const source = resolve(sourcePath);
+  let input = source;
+  let expectedRows: number | undefined;
+  let expectedSourceBytes: number | undefined;
+  let expectedSourceDigest: `sha256:${string}` | undefined;
+  let expectedPayloadBytes: number | undefined;
+  let expectedPayloadDigest: `sha256:${string}` | undefined;
+  let containmentRoot: string | undefined;
+  if (source.endsWith(".manifest.json")) {
+    const manifestHandle = await openStableRegularFile(source, undefined, "manifest");
+    let manifest: LegacyV1SegmentManifest;
+    try {
+      const stat = await manifestHandle.stat();
+      if (stat.size > bounded.max_manifest_bytes) throw new Error("legacy_v1_manifest_too_large");
+      manifest = validateLegacyV1SegmentManifest(
+        JSON.parse(await manifestHandle.readFile("utf8")) as unknown,
+      );
+    } finally {
+      await manifestHandle.close();
+    }
+    const directory = await realpath(dirname(source));
+    containmentRoot = directory;
+    input = resolve(directory, manifest.payload.path);
+    expectedRows = manifest.source.row_count;
+    expectedSourceBytes = manifest.source.bytes;
+    expectedSourceDigest = manifest.source.digest;
+    expectedPayloadBytes = manifest.payload.bytes;
+    expectedPayloadDigest = manifest.payload.digest;
+  }
+
+  const compressed = source.endsWith(".manifest.json") || source.endsWith(".gz");
+  const kind = source.endsWith(".manifest.json") ? "payload" : "segment";
+  const inputHandle = await openStableRegularFile(input, containmentRoot, kind);
+  try {
+    const inputStat = await inputHandle.stat();
+    if (source.endsWith(".manifest.json")) {
+      if (inputStat.size !== expectedPayloadBytes)
+        throw new Error("legacy_v1_payload_length_mismatch");
+      if (inputStat.size > bounded.max_payload_bytes)
+        throw new Error("legacy_v1_payload_too_large");
+      if ((await hashHandle(input, inputHandle)) !== expectedPayloadDigest) {
+        throw new Error("legacy_v1_payload_digest_mismatch");
+      }
+    }
+    limits.fault?.("after_source_digest", input);
+    const file = createReadStream(input, { fd: inputHandle.fd, autoClose: false, start: 0 });
+    const stream = compressed ? file.pipe(createGunzip()) : file;
+    let pending: Buffer<ArrayBufferLike> = Buffer.alloc(0);
+    let rows = 0;
+    let decompressed = 0;
+    const sourceHash = createHash("sha256");
+    for await (const current of stream) {
+      const chunk = current as Buffer;
+      decompressed += chunk.length;
+      sourceHash.update(chunk);
+      if (decompressed > bounded.max_decompressed_bytes) {
+        throw new Error("legacy_v1_decompression_bound_exceeded");
+      }
+      pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]);
+      if (pending.length > bounded.max_row_bytes && !pending.includes(0x0a)) {
+        throw new Error("legacy_v1_row_bound_exceeded");
+      }
+      let newline = pending.indexOf(0x0a);
+      while (newline >= 0) {
+        const row = pending.subarray(0, newline);
+        pending = pending.subarray(newline + 1);
+        if (row.length === 0) throw new Error("legacy_v1_empty_row");
+        if (row.length > bounded.max_row_bytes) throw new Error("legacy_v1_row_bound_exceeded");
+        rows += 1;
+        if (rows > bounded.max_rows) throw new Error("legacy_v1_row_count_bound_exceeded");
+        yield { row_number: rows, bytes: Buffer.from(row) };
+        newline = pending.indexOf(0x0a);
+      }
+    }
+    if (pending.length !== 0) throw new Error("legacy_v1_segment_unterminated_or_truncated");
+    if (expectedRows !== undefined && rows !== expectedRows)
+      throw new Error("legacy_v1_row_count_mismatch");
+    if (expectedSourceBytes !== undefined && decompressed !== expectedSourceBytes) {
+      throw new Error("legacy_v1_source_length_mismatch");
+    }
+    const digest = `sha256:${sourceHash.digest("hex")}`;
+    if (expectedSourceDigest !== undefined && digest !== expectedSourceDigest) {
+      throw new Error("legacy_v1_source_digest_mismatch");
+    }
+  } finally {
+    await inputHandle.close();
+  }
+}
+
+async function hashHandle(
+  path: string,
+  handle: Awaited<ReturnType<typeof open>>,
+): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  const stream = createReadStream(path, { fd: handle.fd, autoClose: false, start: 0 });
+  for await (const current of stream) hash.update(current as Buffer);
+  return `sha256:${hash.digest("hex")}`;
+}
+
+async function openStableRegularFile(
+  path: string,
+  containmentRoot: string | undefined,
+  kind: "manifest" | "payload" | "segment",
+) {
+  const before = await lstat(path);
+  if (!before.isFile() || before.isSymbolicLink()) throw new Error(`legacy_v1_${kind}_not_regular`);
+  const beforeReal = await realpath(path);
+  if (containmentRoot) assertContained(containmentRoot, beforeReal);
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const opened = await handle.stat();
+    const after = await lstat(path);
+    const afterReal = await realpath(path);
+    if (
+      !opened.isFile() ||
+      !after.isFile() ||
+      after.isSymbolicLink() ||
+      opened.dev !== before.dev ||
+      opened.ino !== before.ino ||
+      opened.dev !== after.dev ||
+      opened.ino !== after.ino ||
+      beforeReal !== afterReal
+    )
+      throw new Error(`legacy_v1_${kind}_changed_during_open`);
+    if (containmentRoot) assertContained(containmentRoot, afterReal);
+    return handle;
+  } catch (error) {
+    await handle.close();
+    throw error;
+  }
+}
+
+function assertContained(root: string, candidate: string): void {
+  const path = relative(root, candidate);
+  if (
+    path === "" ||
+    (path !== ".." &&
+      !path.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) &&
+      !isAbsolute(path))
+  )
+    return;
+  throw new Error("legacy_v1_payload_escape");
+}

package/src/core/events/v3/authority-outbox.ts

@@ -14,9 +14,15 @@ import { join } from "node:path";
 import { fsyncParentDirectory } from "../../workflow/durable-record.ts";
 import { canonicalJsonV3, sha256V3 } from "./canonical.ts";
 import type { EventV3 } from "./contract.ts";
+import { liveGenesisIdV3 } from "./control.ts";
 import { assertEventV3 } from "./validate.ts";
 import { ensureEventV3Layout, eventV3Paths, writeEventV3 } from "./writer.ts";
 
+export interface AuthorityEpochFenceV3 {
+  /** Genesis id of the epoch the transaction's event was produced for. */
+  expectedGenesisId?: `gex_${string}`;
+}
+
 const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/;
 const INSTANCE_PATTERN = /^inst_[a-zA-Z0-9._-]{1,128}$/;
 const TX_PATTERN = /^txn_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
@@ -146,9 +152,13 @@ export function buildAuthorityTransactionV3(
 export function publishAuthorityTransactionV3(
   coordRoot: string,
   transaction: AuthorityTransactionV3,
+  fence: AuthorityEpochFenceV3 = {},
 ): AuthorityTransactionV3 {
   validateTransaction(transaction);
   const paths = ensureEventV3Layout(coordRoot);
+  if (fence.expectedGenesisId && liveGenesisIdV3(coordRoot) !== fence.expectedGenesisId) {
+    throw new Error("authority transaction epoch was replaced before publication");
+  }
   const readyPath = authorityReadyPath(paths.authorityOutbox, transaction.transaction_id);
   const serialized = `${canonicalJsonV3(transaction)}\n`;
   if (existsSync(readyPath)) {
@@ -173,6 +183,7 @@ export function reconcileAuthorityTransactionV3(
   coordRoot: string,
   transactionId: string,
   reconciler: AuthorityReconcilerV3,
+  fence: AuthorityEpochFenceV3 = {},
 ): AuthorityReceiptV3 {
   assertTransactionId(transactionId);
   const paths = ensureEventV3Layout(coordRoot);
@@ -191,7 +202,12 @@ export function reconcileAuthorityTransactionV3(
   }
   const event = JSON.parse(transaction.event_row) as unknown;
   assertEventV3(event);
-  writeEventV3(coordRoot, event);
+  const durability = writeEventV3(coordRoot, event, {
+    ...(fence.expectedGenesisId ? { expectedGenesisId: fence.expectedGenesisId } : {}),
+  });
+  if (durability.state === "epoch_replaced") {
+    throw new Error("authority transaction epoch was replaced before its event committed");
+  }
   const receipt: AuthorityReceiptV3 = {
     format: "harnery-event-v3-authority-receipt",
     format_version: 1,
@@ -222,6 +238,11 @@ export function listPendingAuthorityTransactionsV3(coordRoot: string): Authority
     .map((name) => readAuthorityTransactionV3(join(outbox, name)));
 }
 
+export function authorityRecoveryIntentPathV3(coordRoot: string, transactionId: string): string {
+  assertTransactionId(transactionId);
+  return join(eventV3Paths(coordRoot).root, "authority-recoveries", `${transactionId}.ready.json`);
+}
+
 export function readAuthorityTransactionV3(path: string): AuthorityTransactionV3 {
   const serialized = readFileSync(path, "utf8");
   let parsed: unknown;

package/src/core/events/v3/base-contract.ts

@@ -100,6 +100,26 @@ export function ObservationV3BaseSchema<T extends TSchema>(value: T) {
   ]);
 }
 
+const RuntimeTelemetryCapabilityValueV3BaseSchema = StrictObject({
+  support: Type.Union([
+    Type.Literal("native"),
+    Type.Literal("derived"),
+    Type.Literal("conditional"),
+  ]),
+  source: SafeToken,
+  completeness: Type.Union([
+    Type.Literal("exact"),
+    Type.Literal("inferred"),
+    Type.Literal("percentage_only"),
+    Type.Literal("lower_bound"),
+    Type.Literal("unknown"),
+  ]),
+});
+
+const RuntimeTelemetryCapabilityObservationV3BaseSchema = ObservationV3BaseSchema(
+  RuntimeTelemetryCapabilityValueV3BaseSchema,
+);
+
 export const RuntimeAttestationV3BaseSchema = StrictObject({
   attestation_id: AttestationId,
   generation_id: GenerationId,
@@ -110,6 +130,27 @@ export const RuntimeAttestationV3BaseSchema = StrictObject({
     StrictObject({ id: SafeToken, version: Type.Optional(SafeToken) }),
   ),
   model: ObservationV3BaseSchema(StrictObject({ provider: SafeToken, id: SafeToken })),
+  /** Runtime tuning identity: the effort dial (and speed tier where the
+   * runtime reports one). `observed` with a missing `effort` never occurs —
+   * a runtime that proved the model has no dial attests `unsupported` with
+   * capability `effort_selection` instead. Values pass through verbatim
+   * (`low`…`max`, `standard`/`fast`); display labels are a renderer concern. */
+  tuning: ObservationV3BaseSchema(
+    StrictObject({
+      effort: Type.Optional(SafeToken),
+      speed: Type.Optional(SafeToken),
+    }),
+  ),
+  /** Effective telemetry support proven for this runtime session. This is
+   * intentionally separate from the static capability profile digest: a
+   * missing or late local source changes evidence, not the gated profile. */
+  telemetry: StrictObject({
+    context_usage: RuntimeTelemetryCapabilityObservationV3BaseSchema,
+    wait_spans: RuntimeTelemetryCapabilityObservationV3BaseSchema,
+    wait_completeness: RuntimeTelemetryCapabilityObservationV3BaseSchema,
+    response_latency: RuntimeTelemetryCapabilityObservationV3BaseSchema,
+    inference_timing: RuntimeTelemetryCapabilityObservationV3BaseSchema,
+  }),
   capability_profile: Type.String({ pattern: "^cap_[a-f0-9]{64}$" }),
   declared_by_event_id: EventId,
 });
@@ -391,13 +432,21 @@ const AuthorityReferenceSchema = StrictObject({
   record_id: Type.Optional(SafeToken),
 });
 
-const MeasurementSchema = StrictObject({
-  used_tokens: Type.Integer({ minimum: 0 }),
-  limit_tokens: Type.Integer({ minimum: 1 }),
-  remaining_tokens: Type.Optional(Type.Integer({ minimum: 0 })),
-  measured_at: Timestamp,
-  method: SafeToken,
-});
+const MeasurementSchema = Type.Union([
+  StrictObject({
+    used_tokens: Type.Integer({ minimum: 0 }),
+    limit_tokens: Type.Integer({ minimum: 1 }),
+    remaining_tokens: Type.Optional(Type.Integer({ minimum: 0 })),
+    measured_at: Timestamp,
+    method: SafeToken,
+  }),
+  StrictObject({
+    used_percent: Type.Number({ minimum: 0, maximum: 100 }),
+    remaining_percent: Type.Optional(Type.Number({ minimum: 0, maximum: 100 })),
+    measured_at: Timestamp,
+    method: SafeToken,
+  }),
+]);
 
 const ArtifactReferenceSchema = StrictObject({
   artifact_id: Type.String({ pattern: "^art_[a-zA-Z0-9._-]{1,128}$" }),