harnery 0.35.0 → 0.36.0

package/src/core/conversations/search-index.ts

@@ -0,0 +1,48 @@
+import type { HarneryConversationRecordV1 } from "./contract.ts";
+import { digest } from "./normalize.ts";
+
+export interface HarneryConversationSearchIndexV1 {
+  schema: "harnery.conversation-search-index/v1";
+  source_digest: `sha256:${string}`;
+  terms: Readonly<Record<string, readonly string[]>>;
+}
+
+export function buildConversationSearchIndex(
+  records: readonly HarneryConversationRecordV1[],
+): HarneryConversationSearchIndexV1 {
+  const terms = new Map<string, Set<string>>();
+  const ordered = [...records].sort((left, right) => left.record_id.localeCompare(right.record_id));
+  for (const record of ordered) {
+    for (const term of record.content.toLocaleLowerCase().match(/[\p{L}\p{N}_-]{2,}/gu) ?? []) {
+      const ids = terms.get(term) ?? new Set<string>();
+      ids.add(record.record_id);
+      terms.set(term, ids);
+    }
+  }
+  return {
+    schema: "harnery.conversation-search-index/v1",
+    source_digest: digest(
+      ordered.map((record) => `${record.record_id}:${record.content_digest}`).join("\n"),
+    ),
+    terms: Object.fromEntries(
+      [...terms]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([term, ids]) => [term, [...ids].sort()]),
+    ),
+  };
+}
+
+export function verifyIndexedRecords(
+  index: HarneryConversationSearchIndexV1,
+  records: readonly HarneryConversationRecordV1[],
+  term: string,
+): HarneryConversationRecordV1[] {
+  const byId = new Map(records.map((record) => [record.record_id, record]));
+  return (index.terms[term.toLocaleLowerCase()] ?? []).map((id) => {
+    const record = byId.get(id);
+    if (!record?.content.toLocaleLowerCase().includes(term.toLocaleLowerCase())) {
+      throw new Error(`stale conversation index hit: ${id}`);
+    }
+    return record;
+  });
+}

package/src/core/events/legacy-storage/canary.ts

@@ -0,0 +1,177 @@
+import { createHash } from "node:crypto";
+import { constants, createReadStream, createWriteStream } from "node:fs";
+import { chmod, lstat, mkdir, open, rename, rm } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createGzip } from "node:zlib";
+import { canonicalJsonV3 } from "../v3/canonical.ts";
+import { verifyLegacyV1HardFence } from "./fence.ts";
+import { type LegacyV1SegmentManifest, validateLegacyV1SegmentManifest } from "./manifest.ts";
+import { streamLegacyV1Rows } from "./reader.ts";
+
+export interface LegacyV1CanaryResult {
+  manifest: LegacyV1SegmentManifest;
+  manifest_path: string;
+  payload_path: string;
+  source_unchanged: true;
+  replacement_enabled: false;
+}
+
+/** Shadow-compress one fenced legacy segment, then prove byte and row parity. */
+export async function writeLegacyV1Canary(input: {
+  coord_root: string;
+  source_filename: string;
+  output_directory: string;
+  minimum_harnery_version: string;
+  created_at: string;
+}): Promise<LegacyV1CanaryResult> {
+  await verifyLegacyV1HardFence(input.coord_root);
+  if (
+    !/^events.+\.ndjson$/.test(input.source_filename) ||
+    input.source_filename.includes("/") ||
+    input.source_filename.includes("\\")
+  ) {
+    throw new Error("legacy_v1_canary_source_name_invalid");
+  }
+  const harnery = resolve(input.coord_root, ".harnery");
+  const source = join(harnery, input.source_filename);
+  const sourceStat = await lstat(source);
+  if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) {
+    throw new Error("legacy_v1_canary_source_not_regular");
+  }
+  const sourceDigest = await hashFile(source);
+  const sourceFacts = await inspectRows(source);
+  const segmentId = `v1s_${createHash("sha256")
+    .update(`${input.source_filename}\0${sourceStat.size}\0${sourceDigest}`)
+    .digest("hex")
+    .slice(0, 32)}` as const;
+  const outputDirectory = resolve(input.output_directory);
+  await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
+  await chmod(outputDirectory, 0o700);
+  const payloadPath = join(outputDirectory, `${segmentId}.ndjson.gz`);
+  const manifestPath = join(outputDirectory, `${segmentId}.manifest.json`);
+  const partialPayload = `${payloadPath}.partial`;
+  const partialManifest = `${manifestPath}.partial`;
+  await rejectExisting([payloadPath, manifestPath, partialPayload, partialManifest]);
+  try {
+    await pipeline(
+      createReadStream(source),
+      createGzip({ level: 9 }),
+      createWriteStream(partialPayload, { flags: "wx", mode: 0o600 }),
+    );
+    await rename(partialPayload, payloadPath);
+    const payloadStat = await lstat(payloadPath);
+    const manifest = validateLegacyV1SegmentManifest({
+      format: "harnery-legacy-v1-segment",
+      format_version: 1,
+      segment_id: segmentId,
+      source: {
+        filename: input.source_filename,
+        bytes: sourceStat.size,
+        digest: sourceDigest,
+        row_count: sourceFacts.rows,
+        minimum_recorded_at: sourceFacts.minimum,
+        maximum_recorded_at: sourceFacts.maximum,
+      },
+      payload: {
+        algorithm: "gzip",
+        path: basename(payloadPath),
+        bytes: payloadStat.size,
+        digest: await hashFile(payloadPath),
+      },
+      minimum_harnery_version: input.minimum_harnery_version,
+      created_at: input.created_at,
+    });
+    const handle = await open(
+      partialManifest,
+      constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL,
+      0o600,
+    );
+    try {
+      await handle.writeFile(`${canonicalJsonV3(manifest)}\n`, "utf8");
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await rename(partialManifest, manifestPath);
+    await proveRowParity(source, manifestPath);
+    const after = await lstat(source);
+    if (after.size !== sourceStat.size || (await hashFile(source)) !== sourceDigest) {
+      throw new Error("legacy_v1_canary_source_changed");
+    }
+    return {
+      manifest,
+      manifest_path: manifestPath,
+      payload_path: payloadPath,
+      source_unchanged: true,
+      replacement_enabled: false,
+    };
+  } catch (error) {
+    await rm(partialPayload, { force: true });
+    await rm(partialManifest, { force: true });
+    await rm(payloadPath, { force: true });
+    await rm(manifestPath, { force: true });
+    throw error;
+  }
+}
+
+export function activateLegacyV1CanaryReplacement(): never {
+  throw new Error("legacy_v1_replacement_activation_disabled");
+}
+
+async function inspectRows(
+  path: string,
+): Promise<{ rows: number; minimum: string | null; maximum: string | null }> {
+  let rows = 0;
+  const timestamps: string[] = [];
+  for await (const row of streamLegacyV1Rows(path)) {
+    rows += 1;
+    const value = JSON.parse(row.bytes.toString("utf8")) as Record<string, unknown>;
+    const candidate =
+      typeof value.at === "string"
+        ? value.at
+        : typeof value.timestamp === "string"
+          ? value.timestamp
+          : value.time &&
+              typeof value.time === "object" &&
+              typeof (value.time as Record<string, unknown>).observed_at === "string"
+            ? ((value.time as Record<string, unknown>).observed_at as string)
+            : undefined;
+    if (candidate && Number.isFinite(Date.parse(candidate)))
+      timestamps.push(new Date(candidate).toISOString());
+  }
+  timestamps.sort();
+  return { rows, minimum: timestamps[0] ?? null, maximum: timestamps.at(-1) ?? null };
+}
+
+async function proveRowParity(loose: string, manifest: string): Promise<void> {
+  const left = streamLegacyV1Rows(loose)[Symbol.asyncIterator]();
+  const right = streamLegacyV1Rows(manifest)[Symbol.asyncIterator]();
+  while (true) {
+    const [looseRow, packedRow] = await Promise.all([left.next(), right.next()]);
+    if (looseRow.done || packedRow.done) {
+      if (looseRow.done !== packedRow.done) throw new Error("legacy_v1_canary_row_count_mismatch");
+      break;
+    }
+    if (!looseRow.value.bytes.equals(packedRow.value.bytes)) {
+      throw new Error(`legacy_v1_canary_row_mismatch:${looseRow.value.row_number}`);
+    }
+  }
+}
+
+async function hashFile(path: string): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  for await (const current of createReadStream(path)) hash.update(current as Buffer);
+  return `sha256:${hash.digest("hex")}`;
+}
+
+async function rejectExisting(paths: string[]): Promise<void> {
+  for (const path of paths) {
+    try {
+      await lstat(path);
+      throw new Error("legacy_v1_canary_output_exists");
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}

package/src/core/events/legacy-storage/fence.ts

@@ -0,0 +1,129 @@
+import { createHash } from "node:crypto";
+import { constants, createReadStream } from "node:fs";
+import { lstat, open, readdir, realpath } from "node:fs/promises";
+import { basename, join, relative, resolve } from "node:path";
+import { canonicalJsonV3 } from "../v3/canonical.ts";
+
+export interface LegacyV1HardFence {
+  manifest_version: 1;
+  kind: "v1_hard_path_fence";
+  terminal_archive: string;
+  terminal_digest: `sha256:${string}`;
+}
+
+export interface VerifiedLegacyV1HardFence {
+  fence_directory: string;
+  marker_path: string;
+  terminal_archive_path: string;
+  marker: LegacyV1HardFence;
+}
+
+const MARKER = "V1-SEALED.json";
+const LEGACY_ACTIVE_NAME = ["events", "ndjson"].join(".");
+
+/** Verify the exact read-only V1 path fence and its terminal sealed segment. */
+export async function verifyLegacyV1HardFence(
+  coordRoot: string,
+): Promise<VerifiedLegacyV1HardFence> {
+  const root = resolve(coordRoot);
+  const harnery = join(root, ".harnery");
+  const harneryStat = await lstat(harnery);
+  if (!harneryStat.isDirectory() || harneryStat.isSymbolicLink()) {
+    throw new Error("legacy_v1_harnery_root_invalid");
+  }
+  const harneryReal = await realpath(harnery);
+  const fence = join(harnery, LEGACY_ACTIVE_NAME);
+  let fenceStat: Awaited<ReturnType<typeof lstat>>;
+  try {
+    fenceStat = await lstat(fence);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      throw new Error("legacy_v1_hard_fence_missing");
+    }
+    throw error;
+  }
+  if (!fenceStat.isDirectory() || fenceStat.isSymbolicLink()) {
+    throw new Error("legacy_v1_active_file_present_or_fence_invalid");
+  }
+  if ((fenceStat.mode & 0o222) !== 0) throw new Error("legacy_v1_hard_fence_is_writable");
+  const fenceReal = await realpath(fence);
+  if (relative(harneryReal, fenceReal).startsWith(".."))
+    throw new Error("legacy_v1_hard_fence_escape");
+  const names = (await readdir(fence)).sort();
+  if (names.length !== 1 || names[0] !== MARKER) throw new Error("legacy_v1_hard_fence_not_exact");
+
+  const markerPath = join(fence, MARKER);
+  const markerStat = await lstat(markerPath);
+  if (!markerStat.isFile() || markerStat.isSymbolicLink()) {
+    throw new Error("legacy_v1_hard_fence_marker_invalid");
+  }
+  if ((markerStat.mode & 0o222) !== 0) throw new Error("legacy_v1_hard_fence_marker_writable");
+  if (markerStat.size > 64 * 1024) throw new Error("legacy_v1_hard_fence_marker_too_large");
+  const markerText = await readFileNoFollow(markerPath);
+  const marker = validateFenceMarker(JSON.parse(markerText) as unknown);
+  if (markerText.trim() !== canonicalJsonV3(marker)) {
+    throw new Error("legacy_v1_hard_fence_marker_noncanonical");
+  }
+  const terminalName = marker.terminal_archive.replace(/^\.harnery\//, "");
+  if (!/^events-[0-9]{4}-[0-9]{2}-[0-9]{2}(?:\.[1-9][0-9]*)?\.ndjson$/.test(terminalName)) {
+    throw new Error("legacy_v1_terminal_archive_name_invalid");
+  }
+  const terminalPath = join(harnery, terminalName);
+  const terminalStat = await lstat(terminalPath);
+  if (!terminalStat.isFile() || terminalStat.isSymbolicLink()) {
+    throw new Error("legacy_v1_terminal_archive_invalid");
+  }
+  if ((await hashFile(terminalPath)) !== marker.terminal_digest) {
+    throw new Error("legacy_v1_terminal_archive_digest_mismatch");
+  }
+  return {
+    fence_directory: fence,
+    marker_path: markerPath,
+    terminal_archive_path: terminalPath,
+    marker,
+  };
+}
+
+function validateFenceMarker(value: unknown): LegacyV1HardFence {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("legacy_v1_hard_fence_marker_invalid");
+  }
+  const marker = value as Record<string, unknown>;
+  if (
+    Object.keys(marker).sort().join("\0") !==
+      ["kind", "manifest_version", "terminal_archive", "terminal_digest"].sort().join("\0") ||
+    marker.manifest_version !== 1 ||
+    marker.kind !== "v1_hard_path_fence" ||
+    typeof marker.terminal_archive !== "string" ||
+    typeof marker.terminal_digest !== "string" ||
+    !/^sha256:[0-9a-f]{64}$/.test(marker.terminal_digest)
+  ) {
+    throw new Error("legacy_v1_hard_fence_marker_invalid");
+  }
+  return marker as unknown as LegacyV1HardFence;
+}
+
+async function readFileNoFollow(path: string): Promise<string> {
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile()) throw new Error("legacy_v1_hard_fence_marker_invalid");
+    return handle.readFile("utf8");
+  } finally {
+    await handle.close();
+  }
+}
+
+async function hashFile(path: string): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const stat = await handle.stat();
+    if (!stat.isFile()) throw new Error(`legacy_v1_segment_not_regular:${basename(path)}`);
+    const stream = createReadStream(path, { fd: handle.fd, autoClose: false });
+    for await (const current of stream) hash.update(current as Buffer);
+  } finally {
+    await handle.close();
+  }
+  return `sha256:${hash.digest("hex")}`;
+}

package/src/core/events/legacy-storage/index.ts

@@ -0,0 +1,5 @@
+export * from "./canary.ts";
+export * from "./fence.ts";
+export * from "./inventory.ts";
+export * from "./manifest.ts";
+export * from "./reader.ts";

package/src/core/events/legacy-storage/inventory.ts

@@ -0,0 +1,51 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir, realpath } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+
+export interface LegacyV1SegmentInventoryEntry {
+  filename: string;
+  path: string;
+  bytes: number;
+  digest: `sha256:${string}`;
+  compressed: boolean;
+}
+
+const LEGACY_ACTIVE_NAME = ["events", "ndjson"].join(".");
+
+/** Enumerate all root events*.ndjson* frozen-history candidates, including manual variants. */
+export async function inventoryLegacyV1Segments(
+  coordRoot: string,
+): Promise<LegacyV1SegmentInventoryEntry[]> {
+  const harnery = resolve(coordRoot, ".harnery");
+  const stat = await lstat(harnery);
+  if (!stat.isDirectory() || stat.isSymbolicLink())
+    throw new Error("legacy_v1_inventory_root_invalid");
+  const rootReal = await realpath(harnery);
+  const result: LegacyV1SegmentInventoryEntry[] = [];
+  for (const name of (await readdir(harnery)).sort()) {
+    if (!/^events.*\.ndjson(?:\.gz)?$/.test(name) || name === LEGACY_ACTIVE_NAME) continue;
+    const path = join(harnery, name);
+    const entryStat = await lstat(path);
+    if (!entryStat.isFile() || entryStat.isSymbolicLink()) {
+      throw new Error(`legacy_v1_inventory_segment_invalid:${name}`);
+    }
+    const pathReal = await realpath(path);
+    if (relative(rootReal, pathReal).startsWith(".."))
+      throw new Error("legacy_v1_inventory_escape");
+    result.push({
+      filename: name,
+      path,
+      bytes: entryStat.size,
+      digest: await hashFile(path),
+      compressed: name.endsWith(".gz"),
+    });
+  }
+  return result;
+}
+
+async function hashFile(path: string): Promise<`sha256:${string}`> {
+  const hash = createHash("sha256");
+  for await (const current of createReadStream(path)) hash.update(current as Buffer);
+  return `sha256:${hash.digest("hex")}`;
+}

package/src/core/events/legacy-storage/manifest.ts

@@ -0,0 +1,136 @@
+export interface LegacyV1SegmentManifest {
+  format: "harnery-legacy-v1-segment";
+  format_version: 1;
+  segment_id: `v1s_${string}`;
+  source: {
+    filename: string;
+    bytes: number;
+    digest: `sha256:${string}`;
+    row_count: number;
+    minimum_recorded_at: string | null;
+    maximum_recorded_at: string | null;
+  };
+  payload: {
+    algorithm: "gzip";
+    path: string;
+    bytes: number;
+    digest: `sha256:${string}`;
+  };
+  minimum_harnery_version: string;
+  created_at: string;
+}
+
+export function validateLegacyV1SegmentManifest(value: unknown): LegacyV1SegmentManifest {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("legacy_v1_manifest_invalid");
+  }
+  const manifest = value as Record<string, unknown>;
+  exact(manifest, [
+    "format",
+    "format_version",
+    "segment_id",
+    "source",
+    "payload",
+    "minimum_harnery_version",
+    "created_at",
+  ]);
+  if (manifest.format !== "harnery-legacy-v1-segment" || manifest.format_version !== 1) {
+    throw new Error("legacy_v1_manifest_version_unsupported");
+  }
+  const segmentId = text(manifest.segment_id);
+  if (!/^v1s_[0-9a-f]{32}$/.test(segmentId)) throw new Error("legacy_v1_segment_id_invalid");
+  const source = object(manifest.source);
+  exact(source, [
+    "filename",
+    "bytes",
+    "digest",
+    "row_count",
+    "minimum_recorded_at",
+    "maximum_recorded_at",
+  ]);
+  const filename = text(source.filename);
+  if (!/^events.+\.ndjson$/.test(filename) || filename.includes("/") || filename.includes("\\")) {
+    throw new Error("legacy_v1_source_filename_invalid");
+  }
+  const minimum = nullableTimestamp(source.minimum_recorded_at);
+  const maximum = nullableTimestamp(source.maximum_recorded_at);
+  if ((minimum === null) !== (maximum === null) || (minimum && maximum && minimum > maximum)) {
+    throw new Error("legacy_v1_manifest_time_range_invalid");
+  }
+  const payload = object(manifest.payload);
+  exact(payload, ["algorithm", "path", "bytes", "digest"]);
+  if (payload.algorithm !== "gzip" || payload.path !== `${segmentId}.ndjson.gz`) {
+    throw new Error("legacy_v1_payload_contract_invalid");
+  }
+  const version = text(manifest.minimum_harnery_version);
+  if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) {
+    throw new Error("legacy_v1_minimum_version_invalid");
+  }
+  return {
+    format: "harnery-legacy-v1-segment",
+    format_version: 1,
+    segment_id: segmentId as `v1s_${string}`,
+    source: {
+      filename,
+      bytes: integer(source.bytes),
+      digest: digest(source.digest),
+      row_count: integer(source.row_count),
+      minimum_recorded_at: minimum,
+      maximum_recorded_at: maximum,
+    },
+    payload: {
+      algorithm: "gzip",
+      path: text(payload.path),
+      bytes: integer(payload.bytes),
+      digest: digest(payload.digest),
+    },
+    minimum_harnery_version: version,
+    created_at: timestamp(manifest.created_at),
+  };
+}
+
+function object(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("legacy_v1_manifest_object_invalid");
+  }
+  return value as Record<string, unknown>;
+}
+
+function exact(value: Record<string, unknown>, keys: string[]): void {
+  if (Object.keys(value).sort().join("\0") !== [...keys].sort().join("\0")) {
+    throw new Error("legacy_v1_manifest_unknown_field");
+  }
+}
+
+function text(value: unknown): string {
+  if (typeof value !== "string") throw new Error("legacy_v1_manifest_string_invalid");
+  return value;
+}
+
+function integer(value: unknown): number {
+  if (!Number.isSafeInteger(value) || (value as number) < 0) {
+    throw new Error("legacy_v1_manifest_integer_invalid");
+  }
+  return value as number;
+}
+
+function digest(value: unknown): `sha256:${string}` {
+  const result = text(value);
+  if (!/^sha256:[0-9a-f]{64}$/.test(result)) throw new Error("legacy_v1_manifest_digest_invalid");
+  return result as `sha256:${string}`;
+}
+
+function timestamp(value: unknown): string {
+  const result = text(value);
+  if (
+    !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(result) ||
+    !Number.isFinite(Date.parse(result))
+  ) {
+    throw new Error("legacy_v1_manifest_timestamp_invalid");
+  }
+  return result;
+}
+
+function nullableTimestamp(value: unknown): string | null {
+  return value === null ? null : timestamp(value);
+}