@aldus-runtime/file-store 0.1.0

package/src/document.ts

@@ -0,0 +1,146 @@
+/**
+ * Validated JSON documents with forward-compatible round-tripping.
+ *
+ * ADR-0004 decision 3 is a MUST on this package: a store MUST preserve unknown properties across
+ * a read-modify-write. The hazard it addresses is quiet. Zod strips unknown properties on parse
+ * (ADR-0002 decision 7), so an older build that reads a manifest written by a newer minor
+ * version, changes one field, and writes it back would delete the newer version's data without
+ * any error, any warning, or any way to notice afterwards.
+ *
+ * A Git-tracked `.aldus/` shared between machines on different builds makes that an ordinary
+ * situation, not an edge case (contract §5.1, §7). So every read keeps the raw parsed JSON
+ * beside the validated value, and every update merges the caller's changes back over the raw.
+ */
+
+import {
+  validateRecord,
+  fromStructuredError,
+  type SchemaCompatibility,
+  type SchemaTypeFor,
+  type VersionedSchemaName,
+} from "@aldus-runtime/core";
+
+import { readFileOrUndefined, writeFileAtomic, type AtomicWriteOptions } from "./atomic.js";
+import { FileStoreErrorCodes, fileStoreError } from "./errors.js";
+
+/** A record as it exists on disk: validated, plus the bytes it was validated from. */
+export interface StoredDocument<T> {
+  /** The validated record. Unknown properties are absent, as Zod strips them. */
+  value: T;
+  /** The parsed JSON exactly as stored, including properties this build does not know about. */
+  raw: Record<string, unknown>;
+  /** How the stored `schemaVersion` relates to this build (ADR-0003). */
+  compatibility: SchemaCompatibility;
+}
+
+/**
+ * Read and validate a versioned record.
+ *
+ * Returns `undefined` when the file does not exist — an absent record is an ordinary state, not
+ * a failure, and a store that threw here would make "does this run exist?" an exception-handling
+ * exercise.
+ *
+ * @throws {AldusError} `ALDUS_RECORD_MALFORMED` if the bytes are not JSON,
+ * `ALDUS_SCHEMA_VERSION_UNSUPPORTED` if the major version is unreadable, or
+ * `ALDUS_SCHEMA_VALIDATION_FAILED` if the record does not satisfy its schema.
+ */
+export async function readDocument<N extends VersionedSchemaName>(
+  path: string,
+  schema: N,
+): Promise<StoredDocument<SchemaTypeFor<N>> | undefined> {
+  const contents = await readFileOrUndefined(path);
+  if (contents === undefined) return undefined;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(contents);
+  } catch {
+    throw fileStoreError(
+      FileStoreErrorCodes.RECORD_MALFORMED,
+      `The stored ${schema} record is not valid JSON. Atomic writes make a torn file impossible, ` +
+        "so this means the file was edited or replaced by something other than the store.",
+      {
+        category: "io",
+        retryable: false,
+        // The file's contents are excluded deliberately: a record may carry redacted-but-
+        // sensitive context (§19.2), and an error is itself durable.
+        details: { path, schema, byteLength: contents.length },
+      },
+    );
+  }
+
+  if (!isPlainObject(parsed)) {
+    throw fileStoreError(
+      FileStoreErrorCodes.RECORD_MALFORMED,
+      `The stored ${schema} record is valid JSON but not an object.`,
+      { category: "io", retryable: false, details: { path, schema } },
+    );
+  }
+
+  const result = validateRecord(schema, parsed);
+  if (!result.ok) throw fromStructuredError(result.error);
+
+  return { value: result.value, raw: parsed, compatibility: result.compatibility };
+}
+
+/** Write a record, replacing any previous contents atomically. */
+export async function writeDocument(
+  path: string,
+  value: unknown,
+  options: AtomicWriteOptions = {},
+): Promise<void> {
+  await writeFileAtomic(path, `${JSON.stringify(value, null, 2)}\n`, options);
+}
+
+/**
+ * Merge a validated update back over the bytes it came from, keeping properties this build does
+ * not know about (ADR-0004 decision 3).
+ *
+ * The three arguments are what make this precise rather than a guess:
+ *
+ * - `raw` — everything that was on disk.
+ * - `original` — the same record after validation, so its keys are exactly the ones this build
+ *   understands.
+ * - `next` — the caller's updated record.
+ *
+ * A key present in `raw` but absent from `original` is unknown to this build, so it is preserved.
+ * A key present in both but absent from `next` was deliberately removed by the caller, so it is
+ * dropped. Without that distinction, preservation would resurrect fields a caller had just
+ * deleted.
+ *
+ * Arrays are merged element-wise **only when the lengths match**, which is the case where index
+ * identity is sound. If the caller added or removed elements, indices no longer denote the same
+ * records and `next` is taken wholesale — losing unknown properties inside those elements rather
+ * than attaching them to the wrong element. That trade is deliberate: silently misattributing a
+ * field is worse than dropping it.
+ */
+export function preserveUnknown(raw: unknown, original: unknown, next: unknown): unknown {
+  if (Array.isArray(raw) && Array.isArray(original) && Array.isArray(next)) {
+    if (raw.length !== original.length || original.length !== next.length) return next;
+    return next.map((element, index) => preserveUnknown(raw[index], original[index], element));
+  }
+
+  if (!isPlainObject(raw) || !isPlainObject(original) || !isPlainObject(next)) return next;
+
+  const merged: Record<string, unknown> = {};
+  for (const key of Object.keys(raw)) {
+    if (!(key in original)) merged[key] = raw[key];
+  }
+  for (const key of Object.keys(next)) {
+    merged[key] =
+      key in raw && key in original
+        ? preserveUnknown(raw[key], original[key], next[key])
+        : next[key];
+  }
+  return merged;
+}
+
+/** Apply {@link preserveUnknown} against a stored document. */
+export function mergeForWrite<T>(document: StoredDocument<T>, next: T): unknown {
+  return preserveUnknown(document.raw, document.value, next);
+}
+
+/** True for a JSON object, excluding arrays and null. */
+export function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/errors.ts

@@ -0,0 +1,65 @@
+/**
+ * Failures specific to file-backed storage.
+ *
+ * Aldus Core deliberately keeps no central error-code registry, so that a package can name a new
+ * failure without forking Core. These codes are this package's contribution; they carry the same
+ * `ALDUS_` prefix and `SCREAMING_SNAKE_CASE` shape so production trace (contract §20) stays
+ * uniform across packages.
+ */
+
+import { AldusError, type ErrorCategory } from "@aldus-runtime/core";
+
+/** Error codes raised by the file store. */
+export const FileStoreErrorCodes = {
+  /** The `.aldus` workspace directory does not exist, or is not a directory. */
+  WORKSPACE_NOT_FOUND: "ALDUS_WORKSPACE_NOT_FOUND",
+  /** A record was requested that has never been written. */
+  RECORD_NOT_FOUND: "ALDUS_RECORD_NOT_FOUND",
+  /** A stored file held bytes that are not valid JSON. */
+  RECORD_MALFORMED: "ALDUS_RECORD_MALFORMED",
+  /**
+   * An event log line in the interior of the file could not be parsed.
+   *
+   * Distinct from a torn tail: a bad line in the middle means bytes were lost or overwritten
+   * inside an append-only file, which is corruption. See {@link FileStoreErrorCodes.EVENT_LOG_TORN_TAIL}.
+   */
+  EVENT_LOG_CORRUPT: "ALDUS_EVENT_LOG_CORRUPT",
+  /**
+   * The final line of an event log was truncated, and the caller asked for strict reads.
+   *
+   * Recoverable by design: a process that died mid-append leaves exactly this, and everything
+   * before the torn line is intact (contract §19.1 "recovery from partial success").
+   */
+  EVENT_LOG_TORN_TAIL: "ALDUS_EVENT_LOG_TORN_TAIL",
+  /** An event was appended out of sequence order (ADR-0005). */
+  EVENT_OUT_OF_SEQUENCE: "ALDUS_EVENT_OUT_OF_SEQUENCE",
+  /** An event was appended whose `eventId` already exists in the log. */
+  EVENT_DUPLICATE: "ALDUS_EVENT_DUPLICATE",
+  /** A lock could not be acquired before the caller's deadline. */
+  LOCK_TIMEOUT: "ALDUS_LOCK_TIMEOUT",
+  /** A lock was released or renewed by something that no longer holds it. */
+  LOCK_LOST: "ALDUS_LOCK_LOST",
+  /**
+   * A lock was re-acquired inside a scope that already holds it.
+   *
+   * File locks are not re-entrant, so this can never succeed: the acquirer is waiting on itself
+   * and would spin until the acquisition deadline. Distinct from {@link LOCK_TIMEOUT}, which
+   * means another session genuinely holds the lock — this one means the caller's own design is
+   * wrong, and no amount of retrying will fix it.
+   */
+  LOCK_REENTRANT: "ALDUS_LOCK_REENTRANT",
+  /** A write was attempted against a record whose identity does not match its location. */
+  RECORD_IDENTITY_MISMATCH: "ALDUS_RECORD_IDENTITY_MISMATCH",
+} as const;
+
+/** @see FileStoreErrorCodes */
+export type FileStoreErrorCode = (typeof FileStoreErrorCodes)[keyof typeof FileStoreErrorCodes];
+
+/** Construct an {@link AldusError} with a file-store code. */
+export function fileStoreError(
+  code: FileStoreErrorCode,
+  message: string,
+  options: { category: ErrorCategory; retryable?: boolean; details?: Record<string, unknown> },
+): AldusError {
+  return new AldusError(code, message, options);
+}

package/src/index.ts

@@ -0,0 +1,96 @@
+/**
+ * `@aldus-runtime/file-store` — file-backed state and event storage for the Aldus runtime.
+ *
+ * Implements architecture contract §22 **WP-02 File state and event store**: atomic manifest
+ * writes, an append-only JSONL event log, file locking, materialized current state, and recovery
+ * from interrupted writes, over the local layout §7 recommends.
+ *
+ * This package stores and retrieves records. It does not interpret them: stage execution is
+ * WP-04, gate evaluation is WP-05, and artifact lineage is WP-03.
+ *
+ * Physical storage stays behind {@link EpisodeStore}, {@link RunStore}, and {@link EventStore}
+ * so that §7's requirement is structural rather than aspirational: the databases, object stores,
+ * and cloud drives §7 lists must remain possible as adapters instead of becoming assumptions.
+ * Core deliberately names none of them, and neither does this package (§4.2).
+ *
+ * @packageDocumentation
+ */
+
+export {
+  appendLineSynced,
+  createExclusive,
+  isAlreadyExists,
+  isNotFound,
+  overwrite,
+  readFileOrUndefined,
+  removeIfPresent,
+  writeFileAtomic,
+  type AtomicWriteHooks,
+  type AtomicWriteOptions,
+} from "./atomic.js";
+
+export { appendToCollection, readCollection, type StoredCollection } from "./collections.js";
+
+export {
+  isPlainObject,
+  mergeForWrite,
+  preserveUnknown,
+  readDocument,
+  writeDocument,
+  type StoredDocument,
+} from "./document.js";
+
+export { FileStoreErrorCodes, fileStoreError, type FileStoreErrorCode } from "./errors.js";
+
+export {
+  parseJsonLines,
+  readJsonLines,
+  toJsonLine,
+  type JsonLinesReadResult,
+  type ReadJsonLinesOptions,
+} from "./jsonl.js";
+
+export {
+  ALDUS_DIRECTORY,
+  EPISODE_LOCK_RESOURCE,
+  RUN_FILES,
+  WorkspaceLayout,
+  runLockResource,
+  type RunFileName,
+} from "./layout.js";
+
+export {
+  DEFAULT_LOCK_RETRY_MS,
+  DEFAULT_LOCK_TIMEOUT_MS,
+  DEFAULT_LOCK_TTL_MS,
+  FileLockManager,
+  type AcquireOptions,
+  type FileLockManagerOptions,
+  type Lease,
+  type LockManager,
+} from "./lock.js";
+
+export type {
+  EpisodeStore,
+  EventReadOptions,
+  EventReadResult,
+  EventStore,
+  RunCollectionName,
+  RunCollectionTypes,
+  RunStore,
+} from "./ports.js";
+
+export {
+  FileEpisodeStore,
+  FileEventStore,
+  FileRunStore,
+  RUN_COLLECTION_SCHEMAS,
+  nextSequenceOf,
+} from "./stores.js";
+
+export {
+  FileWorkspace,
+  initWorkspace,
+  openWorkspace,
+  type OpenWorkspaceOptions,
+} from "./workspace.js";

package/src/jsonl.ts

@@ -0,0 +1,149 @@
+/**
+ * Append-only JSON Lines reading.
+ *
+ * Architecture contract §6.4 requires every state mutation to emit an immutable event, and §7
+ * stores those events in `events.jsonl`. §19.1 requires "recovery from partial success".
+ *
+ * The distinction this module draws is the whole point of it:
+ *
+ * - A **torn tail** is the last line of the file being incomplete. That is what a process killed
+ *   mid-append leaves behind, it is expected, and everything before it is intact. Discarding it
+ *   with a report is recovery.
+ * - A **corrupt interior line** is an unparseable line with complete lines after it. That cannot
+ *   result from an interrupted append, because appends only ever extend the file. It means bytes
+ *   inside an append-only log were lost or overwritten, and reporting it as a recoverable
+ *   truncation would be a lie that silently drops an audit record.
+ *
+ * Treating the second case as the first is the failure mode worth engineering against: it would
+ * turn "your audit log is damaged" into "everything is fine, minus one event you will never
+ * learn about".
+ */
+
+import { readFileOrUndefined } from "./atomic.js";
+import { FileStoreErrorCodes, fileStoreError } from "./errors.js";
+
+/** What a JSON Lines read found. */
+export interface JsonLinesReadResult {
+  /** Successfully parsed values, in file order. */
+  values: unknown[];
+  /**
+   * The raw text of a truncated final line, when one was found.
+   *
+   * Surfaced rather than swallowed so a caller can log it, and so a test can assert recovery
+   * happened rather than assert that nothing went wrong.
+   */
+  tornTail?: string;
+  /** 1-based line numbers that were blank and therefore carried no record. */
+  blankLines: number[];
+}
+
+/** Options for {@link readJsonLines}. */
+export interface ReadJsonLinesOptions {
+  /**
+   * Fail on a torn tail instead of recovering from it.
+   *
+   * Default `false`. A reader inspecting state wants recovery; a tool auditing log integrity
+   * wants to know. Both are legitimate, so it is the caller's choice rather than a policy baked
+   * into the reader.
+   */
+  strictTail?: boolean;
+  /** Included in error details so a failure names the file it came from. */
+  path?: string;
+}
+
+/**
+ * Parse a JSON Lines file.
+ *
+ * Returns an empty result when the file does not exist: a Run with no events yet is an ordinary
+ * state, not an error.
+ *
+ * @throws {AldusError} `ALDUS_EVENT_LOG_CORRUPT` when an interior line cannot be parsed, or
+ * `ALDUS_EVENT_LOG_TORN_TAIL` when the tail is torn and `strictTail` is set.
+ */
+export async function readJsonLines(
+  path: string,
+  options: ReadJsonLinesOptions = {},
+): Promise<JsonLinesReadResult> {
+  const contents = await readFileOrUndefined(path);
+  if (contents === undefined || contents.length === 0) {
+    return { values: [], blankLines: [] };
+  }
+  return parseJsonLines(contents, { ...options, path: options.path ?? path });
+}
+
+/** Parse JSON Lines text. Separated from IO so the recovery rules can be tested directly. */
+export function parseJsonLines(
+  contents: string,
+  options: ReadJsonLinesOptions = {},
+): JsonLinesReadResult {
+  // An empty file is an empty log, not a blank line: `"".split("\n")` yields `[""]`, which would
+  // otherwise be reported as a blank line that was never written.
+  if (contents.length === 0) return { values: [], blankLines: [] };
+
+  const endsWithNewline = contents.endsWith("\n");
+  const rawLines = contents.split("\n");
+  // A well-formed file ends with a newline, which `split` renders as a trailing empty element.
+  // Dropping it is what makes "the last element" mean "the possibly-torn line".
+  if (endsWithNewline) rawLines.pop();
+
+  const values: unknown[] = [];
+  const blankLines: number[] = [];
+  let tornTail: string | undefined;
+
+  for (let index = 0; index < rawLines.length; index += 1) {
+    const line = rawLines[index] ?? "";
+    const isLastLine = index === rawLines.length - 1;
+
+    // A blank line carries no record and cannot represent a lost event, so it is skipped rather
+    // than treated as corruption — but it is reported, so nothing is silently ignored.
+    if (line.trim().length === 0) {
+      blankLines.push(index + 1);
+      continue;
+    }
+
+    try {
+      values.push(JSON.parse(line));
+    } catch {
+      // Only the final line of a file with no trailing newline can be a torn append. Anything
+      // else is damage inside an append-only file.
+      if (isLastLine && !endsWithNewline) {
+        if (options.strictTail === true) {
+          throw fileStoreError(
+            FileStoreErrorCodes.EVENT_LOG_TORN_TAIL,
+            "The final line of the event log is truncated, which is what an interrupted append " +
+              "leaves behind. Every line before it is intact.",
+            {
+              category: "io",
+              retryable: false,
+              details: { path: options.path, line: index + 1, byteLength: line.length },
+            },
+          );
+        }
+        tornTail = line;
+        continue;
+      }
+
+      throw fileStoreError(
+        FileStoreErrorCodes.EVENT_LOG_CORRUPT,
+        "An event log line could not be parsed, and it is not the final line. An append-only " +
+          "log can only ever be damaged at its tail, so this means bytes inside the log were " +
+          "lost or overwritten.",
+        {
+          category: "io",
+          retryable: false,
+          // The line's contents are deliberately excluded: an event may carry redacted-but-
+          // sensitive context (§19.2), and an error is itself a durable record.
+          details: { path: options.path, line: index + 1, byteLength: line.length },
+        },
+      );
+    }
+  }
+
+  return tornTail === undefined ? { values, blankLines } : { values, blankLines, tornTail };
+}
+
+/** Serialise a value to a single JSON Lines entry. Rejects embedded newlines by construction. */
+export function toJsonLine(value: unknown): string {
+  // `JSON.stringify` escapes newlines inside strings, so one value is always exactly one line.
+  return JSON.stringify(value);
+}

package/src/layout.ts

@@ -0,0 +1,105 @@
+/**
+ * Workspace layout.
+ *
+ * Paths follow architecture contract §7's recommended local layout verbatim. They are collected
+ * here rather than spread through the stores because §8.1 states that a path MUST NOT be treated
+ * as identity: keeping every path construction in one module makes it visible that paths are a
+ * storage detail this adapter owns, and that no record's identity is derived from one.
+ */
+
+import { join } from "node:path";
+
+/** Directory name holding Aldus state inside a workspace (contract §7). */
+export const ALDUS_DIRECTORY = ".aldus";
+
+/** File names inside a run directory (contract §7). */
+export const RUN_FILES = {
+  manifest: "run.json",
+  events: "events.jsonl",
+  artifacts: "artifacts.json",
+  approvals: "approvals.json",
+  costs: "costs.json",
+  release: "release.json",
+} as const;
+
+/** @see RUN_FILES */
+export type RunFileName = keyof typeof RUN_FILES;
+
+/**
+ * Resolves the contract §7 paths for one workspace.
+ *
+ * A run ID becomes a directory name, so it is validated before use: an identifier containing a
+ * path separator or `..` would escape the workspace, and identifiers can originate from files
+ * another machine wrote.
+ */
+export class WorkspaceLayout {
+  readonly root: string;
+  readonly aldusDirectory: string;
+
+  constructor(workspaceRoot: string) {
+    this.root = workspaceRoot;
+    this.aldusDirectory = join(workspaceRoot, ALDUS_DIRECTORY);
+  }
+
+  /** `.aldus/episode.json` — the durable content identity (contract §6.1, §7). */
+  episodePath(): string {
+    return join(this.aldusDirectory, "episode.json");
+  }
+
+  /** `.aldus/runs` */
+  runsDirectory(): string {
+    return join(this.aldusDirectory, "runs");
+  }
+
+  /** `.aldus/runs/{run-id}` */
+  runDirectory(runId: string): string {
+    return join(this.runsDirectory(), assertPathSafe(runId));
+  }
+
+  /** A named file inside a run directory (contract §7). */
+  runFilePath(runId: string, file: RunFileName): string {
+    return join(this.runDirectory(runId), RUN_FILES[file]);
+  }
+
+  /**
+   * `.aldus/locks` — lockfiles, which are machine-local runtime state.
+   *
+   * Deliberately a sibling of `runs/` rather than a file inside each run directory: §7's run
+   * directory lists exactly six files, and a lockfile appearing among Git-tracked state would
+   * invite committing another machine's PID.
+   */
+  locksDirectory(): string {
+    return join(this.aldusDirectory, "locks");
+  }
+}
+
+/** Resource name for the workspace-wide Episode record. */
+export const EPISODE_LOCK_RESOURCE = "episode";
+
+/** Resource name locking one Run. */
+export function runLockResource(runId: string): string {
+  return `run-${assertPathSafe(runId)}`;
+}
+
+/**
+ * Reject an identifier that could escape its directory.
+ *
+ * Run IDs are minted by Core as `run_<ULID>`, but a workspace is shared and Git-tracked, so an
+ * identifier read from disk is untrusted input.
+ */
+function assertPathSafe(identifier: string): string {
+  if (
+    identifier.length === 0 ||
+    identifier === "." ||
+    identifier === ".." ||
+    identifier.includes("/") ||
+    identifier.includes("\\") ||
+    identifier.includes("\0")
+  ) {
+    throw new Error(
+      `Identifier ${JSON.stringify(identifier)} cannot be used as a directory name: it is empty, ` +
+        "a relative path segment, or contains a path separator.",
+    );
+  }
+  return identifier;
+}