@arki/journal 0.0.0-stage → 0.1.0

package/src/pg.ts

@@ -0,0 +1,107 @@
+import type { PgSchema, PgTableFn } from 'drizzle-orm/pg-core';
+import { sql } from 'drizzle-orm';
+import {
+  bigint,
+  bigserial,
+  index,
+  integer,
+  jsonb,
+  pgTable,
+  text,
+  timestamp,
+  unique,
+  varchar,
+} from 'drizzle-orm/pg-core';
+
+/**
+ * The two journal tables. Call this once in the app's schema module so
+ * `drizzle-kit generate` creates them. The library never creates tables at
+ * runtime.
+ */
+export function journalTables(schema?: PgSchema) {
+  const table: PgTableFn<string | undefined> = schema === undefined ? pgTable : schema.table;
+
+  const events = table(
+    'journal_events',
+    {
+      /** Global order. `rerun` walks the log by this. */
+      seq: bigserial('seq', { mode: 'number' }).primaryKey(),
+      /** `evt_xxx`. */
+      id: varchar('id').notNull().unique(),
+      kind: varchar('kind', { length: 64 }).notNull(),
+      entityId: varchar('entity_id', { length: 256 }).notNull(),
+      /** 1, 2, 3 per entity. */
+      version: integer('version').notNull(),
+      type: varchar('type', { length: 128 }).notNull(),
+      data: jsonb('data').notNull(),
+      meta: jsonb('meta').notNull(),
+      recordedAt: timestamp('recorded_at', { withTimezone: true, mode: 'date' }).notNull().defaultNow(),
+      // Stamped by the library, never by the app. Null on rows written before these columns existed.
+      /** The `cmd_` id of the command run that wrote this event. Null for a bare `journal.write`. */
+      commandId: varchar('command_id'),
+      /** Inside an effect's `ctx.run`: the seq of the event the job was for. Else null. */
+      causeSeq: bigint('cause_seq', { mode: 'number' }),
+      /** Shared by every event of one chain: inherited from the cause event or the parent run, else new per root run. */
+      correlationId: varchar('correlation_id'),
+      /** Set by `forget` when `data` was rewritten to the event's erased form. */
+      erasedAt: timestamp('erased_at', { withTimezone: true, mode: 'date' }),
+      /** The event's schema version when `data` was written. Older rows are upcast on read. */
+      schemaVersion: integer('schema_version').notNull().default(1),
+    },
+    t => [
+      unique('journal_events_kind_entity_id_version_unique').on(t.kind, t.entityId, t.version),
+      index('journal_events_correlation_id_idx').on(t.correlationId),
+    ],
+  );
+
+  const jobs = table(
+    'journal_jobs',
+    {
+      /** `job_xxx`. Pass it on as an idempotency key. */
+      id: varchar('id').primaryKey(),
+      eventSeq: bigint('event_seq', { mode: 'number' })
+        .notNull()
+        .references(() => events.seq),
+      effect: varchar('effect', { length: 128 }).notNull(),
+      /** Higher runs first among due jobs. From the effect's schedule. */
+      priority: integer('priority').notNull().default(0),
+      attempts: integer('attempts').notNull().default(0),
+      availableAt: timestamp('available_at', { withTimezone: true, mode: 'date' }).notNull().defaultNow(),
+      leaseUntil: timestamp('lease_until', { withTimezone: true, mode: 'date' }),
+      leaseToken: varchar('lease_token'),
+      lastError: text('last_error'),
+      /** Set when attempts run out. A human clears it or deletes the row. */
+      deadAt: timestamp('dead_at', { withTimezone: true, mode: 'date' }),
+    },
+    t => [
+      unique('journal_jobs_event_seq_effect_unique').on(t.eventSeq, t.effect),
+      index('journal_jobs_due_idx')
+        .on(t.priority.desc(), t.availableAt)
+        .where(sql`${t.deadAt} is null`),
+    ],
+  );
+
+  const receipts = table('journal_receipts', {
+    /** The app's idempotency key. Scope it to the caller when it comes from outside (tenant, actor, source). */
+    key: varchar('key', { length: 256 }).primaryKey(),
+    command: varchar('command', { length: 128 }).notNull(),
+    /** The parsed input, after a JSON round trip. A repeat with another input is a conflict. */
+    input: jsonb('input').notNull(),
+    /** `{ value }` once the run finished; `{}` for a command that returned nothing; null while it runs. */
+    result: jsonb('result').$type<{ readonly value?: unknown }>(),
+    commandId: varchar('command_id').notNull(),
+    recordedAt: timestamp('recorded_at', { withTimezone: true, mode: 'date' }).notNull().defaultNow(),
+  });
+
+  const cursors = table('journal_cursors', {
+    /** The async projection's name. */
+    name: varchar('name', { length: 128 }).primaryKey(),
+    /** Global scan progress: every event with a seq at or below this was looked at, matching or not. */
+    seq: bigint('seq', { mode: 'number' }).notNull().default(0),
+    updatedAt: timestamp('updated_at', { withTimezone: true, mode: 'date' }).notNull().defaultNow(),
+  });
+
+  return { events, jobs, receipts, cursors };
+}
+
+export type JournalTables = ReturnType<typeof journalTables>;

package/src/storage.ts

@@ -0,0 +1,277 @@
+/**
+ * The entity row and stored values: load and save the row, exact-id checks, value
+ * comparison, the JSON round-trip check, freezing, and the root-database guard.
+ * Imports only leaf files (`types`, `errors`). Must not import `journal`, `api`,
+ * `write`, `command`, `history` or `worker`.
+ */
+import type { z } from 'zod';
+import { eq, getTableColumns, is } from 'drizzle-orm';
+import { PgAsyncTransaction } from 'drizzle-orm/pg-core';
+
+import type {
+  AnyEntity,
+  AnyEventDef,
+  DbHandle,
+  EntityDef,
+  EntityId,
+  EntityTable,
+  ReducerRow,
+  RootDb,
+  Row,
+} from './types.js';
+import { JournalError } from './errors.js';
+
+/** A select row has every column of its table. The check is real; the guard narrows. */
+export function isRowOf<Tbl extends EntityTable>(table: Tbl, value: object): value is Row<Tbl> {
+  return Object.keys(getTableColumns(table)).every(column => column in value);
+}
+
+type EntityRowSource = Pick<AnyEntity, 'table' | 'keyColumn'>;
+
+export async function loadRow(
+  db: DbHandle,
+  entity: EntityRowSource,
+  id: string,
+  lock: boolean,
+): Promise<object | null> {
+  const query = db.select().from(entity.table).where(eq(entity.keyColumn, id)).limit(1);
+  const rows = lock ? await query.for('update') : await query;
+  return rows[0] ?? null;
+}
+
+/**
+ * The id the caller passes must be exactly the key the database stores. A database can treat
+ * two spellings as one key (a `uuid` ignores case), but the log is keyed by the raw string,
+ * so a second spelling would split one entity's history in two.
+ */
+const CANONICAL_UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/**
+ * With no row to compare against (a create and delete in one write, a replay of a missing row),
+ * `assertStoredId` sees nothing, so the id is checked up front, before any database access: not
+ * empty, within the key's length limit in code points (the column or the log would cut it), and,
+ * for a `uuid` key, lower-case and hyphenated.
+ */
+export function assertExactId(entity: Pick<AnyEntity, 'kind' | 'uuidKey' | 'keyLimit'>, id: string): void {
+  const fail = (reason: string): never => {
+    throw new JournalError('ID_MISMATCH', entity.kind, id, reason);
+  };
+  if (id === '') fail('the id is empty');
+  // (Surrogates are U+D800 to U+DFFF.) A lone surrogate is encoded by the driver as U+FFFD, so it would reach the history of another id.
+  // Iterating by code point yields a paired surrogate as one astral character, never as a surrogate.
+  for (const ch of id) {
+    const point = ch.codePointAt(0) ?? 0;
+    if (point >= 55_296 && point <= 57_343) {
+      fail(
+        'the id contains an unpaired surrogate, so it is not well-formed text and the database would store a different key',
+      );
+    }
+  }
+  const length = [...id].length;
+  if (length > entity.keyLimit.max) {
+    const owner = entity.keyLimit.source === 'column' ? "the key column's" : "the journal's";
+    fail(`the id is ${length} characters but the limit is ${entity.keyLimit.max}, ${owner} limit`);
+  }
+  if (entity.uuidKey && !CANONICAL_UUID.test(id)) {
+    fail(`${JSON.stringify(id)} is not a canonical uuid; uuid keys must be lower-case and hyphenated`);
+  }
+}
+
+export function assertStoredId(entity: Pick<AnyEntity, 'kind' | 'key'>, row: object | null, id: string): void {
+  if (row === null) return;
+  const stored: unknown = Reflect.get(row, entity.key);
+  if (stored !== id) {
+    throw new JournalError(
+      'ID_MISMATCH',
+      entity.kind,
+      id,
+      `the stored key is ${JSON.stringify(stored)} but the id given is ${JSON.stringify(id)}; use the stored form`,
+    );
+  }
+}
+
+/** Insert, update or delete the row so it matches `next`. Returns the stored row. */
+export async function saveReplayed<Tbl extends EntityTable, Key extends string>(
+  db: DbHandle,
+  entity: EntityDef<string, Tbl, AnyEventDef, unknown, Key>,
+  id: EntityId<Tbl, Key>,
+  current: object | null,
+  next: ReducerRow<Tbl, Key> | null,
+): Promise<Row<Tbl> | null> {
+  const erased: EntityTable = entity.table;
+  if (next === null) {
+    if (current !== null) await db.delete(erased).where(eq(entity.keyColumn, id));
+    return null;
+  }
+  const saved =
+    current === null
+      ? await db.insert(erased).values(next).returning()
+      : await db.update(erased).set(next).where(eq(entity.keyColumn, id)).returning();
+  const row = saved[0];
+  if (row === undefined || !isRowOf(entity.table, row)) {
+    throw new Error(`journal: saving ${id} returned no complete row`);
+  }
+  return row;
+}
+
+export function toRecord(value: object): Record<string, unknown> {
+  return { ...value };
+}
+
+export function showValue(value: unknown): string {
+  if (value instanceof Date) return Number.isNaN(value.getTime()) ? 'Invalid Date' : value.toISOString();
+  if (typeof value === 'bigint') return `${value}n`;
+  if (typeof value === 'object' && value !== null) {
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return Object.prototype.toString.call(value);
+    }
+  }
+  return typeof value === 'string' ? JSON.stringify(value) : String(value);
+}
+
+/**
+ * The row the database returned must hold, for each of `columns`, the value the reducers produced.
+ * Replay recomputes from reducer output, so a value the database rounds, trims or alters would make
+ * replay and the table drift apart. Uses the same comparison as `verify`.
+ */
+export function assertStoredValues(
+  entity: Pick<AnyEntity, 'kind'>,
+  id: string,
+  produced: object,
+  saved: object,
+  columns: Iterable<string>,
+): void {
+  const want = toRecord(produced);
+  const have = toRecord(saved);
+  for (const column of columns) {
+    if (!sameValue(want[column], have[column])) {
+      throw new JournalError(
+        'VALUE_MISMATCH',
+        entity.kind,
+        id,
+        `column "${column}": the reducer produced ${showValue(want[column])} but the database stored ${showValue(have[column])}`,
+      );
+    }
+  }
+}
+
+/** Same own enumerable keys and same values. A key holding `undefined` is the same as a missing key, as in JSON. */
+function sameKeys(
+  left: Record<string, unknown>,
+  right: Record<string, unknown>,
+  same: (a: unknown, b: unknown) => boolean,
+): boolean {
+  const keys = Object.keys(left).filter(key => left[key] !== undefined);
+  const rightKeys = Object.keys(right).filter(key => right[key] !== undefined);
+  return keys.length === rightKeys.length && keys.every(key => same(left[key], right[key]));
+}
+
+/** An object literal shape: prototype `Object.prototype` or `null`. A `Set`, `Map`, `Date` or class instance is not one. */
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  if (typeof value !== 'object' || value === null) return false;
+  const proto: unknown = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+
+/**
+ * Freeze a plain object or array and everything inside it. A `Date` is frozen
+ * itself; its internals are not touched. Anything else is left alone.
+ */
+export function deepFreeze<T>(value: T): T {
+  if (Array.isArray(value)) {
+    for (const item of value) deepFreeze(item);
+    Object.freeze(value);
+  } else if (isPlainObject(value)) {
+    for (const item of Object.values(value)) deepFreeze(item);
+    Object.freeze(value);
+  } else if (value instanceof Date) {
+    Object.freeze(value);
+  }
+  return value;
+}
+
+/** Same length, and every index from 0 up compared. A hole is not equal to a value, `null` and `undefined` included. */
+function sameArray(a: readonly unknown[], b: readonly unknown[], same: (x: unknown, y: unknown) => boolean): boolean {
+  if (a.length !== b.length) return false;
+  for (let index = 0; index < a.length; index++) {
+    if (index in a !== index in b) return false;
+    if (!same(a[index], b[index])) return false;
+  }
+  return true;
+}
+
+/**
+ * Deep equality for stored rows: `verify` compares what replay produced with what the table holds.
+ * Strict: primitives, `Date`, value objects, arrays and plain objects only. A value object is a class
+ * instance with an `equals` method, such as a Temporal value from a custom column: it equals another
+ * instance of the same class when its `equals` says so. A `Map`, `Set`, other class instance or typed
+ * array is never equal to anything, because JSON storage would lose its contents.
+ */
+export function sameValue(a: unknown, b: unknown): boolean {
+  if (typeof a === 'function' || typeof a === 'symbol') return false;
+  if (typeof a !== 'object' || a === null) return Object.is(a, b);
+  if (a instanceof Date) return b instanceof Date && a.getTime() === b.getTime();
+  if (Array.isArray(a)) return Array.isArray(b) && sameArray(a, b, sameValue);
+  if (isValueObject(a)) return Object.getPrototypeOf(a) === Object.getPrototypeOf(b) && a.equals(b) === true;
+  if (isPlainObject(a) && isPlainObject(b)) return sameKeys(a, b, sameValue);
+  return false;
+}
+
+/**
+ * A class instance that defines its own equality, like Temporal's `Instant` or `PlainDate`. A column
+ * whose driver rebuilds the same class on read round-trips it; a lossy column hands back another type
+ * or another value, and `equals` then reports the difference.
+ */
+function isValueObject(value: object): value is { equals(other: unknown): unknown } {
+  return !isPlainObject(value) && 'equals' in value && typeof value.equals === 'function';
+}
+
+/**
+ * Deep equality for a value that must survive JSON storage: primitives, arrays and
+ * plain objects only. A `Set` has no enumerable keys and would otherwise equal `{}`.
+ */
+function sameJson(a: unknown, b: unknown): boolean {
+  if (Object.is(a, b)) return true;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    return sameArray(a, b, sameJson);
+  }
+  if (isPlainObject(a) && isPlainObject(b)) return sameKeys(a, b, sameJson);
+  return false;
+}
+
+/**
+ * Throws `UNSTABLE_SCHEMA` unless the value survives JSON storage and a second parse
+ * unchanged. Replay depends on this. A value JSON cannot encode (a `bigint`, a cycle)
+ * is the same failure, with the serializer's error as `cause`.
+ */
+export function assertRoundTrips(schema: z.ZodType, value: unknown, kind: string, id: string, what: string): void {
+  let stable: boolean;
+  try {
+    // Not structuredClone: the point is what JSON storage does to the value (Dates become strings).
+    const text = JSON.stringify(value);
+    const stored: unknown = JSON.parse(text);
+    const again = schema.safeParse(stored);
+    stable = again.success && sameJson(again.data, value);
+  } catch (error) {
+    throw new JournalError('UNSTABLE_SCHEMA', kind, id, `${what}: the value cannot be stored as JSON`, {
+      cause: error,
+    });
+  }
+  if (!stable)
+    throw new JournalError(
+      'UNSTABLE_SCHEMA',
+      kind,
+      id,
+      `${what}: the schema must accept its own stored output unchanged`,
+    );
+}
+
+export function assertRootDb(db: RootDb, verb: string): void {
+  if (is(db, PgAsyncTransaction)) {
+    throw new Error(
+      `journal: ${verb} needs the root database, not a transaction: effects would run before the outer transaction commits`,
+    );
+  }
+}

package/src/test-support.ts

@@ -0,0 +1,82 @@
+/**
+ * Shared fixture for the package tests: an in-memory PGlite, the journal
+ * tables, and a small `orders` domain. Excluded from the build.
+ */
+import { integer, pgTable, varchar } from 'drizzle-orm/pg-core';
+import { z } from 'zod';
+
+import { initJournal } from './journal.js';
+import { journalTables } from './pg.js';
+import { openJournalDb } from './testing.js';
+
+export type OrderId = `ord_${string}`;
+export const orderId = (n: number): OrderId => `ord_${n}`;
+
+export const orders = pgTable('orders', {
+  id: varchar('id', { length: 256 }).primaryKey().$type<OrderId>(),
+  customerId: varchar('customer_id', { length: 256 }).notNull(),
+  total: integer('total').notNull(),
+  status: varchar('status', { enum: ['placed', 'paid', 'cancelled'] }).notNull(),
+  note: varchar('note', { length: 256 }).notNull().default('none'),
+  version: integer('version').notNull().default(0),
+});
+
+export const customerOrderCounts = pgTable('customer_order_counts', {
+  customerId: varchar('customer_id', { length: 256 }).primaryKey(),
+  placed: integer('placed').notNull().default(0),
+});
+
+/** A table whose key is not called `id`. */
+export type AccountId = `acc_${string}`;
+export const accountId = (n: number): AccountId => `acc_${n}`;
+
+export const profiles = pgTable('profiles', {
+  accountId: varchar('account_id', { length: 256 }).primaryKey().$type<AccountId>(),
+  nick: varchar('nick', { length: 64 }).notNull(),
+  moveCount: integer('move_count').notNull().default(0),
+  version: integer('version').notNull().default(0),
+});
+
+export const tables = journalTables();
+
+export const j = initJournal({ meta: z.object({ actorId: z.string().min(1) }) });
+
+export const OrderPlaced = j.event(
+  'OrderPlaced',
+  z.object({ customerId: z.string().min(1), total: z.int().positive() }),
+);
+export const OrderPaid = j.event('OrderPaid', z.object({ amount: z.int().positive() }));
+/** A note is free text a person wrote, so it has an erased form. */
+export const OrderNoted = j.event('OrderNoted', z.object({ note: z.string().min(1) }), {
+  erased: () => ({ note: '[erased]' }),
+});
+export const OrderCancelled = j.event('OrderCancelled', z.object({ reason: z.string().min(1) }));
+export const Unrelated = j.event('Unrelated', z.object({}));
+
+export const order = j
+  .entity('order', orders)
+  .create(OrderPlaced, e => ({ customerId: e.data.customerId, total: e.data.total, status: 'placed' }))
+  // Spread style: still allowed, the patch is merged onto the state.
+  .on(OrderPaid, row => ({ ...row, status: 'paid' }))
+  // Patch style: only the column that changed.
+  .on(OrderNoted, (_row, e) => ({ note: e.data.note }))
+  .on(OrderCancelled, () => null);
+
+export const ProfileOpened = j.event('ProfileOpened', z.object({ nick: z.string().min(1) }));
+export const ProfileMoved = j.event('ProfileMoved', z.object({}));
+export const ProfileClosed = j.event('ProfileClosed', z.object({}));
+/** May be the first event of a profile or a later one. */
+export const ProfileFlagged = j.event('ProfileFlagged', z.object({ nick: z.string().min(1) }));
+
+export const profile = j
+  .entity('profile', profiles, { key: 'accountId' })
+  .create(ProfileOpened, e => ({ nick: e.data.nick, moveCount: 0 }))
+  .create(ProfileFlagged, e => ({ nick: e.data.nick, moveCount: 0 }))
+  // Every create sets `moveCount`, so it is a number here: no `?? 0`.
+  .on(ProfileMoved, row => ({ moveCount: row.moveCount + 1 }))
+  .on(ProfileFlagged, (_row, e) => ({ nick: e.data.nick }))
+  .on(ProfileClosed, () => null);
+
+export const openDb = () => openJournalDb({ ...tables, orders, customerOrderCounts, profiles });
+
+export const meta = { actorId: 'usr_test' };

package/src/testing.ts

@@ -0,0 +1,23 @@
+/**
+ * Test helper: an in-memory PGlite with a schema pushed, the fixture every
+ * journal test rebuilds by hand. `@electric-sql/pglite` and `drizzle-kit` are
+ * optional peers. Import this module only from tests. This is the one file
+ * that may import them.
+ */
+import { PGlite } from '@electric-sql/pglite';
+import { pushSchema } from 'drizzle-kit/api-postgres';
+import { drizzle } from 'drizzle-orm/pglite';
+
+/**
+ * Open a fresh in-memory database with `schema` (the journal tables and the
+ * app's tables) pushed. Returns the Drizzle database and the PGlite client;
+ * call `client.close()` when the test file ends.
+ */
+export async function openJournalDb(schema: Record<string, unknown>) {
+  const client = new PGlite('memory://');
+  await client.waitReady;
+  const db = drizzle({ client });
+  const { apply } = await pushSchema(schema, db);
+  await apply();
+  return { db, client };
+}

package/src/transaction-guard.ts

@@ -0,0 +1,41 @@
+/**
+ * The guard that allows one journal operation per transaction at a time.
+ * Its `WeakMap` is the only module-level state in the library, and it must stay
+ * module-scoped: see the comment on it.
+ * Imports nothing from this package. `write` and `command` import it.
+ */
+import { is } from 'drizzle-orm';
+import { PgAsyncTransaction } from 'drizzle-orm/pg-core';
+
+/** Passed to `then` twice: waits for a promise to settle, whether it resolves or rejects. */
+const ignoreSettlement = (): undefined => undefined;
+
+/**
+ * Savepoints are a stack. Two journal operations open on one transaction at the
+ * same time release each other's savepoint, so a caught failure of one could
+ * erase what the other wrote. A transaction therefore holds one operation at a
+ * time. A second one is refused: it waits for the first to settle (so the
+ * caller's rollback undoes it whole), then rejects. A child runs on its own
+ * savepoint handle, so nesting is fine. The root database opens a separate
+ * transaction per operation and is never busy.
+ *
+ * The map is module-scoped on purpose. Two journals can run on one transaction,
+ * and then they share its savepoint stack, so they must share this guard too.
+ * Do not move it into the journal factory.
+ */
+const inFlight = new WeakMap<object, Promise<unknown>>();
+export function oneAtATime<T>(handle: object, what: string, work: () => Promise<T>): Promise<T> {
+  if (!is(handle, PgAsyncTransaction)) return work();
+  const busy = inFlight.get(handle);
+  if (busy !== undefined) {
+    return busy.then(ignoreSettlement, ignoreSettlement).then(() => {
+      throw new Error(
+        `journal: ${what} started on a transaction that already has a journal operation in flight; await operations on one transaction one at a time`,
+      );
+    });
+  }
+  // eslint-disable-next-line drizzle/enforce-delete-with-where -- WeakMap#delete, not a Drizzle query
+  const running = work().finally(() => inFlight.delete(handle));
+  inFlight.set(handle, running);
+  return running;
+}