@zmdb/orm 1.0.0-beta.1

package/src/loaders/index.ts

@@ -0,0 +1,256 @@
+import { type DeclaredTable, type Entity, type PrimaryKeyOf } from '@zmdb/schema';
+import { type Populated, type RelationKeys, type RelationPath } from '@zmdb/schema/derive';
+
+import { type BaseRepository, type ReadOptions } from '../index.js';
+
+/** One explicit, request-lifetime loader for a repository's primary key. */
+export interface EntityLoader<T extends DeclaredTable> {
+  load(id: PrimaryKeyOf<T>): Promise<Entity<T> | undefined>;
+}
+
+/** The populated value attached under one declared relation name. */
+export type RelationValueOf<T extends DeclaredTable, K extends RelationKeys<T>> = Populated<T, K>[K];
+
+/** One explicit, request-lifetime loader for a declared relation. */
+export interface RelationLoader<T extends DeclaredTable, K extends RelationKeys<T>> {
+  load(parent: Entity<T>): Promise<RelationValueOf<T, K>>;
+}
+
+export interface LoaderScope {
+  loaderFor<T extends DeclaredTable>(repository: BaseRepository<T>): EntityLoader<T>;
+  relationLoader<T extends DeclaredTable, K extends RelationKeys<T> & string>(
+    repository: BaseRepository<T>,
+    relation: K,
+  ): RelationLoader<T, K>;
+  populate<T extends DeclaredTable, K extends string>(
+    repository: BaseRepository<T>,
+    rows: readonly Entity<T>[],
+    paths: readonly (K & RelationPath<T, K>)[],
+    options?: ReadOptions,
+  ): Promise<readonly Populated<T, RelationPath<T, K>>[]>;
+  populate<T extends DeclaredTable, K extends string>(
+    repository: BaseRepository<T>,
+    row: Entity<T>,
+    paths: readonly (K & RelationPath<T, K>)[],
+    options?: ReadOptions,
+  ): Promise<Populated<T, RelationPath<T, K>>>;
+}
+
+interface PopulateRequest {
+  readonly rows: readonly object[];
+  readonly resolve: (rows: readonly object[]) => void;
+  readonly reject: (reason: unknown) => void;
+}
+
+/**
+ * Internal repository hooks. They are symbols so the loader can reuse the
+ * repository's schema, decoding, relation resolution and dialect limits without
+ * widening the public method surface with a second family of read operations.
+ */
+export const LOADER_FOR_SCOPE = Symbol('zmdb.loaderForScope');
+export const RELATION_LOADER_FOR_SCOPE = Symbol('zmdb.relationLoaderForScope');
+export const LOADER_ENTITY_KEY = Symbol('zmdb.loaderEntityKey');
+export const LOADER_ENTITY_BATCH = Symbol('zmdb.loaderEntityBatch');
+export const LOADER_RELATION_KEY = Symbol('zmdb.loaderRelationKey');
+export const LOADER_RELATION_BATCH = Symbol('zmdb.loaderRelationBatch');
+
+interface BatchWaiter<Value> {
+  readonly resolve: (value: Value) => void;
+  readonly reject: (reason: unknown) => void;
+}
+
+interface BatchEntry<Input, Value> {
+  readonly input: Input;
+  readonly waiters: BatchWaiter<Value>[];
+}
+
+interface BatchLoader<Input, Value> {
+  load(input: Input): Promise<Value>;
+}
+
+/**
+ * One-microtask batching with a request-lifetime result map.
+ *
+ * `inFlight` matters for a duplicate that arrives after dispatch but before the
+ * driver answers: it joins the dispatched entry instead of opening a second
+ * query. Errors are deliberately not cached, so a later call may retry.
+ */
+function createBatchLoader<Input, Value>(
+  keyOf: (input: Input) => string,
+  dispatch: (inputs: readonly Input[]) => Promise<readonly Value[]>,
+  copy: (value: Value) => Value,
+): BatchLoader<Input, Value> {
+  const cache = new Map<string, { readonly value: Value }>();
+  let pending = new Map<string, BatchEntry<Input, Value>>();
+  const inFlight = new Map<string, BatchEntry<Input, Value>>();
+  let scheduled = false;
+
+  const flush = async (): Promise<void> => {
+    scheduled = false;
+    const batch = pending;
+    pending = new Map();
+    const entries = [...batch.entries()];
+    for (const [key, entry] of entries) inFlight.set(key, entry);
+
+    try {
+      const values = await dispatch(entries.map(([, entry]) => entry.input));
+      if (values.length !== entries.length) {
+        throw new Error(`loader batch returned ${values.length} result(s) for ${entries.length} key(s)`);
+      }
+
+      const results = values.map(value => ({ value }));
+      for (let index = 0; index < entries.length; index++) {
+        const keyed = entries[index];
+        const result = results[index];
+        if (!keyed || !result) throw new Error(`loader batch omitted result ${index}`);
+        const [key, entry] = keyed;
+        cache.set(key, result);
+        for (const waiter of entry.waiters) waiter.resolve(copy(result.value));
+      }
+    } catch (error) {
+      for (const [, entry] of entries) {
+        for (const waiter of entry.waiters) waiter.reject(error);
+      }
+    } finally {
+      for (const [key] of entries) inFlight.delete(key);
+    }
+  };
+
+  const enqueue = (input: Input): Promise<Value> => {
+    const key = keyOf(input);
+    const cached = cache.get(key);
+    if (cached) return Promise.resolve(copy(cached.value));
+
+    return new Promise<Value>((resolve, reject) => {
+      const active = inFlight.get(key) ?? pending.get(key);
+      if (active) {
+        active.waiters.push({ resolve, reject });
+        return;
+      }
+
+      pending.set(key, { input, waiters: [{ resolve, reject }] });
+      if (!scheduled) {
+        scheduled = true;
+        queueMicrotask(() => void flush());
+      }
+    });
+  };
+
+  return {
+    async load(input) {
+      return enqueue(input);
+    },
+  };
+}
+
+function copyEntity<Row extends object>(row: Row | undefined): Row | undefined {
+  return row === undefined ? undefined : { ...row };
+}
+
+function copyRelation<Value extends object | readonly object[] | null>(value: Value): Value;
+function copyRelation(value: object | readonly object[] | null): object | readonly object[] | null {
+  if (value === null) return null;
+  if (Array.isArray(value)) return value.map(item => ({ ...item }));
+  return { ...value };
+}
+
+export function createEntityLoader<T extends DeclaredTable>(repository: BaseRepository<T>): EntityLoader<T> {
+  return createBatchLoader(
+    id => repository[LOADER_ENTITY_KEY](id),
+    ids => repository[LOADER_ENTITY_BATCH](ids),
+    copyEntity,
+  );
+}
+
+export function createRelationLoader<T extends DeclaredTable, K extends RelationKeys<T> & string>(
+  repository: BaseRepository<T>,
+  relation: K,
+): RelationLoader<T, K> {
+  return createBatchLoader(
+    parent => repository[LOADER_RELATION_KEY](parent, relation),
+    parents => repository[LOADER_RELATION_BATCH](parents, relation),
+    copyRelation,
+  );
+}
+
+/**
+ * Construct this at the request boundary and pass it explicitly. There is no
+ * default, module-global or ambient scope.
+ */
+export function createLoaderScope(): LoaderScope {
+  const token = {};
+  const pending = new WeakMap<object, Map<string, Map<ReadOptions | undefined, PopulateRequest[]>>>();
+
+  function populate<T extends DeclaredTable, K extends string>(
+    repository: BaseRepository<T>,
+    rows: readonly Entity<T>[],
+    paths: readonly (K & RelationPath<T, K>)[],
+    options?: ReadOptions,
+  ): Promise<readonly Populated<T, RelationPath<T, K>>[]>;
+  function populate<T extends DeclaredTable, K extends string>(
+    repository: BaseRepository<T>,
+    row: Entity<T>,
+    paths: readonly (K & RelationPath<T, K>)[],
+    options?: ReadOptions,
+  ): Promise<Populated<T, RelationPath<T, K>>>;
+  function populate<T extends DeclaredTable, K extends string>(
+    repository: BaseRepository<T>,
+    rowOrRows: Entity<T> | readonly Entity<T>[],
+    paths: readonly (K & RelationPath<T, K>)[],
+    options?: ReadOptions,
+  ): Promise<Populated<T, RelationPath<T, K>> | readonly Populated<T, RelationPath<T, K>>[]> {
+    const allPaths = [...new Set(paths)].toSorted();
+    const canonical = allPaths.filter(path => !allPaths.some(other => other.startsWith(`${path}.`)));
+    const key = JSON.stringify(canonical);
+    let byPath = pending.get(repository);
+    if (byPath === undefined) {
+      byPath = new Map();
+      pending.set(repository, byPath);
+    }
+    let byOptions = byPath.get(key);
+    if (byOptions === undefined) {
+      byOptions = new Map();
+      byPath.set(key, byOptions);
+    }
+    const isMany = Array.isArray(rowOrRows);
+    return new Promise((resolve, reject) => {
+      const request: PopulateRequest = {
+        rows: isMany ? rowOrRows : [rowOrRows],
+        // The grouping key preserves repository identity, so every request has this T.
+        resolve: rows =>
+          resolve(
+            (isMany ? rows : rows[0]) as Populated<T, RelationPath<T, K>> | readonly Populated<T, RelationPath<T, K>>[],
+          ),
+        reject,
+      };
+      const existing = byOptions.get(options);
+      if (existing !== undefined) {
+        existing.push(request);
+        return;
+      }
+      const requests = [request];
+      byOptions.set(options, requests);
+      queueMicrotask(async () => {
+        byOptions.delete(options);
+        if (byOptions.size === 0) byPath.delete(key);
+        try {
+          const roots = requests.flatMap(entry => entry.rows) as readonly Entity<T>[];
+          const rows = await repository.populate<K>(roots, canonical, options);
+          let offset = 0;
+          for (const entry of requests) {
+            entry.resolve(rows.slice(offset, offset + entry.rows.length));
+            offset += entry.rows.length;
+          }
+        } catch (error) {
+          for (const entry of requests) entry.reject(error);
+        }
+      });
+    });
+  }
+
+  return {
+    loaderFor: repository => repository[LOADER_FOR_SCOPE](token),
+    relationLoader: (repository, relation) => repository[RELATION_LOADER_FOR_SCOPE](token, relation),
+    populate,
+  };
+}

package/src/outbox/index.ts

@@ -0,0 +1,409 @@
+import { type CoreSchema } from '@zmdb/schema';
+import { schemaFromIR } from '@zmdb/schema/ir';
+import { type HasDefault, type PrimaryKey, type Sql, type Table } from '@zmdb/schema/tags';
+import { createQueryCompiler, type DialectTarget } from '@zmdb/sql';
+
+import { type Driver } from '../index.js';
+import { type TransactionContext } from '../transactions/index.js';
+import {
+  outboxCandidatesQuery,
+  outboxClaimQuery,
+  outboxMarkDeadQuery,
+  outboxMarkDeliveredQuery,
+  outboxMarkRetryQuery,
+  outboxReadBackQuery,
+} from './sql.js';
+
+function requiredDialect(dialect: DialectTarget | undefined, owner: string) {
+  if (dialect === undefined) throw new TypeError(`${owner} requires an explicit database dialect object`);
+  return dialect;
+}
+
+import type { OutboxStatus } from './sql.js';
+export * from './sql.js';
+
+export interface OutboxRow extends Table<'zmdb_outbox'> {
+  id: string & Sql<'text'> & PrimaryKey;
+  topic: string & Sql<'text'>;
+  payload: string & Sql<'text'>;
+  status: OutboxStatus & Sql<'jsonEnum'> & HasDefault;
+  attempts: number & Sql<'integer'> & HasDefault;
+  createdAt: Date & Sql<'timestamp'> & HasDefault;
+  leaseOwner: string & Sql<'text'> & HasDefault;
+  leaseUntil: Date & Sql<'timestamp'> & HasDefault;
+  deliveredAt: (Date & Sql<'timestamp'>) | null;
+  lastError: (string & Sql<'text'>) | null;
+}
+
+const EMPTY_COLUMN = {
+  primaryKey: false,
+  serial: false,
+  unique: false,
+  sensitive: false,
+  constraints: {},
+  rules: [],
+} as const;
+
+/**
+ * The built-in physical declaration applications include in schema snapshots.
+ *
+ * The public row type uses app-facing camelCase, while this value names the
+ * snake_case columns emitted by `outboxMigration` and used by the dispatcher.
+ */
+export const OutboxSchema: CoreSchema<string> = schemaFromIR({
+  table: 'zmdb_outbox',
+  physicalTable: 'zmdb_outbox',
+  primaryKey: ['id'],
+  relations: [],
+  foreignKeys: [],
+  columns: [
+    {
+      ...EMPTY_COLUMN,
+      name: 'id',
+      physicalName: 'id',
+      sql: 'text',
+      nullable: false,
+      primaryKey: true,
+      hasDefault: false,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'topic',
+      physicalName: 'topic',
+      sql: 'text',
+      nullable: false,
+      hasDefault: false,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'payload',
+      physicalName: 'payload',
+      sql: 'text',
+      nullable: false,
+      hasDefault: false,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'status',
+      physicalName: 'status',
+      sql: 'jsonEnum',
+      nullable: false,
+      hasDefault: true,
+      enum: ['dead', 'delivered', 'pending'],
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'attempts',
+      physicalName: 'attempts',
+      sql: 'integer',
+      nullable: false,
+      hasDefault: true,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'created_at',
+      physicalName: 'created_at',
+      sql: 'timestamp',
+      nullable: false,
+      hasDefault: true,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'lease_owner',
+      physicalName: 'lease_owner',
+      sql: 'text',
+      nullable: false,
+      hasDefault: true,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'lease_until',
+      physicalName: 'lease_until',
+      sql: 'timestamp',
+      nullable: false,
+      hasDefault: true,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'delivered_at',
+      physicalName: 'delivered_at',
+      sql: 'timestamp',
+      nullable: true,
+      hasDefault: false,
+    },
+    {
+      ...EMPTY_COLUMN,
+      name: 'last_error',
+      physicalName: 'last_error',
+      sql: 'text',
+      nullable: true,
+      hasDefault: false,
+    },
+  ],
+});
+
+export interface OutboxWriter {
+  write(topic: string, payload: string): Promise<string>;
+}
+
+export function outboxWriter(tx: TransactionContext): OutboxWriter {
+  const dialect = requiredDialect(tx.dialect, 'outboxWriter');
+  return {
+    async write(topic, payload) {
+      const id = globalThis.crypto.randomUUID();
+      await tx.execute(
+        createQueryCompiler(dialect)
+          .insertInto('zmdb_outbox')
+          .values({
+            id,
+            topic,
+            payload,
+            status: 'pending',
+            attempts: 0,
+            created_at: new Date(),
+            lease_owner: '',
+            lease_until: new Date(0),
+            delivered_at: null,
+            last_error: null,
+          })
+          .compile(),
+      );
+      return id;
+    },
+  };
+}
+
+export interface DeadOutboxRow {
+  readonly id: string;
+  readonly topic: string;
+  readonly payload: string;
+  readonly attempts: number;
+  readonly lastError: string | null;
+}
+
+export interface OutboxDispatcherOptions {
+  readonly driver: Driver;
+  readonly publish: (topic: string, payload: string) => Promise<void>;
+  readonly batch?: number;
+  readonly leaseMs?: number;
+  readonly idleMs?: number;
+  readonly maxIdleMs?: number;
+  readonly maxAttempts?: number;
+  readonly backoffMs?: (attempts: number) => number;
+  readonly onDead?: (row: DeadOutboxRow) => void | Promise<void>;
+}
+
+export interface OutboxDispatcher {
+  runOnce(): Promise<{ readonly claimed: number; readonly delivered: number; readonly failed: number }>;
+  start(): void;
+  onModuleInit(): void;
+  onShutdown(): Promise<void>;
+}
+
+interface ClaimedRow {
+  readonly id: string;
+  readonly topic: string;
+  readonly payload: string;
+  readonly attempts: number;
+}
+
+function claimedRow(row: Readonly<Record<string, unknown>>): ClaimedRow {
+  const { id, topic, payload, attempts } = row;
+  if (
+    typeof id !== 'string' ||
+    typeof topic !== 'string' ||
+    typeof payload !== 'string' ||
+    typeof attempts !== 'number' ||
+    !Number.isInteger(attempts) ||
+    attempts < 0
+  ) {
+    throw new Error('@zmdb/orm: invalid outbox row returned by the driver');
+  }
+  return { id, topic, payload, attempts };
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function deadRowFromInvalid(
+  raw: Readonly<Record<string, unknown>>,
+  attempts: number,
+  lastError: string,
+): DeadOutboxRow {
+  return {
+    id: typeof raw['id'] === 'string' ? raw['id'] : String(raw['id']),
+    topic: typeof raw['topic'] === 'string' ? raw['topic'] : String(raw['topic']),
+    payload: typeof raw['payload'] === 'string' ? raw['payload'] : String(raw['payload']),
+    attempts,
+    lastError,
+  };
+}
+
+function positiveInteger(name: string, value: number): number {
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`@zmdb/orm: ${name} must be a positive safe integer`);
+  }
+  return value;
+}
+
+const MAX_TIMER_MS = 2_147_483_647;
+
+function timerDelay(name: string, value: number): number {
+  const delay = positiveInteger(name, value);
+  if (delay > MAX_TIMER_MS) {
+    throw new Error(`@zmdb/orm: ${name} must not exceed ${MAX_TIMER_MS}`);
+  }
+  return delay;
+}
+
+export function createOutboxDispatcher(options: OutboxDispatcherOptions): OutboxDispatcher {
+  const dialect = requiredDialect(options.driver.dialect, 'createOutboxDispatcher');
+  const batch = positiveInteger('batch', options.batch ?? 100);
+  const leaseMs = positiveInteger('leaseMs', options.leaseMs ?? 30_000);
+  const idleMs = timerDelay('idleMs', options.idleMs ?? 1_000);
+  const maxIdleMs = timerDelay('maxIdleMs', options.maxIdleMs ?? 30_000);
+  const maxAttempts = positiveInteger('maxAttempts', options.maxAttempts ?? 10);
+  if (maxIdleMs < idleMs) {
+    throw new Error('@zmdb/orm: maxIdleMs must be greater than or equal to idleMs');
+  }
+  const backoffMs = options.backoffMs ?? (attempts => Math.min(2 ** attempts * 1_000, 300_000));
+
+  let stopped = false;
+  let started = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const activePasses = new Set<
+    Promise<{ readonly claimed: number; readonly delivered: number; readonly failed: number }>
+  >();
+  let nextIdleMs = idleMs;
+
+  const runPass = async () => {
+    const now = new Date();
+    const candidates = await options.driver.execute(outboxCandidatesQuery(dialect, { now, batch }));
+    const ids = candidates.flatMap(row => (typeof row['id'] === 'string' ? [row['id']] : []));
+    if (ids.length === 0) return { claimed: 0, delivered: 0, failed: 0 } as const;
+
+    const token = globalThis.crypto.randomUUID();
+    const leaseUntil = new Date(now.getTime() + leaseMs);
+    await options.driver.execute(outboxClaimQuery(dialect, { now, token, leaseUntil, ids }));
+    const candidateOrder = new Map(ids.map((id, index) => [id, index]));
+    const rows = (await options.driver.execute(outboxReadBackQuery(dialect, { token }))).toSorted((left, right) => {
+      const leftIndex = typeof left['id'] === 'string' ? candidateOrder.get(left['id']) : undefined;
+      const rightIndex = typeof right['id'] === 'string' ? candidateOrder.get(right['id']) : undefined;
+      return (leftIndex ?? ids.length) - (rightIndex ?? ids.length);
+    });
+
+    let delivered = 0;
+    let failed = 0;
+    for (const raw of rows) {
+      let row: ClaimedRow;
+      try {
+        row = claimedRow(raw);
+      } catch (error) {
+        failed += 1;
+        const id = raw['id'];
+        const previousAttempts = raw['attempts'];
+        if (typeof id !== 'string' || typeof previousAttempts !== 'number' || !Number.isInteger(previousAttempts)) {
+          throw error;
+        }
+        const attempts = previousAttempts + 1;
+        const lastError = errorMessage(error);
+        await options.driver.execute(outboxMarkDeadQuery(dialect, { id, token, attempts, lastError }));
+        await options.onDead?.(deadRowFromInvalid(raw, attempts, lastError));
+        continue;
+      }
+      const attempts = row.attempts + 1;
+      let publishError: unknown;
+      try {
+        await options.publish(row.topic, row.payload);
+      } catch (error) {
+        publishError = error;
+      }
+      if (publishError === undefined) {
+        await options.driver.execute(
+          outboxMarkDeliveredQuery(dialect, {
+            id: row.id,
+            token,
+            deliveredAt: new Date(),
+            attempts,
+          }),
+        );
+        delivered += 1;
+      } else {
+        failed += 1;
+        const lastError = errorMessage(publishError);
+        if (attempts >= maxAttempts) {
+          await options.driver.execute(outboxMarkDeadQuery(dialect, { id: row.id, token, attempts, lastError }));
+          await options.onDead?.({ ...row, attempts, lastError });
+        } else {
+          const delay = backoffMs(attempts);
+          if (!Number.isFinite(delay) || delay < 0) {
+            throw new Error('@zmdb/orm: backoffMs must return a finite non-negative number');
+          }
+          await options.driver.execute(
+            outboxMarkRetryQuery(dialect, {
+              id: row.id,
+              token,
+              attempts,
+              lastError,
+              leaseUntil: new Date(Date.now() + delay),
+            }),
+          );
+        }
+      }
+    }
+    return { claimed: rows.length, delivered, failed };
+  };
+
+  const runOnce = (): Promise<{ readonly claimed: number; readonly delivered: number; readonly failed: number }> => {
+    if (stopped) return Promise.resolve({ claimed: 0, delivered: 0, failed: 0 });
+    const pass = runPass();
+    activePasses.add(pass);
+    void pass.then(
+      () => activePasses.delete(pass),
+      () => activePasses.delete(pass),
+    );
+    return pass;
+  };
+
+  const schedule = (delay: number): void => {
+    timer = setTimeout(() => {
+      if (stopped) return;
+      const pass = runOnce();
+      void pass.then(
+        report => {
+          if (stopped) return;
+          if (report.claimed > 0) {
+            nextIdleMs = idleMs;
+            schedule(report.claimed >= batch ? 0 : nextIdleMs);
+          } else {
+            nextIdleMs = Math.min(nextIdleMs * 2, maxIdleMs);
+            schedule(nextIdleMs);
+          }
+        },
+        () => {
+          if (!stopped) schedule(nextIdleMs);
+        },
+      );
+    }, delay);
+  };
+
+  const start = (): void => {
+    if (started || stopped) return;
+    started = true;
+    schedule(idleMs);
+  };
+
+  return {
+    runOnce,
+    start,
+    onModuleInit: start,
+    async onShutdown() {
+      stopped = true;
+      if (timer !== undefined) clearTimeout(timer);
+      while (activePasses.size > 0) {
+        await Promise.allSettled(activePasses);
+      }
+    },
+  };
+}