@arki/journal 0.0.0-stage → 0.1.1

package/src/command.ts

@@ -0,0 +1,242 @@
+/**
+ * Command execution: parse the input, open the transaction, give the handler a
+ * context bound to it, and refuse to close while an operation is still running.
+ * May import `api`, `write` (types), `storage`, `transaction-guard` and leaf files.
+ * Must not import `journal`, `history` or `worker`.
+ */
+import type { z } from 'zod';
+import { eq, is } from 'drizzle-orm';
+import { PgAsyncTransaction } from 'drizzle-orm/pg-core';
+
+import type { Command, CommandContext, RunOptions } from './api.js';
+import type { HookPointInternal } from './hooks.js';
+import type { JournalTables } from './pg.js';
+import type { CommandHookContext, DbHandle, Trace, TransactionSource } from './types.js';
+import type { Writer } from './write.js';
+import { debugCommand } from './debug.js';
+import { JournalError } from './errors.js';
+import { commandId, correlationId } from './id.js';
+import { assertRoundTrips, sameValue } from './storage.js';
+import { oneAtATime } from './transaction-guard.js';
+
+/** What JSON storage keeps of `value`. `undefined` has no JSON form, so it comes back as `undefined` here too. */
+const throughJson = (value: unknown): unknown =>
+  // eslint-disable-next-line unicorn/prefer-structured-clone -- JSON semantics on purpose: this is what the column stores
+  value === undefined ? undefined : JSON.parse(JSON.stringify(value));
+
+/** What command execution needs from the journal. */
+export type CommandDeps<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>> = {
+  readonly metaSchema: Meta;
+  readonly writeWithValidatedMeta: Writer<Meta, Tx>['writeWithValidatedMeta'];
+  /** Wakes this journal's workers. Called after a root-database run that queued a job. */
+  readonly wake: () => void;
+  /** The command hooks. Frozen by the first run. */
+  readonly hooks: HookPointInternal<CommandHookContext<z.output<Meta>>>;
+  readonly tables: Pick<JournalTables, 'receipts'>;
+};
+
+export function createCommandRunner<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>>(
+  deps: CommandDeps<Meta, Tx>,
+) {
+  type MetaOut = z.output<Meta>;
+  const { metaSchema, writeWithValidatedMeta, hooks } = deps;
+  const receipts = deps.tables.receipts;
+
+  /**
+   * Take the key for this run, or find the run that holds it. Returns `{ replay }` with the
+   * stored result when a finished run holds the key with the same command and input. The
+   * insert waits on an uncommitted holder, so a concurrent repeat sees its outcome.
+   */
+  const claimReceipt = async (
+    tx: DbHandle,
+    key: string,
+    name: string,
+    input: unknown,
+    own: string,
+  ): Promise<{ readonly replay: unknown } | undefined> => {
+    const inserted = await tx
+      .insert(receipts)
+      .values({ key, command: name, input: throughJson(input) ?? null, commandId: own })
+      .onConflictDoNothing()
+      .returning({ key: receipts.key });
+    if (inserted.length > 0) return undefined;
+    const [held] = await tx.select().from(receipts).where(eq(receipts.key, key));
+    if (held === undefined)
+      throw new Error(`journal: idempotency key ${JSON.stringify(key)} vanished while it was being claimed`);
+    // Only a run inside this transaction can see an unfinished receipt: a concurrent one waited on the insert.
+    if (held.result === null)
+      throw new Error(
+        `journal: idempotency key ${JSON.stringify(key)} is held by a run of "${held.command}" that has not finished; a key cannot be reused inside its own transaction`,
+      );
+    if (held.command !== name)
+      throw new JournalError(
+        'CONFLICT',
+        'receipt',
+        key,
+        `the idempotency key is held by command "${held.command}", not "${name}"`,
+      );
+    if (!sameValue(held.input, throughJson(input) ?? null))
+      throw new JournalError(
+        'CONFLICT',
+        'receipt',
+        key,
+        `the idempotency key was used by command "${name}" with a different input`,
+      );
+    return { replay: 'value' in held.result ? held.result.value : undefined };
+  };
+
+  /** Store the result under the key. It must survive JSON, because the replay returns the stored form. */
+  const storeReceipt = async (tx: DbHandle, key: string, name: string, value: unknown): Promise<void> => {
+    const stored = throughJson(value);
+    if (!sameValue(stored, value))
+      throw new JournalError(
+        'UNSTABLE_SCHEMA',
+        'command',
+        name,
+        `the result of command "${name}" does not survive JSON storage; a command run with an idempotency key must return plain data, because a repeat returns the stored result`,
+      );
+    await tx
+      .update(receipts)
+      .set({ result: stored === undefined ? {} : { value: stored } })
+      .where(eq(receipts.key, key));
+  };
+
+  /**
+   * `journal.run` and `context.run` share this. `getValidatedMeta` gives the parsed meta:
+   * the root call parses it, a child inherits it. It runs inside the one-at-a-time
+   * guard, so a bad meta on a busy transaction waits like any other rejection.
+   * `onJobs` hears how many jobs each write queued, in this command and its children.
+   * `parent` is the trace this run continues: the chain's correlation id and cause are
+   * inherited, the command id is this run's own.
+   */
+  const runCommand = <S extends z.ZodType, R>(
+    source: TransactionSource<Tx>,
+    command: Command<S, R, MetaOut, Tx>,
+    input: z.input<S>,
+    getValidatedMeta: () => Promise<MetaOut>,
+    onJobs: (count: number) => void,
+    parent: Omit<Trace, 'commandId'>,
+    key: string | undefined,
+  ): Promise<Awaited<R>> =>
+    oneAtATime(source, `command "${command.name}"`, async (): Promise<Awaited<R>> => {
+      hooks.freeze();
+      const meta = await getValidatedMeta();
+      const parsed = await command.input.parseAsync(input);
+      const own = commandId();
+      const trace: Trace = { ...parent, commandId: own };
+      debugCommand.debug('{commandName}: input ok ({own}, {correlationId})', {
+        commandName: command.name,
+        own,
+        correlationId: trace.correlationId,
+      });
+      return source.transaction(async (tx): Promise<Awaited<R>> => {
+        // Every operation the handler starts through this context. The transaction
+        // must not close while one is still running: its work would land outside.
+        const started: Promise<unknown>[] = [];
+        let unfinished = 0;
+        const track = <T>(operation: Promise<T>): Promise<T> => {
+          started.push(operation);
+          unfinished += 1;
+          const done = () => {
+            unfinished -= 1;
+          };
+          void operation.then(done, done);
+          return operation;
+        };
+        const context: CommandContext<MetaOut, Tx> = {
+          tx,
+          meta,
+          commandId: own,
+          correlationId: trace.correlationId,
+          write: (entity, id, events, opts) =>
+            track(writeWithValidatedMeta(tx, entity, id, () => meta, events, trace, onJobs, opts)),
+          run: (child, childInput, childOpts) =>
+            track(
+              runCommand(tx, child, childInput, () => Promise.resolve(meta), onJobs, trace, childOpts?.idempotencyKey),
+            ),
+        };
+        let result: Awaited<R>;
+        try {
+          // Inside the transaction: a hook that throws rolls the command back cleanly.
+          result = await hooks.run(
+            {
+              name: command.name,
+              input: parsed,
+              meta,
+              commandId: own,
+              correlationId: trace.correlationId,
+              idempotencyKey: key ?? null,
+            },
+            async (): Promise<Awaited<R>> => {
+              if (key === undefined) return await command.handler(parsed, context);
+              const held = await claimReceipt(tx, key, command.name, parsed, own);
+              if (held !== undefined) {
+                debugCommand.debug('{commandName}: replayed from receipt {key}', { commandName: command.name, key });
+                // The receipt holds what this same command returned earlier, in the JSON form
+                // `storeReceipt` checked against the live value. This is the one cast in the library.
+                return held.replay as Awaited<R>;
+              }
+              const value = await command.handler(parsed, context);
+              await storeReceipt(tx, key, command.name, value);
+              return value;
+            },
+          );
+        } catch (error) {
+          // The handler's own error is the one to report; the rollback undoes the rest.
+          await Promise.allSettled(started);
+          throw error;
+        }
+        if (unfinished > 0) {
+          const count = unfinished;
+          await Promise.allSettled(started);
+          throw new Error(
+            `journal: command "${command.name}" returned with ${count} journal operation${count === 1 ? '' : 's'} still running; await every write and run before the handler returns`,
+          );
+        }
+        debugCommand.debug('{commandName}: done', { commandName: command.name });
+        return result;
+      });
+    });
+
+  /**
+   * `journal.run`: parses `meta` once for the whole command. `cause` is set only by an
+   * effect's `ctx.run`: the event the job was for. Its correlation id carries over unless
+   * `opts.correlationId` says otherwise; with neither, the run starts a new chain.
+   */
+  const run = <S extends z.ZodType, R>(
+    source: TransactionSource<Tx>,
+    command: Command<S, R, MetaOut, Tx>,
+    input: NoInfer<z.input<S>>,
+    meta: z.input<Meta>,
+    opts?: RunOptions,
+    cause?: { readonly seq: number; readonly correlationId: string | null },
+  ): Promise<Awaited<R>> => {
+    let queued = 0;
+    return runCommand(
+      source,
+      command,
+      input,
+      async () => {
+        const parsed = await metaSchema.parseAsync(meta);
+        // No entity yet: the error names the command instead.
+        assertRoundTrips(metaSchema, parsed, 'command', command.name, 'meta');
+        return parsed;
+      },
+      count => {
+        queued += count;
+      },
+      {
+        causeSeq: cause?.seq ?? null,
+        correlationId: opts?.correlationId ?? cause?.correlationId ?? correlationId(),
+      },
+      opts?.idempotencyKey,
+    ).then(result => {
+      // The root database has committed by now. Inside a caller's transaction
+      // only the caller knows when the commit happens, so the caller wakes.
+      if (queued > 0 && !is(source, PgAsyncTransaction)) deps.wake();
+      return result;
+    });
+  };
+
+  return run;
+}

package/src/debug.ts

@@ -0,0 +1,8 @@
+import { getLogger } from '@arki/log';
+
+/** Enable with `DEBUG=arki:journal:*`. */
+export const debugWrite = getLogger('arki:journal:write');
+export const debugCommand = getLogger('arki:journal:command');
+export const debugReplay = getLogger('arki:journal:replay');
+export const debugDrain = getLogger('arki:journal:drain');
+export const debugWorker = getLogger('arki:journal:worker');

package/src/dot.ts

@@ -0,0 +1,296 @@
+/**
+ * `@arki/dot` adapter.
+ *
+ * - `journal(...)` is the plugin. It publishes the journal as `services.journal`
+ *   and runs one worker per plugin instance between `start` and `stop`.
+ * - `journalHandlers(...)` is the feature slice. A feature lists its projections
+ *   and effects with it, and `journal({ j, tables, features })` collects them.
+ *
+ * The app's `db` plugin must publish the root Drizzle database under the `db` key.
+ *
+ * `@arki/dot` is an optional peer. Import this module only inside a Dot app.
+ * Feature slices are read by shape: this file never imports `@arki/feature`.
+ */
+import type { z } from 'zod';
+
+import type { Plugin, Service, Token } from '@arki/dot/plugin';
+import { DotPluginError, plugin, service } from '@arki/dot/plugin';
+
+import type { Journal, JournalDefinitions } from './api.js';
+import type { JournalTables } from './pg.js';
+import type {
+  DbHandle,
+  Effect,
+  JournalWorker,
+  Needs,
+  Projection,
+  RootDb,
+  Services,
+  ServicesFor,
+  Tap,
+  TransactionSource,
+  WakeListener,
+} from './types.js';
+import { debugWorker } from './debug.js';
+import { JournalError } from './errors.js';
+import { bindEffect, bindTap } from './handlers.js';
+
+export const JOURNAL_ACTION_META_SCHEMA = '@arki/journal/action-meta@1';
+
+export const JOURNAL_PLUGIN_ERROR_CODES = { badSlice: 'JOURNAL_PLUGIN_E001' } as const;
+
+/** One effect in the app manifest: async work that the journal's job table feeds in. */
+export type JournalDotAction = {
+  readonly id: string;
+  readonly binding: 'journal';
+  readonly direction: 'in';
+  readonly address: string;
+  readonly metaSchema: typeof JOURNAL_ACTION_META_SCHEMA;
+  readonly meta: { readonly entities: string[]; readonly events: string[] };
+};
+
+/** What one feature hands to the journal. Its effects already hold the feature's services. */
+export type JournalHandlers<Meta = unknown, Tx = DbHandle, AppDb = RootDb> = {
+  readonly projections: readonly Projection<Meta, Tx>[];
+  readonly effects: readonly Effect<Meta, AppDb>[];
+  readonly taps: readonly Tap<Meta, AppDb>[];
+};
+
+/**
+ * The `@arki/feature` slice contract a `journalHandlers(...)` declaration produces.
+ * `TServices` is what the feature's effects need: the feature must declare it in `needs`.
+ */
+export type JournalFeatureSlice<Meta = unknown, Tx = DbHandle, AppDb = RootDb, TServices = unknown> = {
+  readonly key: 'journal';
+  readonly actions: readonly JournalDotAction[];
+  readonly resolve: (services: TServices) => JournalHandlers<Meta, Tx, AppDb>;
+};
+
+function effectAction(effect: {
+  readonly name: string;
+  readonly entities: readonly { readonly kind: string }[];
+  readonly types: ReadonlySet<string>;
+}): JournalDotAction {
+  return {
+    id: `${effect.name}.run`,
+    binding: 'journal',
+    direction: 'in',
+    address: effect.name,
+    metaSchema: JOURNAL_ACTION_META_SCHEMA,
+    meta: { entities: effect.entities.map(entity => entity.kind), events: [...effect.types] },
+  };
+}
+
+/**
+ * Declare a feature's projections and effects as a boot-time slice
+ * (`defineFeature(..., { use: [journalHandlers({ projections, effects })] })`).
+ * The payload lands under `journal` in the feature's slice token, where
+ * `journal({ j, tables, features })` collects it.
+ *
+ * An effect with needs (`j.effect(name, entity, { mailer: need<Mailer>() }, on)`) gets
+ * them from the booted services of this feature, by name. The feature must declare
+ * each one in its own `needs`: the slice's `resolve` does not type-check otherwise.
+ * Two features may use the same need name for two different services.
+ *
+ * Actions: one per effect. Projections add none: they run inside the write, not
+ * at an app boundary.
+ */
+export function journalHandlers<
+  Meta = unknown,
+  Tx = DbHandle,
+  AppDb = RootDb,
+  const E extends readonly Effect<Meta, AppDb, unknown, Needs>[] = readonly [],
+  const T extends readonly Tap<Meta, AppDb, Needs>[] = readonly [],
+>(handlers: {
+  readonly projections?: readonly Projection<Meta, Tx>[];
+  readonly effects?: E;
+  readonly taps?: T;
+}): JournalFeatureSlice<Meta, Tx, AppDb, ServicesFor<E> & ServicesFor<T>> {
+  const { projections = [], effects = [], taps = [] } = handlers;
+  return {
+    key: 'journal',
+    actions: effects.map(effect => effectAction(effect)),
+    // The kernel hands the feature's services over untyped, so they are read by name here.
+    resolve: (services: unknown) => ({
+      projections,
+      effects: effects.map(effect => bindEffect(effect, isServices(services) ? services : {})),
+      taps: taps.map(tap => bindTap(tap, isServices(services) ? services : {})),
+    }),
+  };
+}
+
+const isServices = (value: unknown): value is Services => typeof value === 'object' && value !== null;
+
+type WorkerOptions = {
+  /** Run the async lane in this process. Default `true`. */
+  readonly worker?: boolean;
+  readonly pollMs?: number;
+  readonly batchSize?: number;
+  readonly leaseSeconds?: number;
+  /** See `journal.worker`: required when the journal has taps, unless `taps: false`. */
+  readonly listen?: WakeListener;
+  readonly taps?: boolean;
+};
+
+/** Direct mode: the app composed the journal itself. */
+export type JournalDirectOptions<
+  Meta extends z.ZodObject,
+  Tx extends TransactionSource<Tx>,
+  AppDb extends RootDb,
+> = WorkerOptions & { readonly journal: Journal<Meta, Tx, AppDb> };
+
+/** Feature mode: the plugin composes the journal from the features' `journalHandlers` slices. */
+export type JournalFeatureOptions<
+  Meta extends z.ZodObject,
+  Tx extends TransactionSource<Tx>,
+  AppDb extends RootDb,
+> = WorkerOptions & {
+  readonly j: Pick<JournalDefinitions<Meta, Tx, AppDb>, 'compose'>;
+  readonly tables: JournalTables;
+  /** `tokens(features)` from `@arki/feature/dot`. A feature without a journal slice adds nothing. */
+  readonly features: readonly Token<object, string>[];
+};
+
+/**
+ * `RootDb` again on purpose: a Drizzle transaction fits `AppDb` by shape, and must not fit here.
+ * `TransactionSource<Tx>`: the worker's database runs the commands of effects, so it opens `Tx`.
+ */
+export type JournalPlugin<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>, AppDb extends RootDb> = Plugin<
+  { db: AppDb & RootDb & TransactionSource<Tx> },
+  { journal: Journal<Meta, Tx, AppDb> }
+>;
+
+const isHandler = (value: unknown): boolean =>
+  typeof value === 'object' &&
+  value !== null &&
+  'name' in value &&
+  typeof value.name === 'string' &&
+  'run' in value &&
+  typeof value.run === 'function';
+
+/** The kernel hands services over untyped, so the slice shape is checked here, once, at boot. */
+function isJournalHandlers(value: unknown): value is JournalHandlers {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'projections' in value &&
+    Array.isArray(value.projections) &&
+    value.projections.every(item => isHandler(item)) &&
+    'effects' in value &&
+    Array.isArray(value.effects) &&
+    value.effects.every(item => isHandler(item)) &&
+    'taps' in value &&
+    Array.isArray(value.taps) &&
+    value.taps.every(item => isHandler(item))
+  );
+}
+
+/**
+ * Throws when another handler of this kind already has the name, and names the
+ * feature tokens that hold the two. `j.compose` makes the same check, but it gets
+ * one flat list and cannot say which feature a handler came from.
+ */
+function claimName(
+  kind: 'effect' | 'projection' | 'tap',
+  owners: Map<string, string>,
+  name: string,
+  feature: string,
+): void {
+  const owner = owners.get(name);
+  if (owner !== undefined) {
+    const where = owner === feature ? `feature "${feature}", twice` : `features "${owner}" and "${feature}"`;
+    throw new JournalError('BAD_DEFINITION', kind, name, `journal: duplicate ${kind} name "${name}" (${where})`);
+  }
+  owners.set(name, feature);
+}
+
+/**
+ * Read each feature's `journal` slice from the booted services, in feature order.
+ * Refuses a projection name or an effect name that is listed twice.
+ */
+function collectHandlers(
+  features: readonly Token<object, string>[],
+  services: Readonly<Record<string, unknown>>,
+): JournalHandlers {
+  const projections: Projection[] = [];
+  const effects: Effect[] = [];
+  const taps: Tap[] = [];
+  const projectionOwners = new Map<string, string>();
+  const effectOwners = new Map<string, string>();
+  const tapOwners = new Map<string, string>();
+  for (const feature of features) {
+    const slices = services[feature.key];
+    const slice = typeof slices === 'object' && slices !== null && 'journal' in slices ? slices.journal : undefined;
+    if (slice === undefined) {
+      debugWorker.debug('[journal] feature "{featureKey}" did not contribute a journal slice', {
+        featureKey: feature.key,
+      });
+      continue;
+    }
+    if (!isJournalHandlers(slice)) {
+      throw new DotPluginError({
+        code: JOURNAL_PLUGIN_ERROR_CODES.badSlice,
+        message: `[journal] feature token "${feature.key}" holds a \`journal\` slice that is not lists of projections, effects and taps.`,
+        remediation: 'Declare the slice with journalHandlers({ projections, effects, taps }) from @arki/journal/dot.',
+        docsUrl: 'https://github.com/arki-labs/arki/blob/HEAD/packages/journal/docs/agent-guide.md#dot-boot-errors',
+      });
+    }
+    for (const { name } of slice.projections) claimName('projection', projectionOwners, name, feature.key);
+    for (const { name } of slice.effects) claimName('effect', effectOwners, name, feature.key);
+    for (const { name } of slice.taps) claimName('tap', tapOwners, name, feature.key);
+    projections.push(...slice.projections);
+    effects.push(...slice.effects);
+    taps.push(...slice.taps);
+  }
+  return { projections, effects, taps };
+}
+
+export function journal<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>, AppDb extends RootDb>(
+  options: JournalDirectOptions<Meta, Tx, AppDb>,
+): JournalPlugin<Meta, Tx, AppDb>;
+export function journal<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>, AppDb extends RootDb>(
+  options: JournalFeatureOptions<Meta, Tx, AppDb>,
+): JournalPlugin<Meta, Tx, AppDb>;
+export function journal<Meta extends z.ZodObject, Tx extends TransactionSource<Tx>, AppDb extends RootDb>(
+  options: JournalDirectOptions<Meta, Tx, AppDb> | JournalFeatureOptions<Meta, Tx, AppDb>,
+) {
+  let composed: Journal<Meta, Tx, AppDb> | undefined;
+  let worker: JournalWorker | undefined;
+  // Each feature token is a need, so the kernel boots the features first and refuses a missing one.
+  const featureNeeds = 'features' in options ? options.features.map(feature => [feature.key, feature]) : [];
+  const needs: { readonly db: Service<AppDb & RootDb & TransactionSource<Tx>> } = {
+    ...Object.fromEntries(featureNeeds),
+    db: service<AppDb & RootDb & TransactionSource<Tx>>(),
+  };
+
+  return plugin({
+    name: 'journal',
+    needs,
+    configure(ctx) {
+      ctx.registerService('journal', 'custom');
+      ctx.declareProvides('journal');
+    },
+    boot(services) {
+      composed =
+        'journal' in options
+          ? options.journal
+          : options.j.compose({ tables: options.tables, ...collectHandlers(options.features, services) });
+      return { journal: composed };
+    },
+    async start({ db }) {
+      if (options.worker === false) return;
+      worker = composed?.worker(db, {
+        pollMs: options.pollMs,
+        batchSize: options.batchSize,
+        leaseSeconds: options.leaseSeconds,
+        listen: options.listen,
+        taps: options.taps,
+      });
+      await worker?.start();
+    },
+    async stop() {
+      await worker?.stop();
+      worker = undefined;
+    },
+  });
+}