@kelpie/server 0.13.0 → 0.15.0

package/src/index.ts

@@ -9,6 +9,22 @@ export type { AppDependencies, AppBindings } from './app.ts'
 export { bootAssembly } from './boot.ts'
 export type { AssemblyBoot } from './boot.ts'
 
+export { startWorker } from './worker.ts'
+
+export { createJobsRuntime } from './runtime/jobs.ts'
+export type { JobsRuntime, JobsRuntimeOptions } from './runtime/jobs.ts'
+export { deadLetterQueueName } from './lib/jobs.ts'
+export type {
+  EnqueueOptions,
+  JobContext,
+  JobDefaults,
+  JobDefinition,
+  JobHandle,
+  JobHandler,
+  JobRegistry,
+  TransactionJobs,
+} from './lib/jobs.ts'
+
 export { WebBundleError, serveWebBundle } from './webBundle.ts'
 export type { WebBundleOptions } from './webBundle.ts'
 
@@ -143,6 +159,7 @@ export type { ModuleEventCatalog } from './runtime/module.ts'
 export { createTransactionScope } from './runtime/transaction.ts'
 export type {
   BufferedEvents,
+  EnqueueOnTransaction,
   Transaction,
   TransactionContext,
   TransactionOptions,

package/src/lib/jobs.ts

@@ -0,0 +1,126 @@
+import type { ZodType } from 'zod'
+
+import type { Logger } from './logger.ts'
+
+/**
+ * The port core runs background work through.
+ *
+ * A module declares a handle at register time with `context.jobs.define(...)`
+ * and enqueues against it from a service. The insert lives inside the same
+ * Drizzle transaction as the write it belongs to, so a rolled-back write
+ * never leaves an orphan job behind. Handlers run out of process — on the
+ * worker entry point, or inline when the API runs its own worker — and see
+ * the same payload the caller passed, validated against the module's Zod
+ * schema before it reaches user code.
+ *
+ * The port itself knows nothing about pg-boss. `runtime/jobs.ts` binds it to
+ * a real provider at boot; tests bind it to a stub. Nothing outside the
+ * runtime imports the provider directly.
+ *
+ * Why a handle rather than a global name map: consumers of `@kelpie/server`
+ * installed from npm lost the `KelpieEventMap` augmentation until every
+ * catalog was pulled in from the entry point (see
+ * `modules/eventCatalogs.ts`). A handle keeps the payload type on the value
+ * the module exports, so `enqueue(handle, data)` typechecks without
+ * declaration merging across the package boundary.
+ */
+
+/**
+ * Marker property carrying the handle's payload type. Never assigned at
+ * runtime; the `?` on the property makes an untagged plain object satisfy the
+ * shape at read time, so a stub handle in a test can be built without
+ * `unsafeCast`.
+ */
+declare const JobHandleDataMarker: unique symbol
+
+/** A typed reference a module exports after `define()`. */
+export interface JobHandle<Data> {
+  readonly name: string
+  readonly [JobHandleDataMarker]?: Data
+}
+
+/**
+ * Defaults every enqueue on this queue inherits, unless the caller overrides
+ * on a per-`enqueue` basis. `localConcurrency` sizes the work loop on the
+ * worker; the rest match pg-boss's `QueueOptions` names.
+ *
+ * `retryDelay` is in seconds. pg-boss's default is 0 (immediate retry).
+ * `retryLimit` counts the retries pg-boss does after the first run; a job
+ * with `retryLimit: 2` runs at most three times.
+ */
+export interface JobDefaults {
+  readonly retryLimit?: number
+  readonly retryDelay?: number
+  readonly retryBackoff?: boolean
+  readonly retryDelayMax?: number
+  readonly expireInSeconds?: number
+  readonly localConcurrency?: number
+}
+
+/** Passed to a job handler when a worker picks the job up. */
+export interface JobContext<Data> {
+  readonly id: string
+  readonly data: Data
+  readonly log: Logger
+  /** Fires when pg-boss decides the job has expired. Long-running handlers should honour it. */
+  readonly signal: AbortSignal
+}
+
+export type JobHandler<Data> = (context: JobContext<Data>) => Promise<void>
+
+export interface JobDefinition<Data> {
+  /**
+   * The queue name pg-boss stores against. Must be unique across the
+   * assembly; a second `define()` with the same name fails boot.
+   *
+   * Prefix with the module id — `webhooks.deliver`, `agent-tasks.dispatch` —
+   * so operators reading `pgboss.job` see who owns it.
+   */
+  readonly name: string
+  /**
+   * Parses the payload at handle time before it reaches user code. A job
+   * whose payload no longer matches fails, retries, and eventually
+   * dead-letters like any other failure.
+   */
+  readonly schema: ZodType<Data>
+  readonly handler: JobHandler<Data>
+  readonly defaults?: JobDefaults
+}
+
+/**
+ * The registration surface exposed to a module through `context.jobs`. Modules
+ * hand a definition in and receive a typed handle to enqueue against.
+ */
+export interface JobRegistry {
+  define<Data>(definition: JobDefinition<Data>): JobHandle<Data>
+}
+
+/**
+ * Options a caller may override per `enqueue`. Deliberately narrow: features
+ * beyond retries, a singleton key, and a deferred start are out of scope until
+ * a consumer needs them (`crm-brief.md` cost/maintainability trade-offs).
+ */
+export interface EnqueueOptions {
+  readonly retryLimit?: number
+  readonly retryBackoff?: boolean
+  readonly singletonKey?: string
+  readonly startAfter?: number | string | Date
+}
+
+/**
+ * Exposed on the transaction context next to `events`. `enqueue` runs its
+ * insert inside the caller's transaction via pg-boss's `fromDrizzle` adapter,
+ * so a rollback discards the job with the write.
+ */
+export interface TransactionJobs {
+  enqueue<Data>(
+    handle: JobHandle<Data>,
+    data: Data,
+    options?: EnqueueOptions,
+  ): Promise<string | null>
+}
+
+/** The name pg-boss stores the dead-letter queue under, given a handle name. */
+export function deadLetterQueueName(jobName: string): string {
+  return `${jobName}.dead`
+}

package/src/runtime/jobs.ts

@@ -0,0 +1,358 @@
+import { sql } from 'drizzle-orm'
+import { PgBoss, fromDrizzle } from 'pg-boss'
+import type { Job } from 'pg-boss'
+
+import { describeThrown } from '../lib/errors.ts'
+import type { Logger } from '../lib/logger.ts'
+import type {
+  EnqueueOptions,
+  JobContext,
+  JobDefinition,
+  JobHandle,
+  JobRegistry,
+} from '../lib/jobs.ts'
+import { deadLetterQueueName } from '../lib/jobs.ts'
+import type { Transaction } from './transaction.ts'
+
+/**
+ * The pg-boss provider that binds the job port defined in `lib/jobs.ts` to a
+ * real Postgres-backed queue.
+ *
+ * Two phases:
+ *
+ * 1. Registration, at boot. Modules call `context.jobs.define(...)` during
+ *    `register`. The registry records the definition (fails boot on a
+ *    duplicate name) and returns the handle. Nothing touches the database
+ *    yet.
+ * 2. `start()`, from the entry point. Opens the pg-boss instance, primes the
+ *    queue cache, and calls `createQueue` for every registered handle plus
+ *    its dead-letter queue. Idempotent (`create_queue` upserts).
+ *
+ * The API and the worker both call `start()`: the API needs a boss instance
+ * to insert jobs on request-scoped transactions; the worker adds a
+ * `startWorking()` step that runs `work()` per handle.
+ *
+ * @see modules.md for how jobs sit next to the event bus.
+ */
+
+/**
+ * The pg-boss schema name. Fixed here rather than configurable per assembly:
+ * self-hosters run one Postgres per Kelpie install and would gain nothing
+ * from renaming the schema.
+ */
+const PGBOSS_SCHEMA = 'pgboss'
+
+const DEFAULT_LOCAL_CONCURRENCY = 5
+
+interface RegisteredJob<Data> {
+  readonly definition: JobDefinition<Data>
+  readonly handle: JobHandle<Data>
+}
+
+export interface JobsRuntimeOptions {
+  readonly connectionString: string
+  readonly logger: Logger
+  /**
+   * Polling floor. LISTEN/NOTIFY wakes a worker immediately when a job is
+   * inserted via `send()`, but pg-boss's own retry re-insert (inside
+   * `fail()`) never notifies, so a retried job is only ever picked up by
+   * this poll — as is a fresh job if the listener never opened. Also used
+   * as pg-boss's `notifyPollingIntervalSeconds`, so NOTIFY-active queues
+   * poll no slower than this either. Default 2s in production; tests pass
+   * a shorter value so a retry runs within the test's own budget.
+   */
+  readonly pollingIntervalSeconds?: number
+  /**
+   * How many workers the runtime spins up per handle when a definition does
+   * not override `defaults.localConcurrency`. Matches pg-boss's default.
+   */
+  readonly defaultLocalConcurrency?: number
+}
+
+export interface JobsRuntime {
+  /** The port modules bind to at register time. */
+  readonly registry: JobRegistry
+  /** True once `register` has been called with this name. Used by tests. */
+  hasHandle(name: string): boolean
+  /**
+   * Applies pg-boss's own schema migration. Idempotent: safe to run on every
+   * boot. Uses a throwaway `PgBoss` in `migrate: true, createSchema: true`
+   * mode, matching the release-command flow the cloud runs.
+   */
+  migrate(): Promise<void>
+  /**
+   * Opens the persistent pg-boss instance and declares every registered
+   * queue plus its dead-letter counterpart. Must run before `enqueueOnTx`
+   * or `startWorking`. `migrate: false` here: `migrate()` above owns that
+   * step, so instances never race the migration.
+   */
+  start(): Promise<void>
+  /**
+   * Starts the `work()` loop for every registered handle. The worker entry
+   * point calls this; the API calls it too unless `--no-worker` is set.
+   * Handler errors are logged and rethrown so pg-boss retries or
+   * dead-letters the job.
+   */
+  startWorking(): Promise<void>
+  /**
+   * Enqueues a job on the caller's transaction via `fromDrizzle(tx, sql)`.
+   * The insert commits with the surrounding write; a rollback discards it.
+   * Rejects when no `handle` was registered for `name` (this would be a
+   * boot-time bug, not a runtime one).
+   */
+  enqueueOnTx<Data>(
+    tx: Transaction,
+    handle: JobHandle<Data>,
+    data: Data,
+    options?: EnqueueOptions,
+  ): Promise<string | null>
+  /**
+   * Drains in-flight work and closes the pg-boss instance. `offWork({wait:
+   * true})` awaits every handler mid-flight; `boss.stop` then tears down
+   * the connection pool.
+   */
+  stop(): Promise<void>
+}
+
+export function createJobsRuntime(options: JobsRuntimeOptions): JobsRuntime {
+  const registered = new Map<string, RegisteredJob<unknown>>()
+  // Tracks which handles have had `createQueue` run and which have had
+  // `work()` called, so a runtime that registers a fresh handle after
+  // `start()` (a test that adds one mid-suite) still gets its queue built
+  // and its worker started without redoing the ones already up.
+  const queuesCreated = new Set<string>()
+  const workersStarted = new Set<string>()
+  let boss: PgBoss | undefined
+
+  function requireBoss(): PgBoss {
+    if (boss === undefined) {
+      throw new Error('jobs runtime used before start(): call jobs.start() at boot')
+    }
+    return boss
+  }
+
+  const registry: JobRegistry = {
+    define<Data>(definition: JobDefinition<Data>): JobHandle<Data> {
+      if (registered.has(definition.name)) {
+        throw new Error(`job "${definition.name}" is defined twice`)
+      }
+
+      const handle: JobHandle<Data> = { name: definition.name }
+      registered.set(definition.name, {
+        definition: definition as JobDefinition<unknown>,
+        handle: handle as JobHandle<unknown>,
+      })
+
+      return handle
+    },
+  }
+
+  async function migrate(): Promise<void> {
+    // A throwaway instance in migrate/createSchema mode. Cheaper than
+    // vendoring the SQL: pg-boss owns its schema, and `getConstructionPlans`
+    // / `getMigrationPlans` produce statements we would just execute in the
+    // same order the library does itself. `supervise: false, schedule: false`
+    // keeps the instance from spawning any background work while it lives.
+    const migrator = new PgBoss({
+      connectionString: options.connectionString,
+      schema: PGBOSS_SCHEMA,
+      migrate: true,
+      createSchema: true,
+      supervise: false,
+      schedule: false,
+    })
+
+    try {
+      await migrator.start()
+    } finally {
+      await migrator.stop({ graceful: false, close: true, timeout: 5_000 })
+    }
+  }
+
+  async function ensureQueuesFor(instance: PgBoss): Promise<void> {
+    for (const { definition } of registered.values()) {
+      if (queuesCreated.has(definition.name)) {
+        continue
+      }
+
+      const deadLetter = deadLetterQueueName(definition.name)
+
+      // Dead letter queue must exist before the main queue references it.
+      await instance.createQueue(deadLetter, {
+        policy: 'standard',
+        notify: false,
+      })
+
+      await instance.createQueue(definition.name, {
+        policy: 'standard',
+        deadLetter,
+        notify: true,
+        ...(definition.defaults?.retryLimit === undefined
+          ? {}
+          : { retryLimit: definition.defaults.retryLimit }),
+        ...(definition.defaults?.retryDelay === undefined
+          ? {}
+          : { retryDelay: definition.defaults.retryDelay }),
+        ...(definition.defaults?.retryBackoff === undefined
+          ? {}
+          : { retryBackoff: definition.defaults.retryBackoff }),
+        ...(definition.defaults?.retryDelayMax === undefined
+          ? {}
+          : { retryDelayMax: definition.defaults.retryDelayMax }),
+        ...(definition.defaults?.expireInSeconds === undefined
+          ? {}
+          : { expireInSeconds: definition.defaults.expireInSeconds }),
+      })
+
+      queuesCreated.add(definition.name)
+    }
+  }
+
+  async function start(): Promise<void> {
+    if (boss === undefined) {
+      const instance = new PgBoss({
+        connectionString: options.connectionString,
+        schema: PGBOSS_SCHEMA,
+        migrate: false,
+        createSchema: false,
+        supervise: false,
+        schedule: false,
+        useListenNotify: true,
+      })
+
+      instance.on('error', (error: unknown) => {
+        options.logger.error('pg-boss error', { error: describeThrown(error) })
+      })
+
+      await instance.start()
+      boss = instance
+    }
+
+    await ensureQueuesFor(boss)
+  }
+
+  async function startWorking(): Promise<void> {
+    const instance = requireBoss()
+    const pollingIntervalSeconds = options.pollingIntervalSeconds ?? 2
+    const defaultLocalConcurrency = options.defaultLocalConcurrency ?? DEFAULT_LOCAL_CONCURRENCY
+
+    for (const { definition } of registered.values()) {
+      if (workersStarted.has(definition.name)) {
+        continue
+      }
+
+      const localConcurrency = definition.defaults?.localConcurrency ?? defaultLocalConcurrency
+
+      await instance.work(
+        definition.name,
+        {
+          batchSize: 1,
+          localConcurrency,
+          pollingIntervalSeconds,
+          notifyPollingIntervalSeconds: pollingIntervalSeconds,
+        },
+        async (batch: Job<unknown>[]) => {
+          const job = batch[0]
+          if (job === undefined) {
+            return
+          }
+
+          const parsed = definition.schema.safeParse(job.data)
+
+          if (!parsed.success) {
+            options.logger.error('job payload rejected by schema', {
+              job: definition.name,
+              id: job.id,
+              issues: parsed.error.issues,
+            })
+            throw new Error(
+              `job "${definition.name}" payload does not match its schema: ${parsed.error.issues
+                .map((issue) => issue.message)
+                .join('; ')}`,
+            )
+          }
+
+          const jobLogger = options.logger.child({ job: definition.name, jobId: job.id })
+          const context: JobContext<unknown> = {
+            id: job.id,
+            data: parsed.data,
+            log: jobLogger,
+            signal: job.signal,
+          }
+
+          try {
+            await definition.handler(context)
+          } catch (error: unknown) {
+            jobLogger.error('job handler failed', { error: describeThrown(error) })
+            throw error
+          }
+        },
+      )
+
+      workersStarted.add(definition.name)
+    }
+  }
+
+  async function enqueueOnTx<Data>(
+    tx: Transaction,
+    handle: JobHandle<Data>,
+    data: Data,
+    enqueueOptions: EnqueueOptions = {},
+  ): Promise<string | null> {
+    const instance = requireBoss()
+    const entry = registered.get(handle.name)
+
+    if (entry === undefined) {
+      throw new Error(`enqueue for unregistered job "${handle.name}"`)
+    }
+
+    return instance.send(
+      handle.name,
+      data as object,
+      {
+        db: fromDrizzle(tx, sql),
+        ...(enqueueOptions.retryLimit === undefined ? {} : { retryLimit: enqueueOptions.retryLimit }),
+        ...(enqueueOptions.retryBackoff === undefined
+          ? {}
+          : { retryBackoff: enqueueOptions.retryBackoff }),
+        ...(enqueueOptions.singletonKey === undefined
+          ? {}
+          : { singletonKey: enqueueOptions.singletonKey }),
+        ...(enqueueOptions.startAfter === undefined ? {} : { startAfter: enqueueOptions.startAfter }),
+      },
+    )
+  }
+
+  async function stop(): Promise<void> {
+    const instance = boss
+    if (instance === undefined) {
+      return
+    }
+
+    for (const name of workersStarted) {
+      try {
+        await instance.offWork(name, { wait: true })
+      } catch (error: unknown) {
+        options.logger.error('offWork failed', {
+          job: name,
+          error: describeThrown(error),
+        })
+      }
+    }
+    workersStarted.clear()
+
+    await instance.stop({ graceful: true, close: true, timeout: 30_000 })
+    boss = undefined
+    queuesCreated.clear()
+  }
+
+  return {
+    registry,
+    hasHandle: (name) => registered.has(name),
+    migrate,
+    start,
+    startWorking,
+    enqueueOnTx,
+    stop,
+  }
+}

package/src/runtime/module.ts

@@ -5,6 +5,7 @@ import type { Actor } from '../lib/actor.ts'
 import type { Database } from '../lib/database.ts'
 import type { EmailSender } from '../lib/email.ts'
 import type { IdFactory } from '../lib/ids.ts'
+import type { JobRegistry } from '../lib/jobs.ts'
 import type { Logger } from '../lib/logger.ts'
 import type { SecretEncryptionConfig } from '../lib/secrets.ts'
 import type { EntitlementRegistry } from './entitlements.ts'
@@ -240,6 +241,17 @@ export interface ModuleContext extends ModuleServices {
    * commits, and must be idempotent.
    */
   readonly events: EventBus
+  /**
+   * Declares a background job. Modules call `context.jobs.define(...)` at
+   * register time and receive a typed handle to enqueue against. The insert
+   * lives inside the caller's transaction (`services.transaction`) so a
+   * rollback discards the job.
+   *
+   * A second `define` with the same name fails boot. Handlers run out of
+   * process on the worker entry point, or inline when the API runs its own
+   * worker.
+   */
+  readonly jobs: JobRegistry
   /**
    * Declare capabilities and check grants. Every check is granted and unlimited
    * until a module registers a provider.

package/src/runtime/registry.ts

@@ -7,6 +7,7 @@ import type { Environment } from '../lib/config.ts'
 import { createLogEmailSender } from '../lib/email.ts'
 import type { EmailMessage, EmailSender } from '../lib/email.ts'
 import { AppError, describeThrown, describeValidationIssue, toErrorDetails } from '../lib/errors.ts'
+import type { JobRegistry } from '../lib/jobs.ts'
 import type { Logger } from '../lib/logger.ts'
 import { createEntitlementRegistry, requireCapability } from './entitlements.ts'
 import type { EntitlementRegistry } from './entitlements.ts'
@@ -241,6 +242,13 @@ export interface ModuleRuntimeOptions {
    * two modules had registered it.
    */
   readonly additionalEmailProviders?: ReadonlyMap<string, EmailSender> | undefined
+  /**
+   * The job registry modules bind to at register time. Bound to the pg-boss
+   * runtime in production (`runtime/jobs.ts`) and to a stub in unit tests
+   * that never define a job. A module that calls `context.jobs.define` when
+   * this is absent gets a boot-time error naming the module.
+   */
+  readonly jobs?: JobRegistry | undefined
   /**
    * The deploy-time module override (`lib/moduleConfig.ts`), parsed. A locked
    * module id wins over whatever a workspace's own settings say.
@@ -349,6 +357,21 @@ interface RegisteredProvider {
   readonly registeredBy: string
 }
 
+/**
+ * Refuses `define` for a module that reached `context.jobs` when the runtime
+ * booted without a jobs registry. Named so a boot log points at the module
+ * whose declaration cannot be honoured, rather than a bare stack trace.
+ */
+function createMissingJobsRegistry(moduleId: string): JobRegistry {
+  return {
+    define() {
+      throw new ModuleBootError([
+        `module "${moduleId}" defines a job but the assembly booted without a jobs runtime`,
+      ])
+    },
+  }
+}
+
 function createModuleContext(
   module: KelpieModule,
   accumulator: Accumulator,
@@ -363,6 +386,7 @@ function createModuleContext(
   return {
     ...options.services,
     email: emailProxy,
+    jobs: options.jobs ?? createMissingJobsRegistry(module.id),
 
     provideExternalSignIn(handler) {
       externalSignIn.install(module.id, handler)

package/src/runtime/transaction.ts

@@ -2,6 +2,11 @@ import type { EventActor, EventTarget, KelpieEvent } from '@kelpie/schemas'
 
 import type { Database } from '../lib/database.ts'
 import type { IdFactory } from '../lib/ids.ts'
+import type {
+  EnqueueOptions,
+  JobHandle,
+  TransactionJobs,
+} from '../lib/jobs.ts'
 import type { Logger } from '../lib/logger.ts'
 import { checkEventCycle, currentEventChain } from './events.ts'
 import type { EventBus, EventName, KelpieEventMap } from './events.ts'
@@ -43,6 +48,13 @@ export interface BufferedEvents {
 export interface TransactionContext {
   readonly tx: Transaction
   readonly events: BufferedEvents
+  /**
+   * Enqueues background work on the caller's transaction. The insert runs
+   * inside `tx` via pg-boss's Drizzle adapter, so a rolled-back scope
+   * discards the job with the write. Unlike `events`, this is not buffered:
+   * the return value carries the pg-boss job id, awaited inside the scope.
+   */
+  readonly jobs: TransactionJobs
 }
 
 export interface TransactionOptions {
@@ -69,6 +81,17 @@ interface BufferedEvent {
 
 const SYSTEM_ACTOR: EventActor = { kind: 'system' }
 
+/**
+ * Inserts a background job on `tx`. Bound to the pg-boss provider in
+ * production (`runtime/jobs.ts`) and to a stub in tests that never enqueue.
+ */
+export type EnqueueOnTransaction = <Data>(
+  tx: Transaction,
+  handle: JobHandle<Data>,
+  data: Data,
+  options?: EnqueueOptions,
+) => Promise<string | null>
+
 export interface TransactionScopeDependencies {
   readonly db: Database
   readonly bus: EventBus
@@ -78,12 +101,27 @@ export interface TransactionScopeDependencies {
   readonly now?: () => Date
   /** Chain-depth cap. Reads `KELPIE_EVENT_MAX_DEPTH`; otherwise 8. */
   readonly maxDepth?: number
+  /**
+   * How `tx.jobs.enqueue(...)` reaches pg-boss. Optional so unit tests that
+   * never enqueue can build the scope without a runtime; a call from a
+   * scope built without one rejects with a boot-time bug message.
+   */
+  readonly enqueueOnTx?: EnqueueOnTransaction
 }
 
 export function createTransactionScope(dependencies: TransactionScopeDependencies): TransactionScope {
   const now = dependencies.now ?? ((): Date => new Date())
   const maxDepth = dependencies.maxDepth ?? readMaxDepthFromEnv() ?? 8
 
+  const enqueueOnTx: EnqueueOnTransaction =
+    dependencies.enqueueOnTx ??
+    (() =>
+      Promise.reject(
+        new Error(
+          'jobs.enqueue used but no jobs runtime was wired into the transaction scope',
+        ),
+      ))
+
   return async function runInTransaction(work, options) {
     const actor = options?.actor ?? SYSTEM_ACTOR
     const workspaceId = options?.workspaceId
@@ -92,6 +130,11 @@ export function createTransactionScope(dependencies: TransactionScopeDependencie
     const result = await dependencies.db.transaction((tx) =>
       work({
         tx,
+        jobs: {
+          enqueue(handle, data, enqueueOptions) {
+            return enqueueOnTx(tx, handle, data, enqueueOptions)
+          },
+        },
         events: {
           emit(name, target, data) {
             if (workspaceId === undefined || workspaceId.length === 0) {