@owlmeans/queue 0.1.18-rc.4 → 0.1.18-rc.40

package/src/types.ts

@@ -0,0 +1,323 @@
+import type { BasicConfig, BasicContext, InitializedService } from '@owlmeans/context'
+import type { PubSubResource, Resource, ResourceRecord } from '@owlmeans/resource'
+import type { AbstractRequest, AbstractResponse } from '@owlmeans/entrypoint'
+import type { JobEventType, JobState } from './consts.js'
+import type { RouteOptions } from '@owlmeans/route'
+
+/**
+ * Everything the monorepo knows about its queues, declared in the shared backend package so that
+ * producer and consumer read the same list. A process becomes a worker by naming queues in
+ * `listen` — nothing in a declaration decides where it runs.
+ */
+export interface QueueConfig {
+  queues?: QueueDeclaration[]
+  /** The queues THIS process consumes. Absent or empty means producer only. */
+  listen?: string[]
+  /** Which db service carries the queues. Defaults to the queue service's own alias. */
+  db?: string
+  defaults?: JobOptions
+  /**
+   * How long a signed job envelope stays acceptable, in seconds. It bounds both the freshness
+   * check and the nonce cache, so a job delayed behind a long backlog is still admitted.
+   */
+  envelopeTtl?: number
+  /**
+   * Recurring jobs. Declared beside the queues, in the shared package, because every process that
+   * listens to a schedule's queue reconciles the broker against this list when its worker starts —
+   * two listeners loading different lists would remove each other's schedules.
+   */
+  schedules?: ScheduleDeclaration[]
+}
+
+/**
+ * A job the broker produces on its own, on an interval or a cron pattern.
+ *
+ * A scheduled run is an ordinary job of a name the queue declares, dispatched to the processor
+ * registered for that name — never to a guarded entrypoint, because nobody's credentials travel
+ * with it. Only a process that listens to the queue creates the scheduler, and the broker produces
+ * each next run when the previous one STARTS processing — a schedule needs a running consumer.
+ */
+export interface ScheduleDeclaration {
+  /**
+   * The schedule's stable identity: the broker keeps one scheduler per id, so changing any other
+   * field updates it in place, while renaming the id removes one schedule and creates another.
+   */
+  id: string
+  queue: string
+  /** A job name the queue declares. */
+  name: string
+  /**
+   * Milliseconds between runs. Exactly one of `every` and `pattern`. Without `startDate` the first
+   * run is produced as soon as the scheduler is created.
+   */
+  every?: number
+  /** A cron pattern (an optional leading seconds field is accepted). Exactly one of the two. */
+  pattern?: string
+  /** The IANA time zone `pattern` is read in. Pattern only. */
+  tz?: string
+  startDate?: Date | number | string
+  /** Once passed, the schedule is treated as absent and a scheduler left from it is removed. */
+  endDate?: Date | number | string
+  /** Stop after this many runs. */
+  limit?: number
+  /**
+   * Run once as soon as the scheduler is created or its declaration changes, then follow the
+   * pattern. Pattern only, and never together with `startDate`. A restart that finds the
+   * declaration unchanged does not run it again.
+   */
+  immediately?: boolean
+  data?: unknown
+  /**
+   * Options every run is enqueued with, over the queue's defaults. `id` and `delay` belong to the
+   * broker for a scheduled run and are refused.
+   */
+  opts?: Omit<JobOptions, 'id' | 'delay'>
+}
+
+export interface QueueDeclaration {
+  name: string
+  /**
+   * The job names this queue accepts. A job the queue does not declare is refused at enqueue
+   * time rather than becoming a job nothing can process.
+   */
+  jobs: string[]
+  worker?: QueueWorkerOptions
+  defaults?: JobOptions
+}
+
+export interface QueueWorkerOptions {
+  concurrency?: number
+  /**
+   * How long a job may hold its lock without reporting progress. A processor that runs longer
+   * than this between `touch()` calls is treated as stalled and its job is handed to someone else.
+   */
+  lockDuration?: number
+  stalledInterval?: number
+  maxStalledCount?: number
+}
+
+export interface JobOptions {
+  attempts?: number
+  backoff?: { type: 'fixed' | 'exponential', delay: number }
+  delay?: number
+  priority?: number
+  /**
+   * A caller-chosen identity for the job. Enqueueing the same id twice while the first is still
+   * around is a no-op, which is what makes an admission step safe to retry.
+   */
+  id?: string
+  removeOnComplete?: boolean | number
+  removeOnFail?: boolean | number
+}
+
+/**
+ * A job as a RECORD — the same shape the resource contract reads and writes, so inspecting a
+ * queue is `list`/`get`/`count` rather than a broker-specific API.
+ */
+export interface JobRecord<D = unknown, R = unknown> extends ResourceRecord {
+  queue: string
+  name: string
+  data: D
+  state?: JobState
+  result?: R
+  error?: string
+  attempts?: number
+  progress?: unknown
+  parentId?: string
+  createdAt?: string
+  startedAt?: string
+  finishedAt?: string
+  opts?: JobOptions
+}
+
+export interface JobEvent<R = unknown> {
+  type: JobEventType
+  id: string
+  queue: string
+  /**
+   * Empty when the job is already gone. A broker's completion event carries only an id, so the
+   * name is read back from the job — and a queue that removes jobs on completion has nothing left
+   * to read. Subscribe for the id and the outcome; do not branch on the name.
+   */
+  name: string
+  result?: R
+  error?: string
+  progress?: unknown
+}
+
+/** A node of a job graph: children run first, siblings run together. */
+export interface FlowSpec<D = unknown> {
+  name: string
+  queue?: string
+  data: D
+  opts?: JobOptions
+  children?: FlowSpec[]
+}
+
+/**
+ * One queue, addressed as a resource.
+ *
+ * `create` enqueues, `get`/`load`/`list`/`count` inspect, `delete`/`take` cancel. What the base
+ * contract cannot express — waiting for an answer, submitting a graph, releasing the connection —
+ * is added here.
+ *
+ * Three places where the base `Resource` contract reads differently against a broker, because a
+ * job is not a row:
+ * - `create` does NOT throw `RecordExists` for a repeated `opts.id`. Re-enqueueing an id that is
+ *   still around returns the existing job, which is the idempotency that makes an admission step
+ *   safe to retry.
+ * - `WriteOptions.ttl` is refused. A job's lifetime is `removeOnComplete`/`removeOnFail` —
+ *   explicit deletion once it has finished, not expiry while it waits.
+ * - `update`/`save` reach only the job's `data`. Everything else about a job belongs to the
+ *   broker, and an implementation refuses rather than pretending to write it.
+ */
+export interface QueueResource<D = unknown, R = unknown>
+  extends Resource<JobRecord<D, R>>, PubSubResource<JobEvent<R>> {
+  queue: string
+  /**
+   * Release the connections this queue holds.
+   *
+   * Watching for completions means a blocking connection, and a process that never releases it
+   * does not exit. Nothing else in the resource contract can express that, so it lives here and a
+   * producer calls it on shutdown.
+   */
+  close: () => Promise<void>
+  /**
+   * Wait for a job's return value.
+   * @throws {QueueTimeout} when the wait elapses — the job itself is left alone.
+   * @throws the processor's own error, rebuilt as its original class.
+   */
+  wait: (id: string, opts?: { timeout?: number }) => Promise<R>
+  /** Submit a graph and get its root back. Children complete before their parent starts. */
+  flow: (root: FlowSpec<D>) => Promise<JobRecord<D, R>>
+  counts: () => Promise<Record<JobState, number>>
+}
+
+/**
+ * What a processor is handed. `data` is the job's payload; everything else is the means to stay
+ * alive and to read what the children reported.
+ */
+export interface JobContext<D = unknown> {
+  id: string
+  name: string
+  queue: string
+  attempt: number
+  data: D
+  signal: AbortSignal
+  /**
+   * The id of the schedule that produced this run (`ScheduleDeclaration.id`). Absent for a job
+   * enqueued any other way.
+   */
+  scheduled?: string
+  /**
+   * Renew the lock. A processor that runs a long loop MUST call it as it goes: the broker judges
+   * liveness by the lock, so silence for longer than `lockDuration` is indistinguishable from a
+   * dead worker, and the job is re-run somewhere else while this one is still working on it.
+   */
+  touch: () => Promise<void>
+  progress: (value: unknown) => Promise<void>
+  /**
+   * What the children returned, keyed by child job id.
+   *
+   * A broker keys them by its own fully qualified job key; an implementation re-keys to the bare
+   * id, so two children of one parent that live in DIFFERENT queues and happen to share an id
+   * collide here. Give the children of a cross-queue flow distinct ids.
+   */
+  children: <T = unknown>() => Promise<Record<string, T>>
+  failedChildren: () => Promise<Record<string, string>>
+}
+
+export interface JobProcessor<D = unknown, R = unknown> {
+  (job: JobContext<D>): Promise<R>
+}
+
+/**
+ * The consuming half. It is registered only in processes that consume: `start()` binds the
+ * queues named in `cfg.listen` and dispatches by job name, to a processor registered here or to
+ * an entrypoint served in this process.
+ */
+export interface QueueWorkerService extends InitializedService {
+  process: <D, R>(queue: string, name: string, processor: JobProcessor<D, R>) => void
+  start: () => Promise<void>
+  stop: () => Promise<void>
+  /** The queues this process actually consumes. Empty in a producer. */
+  listening: () => string[]
+  /**
+   * Bind the application's reactions to job lifecycle events. Registering more than once merges,
+   * so an application composed of several parts can each contribute what it cares about.
+   *
+   * It belongs on the contract rather than on a driver because `onJobDead` is where an
+   * application COMPENSATES — releases the lock the admission step took, marks the record failed —
+   * and that behaviour must not have to be rewritten when the broker changes.
+   */
+  hooks: (hooks: QueueHooks) => void
+}
+
+/**
+ * The envelope a queued entrypoint call travels in. It is the request, minus everything that only
+ * means something to HTTP — the transport rebuilds a request from it on the far side.
+ */
+export interface JobEnvelope {
+  alias: string
+  params?: Record<string, unknown>
+  body?: unknown
+  query?: Record<string, unknown>
+  headers?: Record<string, string | undefined>
+  /** When the producer enqueued it — freshness is judged here, not at processing time. */
+  enqueuedAt?: string
+}
+
+export interface QueueJobMeta {
+  id: string
+  name: string
+  queue: string
+  attempt: number
+  /** Renew the broker lock for protocol handlers whose work can outlive one lock duration. */
+  touch: () => Promise<void>
+}
+
+/** What a processor returns for an entrypoint job, so the caller can rebuild the reply. */
+export interface JobReply<T = unknown> {
+  value?: T
+  outcome?: string
+  error?: unknown
+}
+
+export interface QueueHooks {
+  wrapHandler?: (job: JobContext, next: () => Promise<unknown>) => Promise<unknown>
+  onJobResult?: (event: JobEvent) => void | Promise<void>
+  onJobStalled?: (meta: { id: string, queue: string, name: string }, phase: 'stalled' | 'failed') => void | Promise<void>
+  /**
+   * The job is finished for good — retries exhausted, or it was dropped. This is where an
+   * application compensates: release the lock the admission step took, mark the record failed.
+   */
+  onJobDead?: (job: JobRecord, reason: string) => void | Promise<void>
+}
+
+export interface QueueDriver {
+  resource: <D, R>(queue: string) => QueueResource<D, R>
+  worker: () => QueueWorkerService
+}
+
+export interface Config extends BasicConfig {
+  queue?: QueueConfig
+}
+
+export interface Context<C extends Config = Config> extends BasicContext<C> {
+}
+
+export interface QueueAppend {
+  jobs: <D = unknown, R = unknown>(queue?: string) => QueueResource<D, R>
+}
+
+export interface QueueTransportRequest extends AbstractRequest {
+}
+
+export interface QueueTransportResponse extends AbstractResponse<unknown> {
+}
+
+/** Queue-owned options stored in a generic route declaration. */
+export interface QueueRouteOptions extends Omit<RouteOptions, 'protocol' | 'protocolOptions'> {
+  queue: string
+  reply?: boolean
+}

package/src/utils.ts

@@ -0,0 +1,5 @@
+import { JobState } from './consts.js'
+
+/** A job has reached a state it will not leave on its own. */
+export const isSettled = (state?: JobState): boolean =>
+  state === JobState.Completed || state === JobState.Failed

package/src/worker.ts

@@ -0,0 +1,73 @@
+import type { Middleware } from '@owlmeans/context'
+import { MiddlewareStage, MiddlewareType, AppType } from '@owlmeans/context'
+import type { CommonEntrypoint } from '@owlmeans/entrypoint'
+import type { Config, Context, JobContext, JobEnvelope, JobProcessor, QueueWorkerService } from './types.js'
+import { handleJob } from './bridge.js'
+import { DEFAULT_ALIAS } from './consts.js'
+import { isQueueRoute, queueRouteOptions } from './route.js'
+
+/**
+ * Every queued entrypoint this process both SERVES and LISTENS to, grouped by queue.
+ *
+ * Both halves are required and they answer different questions. Serving is about code — the alias
+ * was bound here, so a handler exists. Listening is about deployment — this process was
+ * configured to consume that queue. A worker that bound queues by what it can serve would make
+ * every deployment of the same binary a worker for everything it happens to import.
+ */
+export const servedJobs = <C extends Config, T extends Context<C>>(
+  context: T
+): Map<string, CommonEntrypoint[]> => {
+  const listen = context.cfg.queue?.listen ?? []
+  const served = new Map<string, CommonEntrypoint[]>()
+
+  context.entrypoints<CommonEntrypoint>().forEach(entrypoint => {
+    const route = entrypoint.route.route
+    if (route.type !== AppType.Backend || !isQueueRoute(route)) {
+      return
+    }
+    if (route.service != null && route.service !== context.cfg.service) {
+      return
+    }
+    const { queue } = queueRouteOptions(route)
+    if (entrypoint.handle == null || !listen.includes(queue)) {
+      return
+    }
+
+    served.set(queue, [...(served.get(queue) ?? []), entrypoint])
+  })
+
+  return served
+}
+
+/**
+ * A processor that runs a queued entrypoint call. The driver dispatches by job name, and an
+ * entrypoint job's name IS its alias — which is what lets one worker carry both entrypoint jobs
+ * and the internal steps an application registers with `process()`.
+ */
+export const entrypointProcessor = <C extends Config, T extends Context<C>>(
+  context: T
+): JobProcessor<JobEnvelope, unknown> =>
+  async (job: JobContext<JobEnvelope>) => await handleJob(context, job)
+
+/**
+ * Start the worker once the context is ready.
+ *
+ * Ready stage, not Loading: processors are registered while the application wires itself up, and
+ * binding the queues before that finished would take jobs this process cannot yet run. A process
+ * that listens to nothing registers no worker at all, so a producer pays nothing for this.
+ */
+export const queueWorkerMiddleware = (alias: string = DEFAULT_ALIAS): Middleware => ({
+  type: MiddlewareType.Context,
+  stage: MiddlewareStage.Ready,
+  apply: async context => {
+    const ctx = context as unknown as Context<Config>
+    if ((ctx.cfg.queue?.listen ?? []).length === 0) {
+      return
+    }
+    if (!ctx.hasService(alias)) {
+      return
+    }
+
+    await ctx.service<QueueWorkerService>(alias).start()
+  }
+})

package/tests/bridge.spec.ts

@@ -0,0 +1,64 @@
+import { describe, test, expect } from 'bun:test'
+import { assertFresh, requestOf } from '../src/bridge.js'
+import { EnvelopeExpired } from '../src/errors.js'
+import type { JobEnvelope } from '../src/types.js'
+
+const envelope = (enqueuedAt?: string): JobEnvelope => ({
+  alias: 'story:code', params: { id: '7' }, body: { prompt: 'x' }, headers: {}, query: {}, enqueuedAt
+})
+
+describe('rebuilding the request', () => {
+  test('the envelope becomes an ordinary request', () => {
+    const request = requestOf(envelope(), '/story/:id/code')
+
+    expect(request.alias).toBe('story:code')
+    expect(request.params).toEqual({ id: '7' })
+    expect(request.body).toEqual({ prompt: 'x' })
+    expect(request.path).toBe('/story/:id/code')
+  })
+
+  /**
+   * A producer may omit any of the optional parts. The handler on the far side reads them without
+   * checking, so they have to arrive as empty containers rather than as undefined.
+   */
+  test('absent parts arrive as empty, never undefined', () => {
+    const request = requestOf({ alias: 'story:code' }, '/x')
+
+    expect(request.params).toEqual({})
+    expect(request.headers).toEqual({})
+    expect(request.query).toEqual({})
+  })
+})
+
+describe('envelope freshness', () => {
+  test('a fresh envelope passes', () => {
+    expect(() => assertFresh(envelope(new Date().toISOString()), 60)).not.toThrow()
+  })
+
+  test('an envelope older than the window is refused', () => {
+    const old = new Date(Date.now() - 120_000).toISOString()
+
+    expect(() => assertFresh(envelope(old), 60)).toThrow(EnvelopeExpired)
+  })
+
+  /**
+   * The window is judged from when the job was ENQUEUED. A job that waited behind a long backlog
+   * was legitimate when it was produced, and judging it on pickup would reject exactly the work a
+   * busy queue delayed — so a generous window has to keep it.
+   */
+  test('a long wait in the queue is not what expiry is for', () => {
+    const enqueued = new Date(Date.now() - 3600_000).toISOString()
+
+    expect(() => assertFresh(envelope(enqueued), 86_400)).not.toThrow()
+  })
+
+  test('no configured window means no check', () => {
+    const ancient = new Date(0).toISOString()
+
+    expect(() => assertFresh(envelope(ancient), undefined)).not.toThrow()
+  })
+
+  test('an envelope without a timestamp is not judged', () => {
+    expect(() => assertFresh(envelope(undefined), 60)).not.toThrow()
+  })
+})

package/tests/config.spec.ts

@@ -0,0 +1,66 @@
+import { describe, test, expect } from 'bun:test'
+import { declareQueue, isListening, listenQueues, queueOf, queueOfJob } from '../src/config.js'
+import { UnknownQueue } from '../src/errors.js'
+import type { Config } from '../src/types.js'
+
+const cfg = (): Config => ({ ready: false, service: 'test', type: 'backend' } as unknown as Config)
+
+describe('declaring queues', () => {
+  test('a declaration records the queue and the jobs it accepts', () => {
+    const config = declareQueue(cfg(), 'work', ['story:code', 'story:gate'])
+
+    expect(queueOf(config, 'work').jobs).toEqual(['story:code', 'story:gate'])
+  })
+
+  /**
+   * Declaring runs from a helper that an application may call once per app it composes, so the
+   * second call must not leave two entries under one name — a duplicate would make `queueOf`
+   * answer with whichever happened to be first.
+   */
+  test('re-declaring the same name replaces rather than appends', () => {
+    let config = declareQueue(cfg(), 'work', ['story:code'])
+    config = declareQueue(config, 'work', ['story:code', 'story:gate'])
+
+    expect(config.queue?.queues).toHaveLength(1)
+    expect(queueOf(config, 'work').jobs).toHaveLength(2)
+  })
+
+  test('asking for a queue nobody declared is an error, not undefined', () => {
+    expect(() => queueOf(cfg(), 'missing')).toThrow(UnknownQueue)
+  })
+
+  test('a job name resolves back to the queue that accepts it', () => {
+    const config = declareQueue(declareQueue(cfg(), 'work', ['story:code']), 'ops', ['files:get'])
+
+    expect(queueOfJob(config, 'files:get')?.name).toBe('ops')
+    expect(queueOfJob(config, 'nothing:declared')).toBeUndefined()
+  })
+})
+
+describe('listening', () => {
+  /**
+   * Declaring a queue must NOT make this process consume it — that separation is what lets one
+   * binary deploy as a producer in one place and a worker in another.
+   */
+  test('declaring a queue does not make the process listen to it', () => {
+    const config = declareQueue(cfg(), 'work', ['story:code'])
+
+    expect(isListening(config, 'work')).toBe(false)
+    expect(config.queue?.listen ?? []).toHaveLength(0)
+  })
+
+  test('listening is additive and free of duplicates', () => {
+    let config = listenQueues(cfg(), 'work')
+    config = listenQueues(config, 'work', 'ops')
+
+    expect(config.queue?.listen).toEqual(['work', 'ops'])
+    expect(isListening(config, 'ops')).toBe(true)
+  })
+
+  test('listening and declaring do not overwrite each other', () => {
+    const config = declareQueue(listenQueues(cfg(), 'work'), 'work', ['story:code'])
+
+    expect(isListening(config, 'work')).toBe(true)
+    expect(queueOf(config, 'work').jobs).toEqual(['story:code'])
+  })
+})

package/tests/protocol.spec.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from 'bun:test'
+import { contract, protocol, schema } from '@owlmeans/entrypoint'
+import { route } from '@owlmeans/route'
+import type { JSONSchemaType } from 'ajv'
+import { enqueueProtocol, job, waitForProtocol, type JobRecord } from '../src/index.js'
+
+interface Input { value: number }
+interface Output { doubled: number }
+const InputSchema = schema<Input>({
+  type: 'object', properties: { value: { type: 'number' } }, required: ['value'], additionalProperties: false,
+} as JSONSchemaType<Input>)
+const OutputSchema = schema<Output>({
+  type: 'object', properties: { doubled: { type: 'number' } }, required: ['doubled'], additionalProperties: false,
+} as JSONSchemaType<Output>)
+const declaration = protocol(
+  route('test:double', '/double', job({ queue: 'test' })),
+  contract(InputSchema, OutputSchema),
+)
+
+const context = (reply: unknown = { value: { doubled: 6 } }) => {
+  const created: Partial<JobRecord>[] = []
+  return {
+    cfg: { queue: { queues: [{ name: 'test', jobs: [declaration.alias] }] } },
+    jobs: () => ({
+      create: async (record: Partial<JobRecord>) => {
+        created.push(record)
+        return { ...record, id: 'job-1' } as JobRecord
+      },
+      wait: async () => reply,
+    }),
+    created,
+  }
+}
+
+describe('protocol-aware queue calls', () => {
+  test('derives the broker address and keeps per-call options', async () => {
+    const ctx = context()
+    const queued = await enqueueProtocol(ctx as never, declaration, { body: { value: 3 } }, {
+      delay: 250, attempts: 3, id: 'projection-1',
+    })
+    expect(queued.queue).toBe('test')
+    expect(ctx.created[0]?.name).toBe(declaration.alias)
+    expect(ctx.created[0]?.opts).toEqual({ delay: 250, attempts: 3, id: 'projection-1' })
+    expect(await waitForProtocol(ctx as never, declaration, queued as never)).toEqual({ doubled: 6 })
+  })
+
+  test('rejects raw aliases and non-queue declarations at runtime', async () => {
+    await expect(enqueueProtocol(context() as never, declaration.alias as never, {} as never)).rejects.toThrow()
+    const http = protocol(route('test:http', '/http'), contract())
+    await expect(enqueueProtocol(context() as never, http, {})).rejects.toThrow()
+  })
+
+  test('rejects a job from another queue or protocol', async () => {
+    await expect(waitForProtocol(context() as never, declaration, {
+      id: 'x', queue: 'other', name: declaration.alias, data: {},
+    } as never)).rejects.toThrow()
+  })
+})