@owlmeans/queue 0.1.18-rc.6 → 0.1.18-rc.8

package/src/bridge.ts

@@ -0,0 +1,109 @@
+import type { AbstractRequest, CommonEntrypoint, GuardService } from '@owlmeans/entrypoint'
+import { EntrypointOutcome, provideResponse } from '@owlmeans/entrypoint'
+import type { Auth } from '@owlmeans/auth'
+import { AuthorizationError } from '@owlmeans/auth'
+import { attachEntity } from '@owlmeans/auth-common'
+import { ResilientError } from '@owlmeans/error'
+import type { Config, Context, JobContext, JobEnvelope, JobReply } from './types.js'
+import { EnvelopeExpired, JobNotServed } from './errors.js'
+
+/**
+ * Rebuild the request the producer described. A queued call is still a call: the same guards, the
+ * same filter, the same handler run against it — only the wire is different.
+ */
+export const requestOf = (envelope: JobEnvelope, path: string): AbstractRequest => ({
+  alias: envelope.alias,
+  params: (envelope.params ?? {}) as AbstractRequest['params'],
+  body: envelope.body as AbstractRequest['body'],
+  headers: (envelope.headers ?? {}) as AbstractRequest['headers'],
+  query: (envelope.query ?? {}) as AbstractRequest['query'],
+  path,
+})
+
+/**
+ * Freshness is judged from when the job was ENQUEUED, not from now.
+ *
+ * A signed envelope proves who produced it, and a replay window keeps a captured one from being
+ * useful forever. But a job can legitimately sit behind a long backlog, and judging it on pickup
+ * would reject exactly the work that a busy queue delayed — so the producer's timestamp is what
+ * the window applies to.
+ *
+ * @throws {EnvelopeExpired}
+ */
+export const assertFresh = (envelope: JobEnvelope, ttl?: number): void => {
+  if (ttl == null || envelope.enqueuedAt == null) {
+    return
+  }
+  const age = (Date.now() - new Date(envelope.enqueuedAt).getTime()) / 1000
+  if (age > ttl) {
+    throw new EnvelopeExpired(`${envelope.alias}: ${Math.round(age)}s > ${ttl}s`)
+  }
+}
+
+/**
+ * Run one queued entrypoint call and describe the outcome.
+ *
+ * It never throws for a DOMAIN failure — the error is marshalled into the returned reply so the
+ * producer can rebuild it as its own class, and so the broker does not count a legitimate refusal
+ * as a job to retry. Infrastructure failures are left to propagate: those are what retries are for.
+ */
+export const handleJob = async <C extends Config, T extends Context<C>>(
+  context: T, job: JobContext<JobEnvelope>
+): Promise<JobReply> => {
+  const envelope = job.data
+
+  try {
+    assertFresh(envelope, context.cfg.queue?.envelopeTtl)
+
+    const entrypoint = context.entrypoint<CommonEntrypoint>(envelope.alias)
+    if (entrypoint.handle == null) {
+      throw new JobNotServed(envelope.alias)
+    }
+
+    const request = requestOf(envelope, entrypoint.path())
+    const response = provideResponse<unknown>()
+
+    const guards = entrypoint.getGuards()
+    if (guards.length > 0) {
+      let matched: GuardService | undefined
+      for (const alias of guards) {
+        const guard: GuardService = context.service(alias)
+        if (await guard.match(request, response)) {
+          matched = guard
+          break
+        }
+      }
+      if (matched == null) {
+        throw new AuthorizationError(`queue:${envelope.alias}`)
+      }
+
+      const auth = provideResponse<Auth>()
+      if (!await matched.handle<boolean>(request, auth)) {
+        throw new AuthorizationError(`queue:${envelope.alias}:${matched.alias}`)
+      }
+      request.auth = auth.value
+
+      const entity = await attachEntity(context, request)
+      if (entity != null) {
+        request.entity = entity
+      }
+    }
+
+    await entrypoint.handle(request, response)
+
+    if (response.error != null) {
+      return { error: ResilientError.marshal(response.error).message }
+    }
+
+    return { value: response.value, outcome: response.outcome ?? EntrypointOutcome.Ok }
+  } catch (e) {
+    // A refusal the caller must see as its own class — `ContentRefused`, `AgentLocked` — travels
+    // in the reply. Only a broker- or connection-level fault should ever reach the retry logic:
+    // retrying a refusal just spends the work again to be refused identically.
+    if (e instanceof ResilientError) {
+      return { error: ResilientError.marshal(e).message }
+    }
+
+    throw e
+  }
+}

package/src/config.ts

@@ -0,0 +1,56 @@
+import type { Config, JobOptions, QueueDeclaration, QueueWorkerOptions } from './types.js'
+import { UnknownQueue } from './errors.js'
+
+/**
+ * Add a queue to the shared configuration. Every process in the monorepo loads the same
+ * declarations — a producer needs them to know a queue exists and what it accepts, a worker needs
+ * them to bind. Declaring twice replaces, so a helper that runs per app is safe to call.
+ */
+export const declareQueue = <C extends Config>(
+  cfg: C, name: string, jobs: string[], opts?: { worker?: QueueWorkerOptions, defaults?: JobOptions }
+): C => {
+  const queue: QueueDeclaration = { name, jobs, ...opts }
+  const queues = cfg.queue?.queues ?? []
+  const existing = queues.findIndex(declared => declared.name === name)
+
+  if (existing < 0) {
+    queues.push(queue)
+  } else {
+    queues[existing] = queue
+  }
+
+  cfg.queue = { ...cfg.queue, queues }
+
+  return cfg
+}
+
+/**
+ * Name the queues this process consumes. It is what turns an otherwise identical binary into a
+ * worker, which is why it lives in the process's own config and never in a declaration.
+ */
+export const listenQueues = <C extends Config>(cfg: C, ...names: string[]): C => {
+  const listen = new Set([...(cfg.queue?.listen ?? []), ...names])
+
+  cfg.queue = { ...cfg.queue, listen: [...listen] }
+
+  return cfg
+}
+
+/**
+ * @throws {UnknownQueue}
+ */
+export const queueOf = <C extends Config>(cfg: C, name: string): QueueDeclaration => {
+  const queue = cfg.queue?.queues?.find(declared => declared.name === name)
+
+  if (queue == null) {
+    throw new UnknownQueue(name)
+  }
+
+  return queue
+}
+
+export const queueOfJob = <C extends Config>(cfg: C, job: string): QueueDeclaration | undefined =>
+  cfg.queue?.queues?.find(declared => declared.jobs.includes(job))
+
+export const isListening = <C extends Config>(cfg: C, name: string): boolean =>
+  cfg.queue?.listen?.includes(name) === true

package/src/consts.ts

@@ -0,0 +1,39 @@
+
+export const DEFAULT_ALIAS = 'queue'
+
+/**
+ * The state a job is in, as the broker reports it. `Waiting` covers both a job queued normally
+ * and one held by an unfinished parent — the difference matters to the broker, never to a caller
+ * asking whether an answer has arrived.
+ */
+export enum JobState {
+  Waiting = 'waiting',
+  Delayed = 'delayed',
+  Active = 'active',
+  Completed = 'completed',
+  Failed = 'failed',
+  Unknown = 'unknown',
+}
+
+/** A job has reached a state it will not leave on its own. */
+export const isSettled = (state?: JobState): boolean =>
+  state === JobState.Completed || state === JobState.Failed
+
+export enum JobEventType {
+  Completed = 'completed',
+  Failed = 'failed',
+  Progress = 'progress',
+}
+
+/**
+ * How long a caller waits for a reply when neither the route nor the request says. Long enough
+ * for work that talks to a model, short enough that a lost job surfaces as an error rather than a
+ * hung request.
+ */
+export const DEFAULT_JOB_TIMEOUT = 60_000
+
+/**
+ * Retries default to one attempt: the jobs here are mostly LLM work, and a blind retry re-spends
+ * the tokens that just failed. A queue whose jobs are cheap and idempotent raises it per queue.
+ */
+export const DEFAULT_ATTEMPTS = 1

package/src/errors.ts

@@ -0,0 +1,106 @@
+import { ResilientError } from '@owlmeans/error'
+
+export class QueueError extends ResilientError {
+  public static override typeName = 'QueueError'
+
+  constructor(message: string = 'error') {
+    super(QueueError.typeName, `queue:${message}`)
+  }
+}
+
+/**
+ * The caller stopped waiting. The job is untouched — it may still be queued, running, or already
+ * finished; only the wait ended. Read the job back to learn which.
+ */
+export class QueueTimeout extends QueueError {
+  public static override typeName = `${QueueError.typeName}Timeout`
+
+  constructor(msg: string) {
+    super(`timeout:${msg}`)
+    this.type = QueueTimeout.typeName
+  }
+}
+
+/** No job with that id in this queue — it never existed, or it was removed after completing. */
+export class UnknownJob extends QueueError {
+  public static override typeName = `${QueueError.typeName}UnknownJob`
+
+  constructor(msg: string) {
+    super(`unknown-job:${msg}`)
+    this.type = UnknownJob.typeName
+  }
+}
+
+/**
+ * A job name the queue does not declare. Declaring every job a queue accepts is what keeps an
+ * enqueue from succeeding into a queue where nothing can ever process it.
+ */
+export class UnknownJobName extends QueueError {
+  public static override typeName = `${QueueError.typeName}UnknownJobName`
+
+  constructor(msg: string) {
+    super(`unknown-job-name:${msg}`)
+    this.type = UnknownJobName.typeName
+  }
+}
+
+/** A queue asked for that no declaration names. */
+export class UnknownQueue extends QueueError {
+  public static override typeName = `${QueueError.typeName}UnknownQueue`
+
+  constructor(msg: string) {
+    super(`unknown-queue:${msg}`)
+    this.type = UnknownQueue.typeName
+  }
+}
+
+/**
+ * This process was asked to consume a queue it does not listen to. Which queues a process
+ * consumes is configuration (`cfg.queue.listen`), so this is a deployment question, not a code
+ * one — the same binary is a producer or a worker depending on it.
+ */
+export class QueueNotListening extends QueueError {
+  public static override typeName = `${QueueError.typeName}NotListening`
+
+  constructor(msg: string) {
+    super(`not-listening:${msg}`)
+    this.type = QueueNotListening.typeName
+  }
+}
+
+/**
+ * The job named an entrypoint this process does not serve. Unlike a missing queue this is not a
+ * declaration problem: the alias exists, but nothing here elevated it, so the job was taken by a
+ * worker that cannot run it.
+ */
+export class JobNotServed extends QueueError {
+  public static override typeName = `${QueueError.typeName}JobNotServed`
+
+  constructor(msg: string) {
+    super(`job-not-served:${msg}`)
+    this.type = JobNotServed.typeName
+  }
+}
+
+/**
+ * The signed envelope is older than the configured lifetime. Freshness is judged from when the
+ * producer enqueued the job, not from when a worker picked it up, so a long backlog does not
+ * invalidate work that was legitimate when it was submitted.
+ */
+export class EnvelopeExpired extends QueueError {
+  public static override typeName = `${QueueError.typeName}EnvelopeExpired`
+
+  constructor(msg: string) {
+    super(`envelope-expired:${msg}`)
+    this.type = EnvelopeExpired.typeName
+  }
+}
+
+ResilientError.registerErrorClass(QueueError)
+ResilientError.registerErrorClass(QueueTimeout)
+ResilientError.registerErrorClass(UnknownJob)
+ResilientError.registerErrorClass(UnknownJobName)
+ResilientError.registerErrorClass(UnknownQueue)
+ResilientError.registerErrorClass(QueueNotListening)
+ResilientError.registerErrorClass(JobNotServed)
+ResilientError.registerErrorClass(EnvelopeExpired)

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './types.js'
+export * from './consts.js'
+export * from './errors.js'
+export * from './config.js'
+export * from './transport.js'
+export * from './bridge.js'
+export * from './worker.js'

package/src/transport.ts

@@ -0,0 +1,82 @@
+import { assertContext, createService } from '@owlmeans/context'
+import type { AbstractRequest, AbstractResponse, EntrypointTransport } from '@owlmeans/entrypoint'
+import { EntrypointOutcome, transportAlias } from '@owlmeans/entrypoint'
+import { RouteProtocols } from '@owlmeans/route'
+import { ResilientError } from '@owlmeans/error'
+import type { CommonEntrypoint } from '@owlmeans/entrypoint'
+import type { Config, Context, JobEnvelope, JobReply, QueueAppend } from './types.js'
+import { DEFAULT_JOB_TIMEOUT } from './consts.js'
+import { UnknownQueue } from './errors.js'
+
+/**
+ * The producing half of the bridge: it turns a call on a QUEUE entrypoint into a job, and the
+ * job's outcome back into a reply.
+ *
+ * Registering it is the whole opt-in. `apiHandler` looks up a transport by the route's protocol,
+ * so a consumer writes `ep.call(...)` against a declaration it shares with every other kind of
+ * entrypoint and never learns which one carried it. That is also why the envelope is built from
+ * the request rather than from the entrypoint: the far side rebuilds a request, and anything only
+ * HTTP understands would not survive the trip anyway.
+ */
+export const makeQueueTransport = (alias: string = transportAlias(RouteProtocols.QUEUE)) => {
+  const location = `queue-transport:${alias}`
+
+  const service = createService<EntrypointTransport>(alias, {
+    protocol: RouteProtocols.QUEUE,
+
+    handle: (async <T>(req: AbstractRequest, res: AbstractResponse<T>) => {
+      const context = assertContext<Config, Context<Config>>(service.ctx as Context<Config>, location)
+      const entrypoint = context.entrypoint<CommonEntrypoint>(req.alias)
+      const route = entrypoint.route.route
+
+      if (route.queue == null) {
+        throw new UnknownQueue(`${req.alias}: route declares no queue`)
+      }
+
+      const jobs = (context as unknown as QueueAppend).jobs<JobEnvelope, JobReply<T>>(route.queue)
+
+      const envelope: JobEnvelope = {
+        alias: req.alias,
+        params: req.params as Record<string, unknown>,
+        body: req.body,
+        query: req.query as Record<string, unknown>,
+        headers: req.headers as Record<string, string | undefined>,
+        enqueuedAt: new Date().toISOString(),
+      }
+
+      const job = await jobs.create({ queue: route.queue, name: req.alias, data: envelope })
+
+      // `reply: false` says the caller is not waiting for the work, only for the promise that it
+      // was taken. `Accepted` plus the job's identity is the whole answer — enough to watch it.
+      if (route.reply === false) {
+        res.resolve({ id: job.id, queue: route.queue } as T, EntrypointOutcome.Accepted)
+
+        return
+      }
+
+      const reply = await jobs.wait(job.id as string, {
+        timeout: req.timeout ?? route.timeout ?? DEFAULT_JOB_TIMEOUT
+      })
+
+      // A processor reports a domain failure by returning it, so it crosses the hop as the class
+      // it was thrown as rather than as a string a caller would have to parse.
+      if (reply?.error != null) {
+        throw ResilientError.ensure(reply.error as Error | string)
+      }
+
+      res.resolve(reply?.value as T, (reply?.outcome as EntrypointOutcome) ?? EntrypointOutcome.Ok)
+    }) as EntrypointTransport['handle']
+  })
+
+  return service
+}
+
+export const appendQueueTransport = <C extends Config, T extends Context<C> = Context<C>>(
+  context: T, alias?: string
+): T => {
+  const service = makeQueueTransport(alias)
+
+  context.registerService(service)
+
+  return context
+}

package/src/types.ts

@@ -0,0 +1,251 @@
+import type { BasicConfig, BasicContext, InitializedService } from '@owlmeans/context'
+import type { PubSubResource, Resource, ResourceRecord } from '@owlmeans/resource'
+import type { AbstractRequest, AbstractResponse } from '@owlmeans/entrypoint'
+import type { JobEventType, JobState } from './consts.js'
+
+/**
+ * Everything the monorepo knows about its queues, declared in the shared backend package so that
+ * producer and consumer read the same list. A process becomes a worker by naming queues in
+ * `listen` — nothing in a declaration decides where it runs.
+ */
+export interface QueueConfig {
+  queues?: QueueDeclaration[]
+  /** The queues THIS process consumes. Absent or empty means producer only. */
+  listen?: string[]
+  /** Which db service carries the queues. Defaults to the queue service's own alias. */
+  db?: string
+  defaults?: JobOptions
+  /**
+   * How long a signed job envelope stays acceptable, in seconds. It bounds both the freshness
+   * check and the nonce cache, so a job delayed behind a long backlog is still admitted.
+   */
+  envelopeTtl?: number
+}
+
+export interface QueueDeclaration {
+  name: string
+  /**
+   * The job names this queue accepts. A job the queue does not declare is refused at enqueue
+   * time rather than becoming a job nothing can process.
+   */
+  jobs: string[]
+  worker?: QueueWorkerOptions
+  defaults?: JobOptions
+}
+
+export interface QueueWorkerOptions {
+  concurrency?: number
+  /**
+   * How long a job may hold its lock without reporting progress. A processor that runs longer
+   * than this between `touch()` calls is treated as stalled and its job is handed to someone else.
+   */
+  lockDuration?: number
+  stalledInterval?: number
+  maxStalledCount?: number
+}
+
+export interface JobOptions {
+  attempts?: number
+  backoff?: { type: 'fixed' | 'exponential', delay: number }
+  delay?: number
+  priority?: number
+  /**
+   * A caller-chosen identity for the job. Enqueueing the same id twice while the first is still
+   * around is a no-op, which is what makes an admission step safe to retry.
+   */
+  id?: string
+  removeOnComplete?: boolean | number
+  removeOnFail?: boolean | number
+}
+
+/**
+ * A job as a RECORD — the same shape the resource contract reads and writes, so inspecting a
+ * queue is `list`/`get`/`count` rather than a broker-specific API.
+ */
+export interface JobRecord<D = unknown, R = unknown> extends ResourceRecord {
+  queue: string
+  name: string
+  data: D
+  state?: JobState
+  result?: R
+  error?: string
+  attempts?: number
+  progress?: unknown
+  parentId?: string
+  createdAt?: string
+  startedAt?: string
+  finishedAt?: string
+  opts?: JobOptions
+}
+
+export interface JobEvent<R = unknown> {
+  type: JobEventType
+  id: string
+  queue: string
+  /**
+   * Empty when the job is already gone. A broker's completion event carries only an id, so the
+   * name is read back from the job — and a queue that removes jobs on completion has nothing left
+   * to read. Subscribe for the id and the outcome; do not branch on the name.
+   */
+  name: string
+  result?: R
+  error?: string
+  progress?: unknown
+}
+
+/** A node of a job graph: children run first, siblings run together. */
+export interface FlowSpec<D = unknown> {
+  name: string
+  queue?: string
+  data: D
+  opts?: JobOptions
+  children?: FlowSpec[]
+}
+
+/**
+ * One queue, addressed as a resource.
+ *
+ * `create` enqueues, `get`/`load`/`list`/`count` inspect, `delete`/`take` cancel. What the base
+ * contract cannot express — waiting for an answer, submitting a graph, releasing the connection —
+ * is added here.
+ *
+ * Three places where the base `Resource` contract reads differently against a broker, because a
+ * job is not a row:
+ * - `create` does NOT throw `RecordExists` for a repeated `opts.id`. Re-enqueueing an id that is
+ *   still around returns the existing job, which is the idempotency that makes an admission step
+ *   safe to retry.
+ * - `WriteOptions.ttl` is refused. A job's lifetime is `removeOnComplete`/`removeOnFail` —
+ *   explicit deletion once it has finished, not expiry while it waits.
+ * - `update`/`save` reach only the job's `data`. Everything else about a job belongs to the
+ *   broker, and an implementation refuses rather than pretending to write it.
+ */
+export interface QueueResource<D = unknown, R = unknown>
+  extends Resource<JobRecord<D, R>>, PubSubResource<JobEvent<R>> {
+  queue: string
+  /**
+   * Release the connections this queue holds.
+   *
+   * Watching for completions means a blocking connection, and a process that never releases it
+   * does not exit. Nothing else in the resource contract can express that, so it lives here and a
+   * producer calls it on shutdown.
+   */
+  close: () => Promise<void>
+  /**
+   * Wait for a job's return value.
+   * @throws {QueueTimeout} when the wait elapses — the job itself is left alone.
+   * @throws the processor's own error, rebuilt as its original class.
+   */
+  wait: (id: string, opts?: { timeout?: number }) => Promise<R>
+  /** Submit a graph and get its root back. Children complete before their parent starts. */
+  flow: (root: FlowSpec<D>) => Promise<JobRecord<D, R>>
+  counts: () => Promise<Record<JobState, number>>
+}
+
+/**
+ * What a processor is handed. `data` is the job's payload; everything else is the means to stay
+ * alive and to read what the children reported.
+ */
+export interface JobContext<D = unknown> {
+  id: string
+  name: string
+  queue: string
+  attempt: number
+  data: D
+  signal: AbortSignal
+  /**
+   * Renew the lock. A processor that runs a long loop MUST call it as it goes: the broker judges
+   * liveness by the lock, so silence for longer than `lockDuration` is indistinguishable from a
+   * dead worker, and the job is re-run somewhere else while this one is still working on it.
+   */
+  touch: () => Promise<void>
+  progress: (value: unknown) => Promise<void>
+  /**
+   * What the children returned, keyed by child job id.
+   *
+   * A broker keys them by its own fully qualified job key; an implementation re-keys to the bare
+   * id, so two children of one parent that live in DIFFERENT queues and happen to share an id
+   * collide here. Give the children of a cross-queue flow distinct ids.
+   */
+  children: <T = unknown>() => Promise<Record<string, T>>
+  failedChildren: () => Promise<Record<string, string>>
+}
+
+export interface JobProcessor<D = unknown, R = unknown> {
+  (job: JobContext<D>): Promise<R>
+}
+
+/**
+ * The consuming half. It is registered only in processes that consume: `start()` binds the
+ * queues named in `cfg.listen` and dispatches by job name, to a processor registered here or to
+ * an entrypoint served in this process.
+ */
+export interface QueueWorkerService extends InitializedService {
+  process: <D, R>(queue: string, name: string, processor: JobProcessor<D, R>) => void
+  start: () => Promise<void>
+  stop: () => Promise<void>
+  /** The queues this process actually consumes. Empty in a producer. */
+  listening: () => string[]
+  /**
+   * Bind the application's reactions to job lifecycle events. Registering more than once merges,
+   * so an application composed of several parts can each contribute what it cares about.
+   *
+   * It belongs on the contract rather than on a driver because `onJobDead` is where an
+   * application COMPENSATES — releases the lock the admission step took, marks the record failed —
+   * and that behaviour must not have to be rewritten when the broker changes.
+   */
+  hooks: (hooks: QueueHooks) => void
+}
+
+/**
+ * The envelope a queued entrypoint call travels in. It is the request, minus everything that only
+ * means something to HTTP — the transport rebuilds a request from it on the far side.
+ */
+export interface JobEnvelope {
+  alias: string
+  params?: Record<string, unknown>
+  body?: unknown
+  query?: Record<string, unknown>
+  headers?: Record<string, string | undefined>
+  /** When the producer enqueued it — freshness is judged here, not at processing time. */
+  enqueuedAt?: string
+}
+
+/** What a processor returns for an entrypoint job, so the caller can rebuild the reply. */
+export interface JobReply<T = unknown> {
+  value?: T
+  outcome?: string
+  error?: unknown
+}
+
+export interface QueueHooks {
+  wrapHandler?: (job: JobContext, next: () => Promise<unknown>) => Promise<unknown>
+  onJobResult?: (event: JobEvent) => void | Promise<void>
+  onJobStalled?: (meta: { id: string, queue: string, name: string }, phase: 'stalled' | 'failed') => void | Promise<void>
+  /**
+   * The job is finished for good — retries exhausted, or it was dropped. This is where an
+   * application compensates: release the lock the admission step took, mark the record failed.
+   */
+  onJobDead?: (job: JobRecord, reason: string) => void | Promise<void>
+}
+
+export interface QueueDriver {
+  resource: <D, R>(queue: string) => QueueResource<D, R>
+  worker: () => QueueWorkerService
+}
+
+export interface Config extends BasicConfig {
+  queue?: QueueConfig
+}
+
+export interface Context<C extends Config = Config> extends BasicContext<C> {
+}
+
+export interface QueueAppend {
+  jobs: <D = unknown, R = unknown>(queue?: string) => QueueResource<D, R>
+}
+
+export interface QueueTransportRequest extends AbstractRequest {
+}
+
+export interface QueueTransportResponse extends AbstractResponse<unknown> {
+}