@owlmeans/queue 0.1.18-rc.22 → 0.1.18-rc.24

package/src/route.ts

@@ -0,0 +1,48 @@
+import { backend } from '@owlmeans/route'
+import type { RouteDeclaration, RouteModel, RouteOptions } from '@owlmeans/route'
+import { UnknownQueue } from './errors.js'
+
+/** The transport identifier owned by the queue package. */
+export const QUEUE_PROTOCOL = 'queue' as const
+
+/** Queue-owned options stored in a generic route declaration. */
+export interface QueueRouteOptions extends Omit<RouteOptions, 'protocol' | 'protocolOptions'> {
+  queue: string
+  reply?: boolean
+}
+
+/** Declare a backend route carried by a queue. */
+export const job = (
+  opts: QueueRouteOptions,
+  secondary?: Omit<QueueRouteOptions, 'queue'>,
+): Partial<RouteOptions> => {
+  const { queue, reply, ...routeOptions } = { ...opts, ...secondary }
+  const declaration = backend(routeOptions)
+  declaration.protocol = QUEUE_PROTOCOL
+  declaration.protocolOptions = { queue, ...(reply == null ? {} : { reply }) }
+  return declaration
+}
+
+/** Whether a declaration belongs to the queue transport. */
+export const isQueueRoute = (route: RouteDeclaration | RouteModel): boolean =>
+  ('route' in route ? route.route : route).protocol === QUEUE_PROTOCOL
+
+/** Read and validate queue-owned route options. */
+export const queueRouteOptions = (
+  route: RouteDeclaration | RouteModel,
+): Pick<QueueRouteOptions, 'queue' | 'reply'> => {
+  const declaration = 'route' in route ? route.route : route
+  if (declaration.protocol !== QUEUE_PROTOCOL
+    || declaration.protocolOptions == null
+    || typeof declaration.protocolOptions !== 'object') {
+    throw new UnknownQueue(`${declaration.alias}: not a queue protocol`)
+  }
+  const options = declaration.protocolOptions as Record<string, unknown>
+  if (typeof options.queue !== 'string' || options.queue.trim() === '') {
+    throw new UnknownQueue(`${declaration.alias}: route declares no queue`)
+  }
+  if (options.reply != null && typeof options.reply !== 'boolean') {
+    throw new UnknownQueue(`${declaration.alias}: invalid queue reply option`)
+  }
+  return { queue: options.queue, reply: options.reply as boolean | undefined }
+}

package/src/transport.ts

@@ -1,12 +1,11 @@
 import { assertContext, createService } from '@owlmeans/context'
 import type { AbstractRequest, AbstractResponse, EntrypointTransport } from '@owlmeans/entrypoint'
 import { EntrypointOutcome, transportAlias } from '@owlmeans/entrypoint'
-import { RouteProtocols } from '@owlmeans/route'
 import { ResilientError } from '@owlmeans/error'
 import type { CommonEntrypoint } from '@owlmeans/entrypoint'
 import type { Config, Context, JobEnvelope, JobReply, QueueAppend } from './types.js'
 import { DEFAULT_JOB_TIMEOUT } from './consts.js'
-import { UnknownQueue } from './errors.js'
+import { QUEUE_PROTOCOL, queueRouteOptions } from './route.js'
 
 /**
  * The producing half of the bridge: it turns a call on a QUEUE entrypoint into a job, and the
@@ -18,22 +17,20 @@ import { UnknownQueue } from './errors.js'
  * the request rather than from the entrypoint: the far side rebuilds a request, and anything only
  * HTTP understands would not survive the trip anyway.
  */
-export const makeQueueTransport = (alias: string = transportAlias(RouteProtocols.QUEUE)) => {
+export const makeQueueTransport = (alias: string = transportAlias(QUEUE_PROTOCOL)) => {
   const location = `queue-transport:${alias}`
 
   const service = createService<EntrypointTransport>(alias, {
-    protocol: RouteProtocols.QUEUE,
+    protocol: QUEUE_PROTOCOL,
 
     handle: (async <T>(req: AbstractRequest, res: AbstractResponse<T>) => {
       const context = assertContext<Config, Context<Config>>(service.ctx as Context<Config>, location)
       const entrypoint = context.entrypoint<CommonEntrypoint>(req.alias)
       const route = entrypoint.route.route
 
-      if (route.queue == null) {
-        throw new UnknownQueue(`${req.alias}: route declares no queue`)
-      }
+      const { queue, reply: waitsForReply } = queueRouteOptions(route)
 
-      const jobs = (context as unknown as QueueAppend).jobs<JobEnvelope, JobReply<T>>(route.queue)
+      const jobs = (context as unknown as QueueAppend).jobs<JobEnvelope, JobReply<T>>(queue)
 
       const envelope: JobEnvelope = {
         alias: req.alias,
@@ -44,12 +41,12 @@ export const makeQueueTransport = (alias: string = transportAlias(RouteProtocols
         enqueuedAt: new Date().toISOString(),
       }
 
-      const job = await jobs.create({ queue: route.queue, name: req.alias, data: envelope })
+      const job = await jobs.create({ queue, name: req.alias, data: envelope })
 
       // `reply: false` says the caller is not waiting for the work, only for the promise that it
       // was taken. `Accepted` plus the job's identity is the whole answer — enough to watch it.
-      if (route.reply === false) {
-        res.resolve({ id: job.id, queue: route.queue } as T, EntrypointOutcome.Accepted)
+      if (waitsForReply === false) {
+        res.resolve({ id: job.id, queue } as T, EntrypointOutcome.Accepted)
 
         return
       }

package/src/types.ts

@@ -20,6 +20,57 @@ export interface QueueConfig {
    * check and the nonce cache, so a job delayed behind a long backlog is still admitted.
    */
   envelopeTtl?: number
+  /**
+   * Recurring jobs. Declared beside the queues, in the shared package, because every process that
+   * listens to a schedule's queue reconciles the broker against this list when its worker starts —
+   * two listeners loading different lists would remove each other's schedules.
+   */
+  schedules?: ScheduleDeclaration[]
+}
+
+/**
+ * A job the broker produces on its own, on an interval or a cron pattern.
+ *
+ * A scheduled run is an ordinary job of a name the queue declares, dispatched to the processor
+ * registered for that name — never to a guarded entrypoint, because nobody's credentials travel
+ * with it. Only a process that listens to the queue creates the scheduler, and the broker produces
+ * each next run when the previous one STARTS processing — a schedule needs a running consumer.
+ */
+export interface ScheduleDeclaration {
+  /**
+   * The schedule's stable identity: the broker keeps one scheduler per id, so changing any other
+   * field updates it in place, while renaming the id removes one schedule and creates another.
+   */
+  id: string
+  queue: string
+  /** A job name the queue declares. */
+  name: string
+  /**
+   * Milliseconds between runs. Exactly one of `every` and `pattern`. Without `startDate` the first
+   * run is produced as soon as the scheduler is created.
+   */
+  every?: number
+  /** A cron pattern (an optional leading seconds field is accepted). Exactly one of the two. */
+  pattern?: string
+  /** The IANA time zone `pattern` is read in. Pattern only. */
+  tz?: string
+  startDate?: Date | number | string
+  /** Once passed, the schedule is treated as absent and a scheduler left from it is removed. */
+  endDate?: Date | number | string
+  /** Stop after this many runs. */
+  limit?: number
+  /**
+   * Run once as soon as the scheduler is created or its declaration changes, then follow the
+   * pattern. Pattern only, and never together with `startDate`. A restart that finds the
+   * declaration unchanged does not run it again.
+   */
+  immediately?: boolean
+  data?: unknown
+  /**
+   * Options every run is enqueued with, over the queue's defaults. `id` and `delay` belong to the
+   * broker for a scheduled run and are refused.
+   */
+  opts?: Omit<JobOptions, 'id' | 'delay'>
 }
 
 export interface QueueDeclaration {
@@ -152,6 +203,11 @@ export interface JobContext<D = unknown> {
   attempt: number
   data: D
   signal: AbortSignal
+  /**
+   * The id of the schedule that produced this run (`ScheduleDeclaration.id`). Absent for a job
+   * enqueued any other way.
+   */
+  scheduled?: string
   /**
    * Renew the lock. A processor that runs a long loop MUST call it as it goes: the broker judges
    * liveness by the lock, so silence for longer than `lockDuration` is indistinguishable from a

package/src/worker.ts

@@ -1,10 +1,10 @@
 import type { Middleware } from '@owlmeans/context'
 import { MiddlewareStage, MiddlewareType, AppType } from '@owlmeans/context'
 import type { CommonEntrypoint } from '@owlmeans/entrypoint'
-import { RouteProtocols } from '@owlmeans/route'
 import type { Config, Context, JobContext, JobEnvelope, JobProcessor, QueueWorkerService } from './types.js'
 import { handleJob } from './bridge.js'
 import { DEFAULT_ALIAS } from './consts.js'
+import { isQueueRoute, queueRouteOptions } from './route.js'
 
 /**
  * Every queued entrypoint this process both SERVES and LISTENS to, grouped by queue.
@@ -22,17 +22,18 @@ export const servedJobs = <C extends Config, T extends Context<C>>(
 
   context.entrypoints<CommonEntrypoint>().forEach(entrypoint => {
     const route = entrypoint.route.route
-    if (route.type !== AppType.Backend || route.protocol !== RouteProtocols.QUEUE) {
+    if (route.type !== AppType.Backend || !isQueueRoute(route)) {
       return
     }
     if (route.service != null && route.service !== context.cfg.service) {
       return
     }
-    if (entrypoint.handle == null || route.queue == null || !listen.includes(route.queue)) {
+    const { queue } = queueRouteOptions(route)
+    if (entrypoint.handle == null || !listen.includes(queue)) {
       return
     }
 
-    served.set(route.queue, [...(served.get(route.queue) ?? []), entrypoint])
+    served.set(queue, [...(served.get(queue) ?? []), entrypoint])
   })
 
   return served

package/tests/protocol.spec.ts

@@ -1,8 +1,8 @@
 import { describe, expect, test } from 'bun:test'
 import { contract, protocol, schema } from '@owlmeans/entrypoint'
-import { job, route } from '@owlmeans/route'
+import { route } from '@owlmeans/route'
 import type { JSONSchemaType } from 'ajv'
-import { enqueueProtocol, waitForProtocol, type JobRecord } from '../src/index.js'
+import { enqueueProtocol, job, waitForProtocol, type JobRecord } from '../src/index.js'
 
 interface Input { value: number }
 interface Output { doubled: number }

package/tests/schedule.spec.ts

@@ -0,0 +1,100 @@
+import { describe, test, expect } from 'bun:test'
+import {
+  assertSchedule, assertSchedules, declareQueue, declareSchedule, queueOf, schedulesOf
+} from '../src/config.js'
+import { ScheduleMisdeclared, UnknownJobName, UnknownQueue } from '../src/errors.js'
+import type { Config, ScheduleDeclaration } from '../src/types.js'
+
+const cfg = (): Config => declareQueue(
+  declareQueue({ ready: false, service: 'test', type: 'backend' } as unknown as Config,
+    'maintenance', ['reconcile', 'sweep']),
+  'work', ['story:code']
+)
+
+const nightly: ScheduleDeclaration = {
+  id: 'nightly-reconcile', queue: 'maintenance', name: 'reconcile', pattern: '17 3 * * *', tz: 'UTC',
+}
+
+const misdeclared = (schedule: Partial<ScheduleDeclaration>): () => void =>
+  () => assertSchedule(cfg(), { ...nightly, ...schedule } as ScheduleDeclaration)
+
+describe('declaring schedules', () => {
+  test('a declaration is recorded and read back by its queue', () => {
+    const config = declareSchedule(cfg(), nightly)
+
+    expect(schedulesOf(config, 'maintenance')).toEqual([nightly])
+    expect(schedulesOf(config, 'work')).toEqual([])
+    expect(schedulesOf(config)).toHaveLength(1)
+  })
+
+  test('re-declaring an id replaces rather than appends', () => {
+    let config = declareSchedule(cfg(), nightly)
+    config = declareSchedule(config, { ...nightly, pattern: '0 4 * * *' })
+
+    expect(schedulesOf(config)).toHaveLength(1)
+    expect(schedulesOf(config)[0].pattern).toBe('0 4 * * *')
+  })
+
+  /**
+   * A scheduled run of a name the queue does not accept would fail on every tick. Refusing at
+   * declaration turns that into one error at boot, and the refused schedule is not recorded.
+   */
+  test('a schedule for an undeclared queue or job name is refused and not recorded', () => {
+    const config = cfg()
+
+    expect(() => declareSchedule(config, { ...nightly, queue: 'missing' })).toThrow(UnknownQueue)
+    expect(() => declareSchedule(config, { ...nightly, name: 'story:code' })).toThrow(UnknownJobName)
+    expect(schedulesOf(config)).toHaveLength(0)
+  })
+
+  test('re-declaring the queue keeps its schedules', () => {
+    const config = declareQueue(declareSchedule(cfg(), nightly), 'maintenance', ['reconcile'])
+
+    expect(queueOf(config, 'maintenance').jobs).toEqual(['reconcile'])
+    expect(schedulesOf(config)).toEqual([nightly])
+  })
+})
+
+describe('asserting a schedule', () => {
+  test('exactly one of every and pattern', () => {
+    expect(misdeclared({ every: 1_000 })).toThrow(/every-or-pattern/)
+    expect(misdeclared({ pattern: undefined, tz: undefined })).toThrow(/every-or-pattern/)
+    expect(misdeclared({ pattern: undefined, tz: undefined, every: 0 })).toThrow(/:every$/)
+    expect(misdeclared({ pattern: undefined, tz: undefined, every: 1.5 })).toThrow(ScheduleMisdeclared)
+  })
+
+  test('tz and immediately belong to a pattern only', () => {
+    expect(misdeclared({ pattern: undefined, every: 1_000 })).toThrow(/:tz$/)
+    expect(misdeclared({ pattern: undefined, tz: undefined, every: 1_000, immediately: true }))
+      .toThrow(/:immediately$/)
+    expect(misdeclared({ immediately: true, startDate: new Date() })).toThrow(/immediately-start-date/)
+    expect(misdeclared({ immediately: true })).not.toThrow()
+  })
+
+  test('the broker owns a scheduled run\'s id and delay', () => {
+    expect(misdeclared({ opts: { id: 'fixed' } as ScheduleDeclaration['opts'] })).toThrow(/opts-id/)
+    expect(misdeclared({ opts: { delay: 10 } as ScheduleDeclaration['opts'] })).toThrow(/opts-delay/)
+    expect(misdeclared({ opts: { attempts: 3 } })).not.toThrow()
+  })
+
+  test('an empty id is refused', () => {
+    expect(misdeclared({ id: ' ' })).toThrow(ScheduleMisdeclared)
+  })
+})
+
+describe('asserting every schedule', () => {
+  test('two schedules under one id are refused', () => {
+    const config = declareSchedule(cfg(), nightly)
+    config.queue?.schedules?.push({ ...nightly, pattern: '0 5 * * *' })
+
+    expect(() => assertSchedules(config)).toThrow(/duplicate/)
+  })
+
+  /** The configuration is plain data; a schedule written into it directly is checked too. */
+  test('a schedule written past declareSchedule is still checked', () => {
+    const config = declareSchedule(cfg(), nightly)
+    config.queue?.schedules?.push({ id: 'orphan', queue: 'maintenance', name: 'gone', every: 1_000 })
+
+    expect(() => assertSchedules(config)).toThrow(UnknownJobName)
+  })
+})