@owlmeans/queue 0.1.18-rc.4 → 0.1.18-rc.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +286 -8
- package/agent-meta/manifest.json +9 -2
- package/agent-meta/skills/queue/SKILL.md +243 -12
- package/agent-meta/skills/scheduled-jobs/SKILL.md +151 -0
- package/build/bridge/types.d.ts +25 -0
- package/build/bridge/types.d.ts.map +1 -0
- package/build/bridge/types.js +2 -0
- package/build/bridge/types.js.map +1 -0
- package/build/bridge.d.ts +7 -0
- package/build/bridge.d.ts.map +1 -0
- package/build/bridge.js +37 -0
- package/build/bridge.js.map +1 -0
- package/build/config.d.ts +34 -0
- package/build/config.d.ts.map +1 -0
- package/build/config.js +59 -0
- package/build/config.js.map +1 -0
- package/build/consts.d.ts +33 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +35 -0
- package/build/consts.js.map +1 -0
- package/build/entrypoint-jobs/types.d.ts +29 -0
- package/build/entrypoint-jobs/types.d.ts.map +1 -0
- package/build/entrypoint-jobs/types.js +2 -0
- package/build/entrypoint-jobs/types.js.map +1 -0
- package/build/entrypoint-jobs.d.ts +5 -0
- package/build/entrypoint-jobs.d.ts.map +1 -0
- package/build/entrypoint-jobs.js +84 -0
- package/build/entrypoint-jobs.js.map +1 -0
- package/build/errors.d.ts +68 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +103 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +18 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +13 -1
- package/build/index.js.map +1 -1
- package/build/protocol/types.d.ts +12 -0
- package/build/protocol/types.d.ts.map +1 -0
- package/build/protocol/types.js +2 -0
- package/build/protocol/types.js.map +1 -0
- package/build/protocol.d.ts +13 -0
- package/build/protocol.d.ts.map +1 -0
- package/build/protocol.js +63 -0
- package/build/protocol.js.map +1 -0
- package/build/queue-config/types.d.ts +29 -0
- package/build/queue-config/types.d.ts.map +1 -0
- package/build/queue-config/types.js +2 -0
- package/build/queue-config/types.js.map +1 -0
- package/build/queue-config.d.ts +9 -0
- package/build/queue-config.d.ts.map +1 -0
- package/build/queue-config.js +83 -0
- package/build/queue-config.js.map +1 -0
- package/build/queue-route/types.d.ts +14 -0
- package/build/queue-route/types.d.ts.map +1 -0
- package/build/queue-route/types.js +2 -0
- package/build/queue-route/types.js.map +1 -0
- package/build/queue-route.d.ts +4 -0
- package/build/queue-route.d.ts.map +1 -0
- package/build/queue-route.js +24 -0
- package/build/queue-route.js.map +1 -0
- package/build/route.d.ts +9 -0
- package/build/route.d.ts.map +1 -0
- package/build/route.js +16 -0
- package/build/route.js.map +1 -0
- package/build/transport.d.ts +15 -0
- package/build/transport.d.ts.map +1 -0
- package/build/transport.js +59 -0
- package/build/transport.js.map +1 -0
- package/build/types.d.ts +312 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/types.local.d.ts +4 -0
- package/build/types.local.d.ts.map +1 -0
- package/build/types.local.js +2 -0
- package/build/types.local.js.map +1 -0
- package/build/utils.d.ts +4 -0
- package/build/utils.d.ts.map +1 -0
- package/build/utils.js +4 -0
- package/build/utils.js.map +1 -0
- package/build/worker.d.ts +10 -0
- package/build/worker.d.ts.map +1 -0
- package/build/worker.js +24 -0
- package/build/worker.js.map +1 -0
- package/package.json +13 -3
- package/src/bridge/types.ts +25 -0
- package/src/bridge.ts +46 -0
- package/src/config.ts +80 -0
- package/src/consts.ts +38 -0
- package/src/entrypoint-jobs/types.ts +29 -0
- package/src/entrypoint-jobs.ts +105 -0
- package/src/errors.ts +121 -0
- package/src/index.ts +18 -0
- package/src/protocol/types.ts +16 -0
- package/src/protocol.ts +95 -0
- package/src/queue-config/types.ts +29 -0
- package/src/queue-config.ts +102 -0
- package/src/queue-route/types.ts +14 -0
- package/src/queue-route.ts +33 -0
- package/src/route.ts +23 -0
- package/src/transport.ts +77 -0
- package/src/types.ts +326 -0
- package/src/utils.ts +5 -0
- package/src/worker.ts +27 -0
- package/tests/bridge.spec.ts +64 -0
- package/tests/config.spec.ts +67 -0
- package/tests/protocol.spec.ts +58 -0
- package/tests/schedule.spec.ts +99 -0
- package/tsconfig.json +1 -0
- package/build/.gitkeep +0 -0
package/src/config.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Config, JobOptions, QueueDeclaration, QueueWorkerOptions, ScheduleDeclaration
|
|
3
|
+
} from './types.js'
|
|
4
|
+
import { queueConfigOf } from './queue-config.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Add a queue to the shared configuration. Every process in the monorepo loads the same
|
|
8
|
+
* declarations — a producer needs them to know a queue exists and what it accepts, a worker needs
|
|
9
|
+
* them to bind. Declaring twice replaces, so a helper that runs per app is safe to call.
|
|
10
|
+
*/
|
|
11
|
+
export const declareQueue = <C extends Config>(
|
|
12
|
+
cfg: C, name: string, jobs: string[], opts?: { worker?: QueueWorkerOptions, defaults?: JobOptions }
|
|
13
|
+
): C => {
|
|
14
|
+
const queue: QueueDeclaration = { name, jobs, ...opts }
|
|
15
|
+
const queues = cfg.queue?.queues ?? []
|
|
16
|
+
const existing = queues.findIndex(declared => declared.name === name)
|
|
17
|
+
|
|
18
|
+
if (existing < 0) {
|
|
19
|
+
queues.push(queue)
|
|
20
|
+
} else {
|
|
21
|
+
queues[existing] = queue
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
cfg.queue = { ...cfg.queue, queues }
|
|
25
|
+
|
|
26
|
+
return cfg
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Name the queues this process consumes. It is what turns an otherwise identical binary into a
|
|
31
|
+
* worker, which is why it lives in the process's own config and never in a declaration.
|
|
32
|
+
*/
|
|
33
|
+
export const listenQueues = <C extends Config>(cfg: C, ...names: string[]): C => {
|
|
34
|
+
const listen = new Set([...(cfg.queue?.listen ?? []), ...names])
|
|
35
|
+
|
|
36
|
+
cfg.queue = { ...cfg.queue, listen: [...listen] }
|
|
37
|
+
|
|
38
|
+
return cfg
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Add a recurring job to the shared configuration. Declare the queue first — a schedule is checked
|
|
43
|
+
* against it here. Declaring an id twice replaces, like `declareQueue`.
|
|
44
|
+
*
|
|
45
|
+
* Declaring is all this does: the driver creates, updates and removes the broker's schedulers when
|
|
46
|
+
* a worker that listens to the queue starts.
|
|
47
|
+
*
|
|
48
|
+
* @throws {ScheduleMisdeclared} / {UnknownQueue} / {UnknownJobName} — see `assertSchedule`.
|
|
49
|
+
*/
|
|
50
|
+
export const declareSchedule = <C extends Config>(cfg: C, schedule: ScheduleDeclaration): C => {
|
|
51
|
+
queueConfigOf(cfg).assertSchedule(schedule)
|
|
52
|
+
|
|
53
|
+
const schedules = cfg.queue?.schedules ?? []
|
|
54
|
+
const existing = schedules.findIndex(declared => declared.id === schedule.id)
|
|
55
|
+
|
|
56
|
+
if (existing < 0) {
|
|
57
|
+
schedules.push(schedule)
|
|
58
|
+
} else {
|
|
59
|
+
schedules[existing] = schedule
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
cfg.queue = { ...cfg.queue, schedules }
|
|
63
|
+
|
|
64
|
+
return cfg
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** @deprecated compat:factory-refactor — use `queueConfigOf(cfg).queueOfJob(…)` */
|
|
68
|
+
export const queueOfJob = <C extends Config>(cfg: C, job: string): QueueDeclaration | undefined =>
|
|
69
|
+
queueConfigOf(cfg).queueOfJob(job)
|
|
70
|
+
|
|
71
|
+
/** @deprecated compat:factory-refactor — use `queueConfigOf(cfg).isListening(…)` */
|
|
72
|
+
export const isListening = <C extends Config>(cfg: C, name: string): boolean =>
|
|
73
|
+
queueConfigOf(cfg).isListening(name)
|
|
74
|
+
|
|
75
|
+
/** @deprecated compat:factory-refactor — use `queueConfigOf(cfg).schedulesOf(…)` */
|
|
76
|
+
export const schedulesOf = <C extends Config>(cfg: C, queue?: string): ScheduleDeclaration[] =>
|
|
77
|
+
queueConfigOf(cfg).schedulesOf(queue)
|
|
78
|
+
|
|
79
|
+
/** @deprecated compat:factory-refactor — use `queueConfigOf(cfg).assertSchedules()` */
|
|
80
|
+
export const assertSchedules = <C extends Config>(cfg: C): void => queueConfigOf(cfg).assertSchedules()
|
package/src/consts.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
|
|
2
|
+
export const DEFAULT_ALIAS = 'queue'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The state a job is in, as the broker reports it. `Waiting` covers both a job queued normally
|
|
6
|
+
* and one held by an unfinished parent — the difference matters to the broker, never to a caller
|
|
7
|
+
* asking whether an answer has arrived.
|
|
8
|
+
*/
|
|
9
|
+
export enum JobState {
|
|
10
|
+
Waiting = 'waiting',
|
|
11
|
+
Delayed = 'delayed',
|
|
12
|
+
Active = 'active',
|
|
13
|
+
Completed = 'completed',
|
|
14
|
+
Failed = 'failed',
|
|
15
|
+
Unknown = 'unknown',
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export enum JobEventType {
|
|
19
|
+
Completed = 'completed',
|
|
20
|
+
Failed = 'failed',
|
|
21
|
+
Progress = 'progress',
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* How long a caller waits for a reply when neither the route nor the request says. Long enough
|
|
26
|
+
* for work that talks to a model, short enough that a lost job surfaces as an error rather than a
|
|
27
|
+
* hung request.
|
|
28
|
+
*/
|
|
29
|
+
export const DEFAULT_JOB_TIMEOUT = 60_000
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Retries default to one attempt: the jobs here are mostly LLM work, and a blind retry re-spends
|
|
33
|
+
* the tokens that just failed. A queue whose jobs are cheap and idempotent raises it per queue.
|
|
34
|
+
*/
|
|
35
|
+
export const DEFAULT_ATTEMPTS = 1
|
|
36
|
+
|
|
37
|
+
/** The transport identifier owned by the queue package. */
|
|
38
|
+
export const QUEUE_PROTOCOL = 'queue' as const
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { CommonEntrypoint } from '@owlmeans/entrypoint'
|
|
2
|
+
import type { JobContext, JobEnvelope, JobProcessor, JobReply } from '../types.js'
|
|
3
|
+
|
|
4
|
+
/** The queued entrypoints one context serves as a worker, and how a job of theirs is run. */
|
|
5
|
+
export interface EntrypointJobsHelper {
|
|
6
|
+
/**
|
|
7
|
+
* Every queued entrypoint this process both SERVES and LISTENS to, grouped by queue.
|
|
8
|
+
*
|
|
9
|
+
* Both halves are required and they answer different questions. Serving is about code — the alias
|
|
10
|
+
* was bound here, so a handler exists. Listening is about deployment — this process was
|
|
11
|
+
* configured to consume that queue. A worker that bound queues by what it can serve would make
|
|
12
|
+
* every deployment of the same binary a worker for everything it happens to import.
|
|
13
|
+
*/
|
|
14
|
+
served: () => Map<string, CommonEntrypoint[]>
|
|
15
|
+
/**
|
|
16
|
+
* A processor that runs a queued entrypoint call. The driver dispatches by job name, and an
|
|
17
|
+
* entrypoint job's name IS its alias — which is what lets one worker carry both entrypoint jobs
|
|
18
|
+
* and the internal steps an application registers with `process()`.
|
|
19
|
+
*/
|
|
20
|
+
processor: () => JobProcessor<JobEnvelope, unknown>
|
|
21
|
+
/**
|
|
22
|
+
* Run one queued entrypoint call and describe the outcome.
|
|
23
|
+
*
|
|
24
|
+
* It never throws for a DOMAIN failure — the error is marshalled into the returned reply so the
|
|
25
|
+
* producer can rebuild it as its own class, and so the broker does not count a legitimate refusal
|
|
26
|
+
* as a job to retry. Infrastructure failures are left to propagate: those are what retries are for.
|
|
27
|
+
*/
|
|
28
|
+
handle: (job: JobContext<JobEnvelope>) => Promise<JobReply>
|
|
29
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { AppType, memoHelper } from '@owlmeans/context'
|
|
2
|
+
import type { CommonEntrypoint, GuardService } from '@owlmeans/entrypoint'
|
|
3
|
+
import { EntrypointOutcome, provideResponse } from '@owlmeans/entrypoint'
|
|
4
|
+
import type { Auth } from '@owlmeans/auth'
|
|
5
|
+
import { AuthorizationError } from '@owlmeans/auth'
|
|
6
|
+
import { ResilientError } from '@owlmeans/error'
|
|
7
|
+
import type { Config, Context, JobContext, JobEnvelope, JobProcessor, JobReply } from './types.js'
|
|
8
|
+
import type { EntrypointJobsHelper } from './entrypoint-jobs/types.js'
|
|
9
|
+
import { JobNotServed } from './errors.js'
|
|
10
|
+
import { queueBridgeHelper } from './bridge.js'
|
|
11
|
+
import { queueRouteHelper } from './queue-route.js'
|
|
12
|
+
import { makeEntityScope } from '@owlmeans/auth-common'
|
|
13
|
+
|
|
14
|
+
export const makeEntrypointJobsHelper = (context: Context<Config>): EntrypointJobsHelper => {
|
|
15
|
+
const served = (): Map<string, CommonEntrypoint[]> => {
|
|
16
|
+
const listen = context.cfg.queue?.listen ?? []
|
|
17
|
+
const served = new Map<string, CommonEntrypoint[]>()
|
|
18
|
+
|
|
19
|
+
context.entrypoints<CommonEntrypoint>().forEach(entrypoint => {
|
|
20
|
+
const route = entrypoint.route.route
|
|
21
|
+
if (route.type !== AppType.Backend || !queueRouteHelper.isQueueRoute(route)) {
|
|
22
|
+
return
|
|
23
|
+
}
|
|
24
|
+
if (route.service != null && route.service !== context.cfg.service) {
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
const { queue } = queueRouteHelper.queueRouteOptions(route)
|
|
28
|
+
if (entrypoint.handle == null || !listen.includes(queue)) {
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
served.set(queue, [...(served.get(queue) ?? []), entrypoint])
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
return served
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const handle = async (job: JobContext<JobEnvelope>): Promise<JobReply> => {
|
|
39
|
+
const envelope = job.data
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
queueBridgeHelper.assertFresh(envelope, context.cfg.queue?.envelopeTtl)
|
|
43
|
+
|
|
44
|
+
const entrypoint = context.entrypoint<CommonEntrypoint>(envelope.alias)
|
|
45
|
+
if (entrypoint.handle == null) {
|
|
46
|
+
throw new JobNotServed(envelope.alias)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const request = queueBridgeHelper.requestOf(envelope, entrypoint.path(), context, {
|
|
50
|
+
id: job.id, name: job.name, queue: job.queue, attempt: job.attempt, touch: job.touch,
|
|
51
|
+
})
|
|
52
|
+
const response = provideResponse<unknown>()
|
|
53
|
+
|
|
54
|
+
const guards = entrypoint.getGuards()
|
|
55
|
+
if (guards.length > 0) {
|
|
56
|
+
let matched: GuardService | undefined
|
|
57
|
+
for (const alias of guards) {
|
|
58
|
+
const guard: GuardService = context.service(alias)
|
|
59
|
+
if (await guard.match(request, response)) {
|
|
60
|
+
matched = guard
|
|
61
|
+
break
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (matched == null) {
|
|
65
|
+
throw new AuthorizationError(`queue:${envelope.alias}`)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const auth = provideResponse<Auth>()
|
|
69
|
+
if (!await matched.handle<boolean>(request, auth)) {
|
|
70
|
+
throw new AuthorizationError(`queue:${envelope.alias}:${matched.alias}`)
|
|
71
|
+
}
|
|
72
|
+
request.auth = auth.value
|
|
73
|
+
|
|
74
|
+
const entity = await makeEntityScope(request).attachEntity(context)
|
|
75
|
+
if (entity != null) {
|
|
76
|
+
request.entity = entity
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
await entrypoint.handle(request, response)
|
|
81
|
+
|
|
82
|
+
if (response.error != null) {
|
|
83
|
+
return { error: ResilientError.marshal(response.error).message }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return { value: response.value, outcome: response.outcome ?? EntrypointOutcome.Ok }
|
|
87
|
+
} catch (e) {
|
|
88
|
+
// A refusal the caller must see as its own class — `ContentRefused`, `AgentLocked` — travels
|
|
89
|
+
// in the reply. Only a broker- or connection-level fault should ever reach the retry logic:
|
|
90
|
+
// retrying a refusal just spends the work again to be refused identically.
|
|
91
|
+
if (e instanceof ResilientError) {
|
|
92
|
+
return { error: ResilientError.marshal(e).message }
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
throw e
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const processor = (): JobProcessor<JobEnvelope, unknown> =>
|
|
100
|
+
async (job: JobContext<JobEnvelope>) => await handle(job)
|
|
101
|
+
|
|
102
|
+
return { served, processor, handle }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export const entrypointJobsOf = memoHelper.oncePer(makeEntrypointJobsHelper)
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { ResilientError } from '@owlmeans/error'
|
|
2
|
+
|
|
3
|
+
export class QueueError extends ResilientError {
|
|
4
|
+
public static override typeName = 'QueueError'
|
|
5
|
+
|
|
6
|
+
constructor(message: string = 'error') {
|
|
7
|
+
super(QueueError.typeName, `queue:${message}`)
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The caller stopped waiting. The job is untouched — it may still be queued, running, or already
|
|
13
|
+
* finished; only the wait ended. Read the job back to learn which.
|
|
14
|
+
*/
|
|
15
|
+
export class QueueTimeout extends QueueError {
|
|
16
|
+
public static override typeName = `${QueueError.typeName}Timeout`
|
|
17
|
+
|
|
18
|
+
constructor(msg: string) {
|
|
19
|
+
super(`timeout:${msg}`)
|
|
20
|
+
this.type = QueueTimeout.typeName
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** No job with that id in this queue — it never existed, or it was removed after completing. */
|
|
25
|
+
export class UnknownJob extends QueueError {
|
|
26
|
+
public static override typeName = `${QueueError.typeName}UnknownJob`
|
|
27
|
+
|
|
28
|
+
constructor(msg: string) {
|
|
29
|
+
super(`unknown-job:${msg}`)
|
|
30
|
+
this.type = UnknownJob.typeName
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* A job name the queue does not declare. Declaring every job a queue accepts is what keeps an
|
|
36
|
+
* enqueue from succeeding into a queue where nothing can ever process it.
|
|
37
|
+
*/
|
|
38
|
+
export class UnknownJobName extends QueueError {
|
|
39
|
+
public static override typeName = `${QueueError.typeName}UnknownJobName`
|
|
40
|
+
|
|
41
|
+
constructor(msg: string) {
|
|
42
|
+
super(`unknown-job-name:${msg}`)
|
|
43
|
+
this.type = UnknownJobName.typeName
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** A queue asked for that no declaration names. */
|
|
48
|
+
export class UnknownQueue extends QueueError {
|
|
49
|
+
public static override typeName = `${QueueError.typeName}UnknownQueue`
|
|
50
|
+
|
|
51
|
+
constructor(msg: string) {
|
|
52
|
+
super(`unknown-queue:${msg}`)
|
|
53
|
+
this.type = UnknownQueue.typeName
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* This process was asked to consume a queue it does not listen to. Which queues a process
|
|
59
|
+
* consumes is configuration (`cfg.queue.listen`), so this is a deployment question, not a code
|
|
60
|
+
* one — the same binary is a producer or a worker depending on it.
|
|
61
|
+
*/
|
|
62
|
+
export class QueueNotListening extends QueueError {
|
|
63
|
+
public static override typeName = `${QueueError.typeName}NotListening`
|
|
64
|
+
|
|
65
|
+
constructor(msg: string) {
|
|
66
|
+
super(`not-listening:${msg}`)
|
|
67
|
+
this.type = QueueNotListening.typeName
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The job named an entrypoint this process does not serve. Unlike a missing queue this is not a
|
|
73
|
+
* declaration problem: the alias exists, but nothing here bound it, so the job was taken by a
|
|
74
|
+
* worker that cannot run it.
|
|
75
|
+
*/
|
|
76
|
+
export class JobNotServed extends QueueError {
|
|
77
|
+
public static override typeName = `${QueueError.typeName}JobNotServed`
|
|
78
|
+
|
|
79
|
+
constructor(msg: string) {
|
|
80
|
+
super(`job-not-served:${msg}`)
|
|
81
|
+
this.type = JobNotServed.typeName
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The signed envelope is older than the configured lifetime. Freshness is judged from when the
|
|
87
|
+
* producer enqueued the job, not from when a worker picked it up, so a long backlog does not
|
|
88
|
+
* invalidate work that was legitimate when it was submitted.
|
|
89
|
+
*/
|
|
90
|
+
export class EnvelopeExpired extends QueueError {
|
|
91
|
+
public static override typeName = `${QueueError.typeName}EnvelopeExpired`
|
|
92
|
+
|
|
93
|
+
constructor(msg: string) {
|
|
94
|
+
super(`envelope-expired:${msg}`)
|
|
95
|
+
this.type = EnvelopeExpired.typeName
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* A schedule the broker could not carry out as written: no id, both or neither of `every` and
|
|
101
|
+
* `pattern`, an option that only means something for the other one, or a job option the broker
|
|
102
|
+
* owns for a scheduled run. The message names the schedule id and the reason.
|
|
103
|
+
*/
|
|
104
|
+
export class ScheduleMisdeclared extends QueueError {
|
|
105
|
+
public static override typeName = `${QueueError.typeName}ScheduleMisdeclared`
|
|
106
|
+
|
|
107
|
+
constructor(msg: string) {
|
|
108
|
+
super(`schedule-misdeclared:${msg}`)
|
|
109
|
+
this.type = ScheduleMisdeclared.typeName
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
ResilientError.registerErrorClass(QueueError)
|
|
114
|
+
ResilientError.registerErrorClass(QueueTimeout)
|
|
115
|
+
ResilientError.registerErrorClass(UnknownJob)
|
|
116
|
+
ResilientError.registerErrorClass(UnknownJobName)
|
|
117
|
+
ResilientError.registerErrorClass(UnknownQueue)
|
|
118
|
+
ResilientError.registerErrorClass(QueueNotListening)
|
|
119
|
+
ResilientError.registerErrorClass(JobNotServed)
|
|
120
|
+
ResilientError.registerErrorClass(EnvelopeExpired)
|
|
121
|
+
ResilientError.registerErrorClass(ScheduleMisdeclared)
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export * from './types.js'
|
|
2
|
+
export type * from './queue-config/types.js'
|
|
3
|
+
export type * from './bridge/types.js'
|
|
4
|
+
export type * from './queue-route/types.js'
|
|
5
|
+
export type * from './protocol/types.js'
|
|
6
|
+
export type * from './entrypoint-jobs/types.js'
|
|
7
|
+
export * from './consts.js'
|
|
8
|
+
export * from './utils.js'
|
|
9
|
+
export * from './errors.js'
|
|
10
|
+
export * from './config.js'
|
|
11
|
+
export * from './queue-config.js'
|
|
12
|
+
export * from './transport.js'
|
|
13
|
+
export * from './bridge.js'
|
|
14
|
+
export * from './entrypoint-jobs.js'
|
|
15
|
+
export * from './worker.js'
|
|
16
|
+
export * from './protocol.js'
|
|
17
|
+
export * from './route.js'
|
|
18
|
+
export * from './queue-route.js'
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { EntrypointProtocolDeclaration, RequestOf, ResponseOf } from '@owlmeans/entrypoint'
|
|
2
|
+
import type { JobEnvelope, JobOptions, JobRecord, JobReply } from '../types.js'
|
|
3
|
+
|
|
4
|
+
/** Enqueueing and awaiting QUEUE protocols in one context, with their exact types kept. */
|
|
5
|
+
export interface QueueProtocolHelper {
|
|
6
|
+
/** Enqueue an immutable QUEUE protocol while preserving its exact request and response types. */
|
|
7
|
+
enqueue: <Protocol extends EntrypointProtocolDeclaration>(
|
|
8
|
+
declaration: Protocol, request: RequestOf<Protocol>, options?: JobOptions,
|
|
9
|
+
) => Promise<JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>>
|
|
10
|
+
/** Wait for a protocol job and unwrap the typed entrypoint reply. */
|
|
11
|
+
waitFor: <Protocol extends EntrypointProtocolDeclaration>(
|
|
12
|
+
declaration: Protocol,
|
|
13
|
+
job: string | JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>,
|
|
14
|
+
options?: { timeout?: number },
|
|
15
|
+
) => Promise<ResponseOf<Protocol>>
|
|
16
|
+
}
|
package/src/protocol.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { type EntrypointProtocolDeclaration, type RequestOf, type ResponseOf, isEntrypointProtocol } from '@owlmeans/entrypoint'
|
|
2
|
+
import { memoHelper } from '@owlmeans/context'
|
|
3
|
+
import { ResilientError } from '@owlmeans/error'
|
|
4
|
+
import { queueConfigOf } from './queue-config.js'
|
|
5
|
+
import { UnknownQueue } from './errors.js'
|
|
6
|
+
import { queueRouteHelper } from './queue-route.js'
|
|
7
|
+
import type { JobEnvelope, JobOptions, JobRecord, JobReply } from './types.js'
|
|
8
|
+
import type { QueueContext } from './types.js'
|
|
9
|
+
import type { QueueProtocolHelper } from './protocol/types.js'
|
|
10
|
+
|
|
11
|
+
export const makeQueueProtocolHelper = (context: QueueContext): QueueProtocolHelper => {
|
|
12
|
+
const queueFor = (declaration: unknown): {
|
|
13
|
+
protocol: EntrypointProtocolDeclaration
|
|
14
|
+
queue: string
|
|
15
|
+
} => {
|
|
16
|
+
if (declaration == null || typeof declaration !== 'object' || !isEntrypointProtocol(declaration)) {
|
|
17
|
+
throw new UnknownQueue('protocol declaration required')
|
|
18
|
+
}
|
|
19
|
+
if (!queueRouteHelper.isQueueRoute(declaration.route)) {
|
|
20
|
+
throw new UnknownQueue(`${declaration.alias}: not a queue protocol`)
|
|
21
|
+
}
|
|
22
|
+
const { queue } = queueRouteHelper.queueRouteOptions(declaration.route)
|
|
23
|
+
const configured = queueConfigOf(context.cfg).queueOf(queue)
|
|
24
|
+
if (!configured.jobs.includes(declaration.alias)) {
|
|
25
|
+
throw new UnknownQueue(`${queue}:${declaration.alias}`)
|
|
26
|
+
}
|
|
27
|
+
return { protocol: declaration, queue }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const enqueue = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
31
|
+
declaration: Protocol,
|
|
32
|
+
request: RequestOf<Protocol>,
|
|
33
|
+
options?: JobOptions,
|
|
34
|
+
): Promise<JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>> => {
|
|
35
|
+
const { protocol, queue } = queueFor(declaration)
|
|
36
|
+
const shaped = (request ?? {}) as unknown as Record<string, unknown>
|
|
37
|
+
const envelope: JobEnvelope = {
|
|
38
|
+
alias: protocol.alias,
|
|
39
|
+
params: shaped.params as Record<string, unknown> | undefined,
|
|
40
|
+
body: shaped.body,
|
|
41
|
+
query: shaped.query as Record<string, unknown> | undefined,
|
|
42
|
+
headers: shaped.headers as Record<string, string | undefined> | undefined,
|
|
43
|
+
enqueuedAt: new Date().toISOString(),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return await context.jobs<JobEnvelope, JobReply<ResponseOf<Protocol>>>(queue).create({
|
|
47
|
+
queue,
|
|
48
|
+
name: protocol.alias,
|
|
49
|
+
data: envelope,
|
|
50
|
+
opts: options,
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const waitFor = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
55
|
+
declaration: Protocol,
|
|
56
|
+
job: string | JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>,
|
|
57
|
+
options?: { timeout?: number },
|
|
58
|
+
): Promise<ResponseOf<Protocol>> => {
|
|
59
|
+
const { protocol, queue } = queueFor(declaration)
|
|
60
|
+
if (typeof job !== 'string' && (job.queue !== queue || job.name !== protocol.alias)) {
|
|
61
|
+
throw new UnknownQueue(`${job.queue}:${job.name}`)
|
|
62
|
+
}
|
|
63
|
+
const id = typeof job === 'string' ? job : job.id
|
|
64
|
+
if (id == null) {
|
|
65
|
+
throw new UnknownQueue(`${queue}:${protocol.alias}: missing job id`)
|
|
66
|
+
}
|
|
67
|
+
const reply = await context.jobs<JobEnvelope, JobReply<ResponseOf<Protocol>>>(queue)
|
|
68
|
+
.wait(id, options)
|
|
69
|
+
if (reply.error != null) {
|
|
70
|
+
throw ResilientError.ensure(reply.error as Error | string)
|
|
71
|
+
}
|
|
72
|
+
return reply.value as ResponseOf<Protocol>
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return { enqueue, waitFor }
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export const queueProtocolOf = memoHelper.oncePer(makeQueueProtocolHelper)
|
|
79
|
+
|
|
80
|
+
/** @deprecated compat:factory-refactor — use `queueProtocolOf(ctx).enqueue(…)` */
|
|
81
|
+
export const enqueueProtocol = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
82
|
+
context: QueueContext,
|
|
83
|
+
declaration: Protocol,
|
|
84
|
+
request: RequestOf<Protocol>,
|
|
85
|
+
options?: JobOptions,
|
|
86
|
+
): Promise<JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>> =>
|
|
87
|
+
await queueProtocolOf(context).enqueue(declaration, request, options)
|
|
88
|
+
|
|
89
|
+
/** @deprecated compat:factory-refactor — use `queueProtocolOf(ctx).waitFor(…)` */
|
|
90
|
+
export const waitForProtocol = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
91
|
+
context: QueueContext,
|
|
92
|
+
declaration: Protocol,
|
|
93
|
+
job: string | JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>,
|
|
94
|
+
options?: { timeout?: number },
|
|
95
|
+
): Promise<ResponseOf<Protocol>> => await queueProtocolOf(context).waitFor(declaration, job, options)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { QueueDeclaration, ScheduleDeclaration } from '../types.js'
|
|
2
|
+
|
|
3
|
+
/** What one configuration declares about its queues and schedules, read and checked. */
|
|
4
|
+
export interface QueueConfigHelper {
|
|
5
|
+
/**
|
|
6
|
+
* @throws {UnknownQueue}
|
|
7
|
+
*/
|
|
8
|
+
queueOf: (name: string) => QueueDeclaration
|
|
9
|
+
queueOfJob: (job: string) => QueueDeclaration | undefined
|
|
10
|
+
isListening: (name: string) => boolean
|
|
11
|
+
/**
|
|
12
|
+
* Whether the broker can carry a schedule out as written. Nothing is recorded or changed.
|
|
13
|
+
*
|
|
14
|
+
* @throws {ScheduleMisdeclared} naming the schedule id and the reason.
|
|
15
|
+
* @throws {UnknownQueue} when the schedule's queue is not declared.
|
|
16
|
+
* @throws {UnknownJobName} when that queue does not accept the schedule's job name — a scheduled
|
|
17
|
+
* run of a name nothing declares would fail on every tick instead of once, at declaration.
|
|
18
|
+
*/
|
|
19
|
+
assertSchedule: (schedule: ScheduleDeclaration) => void
|
|
20
|
+
/** The declared schedules, of one queue when it is named. */
|
|
21
|
+
schedulesOf: (queue?: string) => ScheduleDeclaration[]
|
|
22
|
+
/**
|
|
23
|
+
* Check every declared schedule, including any written into the configuration without
|
|
24
|
+
* `declareSchedule`, and refuse two under one id.
|
|
25
|
+
*
|
|
26
|
+
* @throws {ScheduleMisdeclared} / {UnknownQueue} / {UnknownJobName} for the first one that fails.
|
|
27
|
+
*/
|
|
28
|
+
assertSchedules: () => void
|
|
29
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { memoHelper } from '@owlmeans/context'
|
|
2
|
+
import type { Config, JobOptions, QueueDeclaration, ScheduleDeclaration } from './types.js'
|
|
3
|
+
import type { QueueConfigHelper } from './queue-config/types.js'
|
|
4
|
+
import { ScheduleMisdeclared, UnknownJobName, UnknownQueue } from './errors.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The queue configuration of one `cfg`. It reads `cfg.queue` on every call, so declarations made
|
|
8
|
+
* after it was built are seen.
|
|
9
|
+
*/
|
|
10
|
+
export const makeQueueConfigHelper = (cfg: Config): QueueConfigHelper => {
|
|
11
|
+
const queueOf = (name: string): QueueDeclaration => {
|
|
12
|
+
const queue = cfg.queue?.queues?.find(declared => declared.name === name)
|
|
13
|
+
|
|
14
|
+
if (queue == null) {
|
|
15
|
+
throw new UnknownQueue(name)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
return queue
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const queueOfJob = (job: string): QueueDeclaration | undefined =>
|
|
22
|
+
cfg.queue?.queues?.find(declared => declared.jobs.includes(job))
|
|
23
|
+
|
|
24
|
+
const isListening = (name: string): boolean =>
|
|
25
|
+
cfg.queue?.listen?.includes(name) === true
|
|
26
|
+
|
|
27
|
+
/** A date option the broker can read — a `Date`, epoch milliseconds or a parsable string. */
|
|
28
|
+
const isInstant = (value: Date | number | string): boolean =>
|
|
29
|
+
Number.isFinite((value instanceof Date ? value : new Date(value)).getTime())
|
|
30
|
+
|
|
31
|
+
const assertSchedule = (schedule: ScheduleDeclaration): void => {
|
|
32
|
+
const { id } = schedule
|
|
33
|
+
if (typeof id !== 'string' || id.trim() === '') {
|
|
34
|
+
throw new ScheduleMisdeclared('(empty):id')
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const queue = queueOf(schedule.queue)
|
|
38
|
+
if (!queue.jobs.includes(schedule.name)) {
|
|
39
|
+
throw new UnknownJobName(`${schedule.queue}:${schedule.name}`)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const every = schedule.every != null
|
|
43
|
+
const pattern = schedule.pattern != null
|
|
44
|
+
if (every === pattern) {
|
|
45
|
+
throw new ScheduleMisdeclared(`${id}:every-or-pattern`)
|
|
46
|
+
}
|
|
47
|
+
if (every && !(Number.isSafeInteger(schedule.every) && (schedule.every as number) > 0)) {
|
|
48
|
+
throw new ScheduleMisdeclared(`${id}:every`)
|
|
49
|
+
}
|
|
50
|
+
if (pattern && (typeof schedule.pattern !== 'string' || schedule.pattern.trim() === '')) {
|
|
51
|
+
throw new ScheduleMisdeclared(`${id}:pattern`)
|
|
52
|
+
}
|
|
53
|
+
// Both only mean something to a cron pattern: an interval has no wall clock to read in a time
|
|
54
|
+
// zone, and its first run is immediate already.
|
|
55
|
+
if (!pattern && schedule.tz != null) {
|
|
56
|
+
throw new ScheduleMisdeclared(`${id}:tz`)
|
|
57
|
+
}
|
|
58
|
+
if (!pattern && schedule.immediately === true) {
|
|
59
|
+
throw new ScheduleMisdeclared(`${id}:immediately`)
|
|
60
|
+
}
|
|
61
|
+
if (schedule.immediately === true && schedule.startDate != null) {
|
|
62
|
+
throw new ScheduleMisdeclared(`${id}:immediately-start-date`)
|
|
63
|
+
}
|
|
64
|
+
if (schedule.limit != null && !(Number.isSafeInteger(schedule.limit) && schedule.limit > 0)) {
|
|
65
|
+
throw new ScheduleMisdeclared(`${id}:limit`)
|
|
66
|
+
}
|
|
67
|
+
if (schedule.startDate != null && !isInstant(schedule.startDate)) {
|
|
68
|
+
throw new ScheduleMisdeclared(`${id}:start-date`)
|
|
69
|
+
}
|
|
70
|
+
if (schedule.endDate != null && !isInstant(schedule.endDate)) {
|
|
71
|
+
throw new ScheduleMisdeclared(`${id}:end-date`)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// The type already omits both; a declaration assembled at runtime still reaches this check. The
|
|
75
|
+
// broker names every run itself and places it on the schedule, so either would be overridden.
|
|
76
|
+
const opts = schedule.opts as JobOptions | undefined
|
|
77
|
+
if (opts?.id != null) {
|
|
78
|
+
throw new ScheduleMisdeclared(`${id}:opts-id`)
|
|
79
|
+
}
|
|
80
|
+
if (opts?.delay != null) {
|
|
81
|
+
throw new ScheduleMisdeclared(`${id}:opts-delay`)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const schedulesOf = (queue?: string): ScheduleDeclaration[] =>
|
|
86
|
+
(cfg.queue?.schedules ?? []).filter(schedule => queue == null || schedule.queue === queue)
|
|
87
|
+
|
|
88
|
+
const assertSchedules = (): void => {
|
|
89
|
+
const seen = new Set<string>()
|
|
90
|
+
for (const schedule of schedulesOf()) {
|
|
91
|
+
assertSchedule(schedule)
|
|
92
|
+
if (seen.has(schedule.id)) {
|
|
93
|
+
throw new ScheduleMisdeclared(`${schedule.id}:duplicate`)
|
|
94
|
+
}
|
|
95
|
+
seen.add(schedule.id)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { queueOf, queueOfJob, isListening, assertSchedule, schedulesOf, assertSchedules }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export const queueConfigOf = memoHelper.oncePer(makeQueueConfigHelper)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { RouteDeclaration, RouteModel } from '@owlmeans/route'
|
|
2
|
+
import type { QueueRouteOptions } from '../types.js'
|
|
3
|
+
|
|
4
|
+
/** Reads a route declaration as one carried by the queue transport. */
|
|
5
|
+
export interface QueueRouteHelper {
|
|
6
|
+
/** Whether a declaration belongs to the queue transport. */
|
|
7
|
+
isQueueRoute: (route: RouteDeclaration | RouteModel) => boolean
|
|
8
|
+
/**
|
|
9
|
+
* Read and validate queue-owned route options.
|
|
10
|
+
*
|
|
11
|
+
* @throws {UnknownQueue} for a route of another protocol, or queue options that do not validate
|
|
12
|
+
*/
|
|
13
|
+
queueRouteOptions: (route: RouteDeclaration | RouteModel) => Pick<QueueRouteOptions, 'queue' | 'reply'>
|
|
14
|
+
}
|