@owlmeans/queue 0.1.18-rc.4 → 0.1.18-rc.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +286 -8
- package/agent-meta/manifest.json +9 -2
- package/agent-meta/skills/queue/SKILL.md +243 -12
- package/agent-meta/skills/scheduled-jobs/SKILL.md +151 -0
- package/build/bridge/types.d.ts +25 -0
- package/build/bridge/types.d.ts.map +1 -0
- package/build/bridge/types.js +2 -0
- package/build/bridge/types.js.map +1 -0
- package/build/bridge.d.ts +7 -0
- package/build/bridge.d.ts.map +1 -0
- package/build/bridge.js +37 -0
- package/build/bridge.js.map +1 -0
- package/build/config.d.ts +34 -0
- package/build/config.d.ts.map +1 -0
- package/build/config.js +59 -0
- package/build/config.js.map +1 -0
- package/build/consts.d.ts +33 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +35 -0
- package/build/consts.js.map +1 -0
- package/build/entrypoint-jobs/types.d.ts +29 -0
- package/build/entrypoint-jobs/types.d.ts.map +1 -0
- package/build/entrypoint-jobs/types.js +2 -0
- package/build/entrypoint-jobs/types.js.map +1 -0
- package/build/entrypoint-jobs.d.ts +5 -0
- package/build/entrypoint-jobs.d.ts.map +1 -0
- package/build/entrypoint-jobs.js +84 -0
- package/build/entrypoint-jobs.js.map +1 -0
- package/build/errors.d.ts +68 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +103 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +18 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +13 -1
- package/build/index.js.map +1 -1
- package/build/protocol/types.d.ts +12 -0
- package/build/protocol/types.d.ts.map +1 -0
- package/build/protocol/types.js +2 -0
- package/build/protocol/types.js.map +1 -0
- package/build/protocol.d.ts +13 -0
- package/build/protocol.d.ts.map +1 -0
- package/build/protocol.js +63 -0
- package/build/protocol.js.map +1 -0
- package/build/queue-config/types.d.ts +29 -0
- package/build/queue-config/types.d.ts.map +1 -0
- package/build/queue-config/types.js +2 -0
- package/build/queue-config/types.js.map +1 -0
- package/build/queue-config.d.ts +9 -0
- package/build/queue-config.d.ts.map +1 -0
- package/build/queue-config.js +83 -0
- package/build/queue-config.js.map +1 -0
- package/build/queue-route/types.d.ts +14 -0
- package/build/queue-route/types.d.ts.map +1 -0
- package/build/queue-route/types.js +2 -0
- package/build/queue-route/types.js.map +1 -0
- package/build/queue-route.d.ts +4 -0
- package/build/queue-route.d.ts.map +1 -0
- package/build/queue-route.js +24 -0
- package/build/queue-route.js.map +1 -0
- package/build/route.d.ts +9 -0
- package/build/route.d.ts.map +1 -0
- package/build/route.js +16 -0
- package/build/route.js.map +1 -0
- package/build/transport.d.ts +15 -0
- package/build/transport.d.ts.map +1 -0
- package/build/transport.js +59 -0
- package/build/transport.js.map +1 -0
- package/build/types.d.ts +312 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/types.local.d.ts +4 -0
- package/build/types.local.d.ts.map +1 -0
- package/build/types.local.js +2 -0
- package/build/types.local.js.map +1 -0
- package/build/utils.d.ts +4 -0
- package/build/utils.d.ts.map +1 -0
- package/build/utils.js +4 -0
- package/build/utils.js.map +1 -0
- package/build/worker.d.ts +10 -0
- package/build/worker.d.ts.map +1 -0
- package/build/worker.js +24 -0
- package/build/worker.js.map +1 -0
- package/package.json +13 -3
- package/src/bridge/types.ts +25 -0
- package/src/bridge.ts +46 -0
- package/src/config.ts +80 -0
- package/src/consts.ts +38 -0
- package/src/entrypoint-jobs/types.ts +29 -0
- package/src/entrypoint-jobs.ts +105 -0
- package/src/errors.ts +121 -0
- package/src/index.ts +18 -0
- package/src/protocol/types.ts +16 -0
- package/src/protocol.ts +95 -0
- package/src/queue-config/types.ts +29 -0
- package/src/queue-config.ts +102 -0
- package/src/queue-route/types.ts +14 -0
- package/src/queue-route.ts +33 -0
- package/src/route.ts +23 -0
- package/src/transport.ts +77 -0
- package/src/types.ts +326 -0
- package/src/utils.ts +5 -0
- package/src/worker.ts +27 -0
- package/tests/bridge.spec.ts +64 -0
- package/tests/config.spec.ts +67 -0
- package/tests/protocol.spec.ts +58 -0
- package/tests/schedule.spec.ts +99 -0
- package/tsconfig.json +1 -0
- package/build/.gitkeep +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { RouteDeclaration, RouteModel } from '@owlmeans/route'
|
|
2
|
+
import { UnknownQueue } from './errors.js'
|
|
3
|
+
import { QUEUE_PROTOCOL } from './consts.js'
|
|
4
|
+
import type { QueueRouteOptions } from './types.js'
|
|
5
|
+
import type { QueueRouteHelper } from './queue-route/types.js'
|
|
6
|
+
|
|
7
|
+
export const createQueueRouteHelper = (): QueueRouteHelper => {
|
|
8
|
+
const isQueueRoute = (route: RouteDeclaration | RouteModel): boolean =>
|
|
9
|
+
('route' in route ? route.route : route).protocol === QUEUE_PROTOCOL
|
|
10
|
+
|
|
11
|
+
const queueRouteOptions = (
|
|
12
|
+
route: RouteDeclaration | RouteModel,
|
|
13
|
+
): Pick<QueueRouteOptions, 'queue' | 'reply'> => {
|
|
14
|
+
const declaration = 'route' in route ? route.route : route
|
|
15
|
+
if (declaration.protocol !== QUEUE_PROTOCOL
|
|
16
|
+
|| declaration.protocolOptions == null
|
|
17
|
+
|| typeof declaration.protocolOptions !== 'object') {
|
|
18
|
+
throw new UnknownQueue(`${declaration.alias}: not a queue protocol`)
|
|
19
|
+
}
|
|
20
|
+
const options = declaration.protocolOptions as Record<string, unknown>
|
|
21
|
+
if (typeof options.queue !== 'string' || options.queue.trim() === '') {
|
|
22
|
+
throw new UnknownQueue(`${declaration.alias}: route declares no queue`)
|
|
23
|
+
}
|
|
24
|
+
if (options.reply != null && typeof options.reply !== 'boolean') {
|
|
25
|
+
throw new UnknownQueue(`${declaration.alias}: invalid queue reply option`)
|
|
26
|
+
}
|
|
27
|
+
return { queue: options.queue, reply: options.reply as boolean | undefined }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return { isQueueRoute, queueRouteOptions }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const queueRouteHelper = createQueueRouteHelper()
|
package/src/route.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { backend, type RouteDeclaration, type RouteModel, type RouteOptions } from '@owlmeans/route'
|
|
2
|
+
import { QUEUE_PROTOCOL } from './consts.js'
|
|
3
|
+
import type { QueueRouteOptions } from './types.js'
|
|
4
|
+
import { queueRouteHelper } from './queue-route.js'
|
|
5
|
+
|
|
6
|
+
/** Declare a backend route carried by a queue. */
|
|
7
|
+
export const job = (
|
|
8
|
+
opts: QueueRouteOptions,
|
|
9
|
+
secondary?: Omit<QueueRouteOptions, 'queue'>,
|
|
10
|
+
): Partial<RouteOptions> => {
|
|
11
|
+
const { queue, reply, ...routeOptions } = { ...opts, ...secondary }
|
|
12
|
+
const declaration = backend(routeOptions)
|
|
13
|
+
declaration.protocol = QUEUE_PROTOCOL
|
|
14
|
+
declaration.protocolOptions = { queue, ...(reply == null ? {} : { reply }) }
|
|
15
|
+
return declaration
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** @deprecated compat:factory-refactor — use `queueRouteHelper.isQueueRoute(…)` */
|
|
19
|
+
export const isQueueRoute = (route: RouteDeclaration | RouteModel): boolean => queueRouteHelper.isQueueRoute(route)
|
|
20
|
+
|
|
21
|
+
/** @deprecated compat:factory-refactor — use `queueRouteHelper.queueRouteOptions(…)` */
|
|
22
|
+
export const queueRouteOptions = (route: RouteDeclaration | RouteModel): Pick<QueueRouteOptions, 'queue' | 'reply'> =>
|
|
23
|
+
queueRouteHelper.queueRouteOptions(route)
|
package/src/transport.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { assertContext, createService } from '@owlmeans/context'
|
|
2
|
+
import { type AbstractRequest, type AbstractResponse, type EntrypointTransport, EntrypointOutcome, transportAlias, type CommonEntrypoint } from '@owlmeans/entrypoint'
|
|
3
|
+
import { ResilientError } from '@owlmeans/error'
|
|
4
|
+
import type { Config, Context, JobEnvelope, JobReply, QueueAppend } from './types.js'
|
|
5
|
+
import { DEFAULT_JOB_TIMEOUT, QUEUE_PROTOCOL } from './consts.js'
|
|
6
|
+
import { queueRouteHelper } from './queue-route.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The producing half of the bridge: it turns a call on a QUEUE entrypoint into a job, and the
|
|
10
|
+
* job's outcome back into a reply.
|
|
11
|
+
*
|
|
12
|
+
* Registering it is the whole opt-in. `apiHandler` looks up a transport by the route's protocol,
|
|
13
|
+
* so a consumer writes `ep.call(...)` against a declaration it shares with every other kind of
|
|
14
|
+
* entrypoint and never learns which one carried it. That is also why the envelope is built from
|
|
15
|
+
* the request rather than from the entrypoint: the far side rebuilds a request, and anything only
|
|
16
|
+
* HTTP understands would not survive the trip anyway.
|
|
17
|
+
*/
|
|
18
|
+
export const makeQueueTransport = (alias: string = transportAlias(QUEUE_PROTOCOL)): EntrypointTransport => {
|
|
19
|
+
const location = `queue-transport:${alias}`
|
|
20
|
+
|
|
21
|
+
const service = createService<EntrypointTransport>(alias, {
|
|
22
|
+
protocol: QUEUE_PROTOCOL,
|
|
23
|
+
|
|
24
|
+
handle: (async <T>(req: AbstractRequest, res: AbstractResponse<T>) => {
|
|
25
|
+
const context = assertContext<Config, Context<Config>>(service.ctx as Context<Config>, location)
|
|
26
|
+
const entrypoint = context.entrypoint<CommonEntrypoint>(req.alias)
|
|
27
|
+
const route = entrypoint.route.route
|
|
28
|
+
|
|
29
|
+
const { queue, reply: waitsForReply } = queueRouteHelper.queueRouteOptions(route)
|
|
30
|
+
|
|
31
|
+
const jobs = (context as unknown as QueueAppend).jobs<JobEnvelope, JobReply<T>>(queue)
|
|
32
|
+
|
|
33
|
+
const envelope: JobEnvelope = {
|
|
34
|
+
alias: req.alias,
|
|
35
|
+
params: req.params as Record<string, unknown>,
|
|
36
|
+
body: req.body,
|
|
37
|
+
query: req.query as Record<string, unknown>,
|
|
38
|
+
headers: req.headers as Record<string, string | undefined>,
|
|
39
|
+
enqueuedAt: new Date().toISOString(),
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const job = await jobs.create({ queue, name: req.alias, data: envelope })
|
|
43
|
+
|
|
44
|
+
// `reply: false` says the caller is not waiting for the work, only for the promise that it
|
|
45
|
+
// was taken. `Accepted` plus the job's identity is the whole answer — enough to watch it.
|
|
46
|
+
if (waitsForReply === false) {
|
|
47
|
+
res.resolve({ id: job.id, queue } as T, EntrypointOutcome.Accepted)
|
|
48
|
+
|
|
49
|
+
return
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const reply = await jobs.wait(job.id as string, {
|
|
53
|
+
timeout: req.timeout ?? route.timeout ?? DEFAULT_JOB_TIMEOUT
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
// A processor reports a domain failure by returning it, so it crosses the hop as the class
|
|
57
|
+
// it was thrown as rather than as a string a caller would have to parse.
|
|
58
|
+
if (reply?.error != null) {
|
|
59
|
+
throw ResilientError.ensure(reply.error as Error | string)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
res.resolve(reply?.value as T, (reply?.outcome as EntrypointOutcome) ?? EntrypointOutcome.Ok)
|
|
63
|
+
}) as EntrypointTransport['handle']
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
return service
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export const appendQueueTransport = <C extends Config, T extends Context<C> = Context<C>>(
|
|
70
|
+
context: T, alias?: string
|
|
71
|
+
): T => {
|
|
72
|
+
const service = makeQueueTransport(alias)
|
|
73
|
+
|
|
74
|
+
context.registerService(service)
|
|
75
|
+
|
|
76
|
+
return context
|
|
77
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import type { BasicConfig, BasicContext, InitializedService } from '@owlmeans/context'
|
|
2
|
+
import type { PubSubResource, Resource, ResourceRecord } from '@owlmeans/resource'
|
|
3
|
+
import type { AbstractRequest, AbstractResponse } from '@owlmeans/entrypoint'
|
|
4
|
+
import type { JobEventType, JobState } from './consts.js'
|
|
5
|
+
import type { RouteOptions } from '@owlmeans/route'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Everything the monorepo knows about its queues, declared in the shared backend package so that
|
|
9
|
+
* producer and consumer read the same list. A process becomes a worker by naming queues in
|
|
10
|
+
* `listen` — nothing in a declaration decides where it runs.
|
|
11
|
+
*/
|
|
12
|
+
export interface QueueConfig {
|
|
13
|
+
queues?: QueueDeclaration[]
|
|
14
|
+
/** The queues THIS process consumes. Absent or empty means producer only. */
|
|
15
|
+
listen?: string[]
|
|
16
|
+
/** Which db service carries the queues. Defaults to the queue service's own alias. */
|
|
17
|
+
db?: string
|
|
18
|
+
defaults?: JobOptions
|
|
19
|
+
/**
|
|
20
|
+
* How long a signed job envelope stays acceptable, in seconds. It bounds both the freshness
|
|
21
|
+
* check and the nonce cache, so a job delayed behind a long backlog is still admitted.
|
|
22
|
+
*/
|
|
23
|
+
envelopeTtl?: number
|
|
24
|
+
/**
|
|
25
|
+
* Recurring jobs. Declared beside the queues, in the shared package, because every process that
|
|
26
|
+
* listens to a schedule's queue reconciles the broker against this list when its worker starts —
|
|
27
|
+
* two listeners loading different lists would remove each other's schedules.
|
|
28
|
+
*/
|
|
29
|
+
schedules?: ScheduleDeclaration[]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A job the broker produces on its own, on an interval or a cron pattern.
|
|
34
|
+
*
|
|
35
|
+
* A scheduled run is an ordinary job of a name the queue declares, dispatched to the processor
|
|
36
|
+
* registered for that name — never to a guarded entrypoint, because nobody's credentials travel
|
|
37
|
+
* with it. Only a process that listens to the queue creates the scheduler, and the broker produces
|
|
38
|
+
* each next run when the previous one STARTS processing — a schedule needs a running consumer.
|
|
39
|
+
*/
|
|
40
|
+
export interface ScheduleDeclaration {
|
|
41
|
+
/**
|
|
42
|
+
* The schedule's stable identity: the broker keeps one scheduler per id, so changing any other
|
|
43
|
+
* field updates it in place, while renaming the id removes one schedule and creates another.
|
|
44
|
+
*/
|
|
45
|
+
id: string
|
|
46
|
+
queue: string
|
|
47
|
+
/** A job name the queue declares. */
|
|
48
|
+
name: string
|
|
49
|
+
/**
|
|
50
|
+
* Milliseconds between runs. Exactly one of `every` and `pattern`. Without `startDate` the first
|
|
51
|
+
* run is produced as soon as the scheduler is created.
|
|
52
|
+
*/
|
|
53
|
+
every?: number
|
|
54
|
+
/** A cron pattern (an optional leading seconds field is accepted). Exactly one of the two. */
|
|
55
|
+
pattern?: string
|
|
56
|
+
/** The IANA time zone `pattern` is read in. Pattern only. */
|
|
57
|
+
tz?: string
|
|
58
|
+
startDate?: Date | number | string
|
|
59
|
+
/** Once passed, the schedule is treated as absent and a scheduler left from it is removed. */
|
|
60
|
+
endDate?: Date | number | string
|
|
61
|
+
/** Stop after this many runs. */
|
|
62
|
+
limit?: number
|
|
63
|
+
/**
|
|
64
|
+
* Run once as soon as the scheduler is created or its declaration changes, then follow the
|
|
65
|
+
* pattern. Pattern only, and never together with `startDate`. A restart that finds the
|
|
66
|
+
* declaration unchanged does not run it again.
|
|
67
|
+
*/
|
|
68
|
+
immediately?: boolean
|
|
69
|
+
data?: unknown
|
|
70
|
+
/**
|
|
71
|
+
* Options every run is enqueued with, over the queue's defaults. `id` and `delay` belong to the
|
|
72
|
+
* broker for a scheduled run and are refused.
|
|
73
|
+
*/
|
|
74
|
+
opts?: Omit<JobOptions, 'id' | 'delay'>
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface QueueDeclaration {
|
|
78
|
+
name: string
|
|
79
|
+
/**
|
|
80
|
+
* The job names this queue accepts. A job the queue does not declare is refused at enqueue
|
|
81
|
+
* time rather than becoming a job nothing can process.
|
|
82
|
+
*/
|
|
83
|
+
jobs: string[]
|
|
84
|
+
worker?: QueueWorkerOptions
|
|
85
|
+
defaults?: JobOptions
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export interface QueueWorkerOptions {
|
|
89
|
+
concurrency?: number
|
|
90
|
+
/**
|
|
91
|
+
* How long a job may hold its lock without reporting progress. A processor that runs longer
|
|
92
|
+
* than this between `touch()` calls is treated as stalled and its job is handed to someone else.
|
|
93
|
+
*/
|
|
94
|
+
lockDuration?: number
|
|
95
|
+
stalledInterval?: number
|
|
96
|
+
maxStalledCount?: number
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface JobOptions {
|
|
100
|
+
attempts?: number
|
|
101
|
+
backoff?: { type: 'fixed' | 'exponential', delay: number }
|
|
102
|
+
delay?: number
|
|
103
|
+
priority?: number
|
|
104
|
+
/**
|
|
105
|
+
* A caller-chosen identity for the job. Enqueueing the same id twice while the first is still
|
|
106
|
+
* around is a no-op, which is what makes an admission step safe to retry.
|
|
107
|
+
*/
|
|
108
|
+
id?: string
|
|
109
|
+
removeOnComplete?: boolean | number
|
|
110
|
+
removeOnFail?: boolean | number
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* A job as a RECORD — the same shape the resource contract reads and writes, so inspecting a
|
|
115
|
+
* queue is `list`/`get`/`count` rather than a broker-specific API.
|
|
116
|
+
*/
|
|
117
|
+
export interface JobRecord<D = unknown, R = unknown> extends ResourceRecord {
|
|
118
|
+
queue: string
|
|
119
|
+
name: string
|
|
120
|
+
data: D
|
|
121
|
+
state?: JobState
|
|
122
|
+
result?: R
|
|
123
|
+
error?: string
|
|
124
|
+
attempts?: number
|
|
125
|
+
progress?: unknown
|
|
126
|
+
parentId?: string
|
|
127
|
+
createdAt?: string
|
|
128
|
+
startedAt?: string
|
|
129
|
+
finishedAt?: string
|
|
130
|
+
opts?: JobOptions
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface JobEvent<R = unknown> {
|
|
134
|
+
type: JobEventType
|
|
135
|
+
id: string
|
|
136
|
+
queue: string
|
|
137
|
+
/**
|
|
138
|
+
* Empty when the job is already gone. A broker's completion event carries only an id, so the
|
|
139
|
+
* name is read back from the job — and a queue that removes jobs on completion has nothing left
|
|
140
|
+
* to read. Subscribe for the id and the outcome; do not branch on the name.
|
|
141
|
+
*/
|
|
142
|
+
name: string
|
|
143
|
+
result?: R
|
|
144
|
+
error?: string
|
|
145
|
+
progress?: unknown
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** A node of a job graph: children run first, siblings run together. */
|
|
149
|
+
export interface FlowSpec<D = unknown> {
|
|
150
|
+
name: string
|
|
151
|
+
queue?: string
|
|
152
|
+
data: D
|
|
153
|
+
opts?: JobOptions
|
|
154
|
+
children?: FlowSpec[]
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* One queue, addressed as a resource.
|
|
159
|
+
*
|
|
160
|
+
* `create` enqueues, `get`/`load`/`list`/`count` inspect, `delete`/`take` cancel. What the base
|
|
161
|
+
* contract cannot express — waiting for an answer, submitting a graph, releasing the connection —
|
|
162
|
+
* is added here.
|
|
163
|
+
*
|
|
164
|
+
* Three places where the base `Resource` contract reads differently against a broker, because a
|
|
165
|
+
* job is not a row:
|
|
166
|
+
* - `create` does NOT throw `RecordExists` for a repeated `opts.id`. Re-enqueueing an id that is
|
|
167
|
+
* still around returns the existing job, which is the idempotency that makes an admission step
|
|
168
|
+
* safe to retry.
|
|
169
|
+
* - `WriteOptions.ttl` is refused. A job's lifetime is `removeOnComplete`/`removeOnFail` —
|
|
170
|
+
* explicit deletion once it has finished, not expiry while it waits.
|
|
171
|
+
* - `update`/`save` reach only the job's `data`. Everything else about a job belongs to the
|
|
172
|
+
* broker, and an implementation refuses rather than pretending to write it.
|
|
173
|
+
*/
|
|
174
|
+
export interface QueueResource<D = unknown, R = unknown>
|
|
175
|
+
extends Resource<JobRecord<D, R>>, PubSubResource<JobEvent<R>> {
|
|
176
|
+
queue: string
|
|
177
|
+
/**
|
|
178
|
+
* Release the connections this queue holds.
|
|
179
|
+
*
|
|
180
|
+
* Watching for completions means a blocking connection, and a process that never releases it
|
|
181
|
+
* does not exit. Nothing else in the resource contract can express that, so it lives here and a
|
|
182
|
+
* producer calls it on shutdown.
|
|
183
|
+
*/
|
|
184
|
+
close: () => Promise<void>
|
|
185
|
+
/**
|
|
186
|
+
* Wait for a job's return value.
|
|
187
|
+
* @throws {QueueTimeout} when the wait elapses — the job itself is left alone.
|
|
188
|
+
* @throws the processor's own error, rebuilt as its original class.
|
|
189
|
+
*/
|
|
190
|
+
wait: (id: string, opts?: { timeout?: number }) => Promise<R>
|
|
191
|
+
/** Submit a graph and get its root back. Children complete before their parent starts. */
|
|
192
|
+
flow: (root: FlowSpec<D>) => Promise<JobRecord<D, R>>
|
|
193
|
+
counts: () => Promise<Record<JobState, number>>
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* What a processor is handed. `data` is the job's payload; everything else is the means to stay
|
|
198
|
+
* alive and to read what the children reported.
|
|
199
|
+
*/
|
|
200
|
+
export interface JobContext<D = unknown> {
|
|
201
|
+
id: string
|
|
202
|
+
name: string
|
|
203
|
+
queue: string
|
|
204
|
+
attempt: number
|
|
205
|
+
data: D
|
|
206
|
+
signal: AbortSignal
|
|
207
|
+
/**
|
|
208
|
+
* The id of the schedule that produced this run (`ScheduleDeclaration.id`). Absent for a job
|
|
209
|
+
* enqueued any other way.
|
|
210
|
+
*/
|
|
211
|
+
scheduled?: string
|
|
212
|
+
/**
|
|
213
|
+
* Renew the lock. A processor that runs a long loop MUST call it as it goes: the broker judges
|
|
214
|
+
* liveness by the lock, so silence for longer than `lockDuration` is indistinguishable from a
|
|
215
|
+
* dead worker, and the job is re-run somewhere else while this one is still working on it.
|
|
216
|
+
*/
|
|
217
|
+
touch: () => Promise<void>
|
|
218
|
+
progress: (value: unknown) => Promise<void>
|
|
219
|
+
/**
|
|
220
|
+
* What the children returned, keyed by child job id.
|
|
221
|
+
*
|
|
222
|
+
* A broker keys them by its own fully qualified job key; an implementation re-keys to the bare
|
|
223
|
+
* id, so two children of one parent that live in DIFFERENT queues and happen to share an id
|
|
224
|
+
* collide here. Give the children of a cross-queue flow distinct ids.
|
|
225
|
+
*/
|
|
226
|
+
children: <T = unknown>() => Promise<Record<string, T>>
|
|
227
|
+
failedChildren: () => Promise<Record<string, string>>
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export interface JobProcessor<D = unknown, R = unknown> {
|
|
231
|
+
(job: JobContext<D>): Promise<R>
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* The consuming half. It is registered only in processes that consume: `start()` binds the
|
|
236
|
+
* queues named in `cfg.listen` and dispatches by job name, to a processor registered here or to
|
|
237
|
+
* an entrypoint served in this process.
|
|
238
|
+
*/
|
|
239
|
+
export interface QueueWorkerService extends InitializedService {
|
|
240
|
+
process: <D, R>(queue: string, name: string, processor: JobProcessor<D, R>) => void
|
|
241
|
+
start: () => Promise<void>
|
|
242
|
+
stop: () => Promise<void>
|
|
243
|
+
/** The queues this process actually consumes. Empty in a producer. */
|
|
244
|
+
listening: () => string[]
|
|
245
|
+
/**
|
|
246
|
+
* Bind the application's reactions to job lifecycle events. Registering more than once merges,
|
|
247
|
+
* so an application composed of several parts can each contribute what it cares about.
|
|
248
|
+
*
|
|
249
|
+
* It belongs on the contract rather than on a driver because `onJobDead` is where an
|
|
250
|
+
* application COMPENSATES — releases the lock the admission step took, marks the record failed —
|
|
251
|
+
* and that behaviour must not have to be rewritten when the broker changes.
|
|
252
|
+
*/
|
|
253
|
+
hooks: (hooks: QueueHooks) => void
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The envelope a queued entrypoint call travels in. It is the request, minus everything that only
|
|
258
|
+
* means something to HTTP — the transport rebuilds a request from it on the far side.
|
|
259
|
+
*/
|
|
260
|
+
export interface JobEnvelope {
|
|
261
|
+
alias: string
|
|
262
|
+
params?: Record<string, unknown>
|
|
263
|
+
body?: unknown
|
|
264
|
+
query?: Record<string, unknown>
|
|
265
|
+
headers?: Record<string, string | undefined>
|
|
266
|
+
/** When the producer enqueued it — freshness is judged here, not at processing time. */
|
|
267
|
+
enqueuedAt?: string
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export interface QueueJobMeta {
|
|
271
|
+
id: string
|
|
272
|
+
name: string
|
|
273
|
+
queue: string
|
|
274
|
+
attempt: number
|
|
275
|
+
/** Renew the broker lock for protocol handlers whose work can outlive one lock duration. */
|
|
276
|
+
touch: () => Promise<void>
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** What a processor returns for an entrypoint job, so the caller can rebuild the reply. */
|
|
280
|
+
export interface JobReply<T = unknown> {
|
|
281
|
+
value?: T
|
|
282
|
+
outcome?: string
|
|
283
|
+
error?: unknown
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export interface QueueHooks {
|
|
287
|
+
wrapHandler?: (job: JobContext, next: () => Promise<unknown>) => Promise<unknown>
|
|
288
|
+
onJobResult?: (event: JobEvent) => void | Promise<void>
|
|
289
|
+
onJobStalled?: (meta: { id: string, queue: string, name: string }, phase: 'stalled' | 'failed') => void | Promise<void>
|
|
290
|
+
/**
|
|
291
|
+
* The job is finished for good — retries exhausted, or it was dropped. This is where an
|
|
292
|
+
* application compensates: release the lock the admission step took, mark the record failed.
|
|
293
|
+
*/
|
|
294
|
+
onJobDead?: (job: JobRecord, reason: string) => void | Promise<void>
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export interface QueueDriver {
|
|
298
|
+
resource: <D, R>(queue: string) => QueueResource<D, R>
|
|
299
|
+
worker: () => QueueWorkerService
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export interface Config extends BasicConfig {
|
|
303
|
+
queue?: QueueConfig
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export interface Context<C extends Config = Config> extends BasicContext<C> {
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export interface QueueAppend {
|
|
310
|
+
jobs: <D = unknown, R = unknown>(queue?: string) => QueueResource<D, R>
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export interface QueueTransportRequest extends AbstractRequest {
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export interface QueueTransportResponse extends AbstractResponse<unknown> {
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** Queue-owned options stored in a generic route declaration. */
|
|
320
|
+
export interface QueueRouteOptions extends Omit<RouteOptions, 'protocol' | 'protocolOptions'> {
|
|
321
|
+
queue: string
|
|
322
|
+
reply?: boolean
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** The context the queue protocol helper is bound to: an app context with the queue appended. */
|
|
326
|
+
export interface QueueContext extends Context<Config>, QueueAppend {}
|
package/src/utils.ts
ADDED
package/src/worker.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Middleware } from '@owlmeans/context'
|
|
2
|
+
import { MiddlewareStage, MiddlewareType } from '@owlmeans/context'
|
|
3
|
+
import type { Config, Context, QueueWorkerService } from './types.js'
|
|
4
|
+
import { DEFAULT_ALIAS } from './consts.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Start the worker once the context is ready.
|
|
8
|
+
*
|
|
9
|
+
* Ready stage, not Loading: processors are registered while the application wires itself up, and
|
|
10
|
+
* binding the queues before that finished would take jobs this process cannot yet run. A process
|
|
11
|
+
* that listens to nothing registers no worker at all, so a producer pays nothing for this.
|
|
12
|
+
*/
|
|
13
|
+
export const queueWorkerMiddleware = (alias: string = DEFAULT_ALIAS): Middleware => ({
|
|
14
|
+
type: MiddlewareType.Context,
|
|
15
|
+
stage: MiddlewareStage.Ready,
|
|
16
|
+
apply: async context => {
|
|
17
|
+
const ctx = context as unknown as Context<Config>
|
|
18
|
+
if ((ctx.cfg.queue?.listen ?? []).length === 0) {
|
|
19
|
+
return
|
|
20
|
+
}
|
|
21
|
+
if (!ctx.hasService(alias)) {
|
|
22
|
+
return
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
await ctx.service<QueueWorkerService>(alias).start()
|
|
26
|
+
}
|
|
27
|
+
})
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { describe, test, expect } from 'bun:test'
|
|
2
|
+
import { queueBridgeHelper } from '../src/bridge.js'
|
|
3
|
+
import { EnvelopeExpired } from '../src/errors.js'
|
|
4
|
+
import type { JobEnvelope } from '../src/types.js'
|
|
5
|
+
|
|
6
|
+
const envelope = (enqueuedAt?: string): JobEnvelope => ({
|
|
7
|
+
alias: 'story:code', params: { id: '7' }, body: { prompt: 'x' }, headers: {}, query: {}, enqueuedAt
|
|
8
|
+
})
|
|
9
|
+
|
|
10
|
+
describe('rebuilding the request', () => {
|
|
11
|
+
test('the envelope becomes an ordinary request', () => {
|
|
12
|
+
const request = queueBridgeHelper.requestOf(envelope(), '/story/:id/code')
|
|
13
|
+
|
|
14
|
+
expect(request.alias).toBe('story:code')
|
|
15
|
+
expect(request.params).toEqual({ id: '7' })
|
|
16
|
+
expect(request.body).toEqual({ prompt: 'x' })
|
|
17
|
+
expect(request.path).toBe('/story/:id/code')
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A producer may omit any of the optional parts. The handler on the far side reads them without
|
|
22
|
+
* checking, so they have to arrive as empty containers rather than as undefined.
|
|
23
|
+
*/
|
|
24
|
+
test('absent parts arrive as empty, never undefined', () => {
|
|
25
|
+
const request = queueBridgeHelper.requestOf({ alias: 'story:code' }, '/x')
|
|
26
|
+
|
|
27
|
+
expect(request.params).toEqual({})
|
|
28
|
+
expect(request.headers).toEqual({})
|
|
29
|
+
expect(request.query).toEqual({})
|
|
30
|
+
})
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
describe('envelope freshness', () => {
|
|
34
|
+
test('a fresh envelope passes', () => {
|
|
35
|
+
expect(() => queueBridgeHelper.assertFresh(envelope(new Date().toISOString()), 60)).not.toThrow()
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
test('an envelope older than the window is refused', () => {
|
|
39
|
+
const old = new Date(Date.now() - 120_000).toISOString()
|
|
40
|
+
|
|
41
|
+
expect(() => queueBridgeHelper.assertFresh(envelope(old), 60)).toThrow(EnvelopeExpired)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The window is judged from when the job was ENQUEUED. A job that waited behind a long backlog
|
|
46
|
+
* was legitimate when it was produced, and judging it on pickup would reject exactly the work a
|
|
47
|
+
* busy queue delayed — so a generous window has to keep it.
|
|
48
|
+
*/
|
|
49
|
+
test('a long wait in the queue is not what expiry is for', () => {
|
|
50
|
+
const enqueued = new Date(Date.now() - 3600_000).toISOString()
|
|
51
|
+
|
|
52
|
+
expect(() => queueBridgeHelper.assertFresh(envelope(enqueued), 86_400)).not.toThrow()
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
test('no configured window means no check', () => {
|
|
56
|
+
const ancient = new Date(0).toISOString()
|
|
57
|
+
|
|
58
|
+
expect(() => queueBridgeHelper.assertFresh(envelope(ancient), undefined)).not.toThrow()
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('an envelope without a timestamp is not judged', () => {
|
|
62
|
+
expect(() => queueBridgeHelper.assertFresh(envelope(undefined), 60)).not.toThrow()
|
|
63
|
+
})
|
|
64
|
+
})
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, test, expect } from 'bun:test'
|
|
2
|
+
import { declareQueue, listenQueues } from '../src/config.js'
|
|
3
|
+
import { queueConfigOf } from '../src/queue-config.js'
|
|
4
|
+
import { UnknownQueue } from '../src/errors.js'
|
|
5
|
+
import type { Config } from '../src/types.js'
|
|
6
|
+
|
|
7
|
+
const cfg = (): Config => ({ ready: false, service: 'test', type: 'backend' } as unknown as Config)
|
|
8
|
+
|
|
9
|
+
describe('declaring queues', () => {
|
|
10
|
+
test('a declaration records the queue and the jobs it accepts', () => {
|
|
11
|
+
const config = declareQueue(cfg(), 'work', ['story:code', 'story:gate'])
|
|
12
|
+
|
|
13
|
+
expect(queueConfigOf(config).queueOf('work').jobs).toEqual(['story:code', 'story:gate'])
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Declaring runs from a helper that an application may call once per app it composes, so the
|
|
18
|
+
* second call must not leave two entries under one name — a duplicate would make `queueOf`
|
|
19
|
+
* answer with whichever happened to be first.
|
|
20
|
+
*/
|
|
21
|
+
test('re-declaring the same name replaces rather than appends', () => {
|
|
22
|
+
let config = declareQueue(cfg(), 'work', ['story:code'])
|
|
23
|
+
config = declareQueue(config, 'work', ['story:code', 'story:gate'])
|
|
24
|
+
|
|
25
|
+
expect(config.queue?.queues).toHaveLength(1)
|
|
26
|
+
expect(queueConfigOf(config).queueOf('work').jobs).toHaveLength(2)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
test('asking for a queue nobody declared is an error, not undefined', () => {
|
|
30
|
+
expect(() => queueConfigOf(cfg()).queueOf('missing')).toThrow(UnknownQueue)
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
test('a job name resolves back to the queue that accepts it', () => {
|
|
34
|
+
const config = declareQueue(declareQueue(cfg(), 'work', ['story:code']), 'ops', ['files:get'])
|
|
35
|
+
|
|
36
|
+
expect(queueConfigOf(config).queueOfJob('files:get')?.name).toBe('ops')
|
|
37
|
+
expect(queueConfigOf(config).queueOfJob('nothing:declared')).toBeUndefined()
|
|
38
|
+
})
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
describe('listening', () => {
|
|
42
|
+
/**
|
|
43
|
+
* Declaring a queue must NOT make this process consume it — that separation is what lets one
|
|
44
|
+
* binary deploy as a producer in one place and a worker in another.
|
|
45
|
+
*/
|
|
46
|
+
test('declaring a queue does not make the process listen to it', () => {
|
|
47
|
+
const config = declareQueue(cfg(), 'work', ['story:code'])
|
|
48
|
+
|
|
49
|
+
expect(queueConfigOf(config).isListening('work')).toBe(false)
|
|
50
|
+
expect(config.queue?.listen ?? []).toHaveLength(0)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
test('listening is additive and free of duplicates', () => {
|
|
54
|
+
let config = listenQueues(cfg(), 'work')
|
|
55
|
+
config = listenQueues(config, 'work', 'ops')
|
|
56
|
+
|
|
57
|
+
expect(config.queue?.listen).toEqual(['work', 'ops'])
|
|
58
|
+
expect(queueConfigOf(config).isListening('ops')).toBe(true)
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('listening and declaring do not overwrite each other', () => {
|
|
62
|
+
const config = declareQueue(listenQueues(cfg(), 'work'), 'work', ['story:code'])
|
|
63
|
+
|
|
64
|
+
expect(queueConfigOf(config).isListening('work')).toBe(true)
|
|
65
|
+
expect(queueConfigOf(config).queueOf('work').jobs).toEqual(['story:code'])
|
|
66
|
+
})
|
|
67
|
+
})
|