@owlmeans/queue 0.1.18-rc.2 → 0.1.18-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -8
- package/agent-meta/manifest.json +2 -2
- package/agent-meta/skills/queue/SKILL.md +198 -12
- package/build/bridge.d.ts +30 -0
- package/build/bridge.d.ts.map +1 -0
- package/build/bridge.js +108 -0
- package/build/bridge.js.map +1 -0
- package/build/config.d.ts +22 -0
- package/build/config.d.ts.map +1 -0
- package/build/config.js +41 -0
- package/build/config.js.map +1 -0
- package/build/consts.d.ts +33 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +35 -0
- package/build/consts.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +90 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +8 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +8 -1
- package/build/index.js.map +1 -1
- package/build/protocol.d.ts +11 -0
- package/build/protocol.d.ts.map +1 -0
- package/build/protocol.js +59 -0
- package/build/protocol.js.map +1 -0
- package/build/transport.d.ts +15 -0
- package/build/transport.d.ts.map +1 -0
- package/build/transport.js +62 -0
- package/build/transport.js.map +1 -0
- package/build/types.d.ts +248 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/worker.d.ts +27 -0
- package/build/worker.d.ts.map +1 -0
- package/build/worker.js +58 -0
- package/build/worker.js.map +1 -0
- package/package.json +12 -2
- package/src/bridge.ts +126 -0
- package/src/config.ts +56 -0
- package/src/consts.ts +39 -0
- package/src/errors.ts +106 -0
- package/src/index.ts +8 -0
- package/src/protocol.ts +81 -0
- package/src/transport.ts +82 -0
- package/src/types.ts +260 -0
- package/src/worker.ts +72 -0
- package/tests/bridge.spec.ts +64 -0
- package/tests/config.spec.ts +66 -0
- package/tests/protocol.spec.ts +58 -0
- package/tsconfig.json +1 -0
- package/build/.gitkeep +0 -0
package/src/errors.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { ResilientError } from '@owlmeans/error'
|
|
2
|
+
|
|
3
|
+
export class QueueError extends ResilientError {
|
|
4
|
+
public static override typeName = 'QueueError'
|
|
5
|
+
|
|
6
|
+
constructor(message: string = 'error') {
|
|
7
|
+
super(QueueError.typeName, `queue:${message}`)
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The caller stopped waiting. The job is untouched — it may still be queued, running, or already
|
|
13
|
+
* finished; only the wait ended. Read the job back to learn which.
|
|
14
|
+
*/
|
|
15
|
+
export class QueueTimeout extends QueueError {
|
|
16
|
+
public static override typeName = `${QueueError.typeName}Timeout`
|
|
17
|
+
|
|
18
|
+
constructor(msg: string) {
|
|
19
|
+
super(`timeout:${msg}`)
|
|
20
|
+
this.type = QueueTimeout.typeName
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** No job with that id in this queue — it never existed, or it was removed after completing. */
|
|
25
|
+
export class UnknownJob extends QueueError {
|
|
26
|
+
public static override typeName = `${QueueError.typeName}UnknownJob`
|
|
27
|
+
|
|
28
|
+
constructor(msg: string) {
|
|
29
|
+
super(`unknown-job:${msg}`)
|
|
30
|
+
this.type = UnknownJob.typeName
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* A job name the queue does not declare. Declaring every job a queue accepts is what keeps an
|
|
36
|
+
* enqueue from succeeding into a queue where nothing can ever process it.
|
|
37
|
+
*/
|
|
38
|
+
export class UnknownJobName extends QueueError {
|
|
39
|
+
public static override typeName = `${QueueError.typeName}UnknownJobName`
|
|
40
|
+
|
|
41
|
+
constructor(msg: string) {
|
|
42
|
+
super(`unknown-job-name:${msg}`)
|
|
43
|
+
this.type = UnknownJobName.typeName
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** A queue asked for that no declaration names. */
|
|
48
|
+
export class UnknownQueue extends QueueError {
|
|
49
|
+
public static override typeName = `${QueueError.typeName}UnknownQueue`
|
|
50
|
+
|
|
51
|
+
constructor(msg: string) {
|
|
52
|
+
super(`unknown-queue:${msg}`)
|
|
53
|
+
this.type = UnknownQueue.typeName
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* This process was asked to consume a queue it does not listen to. Which queues a process
|
|
59
|
+
* consumes is configuration (`cfg.queue.listen`), so this is a deployment question, not a code
|
|
60
|
+
* one — the same binary is a producer or a worker depending on it.
|
|
61
|
+
*/
|
|
62
|
+
export class QueueNotListening extends QueueError {
|
|
63
|
+
public static override typeName = `${QueueError.typeName}NotListening`
|
|
64
|
+
|
|
65
|
+
constructor(msg: string) {
|
|
66
|
+
super(`not-listening:${msg}`)
|
|
67
|
+
this.type = QueueNotListening.typeName
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The job named an entrypoint this process does not serve. Unlike a missing queue this is not a
|
|
73
|
+
* declaration problem: the alias exists, but nothing here bound it, so the job was taken by a
|
|
74
|
+
* worker that cannot run it.
|
|
75
|
+
*/
|
|
76
|
+
export class JobNotServed extends QueueError {
|
|
77
|
+
public static override typeName = `${QueueError.typeName}JobNotServed`
|
|
78
|
+
|
|
79
|
+
constructor(msg: string) {
|
|
80
|
+
super(`job-not-served:${msg}`)
|
|
81
|
+
this.type = JobNotServed.typeName
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The signed envelope is older than the configured lifetime. Freshness is judged from when the
|
|
87
|
+
* producer enqueued the job, not from when a worker picked it up, so a long backlog does not
|
|
88
|
+
* invalidate work that was legitimate when it was submitted.
|
|
89
|
+
*/
|
|
90
|
+
export class EnvelopeExpired extends QueueError {
|
|
91
|
+
public static override typeName = `${QueueError.typeName}EnvelopeExpired`
|
|
92
|
+
|
|
93
|
+
constructor(msg: string) {
|
|
94
|
+
super(`envelope-expired:${msg}`)
|
|
95
|
+
this.type = EnvelopeExpired.typeName
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
ResilientError.registerErrorClass(QueueError)
|
|
100
|
+
ResilientError.registerErrorClass(QueueTimeout)
|
|
101
|
+
ResilientError.registerErrorClass(UnknownJob)
|
|
102
|
+
ResilientError.registerErrorClass(UnknownJobName)
|
|
103
|
+
ResilientError.registerErrorClass(UnknownQueue)
|
|
104
|
+
ResilientError.registerErrorClass(QueueNotListening)
|
|
105
|
+
ResilientError.registerErrorClass(JobNotServed)
|
|
106
|
+
ResilientError.registerErrorClass(EnvelopeExpired)
|
package/src/index.ts
CHANGED
package/src/protocol.ts
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import type { EntrypointProtocolDeclaration, RequestOf, ResponseOf } from '@owlmeans/entrypoint'
|
|
2
|
+
import { isEntrypointProtocol } from '@owlmeans/entrypoint'
|
|
3
|
+
import { ResilientError } from '@owlmeans/error'
|
|
4
|
+
import { RouteProtocols } from '@owlmeans/route'
|
|
5
|
+
import { queueOf } from './config.js'
|
|
6
|
+
import { UnknownQueue } from './errors.js'
|
|
7
|
+
import type {
|
|
8
|
+
Config, Context, JobEnvelope, JobOptions, JobRecord, JobReply, QueueAppend,
|
|
9
|
+
} from './types.js'
|
|
10
|
+
|
|
11
|
+
type QueueContext = Context<Config> & QueueAppend
|
|
12
|
+
|
|
13
|
+
const queueFor = (context: QueueContext, declaration: unknown): {
|
|
14
|
+
protocol: EntrypointProtocolDeclaration
|
|
15
|
+
queue: string
|
|
16
|
+
} => {
|
|
17
|
+
if (declaration == null || typeof declaration !== 'object' || !isEntrypointProtocol(declaration)) {
|
|
18
|
+
throw new UnknownQueue('protocol declaration required')
|
|
19
|
+
}
|
|
20
|
+
if (declaration.route.route.protocol !== RouteProtocols.QUEUE) {
|
|
21
|
+
throw new UnknownQueue(`${declaration.alias}: not a QUEUE protocol`)
|
|
22
|
+
}
|
|
23
|
+
const queue = declaration.route.route.queue
|
|
24
|
+
if (queue == null) {
|
|
25
|
+
throw new UnknownQueue(`${declaration.alias}: route declares no queue`)
|
|
26
|
+
}
|
|
27
|
+
const configured = queueOf(context.cfg, queue)
|
|
28
|
+
if (!configured.jobs.includes(declaration.alias)) {
|
|
29
|
+
throw new UnknownQueue(`${queue}:${declaration.alias}`)
|
|
30
|
+
}
|
|
31
|
+
return { protocol: declaration, queue }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Enqueue an immutable QUEUE protocol while preserving its exact request and response types. */
|
|
35
|
+
export const enqueueProtocol = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
36
|
+
context: QueueContext,
|
|
37
|
+
declaration: Protocol,
|
|
38
|
+
request: RequestOf<Protocol>,
|
|
39
|
+
options?: JobOptions,
|
|
40
|
+
): Promise<JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>> => {
|
|
41
|
+
const { protocol, queue } = queueFor(context, declaration)
|
|
42
|
+
const shaped = (request ?? {}) as unknown as Record<string, unknown>
|
|
43
|
+
const envelope: JobEnvelope = {
|
|
44
|
+
alias: protocol.alias,
|
|
45
|
+
params: shaped.params as Record<string, unknown> | undefined,
|
|
46
|
+
body: shaped.body,
|
|
47
|
+
query: shaped.query as Record<string, unknown> | undefined,
|
|
48
|
+
headers: shaped.headers as Record<string, string | undefined> | undefined,
|
|
49
|
+
enqueuedAt: new Date().toISOString(),
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return await context.jobs<JobEnvelope, JobReply<ResponseOf<Protocol>>>(queue).create({
|
|
53
|
+
queue,
|
|
54
|
+
name: protocol.alias,
|
|
55
|
+
data: envelope,
|
|
56
|
+
opts: options,
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Wait for a protocol job and unwrap the typed entrypoint reply. */
|
|
61
|
+
export const waitForProtocol = async <Protocol extends EntrypointProtocolDeclaration>(
|
|
62
|
+
context: QueueContext,
|
|
63
|
+
declaration: Protocol,
|
|
64
|
+
job: string | JobRecord<JobEnvelope, JobReply<ResponseOf<Protocol>>>,
|
|
65
|
+
options?: { timeout?: number },
|
|
66
|
+
): Promise<ResponseOf<Protocol>> => {
|
|
67
|
+
const { protocol, queue } = queueFor(context, declaration)
|
|
68
|
+
if (typeof job !== 'string' && (job.queue !== queue || job.name !== protocol.alias)) {
|
|
69
|
+
throw new UnknownQueue(`${job.queue}:${job.name}`)
|
|
70
|
+
}
|
|
71
|
+
const id = typeof job === 'string' ? job : job.id
|
|
72
|
+
if (id == null) {
|
|
73
|
+
throw new UnknownQueue(`${queue}:${protocol.alias}: missing job id`)
|
|
74
|
+
}
|
|
75
|
+
const reply = await context.jobs<JobEnvelope, JobReply<ResponseOf<Protocol>>>(queue)
|
|
76
|
+
.wait(id, options)
|
|
77
|
+
if (reply.error != null) {
|
|
78
|
+
throw ResilientError.ensure(reply.error as Error | string)
|
|
79
|
+
}
|
|
80
|
+
return reply.value as ResponseOf<Protocol>
|
|
81
|
+
}
|
package/src/transport.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { assertContext, createService } from '@owlmeans/context'
|
|
2
|
+
import type { AbstractRequest, AbstractResponse, EntrypointTransport } from '@owlmeans/entrypoint'
|
|
3
|
+
import { EntrypointOutcome, transportAlias } from '@owlmeans/entrypoint'
|
|
4
|
+
import { RouteProtocols } from '@owlmeans/route'
|
|
5
|
+
import { ResilientError } from '@owlmeans/error'
|
|
6
|
+
import type { CommonEntrypoint } from '@owlmeans/entrypoint'
|
|
7
|
+
import type { Config, Context, JobEnvelope, JobReply, QueueAppend } from './types.js'
|
|
8
|
+
import { DEFAULT_JOB_TIMEOUT } from './consts.js'
|
|
9
|
+
import { UnknownQueue } from './errors.js'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The producing half of the bridge: it turns a call on a QUEUE entrypoint into a job, and the
|
|
13
|
+
* job's outcome back into a reply.
|
|
14
|
+
*
|
|
15
|
+
* Registering it is the whole opt-in. `apiHandler` looks up a transport by the route's protocol,
|
|
16
|
+
* so a consumer writes `ep.call(...)` against a declaration it shares with every other kind of
|
|
17
|
+
* entrypoint and never learns which one carried it. That is also why the envelope is built from
|
|
18
|
+
* the request rather than from the entrypoint: the far side rebuilds a request, and anything only
|
|
19
|
+
* HTTP understands would not survive the trip anyway.
|
|
20
|
+
*/
|
|
21
|
+
export const makeQueueTransport = (alias: string = transportAlias(RouteProtocols.QUEUE)) => {
|
|
22
|
+
const location = `queue-transport:${alias}`
|
|
23
|
+
|
|
24
|
+
const service = createService<EntrypointTransport>(alias, {
|
|
25
|
+
protocol: RouteProtocols.QUEUE,
|
|
26
|
+
|
|
27
|
+
handle: (async <T>(req: AbstractRequest, res: AbstractResponse<T>) => {
|
|
28
|
+
const context = assertContext<Config, Context<Config>>(service.ctx as Context<Config>, location)
|
|
29
|
+
const entrypoint = context.entrypoint<CommonEntrypoint>(req.alias)
|
|
30
|
+
const route = entrypoint.route.route
|
|
31
|
+
|
|
32
|
+
if (route.queue == null) {
|
|
33
|
+
throw new UnknownQueue(`${req.alias}: route declares no queue`)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const jobs = (context as unknown as QueueAppend).jobs<JobEnvelope, JobReply<T>>(route.queue)
|
|
37
|
+
|
|
38
|
+
const envelope: JobEnvelope = {
|
|
39
|
+
alias: req.alias,
|
|
40
|
+
params: req.params as Record<string, unknown>,
|
|
41
|
+
body: req.body,
|
|
42
|
+
query: req.query as Record<string, unknown>,
|
|
43
|
+
headers: req.headers as Record<string, string | undefined>,
|
|
44
|
+
enqueuedAt: new Date().toISOString(),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const job = await jobs.create({ queue: route.queue, name: req.alias, data: envelope })
|
|
48
|
+
|
|
49
|
+
// `reply: false` says the caller is not waiting for the work, only for the promise that it
|
|
50
|
+
// was taken. `Accepted` plus the job's identity is the whole answer — enough to watch it.
|
|
51
|
+
if (route.reply === false) {
|
|
52
|
+
res.resolve({ id: job.id, queue: route.queue } as T, EntrypointOutcome.Accepted)
|
|
53
|
+
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const reply = await jobs.wait(job.id as string, {
|
|
58
|
+
timeout: req.timeout ?? route.timeout ?? DEFAULT_JOB_TIMEOUT
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
// A processor reports a domain failure by returning it, so it crosses the hop as the class
|
|
62
|
+
// it was thrown as rather than as a string a caller would have to parse.
|
|
63
|
+
if (reply?.error != null) {
|
|
64
|
+
throw ResilientError.ensure(reply.error as Error | string)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
res.resolve(reply?.value as T, (reply?.outcome as EntrypointOutcome) ?? EntrypointOutcome.Ok)
|
|
68
|
+
}) as EntrypointTransport['handle']
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
return service
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export const appendQueueTransport = <C extends Config, T extends Context<C> = Context<C>>(
|
|
75
|
+
context: T, alias?: string
|
|
76
|
+
): T => {
|
|
77
|
+
const service = makeQueueTransport(alias)
|
|
78
|
+
|
|
79
|
+
context.registerService(service)
|
|
80
|
+
|
|
81
|
+
return context
|
|
82
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import type { BasicConfig, BasicContext, InitializedService } from '@owlmeans/context'
|
|
2
|
+
import type { PubSubResource, Resource, ResourceRecord } from '@owlmeans/resource'
|
|
3
|
+
import type { AbstractRequest, AbstractResponse } from '@owlmeans/entrypoint'
|
|
4
|
+
import type { JobEventType, JobState } from './consts.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Everything the monorepo knows about its queues, declared in the shared backend package so that
|
|
8
|
+
* producer and consumer read the same list. A process becomes a worker by naming queues in
|
|
9
|
+
* `listen` — nothing in a declaration decides where it runs.
|
|
10
|
+
*/
|
|
11
|
+
export interface QueueConfig {
|
|
12
|
+
queues?: QueueDeclaration[]
|
|
13
|
+
/** The queues THIS process consumes. Absent or empty means producer only. */
|
|
14
|
+
listen?: string[]
|
|
15
|
+
/** Which db service carries the queues. Defaults to the queue service's own alias. */
|
|
16
|
+
db?: string
|
|
17
|
+
defaults?: JobOptions
|
|
18
|
+
/**
|
|
19
|
+
* How long a signed job envelope stays acceptable, in seconds. It bounds both the freshness
|
|
20
|
+
* check and the nonce cache, so a job delayed behind a long backlog is still admitted.
|
|
21
|
+
*/
|
|
22
|
+
envelopeTtl?: number
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface QueueDeclaration {
|
|
26
|
+
name: string
|
|
27
|
+
/**
|
|
28
|
+
* The job names this queue accepts. A job the queue does not declare is refused at enqueue
|
|
29
|
+
* time rather than becoming a job nothing can process.
|
|
30
|
+
*/
|
|
31
|
+
jobs: string[]
|
|
32
|
+
worker?: QueueWorkerOptions
|
|
33
|
+
defaults?: JobOptions
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface QueueWorkerOptions {
|
|
37
|
+
concurrency?: number
|
|
38
|
+
/**
|
|
39
|
+
* How long a job may hold its lock without reporting progress. A processor that runs longer
|
|
40
|
+
* than this between `touch()` calls is treated as stalled and its job is handed to someone else.
|
|
41
|
+
*/
|
|
42
|
+
lockDuration?: number
|
|
43
|
+
stalledInterval?: number
|
|
44
|
+
maxStalledCount?: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface JobOptions {
|
|
48
|
+
attempts?: number
|
|
49
|
+
backoff?: { type: 'fixed' | 'exponential', delay: number }
|
|
50
|
+
delay?: number
|
|
51
|
+
priority?: number
|
|
52
|
+
/**
|
|
53
|
+
* A caller-chosen identity for the job. Enqueueing the same id twice while the first is still
|
|
54
|
+
* around is a no-op, which is what makes an admission step safe to retry.
|
|
55
|
+
*/
|
|
56
|
+
id?: string
|
|
57
|
+
removeOnComplete?: boolean | number
|
|
58
|
+
removeOnFail?: boolean | number
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* A job as a RECORD — the same shape the resource contract reads and writes, so inspecting a
|
|
63
|
+
* queue is `list`/`get`/`count` rather than a broker-specific API.
|
|
64
|
+
*/
|
|
65
|
+
export interface JobRecord<D = unknown, R = unknown> extends ResourceRecord {
|
|
66
|
+
queue: string
|
|
67
|
+
name: string
|
|
68
|
+
data: D
|
|
69
|
+
state?: JobState
|
|
70
|
+
result?: R
|
|
71
|
+
error?: string
|
|
72
|
+
attempts?: number
|
|
73
|
+
progress?: unknown
|
|
74
|
+
parentId?: string
|
|
75
|
+
createdAt?: string
|
|
76
|
+
startedAt?: string
|
|
77
|
+
finishedAt?: string
|
|
78
|
+
opts?: JobOptions
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface JobEvent<R = unknown> {
|
|
82
|
+
type: JobEventType
|
|
83
|
+
id: string
|
|
84
|
+
queue: string
|
|
85
|
+
/**
|
|
86
|
+
* Empty when the job is already gone. A broker's completion event carries only an id, so the
|
|
87
|
+
* name is read back from the job — and a queue that removes jobs on completion has nothing left
|
|
88
|
+
* to read. Subscribe for the id and the outcome; do not branch on the name.
|
|
89
|
+
*/
|
|
90
|
+
name: string
|
|
91
|
+
result?: R
|
|
92
|
+
error?: string
|
|
93
|
+
progress?: unknown
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** A node of a job graph: children run first, siblings run together. */
|
|
97
|
+
export interface FlowSpec<D = unknown> {
|
|
98
|
+
name: string
|
|
99
|
+
queue?: string
|
|
100
|
+
data: D
|
|
101
|
+
opts?: JobOptions
|
|
102
|
+
children?: FlowSpec[]
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* One queue, addressed as a resource.
|
|
107
|
+
*
|
|
108
|
+
* `create` enqueues, `get`/`load`/`list`/`count` inspect, `delete`/`take` cancel. What the base
|
|
109
|
+
* contract cannot express — waiting for an answer, submitting a graph, releasing the connection —
|
|
110
|
+
* is added here.
|
|
111
|
+
*
|
|
112
|
+
* Three places where the base `Resource` contract reads differently against a broker, because a
|
|
113
|
+
* job is not a row:
|
|
114
|
+
* - `create` does NOT throw `RecordExists` for a repeated `opts.id`. Re-enqueueing an id that is
|
|
115
|
+
* still around returns the existing job, which is the idempotency that makes an admission step
|
|
116
|
+
* safe to retry.
|
|
117
|
+
* - `WriteOptions.ttl` is refused. A job's lifetime is `removeOnComplete`/`removeOnFail` —
|
|
118
|
+
* explicit deletion once it has finished, not expiry while it waits.
|
|
119
|
+
* - `update`/`save` reach only the job's `data`. Everything else about a job belongs to the
|
|
120
|
+
* broker, and an implementation refuses rather than pretending to write it.
|
|
121
|
+
*/
|
|
122
|
+
export interface QueueResource<D = unknown, R = unknown>
|
|
123
|
+
extends Resource<JobRecord<D, R>>, PubSubResource<JobEvent<R>> {
|
|
124
|
+
queue: string
|
|
125
|
+
/**
|
|
126
|
+
* Release the connections this queue holds.
|
|
127
|
+
*
|
|
128
|
+
* Watching for completions means a blocking connection, and a process that never releases it
|
|
129
|
+
* does not exit. Nothing else in the resource contract can express that, so it lives here and a
|
|
130
|
+
* producer calls it on shutdown.
|
|
131
|
+
*/
|
|
132
|
+
close: () => Promise<void>
|
|
133
|
+
/**
|
|
134
|
+
* Wait for a job's return value.
|
|
135
|
+
* @throws {QueueTimeout} when the wait elapses — the job itself is left alone.
|
|
136
|
+
* @throws the processor's own error, rebuilt as its original class.
|
|
137
|
+
*/
|
|
138
|
+
wait: (id: string, opts?: { timeout?: number }) => Promise<R>
|
|
139
|
+
/** Submit a graph and get its root back. Children complete before their parent starts. */
|
|
140
|
+
flow: (root: FlowSpec<D>) => Promise<JobRecord<D, R>>
|
|
141
|
+
counts: () => Promise<Record<JobState, number>>
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* What a processor is handed. `data` is the job's payload; everything else is the means to stay
|
|
146
|
+
* alive and to read what the children reported.
|
|
147
|
+
*/
|
|
148
|
+
export interface JobContext<D = unknown> {
|
|
149
|
+
id: string
|
|
150
|
+
name: string
|
|
151
|
+
queue: string
|
|
152
|
+
attempt: number
|
|
153
|
+
data: D
|
|
154
|
+
signal: AbortSignal
|
|
155
|
+
/**
|
|
156
|
+
* Renew the lock. A processor that runs a long loop MUST call it as it goes: the broker judges
|
|
157
|
+
* liveness by the lock, so silence for longer than `lockDuration` is indistinguishable from a
|
|
158
|
+
* dead worker, and the job is re-run somewhere else while this one is still working on it.
|
|
159
|
+
*/
|
|
160
|
+
touch: () => Promise<void>
|
|
161
|
+
progress: (value: unknown) => Promise<void>
|
|
162
|
+
/**
|
|
163
|
+
* What the children returned, keyed by child job id.
|
|
164
|
+
*
|
|
165
|
+
* A broker keys them by its own fully qualified job key; an implementation re-keys to the bare
|
|
166
|
+
* id, so two children of one parent that live in DIFFERENT queues and happen to share an id
|
|
167
|
+
* collide here. Give the children of a cross-queue flow distinct ids.
|
|
168
|
+
*/
|
|
169
|
+
children: <T = unknown>() => Promise<Record<string, T>>
|
|
170
|
+
failedChildren: () => Promise<Record<string, string>>
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export interface JobProcessor<D = unknown, R = unknown> {
|
|
174
|
+
(job: JobContext<D>): Promise<R>
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* The consuming half. It is registered only in processes that consume: `start()` binds the
|
|
179
|
+
* queues named in `cfg.listen` and dispatches by job name, to a processor registered here or to
|
|
180
|
+
* an entrypoint served in this process.
|
|
181
|
+
*/
|
|
182
|
+
export interface QueueWorkerService extends InitializedService {
|
|
183
|
+
process: <D, R>(queue: string, name: string, processor: JobProcessor<D, R>) => void
|
|
184
|
+
start: () => Promise<void>
|
|
185
|
+
stop: () => Promise<void>
|
|
186
|
+
/** The queues this process actually consumes. Empty in a producer. */
|
|
187
|
+
listening: () => string[]
|
|
188
|
+
/**
|
|
189
|
+
* Bind the application's reactions to job lifecycle events. Registering more than once merges,
|
|
190
|
+
* so an application composed of several parts can each contribute what it cares about.
|
|
191
|
+
*
|
|
192
|
+
* It belongs on the contract rather than on a driver because `onJobDead` is where an
|
|
193
|
+
* application COMPENSATES — releases the lock the admission step took, marks the record failed —
|
|
194
|
+
* and that behaviour must not have to be rewritten when the broker changes.
|
|
195
|
+
*/
|
|
196
|
+
hooks: (hooks: QueueHooks) => void
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* The envelope a queued entrypoint call travels in. It is the request, minus everything that only
|
|
201
|
+
* means something to HTTP — the transport rebuilds a request from it on the far side.
|
|
202
|
+
*/
|
|
203
|
+
export interface JobEnvelope {
|
|
204
|
+
alias: string
|
|
205
|
+
params?: Record<string, unknown>
|
|
206
|
+
body?: unknown
|
|
207
|
+
query?: Record<string, unknown>
|
|
208
|
+
headers?: Record<string, string | undefined>
|
|
209
|
+
/** When the producer enqueued it — freshness is judged here, not at processing time. */
|
|
210
|
+
enqueuedAt?: string
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export interface QueueJobMeta {
|
|
214
|
+
id: string
|
|
215
|
+
name: string
|
|
216
|
+
queue: string
|
|
217
|
+
attempt: number
|
|
218
|
+
/** Renew the broker lock for protocol handlers whose work can outlive one lock duration. */
|
|
219
|
+
touch: () => Promise<void>
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** What a processor returns for an entrypoint job, so the caller can rebuild the reply. */
|
|
223
|
+
export interface JobReply<T = unknown> {
|
|
224
|
+
value?: T
|
|
225
|
+
outcome?: string
|
|
226
|
+
error?: unknown
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export interface QueueHooks {
|
|
230
|
+
wrapHandler?: (job: JobContext, next: () => Promise<unknown>) => Promise<unknown>
|
|
231
|
+
onJobResult?: (event: JobEvent) => void | Promise<void>
|
|
232
|
+
onJobStalled?: (meta: { id: string, queue: string, name: string }, phase: 'stalled' | 'failed') => void | Promise<void>
|
|
233
|
+
/**
|
|
234
|
+
* The job is finished for good — retries exhausted, or it was dropped. This is where an
|
|
235
|
+
* application compensates: release the lock the admission step took, mark the record failed.
|
|
236
|
+
*/
|
|
237
|
+
onJobDead?: (job: JobRecord, reason: string) => void | Promise<void>
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export interface QueueDriver {
|
|
241
|
+
resource: <D, R>(queue: string) => QueueResource<D, R>
|
|
242
|
+
worker: () => QueueWorkerService
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export interface Config extends BasicConfig {
|
|
246
|
+
queue?: QueueConfig
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export interface Context<C extends Config = Config> extends BasicContext<C> {
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export interface QueueAppend {
|
|
253
|
+
jobs: <D = unknown, R = unknown>(queue?: string) => QueueResource<D, R>
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export interface QueueTransportRequest extends AbstractRequest {
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export interface QueueTransportResponse extends AbstractResponse<unknown> {
|
|
260
|
+
}
|
package/src/worker.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { Middleware } from '@owlmeans/context'
|
|
2
|
+
import { MiddlewareStage, MiddlewareType, AppType } from '@owlmeans/context'
|
|
3
|
+
import type { CommonEntrypoint } from '@owlmeans/entrypoint'
|
|
4
|
+
import { RouteProtocols } from '@owlmeans/route'
|
|
5
|
+
import type { Config, Context, JobContext, JobEnvelope, JobProcessor, QueueWorkerService } from './types.js'
|
|
6
|
+
import { handleJob } from './bridge.js'
|
|
7
|
+
import { DEFAULT_ALIAS } from './consts.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Every queued entrypoint this process both SERVES and LISTENS to, grouped by queue.
|
|
11
|
+
*
|
|
12
|
+
* Both halves are required and they answer different questions. Serving is about code — the alias
|
|
13
|
+
* was bound here, so a handler exists. Listening is about deployment — this process was
|
|
14
|
+
* configured to consume that queue. A worker that bound queues by what it can serve would make
|
|
15
|
+
* every deployment of the same binary a worker for everything it happens to import.
|
|
16
|
+
*/
|
|
17
|
+
export const servedJobs = <C extends Config, T extends Context<C>>(
|
|
18
|
+
context: T
|
|
19
|
+
): Map<string, CommonEntrypoint[]> => {
|
|
20
|
+
const listen = context.cfg.queue?.listen ?? []
|
|
21
|
+
const served = new Map<string, CommonEntrypoint[]>()
|
|
22
|
+
|
|
23
|
+
context.entrypoints<CommonEntrypoint>().forEach(entrypoint => {
|
|
24
|
+
const route = entrypoint.route.route
|
|
25
|
+
if (route.type !== AppType.Backend || route.protocol !== RouteProtocols.QUEUE) {
|
|
26
|
+
return
|
|
27
|
+
}
|
|
28
|
+
if (route.service != null && route.service !== context.cfg.service) {
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
if (entrypoint.handle == null || route.queue == null || !listen.includes(route.queue)) {
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
served.set(route.queue, [...(served.get(route.queue) ?? []), entrypoint])
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
return served
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A processor that runs a queued entrypoint call. The driver dispatches by job name, and an
|
|
43
|
+
* entrypoint job's name IS its alias — which is what lets one worker carry both entrypoint jobs
|
|
44
|
+
* and the internal steps an application registers with `process()`.
|
|
45
|
+
*/
|
|
46
|
+
export const entrypointProcessor = <C extends Config, T extends Context<C>>(
|
|
47
|
+
context: T
|
|
48
|
+
): JobProcessor<JobEnvelope, unknown> =>
|
|
49
|
+
async (job: JobContext<JobEnvelope>) => await handleJob(context, job)
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Start the worker once the context is ready.
|
|
53
|
+
*
|
|
54
|
+
* Ready stage, not Loading: processors are registered while the application wires itself up, and
|
|
55
|
+
* binding the queues before that finished would take jobs this process cannot yet run. A process
|
|
56
|
+
* that listens to nothing registers no worker at all, so a producer pays nothing for this.
|
|
57
|
+
*/
|
|
58
|
+
export const queueWorkerMiddleware = (alias: string = DEFAULT_ALIAS): Middleware => ({
|
|
59
|
+
type: MiddlewareType.Context,
|
|
60
|
+
stage: MiddlewareStage.Ready,
|
|
61
|
+
apply: async context => {
|
|
62
|
+
const ctx = context as unknown as Context<Config>
|
|
63
|
+
if ((ctx.cfg.queue?.listen ?? []).length === 0) {
|
|
64
|
+
return
|
|
65
|
+
}
|
|
66
|
+
if (!ctx.hasService(alias)) {
|
|
67
|
+
return
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
await ctx.service<QueueWorkerService>(alias).start()
|
|
71
|
+
}
|
|
72
|
+
})
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { describe, test, expect } from 'bun:test'
|
|
2
|
+
import { assertFresh, requestOf } from '../src/bridge.js'
|
|
3
|
+
import { EnvelopeExpired } from '../src/errors.js'
|
|
4
|
+
import type { JobEnvelope } from '../src/types.js'
|
|
5
|
+
|
|
6
|
+
const envelope = (enqueuedAt?: string): JobEnvelope => ({
|
|
7
|
+
alias: 'story:code', params: { id: '7' }, body: { prompt: 'x' }, headers: {}, query: {}, enqueuedAt
|
|
8
|
+
})
|
|
9
|
+
|
|
10
|
+
describe('rebuilding the request', () => {
|
|
11
|
+
test('the envelope becomes an ordinary request', () => {
|
|
12
|
+
const request = requestOf(envelope(), '/story/:id/code')
|
|
13
|
+
|
|
14
|
+
expect(request.alias).toBe('story:code')
|
|
15
|
+
expect(request.params).toEqual({ id: '7' })
|
|
16
|
+
expect(request.body).toEqual({ prompt: 'x' })
|
|
17
|
+
expect(request.path).toBe('/story/:id/code')
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A producer may omit any of the optional parts. The handler on the far side reads them without
|
|
22
|
+
* checking, so they have to arrive as empty containers rather than as undefined.
|
|
23
|
+
*/
|
|
24
|
+
test('absent parts arrive as empty, never undefined', () => {
|
|
25
|
+
const request = requestOf({ alias: 'story:code' }, '/x')
|
|
26
|
+
|
|
27
|
+
expect(request.params).toEqual({})
|
|
28
|
+
expect(request.headers).toEqual({})
|
|
29
|
+
expect(request.query).toEqual({})
|
|
30
|
+
})
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
describe('envelope freshness', () => {
|
|
34
|
+
test('a fresh envelope passes', () => {
|
|
35
|
+
expect(() => assertFresh(envelope(new Date().toISOString()), 60)).not.toThrow()
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
test('an envelope older than the window is refused', () => {
|
|
39
|
+
const old = new Date(Date.now() - 120_000).toISOString()
|
|
40
|
+
|
|
41
|
+
expect(() => assertFresh(envelope(old), 60)).toThrow(EnvelopeExpired)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The window is judged from when the job was ENQUEUED. A job that waited behind a long backlog
|
|
46
|
+
* was legitimate when it was produced, and judging it on pickup would reject exactly the work a
|
|
47
|
+
* busy queue delayed — so a generous window has to keep it.
|
|
48
|
+
*/
|
|
49
|
+
test('a long wait in the queue is not what expiry is for', () => {
|
|
50
|
+
const enqueued = new Date(Date.now() - 3600_000).toISOString()
|
|
51
|
+
|
|
52
|
+
expect(() => assertFresh(envelope(enqueued), 86_400)).not.toThrow()
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
test('no configured window means no check', () => {
|
|
56
|
+
const ancient = new Date(0).toISOString()
|
|
57
|
+
|
|
58
|
+
expect(() => assertFresh(envelope(ancient), undefined)).not.toThrow()
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('an envelope without a timestamp is not judged', () => {
|
|
62
|
+
expect(() => assertFresh(envelope(undefined), 60)).not.toThrow()
|
|
63
|
+
})
|
|
64
|
+
})
|