@fullstackhouse/open-mercato-durable-work 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -0
- package/dist/core/errors.js +81 -0
- package/dist/core/errors.js.map +7 -0
- package/dist/core/ids.js +30 -0
- package/dist/core/ids.js.map +7 -0
- package/dist/core/reconciler.js +149 -0
- package/dist/core/reconciler.js.map +7 -0
- package/dist/core/registry.js +72 -0
- package/dist/core/registry.js.map +7 -0
- package/dist/core/run-slice.js +210 -0
- package/dist/core/run-slice.js.map +7 -0
- package/dist/core/schema.js +100 -0
- package/dist/core/schema.js.map +7 -0
- package/dist/core/service.js +161 -0
- package/dist/core/service.js.map +7 -0
- package/dist/core/store.js +516 -0
- package/dist/core/store.js.map +7 -0
- package/dist/core/terminal.js +53 -0
- package/dist/core/terminal.js.map +7 -0
- package/dist/core/types.js +1 -0
- package/dist/core/types.js.map +7 -0
- package/dist/core/worker.js +111 -0
- package/dist/core/worker.js.map +7 -0
- package/dist/index.js +99 -0
- package/dist/index.js.map +7 -0
- package/dist/modules/durable_work/acl.js +10 -0
- package/dist/modules/durable_work/acl.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/[id]/redrive.js +20 -0
- package/dist/modules/durable_work/api/jobs/[id]/redrive.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/[id]/route.js +26 -0
- package/dist/modules/durable_work/api/jobs/[id]/route.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/route.js +24 -0
- package/dist/modules/durable_work/api/jobs/route.js.map +7 -0
- package/dist/modules/durable_work/cli.js +72 -0
- package/dist/modules/durable_work/cli.js.map +7 -0
- package/dist/modules/durable_work/data/entities.js +163 -0
- package/dist/modules/durable_work/data/entities.js.map +7 -0
- package/dist/modules/durable_work/di.js +34 -0
- package/dist/modules/durable_work/di.js.map +7 -0
- package/dist/modules/durable_work/events.js +26 -0
- package/dist/modules/durable_work/events.js.map +7 -0
- package/dist/modules/durable_work/index.js +17 -0
- package/dist/modules/durable_work/index.js.map +7 -0
- package/dist/modules/durable_work/lib/route-helpers.js +66 -0
- package/dist/modules/durable_work/lib/route-helpers.js.map +7 -0
- package/dist/modules/durable_work/migrations/Migration20260908120000.js +17 -0
- package/dist/modules/durable_work/migrations/Migration20260908120000.js.map +7 -0
- package/dist/modules/durable_work/setup.js +12 -0
- package/dist/modules/durable_work/setup.js.map +7 -0
- package/dist/om/config.js +49 -0
- package/dist/om/config.js.map +7 -0
- package/dist/om/progress-mirror.js +49 -0
- package/dist/om/progress-mirror.js.map +7 -0
- package/dist/om/sql-executor-mikro.js +48 -0
- package/dist/om/sql-executor-mikro.js.map +7 -0
- package/dist/transport/bullmq.js +144 -0
- package/dist/transport/bullmq.js.map +7 -0
- package/dist/transport/conformance.js +177 -0
- package/dist/transport/conformance.js.map +7 -0
- package/dist/transport/memory.js +139 -0
- package/dist/transport/memory.js.map +7 -0
- package/dist/transport/pgboss.js +176 -0
- package/dist/transport/pgboss.js.map +7 -0
- package/dist/transport/types.js +1 -0
- package/dist/transport/types.js.map +7 -0
- package/generated/entities/durable_work_job/index.ts +42 -0
- package/generated/entities.ids.generated.ts +9 -0
- package/package.json +145 -0
- package/src/core/__tests__/registry.test.ts +43 -0
- package/src/core/errors.ts +104 -0
- package/src/core/ids.ts +58 -0
- package/src/core/reconciler.ts +242 -0
- package/src/core/registry.ts +199 -0
- package/src/core/run-slice.ts +343 -0
- package/src/core/schema.ts +114 -0
- package/src/core/service.ts +222 -0
- package/src/core/store.ts +786 -0
- package/src/core/terminal.ts +107 -0
- package/src/core/types.ts +120 -0
- package/src/core/worker.ts +169 -0
- package/src/index.ts +100 -0
- package/src/modules/durable_work/__integration__/TC-DW-001.spec.ts +51 -0
- package/src/modules/durable_work/__tests__/metadata.test.ts +13 -0
- package/src/modules/durable_work/__tests__/schema-agreement.test.ts +52 -0
- package/src/modules/durable_work/acl.ts +6 -0
- package/src/modules/durable_work/api/jobs/[id]/redrive.ts +27 -0
- package/src/modules/durable_work/api/jobs/[id]/route.ts +27 -0
- package/src/modules/durable_work/api/jobs/route.ts +26 -0
- package/src/modules/durable_work/cli.ts +91 -0
- package/src/modules/durable_work/data/entities.ts +158 -0
- package/src/modules/durable_work/di.ts +41 -0
- package/src/modules/durable_work/events.ts +30 -0
- package/src/modules/durable_work/index.ts +16 -0
- package/src/modules/durable_work/lib/route-helpers.ts +83 -0
- package/src/modules/durable_work/migrations/Migration20260908120000.ts +24 -0
- package/src/modules/durable_work/setup.ts +10 -0
- package/src/om/__tests__/sql-executor-mikro.test.ts +83 -0
- package/src/om/config.ts +65 -0
- package/src/om/progress-mirror.ts +80 -0
- package/src/om/sql-executor-mikro.ts +104 -0
- package/src/transport/bullmq.ts +213 -0
- package/src/transport/conformance.ts +218 -0
- package/src/transport/memory.ts +191 -0
- package/src/transport/pgboss.ts +250 -0
- package/src/transport/types.ts +81 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// One-way mirror onto core's `progress_jobs`, so the existing progress UI keeps working.
|
|
2
|
+
//
|
|
3
|
+
// The direction matters and is the whole design (ADR 0002): the durable job row is the
|
|
4
|
+
// authority for liveness, and `progress_jobs` is presentation. Nothing here is ever read back
|
|
5
|
+
// to make a decision. A progress row can lag, or be briefly wrong, without any consequence
|
|
6
|
+
// beyond what a user sees for a moment.
|
|
7
|
+
|
|
8
|
+
import type { DurableJob, Scope } from '../core/types'
|
|
9
|
+
|
|
10
|
+
/** The subset of core's ProgressService this bridge uses. Structural rather than imported so
|
|
11
|
+
* the package does not take a hard dependency on a service it only writes to. */
|
|
12
|
+
export type ProgressServiceLike = {
|
|
13
|
+
startJob?(id: string, ctx: unknown): Promise<unknown>
|
|
14
|
+
updateProgress?(id: string, patch: Record<string, unknown>, ctx: unknown): Promise<unknown>
|
|
15
|
+
touchJobHeartbeat?(id: string, ctx: unknown): Promise<unknown>
|
|
16
|
+
completeJob?(id: string, input: Record<string, unknown>, ctx: unknown): Promise<unknown>
|
|
17
|
+
failJob?(id: string, input: Record<string, unknown>, ctx: unknown): Promise<unknown>
|
|
18
|
+
markCancelled?(id: string, ctx: unknown): Promise<unknown>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type ProgressMirror = {
|
|
22
|
+
onStarted(job: DurableJob, scope: Scope): Promise<void>
|
|
23
|
+
onProgress(job: DurableJob, scope: Scope): Promise<void>
|
|
24
|
+
onTerminal(job: DurableJob, scope: Scope): Promise<void>
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Every call is best-effort.
|
|
29
|
+
*
|
|
30
|
+
* A mirror that could fail a job would make the presentation layer able to stop the work,
|
|
31
|
+
* which is exactly backwards. The cost of swallowing is a stale progress card; the cost of
|
|
32
|
+
* not swallowing is a sync run failed by a UI table.
|
|
33
|
+
*/
|
|
34
|
+
export function createProgressMirror(progress: ProgressServiceLike): ProgressMirror {
|
|
35
|
+
const quietly = async (fn: () => Promise<unknown> | undefined) => {
|
|
36
|
+
try {
|
|
37
|
+
await fn()
|
|
38
|
+
} catch {
|
|
39
|
+
/* presentation only — never allowed to affect the job */
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
async onStarted(job, scope) {
|
|
45
|
+
if (!job.progressJobId) return
|
|
46
|
+
await quietly(() => progress.startJob?.(job.progressJobId!, scope))
|
|
47
|
+
},
|
|
48
|
+
async onProgress(job, scope) {
|
|
49
|
+
if (!job.progressJobId) return
|
|
50
|
+
await quietly(() =>
|
|
51
|
+
progress.updateProgress?.(
|
|
52
|
+
job.progressJobId!,
|
|
53
|
+
{
|
|
54
|
+
processedCount: job.processedCount,
|
|
55
|
+
totalCount: job.totalCount ?? undefined,
|
|
56
|
+
// Says why a healthy job looks idle. Core's read path fails a progress row whose
|
|
57
|
+
// heartbeat is older than a minute, and a durable job waiting out a retry backoff
|
|
58
|
+
// legitimately trips that; without this the UI's only story is "it broke".
|
|
59
|
+
message: job.nextRunAt && job.nextRunAt.getTime() > Date.now() ? 'waiting for redelivery' : undefined,
|
|
60
|
+
},
|
|
61
|
+
scope,
|
|
62
|
+
),
|
|
63
|
+
)
|
|
64
|
+
},
|
|
65
|
+
async onTerminal(job, scope) {
|
|
66
|
+
if (!job.progressJobId) return
|
|
67
|
+
if (job.status === 'completed') {
|
|
68
|
+
await quietly(() => progress.completeJob?.(job.progressJobId!, { processedCount: job.processedCount }, scope))
|
|
69
|
+
return
|
|
70
|
+
}
|
|
71
|
+
if (job.status === 'cancelled') {
|
|
72
|
+
await quietly(() => progress.markCancelled?.(job.progressJobId!, scope))
|
|
73
|
+
return
|
|
74
|
+
}
|
|
75
|
+
await quietly(() =>
|
|
76
|
+
progress.failJob?.(job.progressJobId!, { errorMessage: job.errorMessage ?? job.errorCode ?? 'failed' }, scope),
|
|
77
|
+
)
|
|
78
|
+
},
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// Bridges the mechanism's `SqlExecutor` onto a MikroORM EntityManager.
|
|
2
|
+
//
|
|
3
|
+
// The mechanism talks SQL, not ORM, because every statement it issues is a compare-and-set
|
|
4
|
+
// whose predicate is the guarantee. But a host's transaction is a MikroORM one, and a job row
|
|
5
|
+
// that has to commit with a domain row has to be inside it. This adapter is how both are true
|
|
6
|
+
// at once: the caller keeps their EntityManager, the mechanism keeps its statements.
|
|
7
|
+
|
|
8
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
9
|
+
|
|
10
|
+
import type { SqlExecutor, SqlTransactor } from '../core/types'
|
|
11
|
+
|
|
12
|
+
type Connection = {
|
|
13
|
+
execute(sql: string, params?: unknown[], method?: 'all' | 'get' | 'run', ctx?: unknown): Promise<unknown>
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
type EntityManagerLike = EntityManager & { getTransactionContext?(): unknown }
|
|
17
|
+
|
|
18
|
+
/** True when the statement returns rows the caller will read. */
|
|
19
|
+
const RETURNS_ROWS = /\breturning\b|^\s*(select|with)\b/i
|
|
20
|
+
|
|
21
|
+
type RunResult = { affectedRows?: number; rowCount?: number }
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Rewrites Postgres's numbered placeholders into the positional ones MikroORM binds with.
|
|
25
|
+
*
|
|
26
|
+
* The statements are written in `$n` form because that is Postgres's own, and because the
|
|
27
|
+
* failure harness runs them through node-postgres unchanged — the SQL that CI exercises is
|
|
28
|
+
* character-for-character the SQL a host runs. MikroORM goes through Knex, which binds `?`
|
|
29
|
+
* positionally, so the translation happens here rather than by writing the statements twice.
|
|
30
|
+
*
|
|
31
|
+
* A placeholder may legitimately appear more than once — the scope predicate reads `$3` twice,
|
|
32
|
+
* to compare an organization and to test it for null — so the parameter list is rebuilt in
|
|
33
|
+
* order of occurrence rather than reused as given.
|
|
34
|
+
*/
|
|
35
|
+
export function toPositional(text: string, params: readonly unknown[]): { text: string; params: unknown[] } {
|
|
36
|
+
const ordered: unknown[] = []
|
|
37
|
+
const rewritten = text.replace(/\$(\d+)/g, (_match, index: string) => {
|
|
38
|
+
const position = Number(index)
|
|
39
|
+
if (position < 1 || position > params.length) {
|
|
40
|
+
throw new Error(`SQL references $${position} but ${params.length} parameter(s) were supplied`)
|
|
41
|
+
}
|
|
42
|
+
ordered.push(params[position - 1])
|
|
43
|
+
return '?'
|
|
44
|
+
})
|
|
45
|
+
return { text: rewritten, params: ordered }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function executorFor(em: EntityManager): SqlExecutor {
|
|
49
|
+
return {
|
|
50
|
+
async query<R = Record<string, unknown>>(text: string, params: readonly unknown[] = []) {
|
|
51
|
+
const manager = em as EntityManagerLike
|
|
52
|
+
const connection = manager.getConnection() as unknown as Connection
|
|
53
|
+
const bound = toPositional(text, params)
|
|
54
|
+
|
|
55
|
+
// The transaction this EntityManager is inside, if any.
|
|
56
|
+
//
|
|
57
|
+
// Without it every statement runs on a pooled connection *outside* the transaction, and
|
|
58
|
+
// the guarantees built on top quietly stop holding: `fencedWrite` no longer rolls back a
|
|
59
|
+
// stale worker's writes, and a terminal transition no longer moves the job row and the
|
|
60
|
+
// domain row together. Nothing fails — it just is not atomic any more, which is the
|
|
61
|
+
// worst possible way for this to be wrong.
|
|
62
|
+
const ctx = manager.getTransactionContext?.()
|
|
63
|
+
|
|
64
|
+
if (RETURNS_ROWS.test(bound.text)) {
|
|
65
|
+
const rows = (await connection.execute(bound.text, bound.params, 'all', ctx)) as R[]
|
|
66
|
+
return { rows, rowCount: rows.length }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// A statement with no RETURNING has no rows to count, and counting them anyway reports
|
|
70
|
+
// zero for an UPDATE that matched. That number is not cosmetic: a domain mirror returns
|
|
71
|
+
// it as `matched`, and `matched: 0` is treated exactly like a throw — so every mirror
|
|
72
|
+
// would look like it had failed while the write it made had already landed.
|
|
73
|
+
const result = (await connection.execute(bound.text, bound.params, 'run', ctx)) as RunResult
|
|
74
|
+
return { rows: [] as R[], rowCount: result?.affectedRows ?? result?.rowCount ?? 0 }
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Wraps an EntityManager as a transactor.
|
|
81
|
+
*
|
|
82
|
+
* Each transaction runs on a *forked* EntityManager. The mechanism's statements must not share
|
|
83
|
+
* an identity map or a flush cycle with whatever the caller is doing — a heartbeat that
|
|
84
|
+
* accidentally flushed a half-built domain entity would be a spectacular way to lose data.
|
|
85
|
+
*/
|
|
86
|
+
export function mikroExecutor(em: EntityManager): SqlTransactor {
|
|
87
|
+
const base = executorFor(em)
|
|
88
|
+
return {
|
|
89
|
+
query: base.query,
|
|
90
|
+
async transaction<T>(fn: (tx: SqlExecutor) => Promise<T>): Promise<T> {
|
|
91
|
+
const forked = em.fork()
|
|
92
|
+
// `transactional` gives the callback an EntityManager carrying the transaction context,
|
|
93
|
+
// which `executorFor` then passes to every statement. Using the outer `em` here instead
|
|
94
|
+
// would silently run them outside the transaction.
|
|
95
|
+
return forked.transactional(async (trx) => fn(executorFor(trx as EntityManager)))
|
|
96
|
+
},
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Wraps an EntityManager the caller has already opened a transaction on, so the mechanism's
|
|
101
|
+
* statements join it rather than opening a second one. */
|
|
102
|
+
export function mikroTx(em: EntityManager): SqlExecutor {
|
|
103
|
+
return executorFor(em)
|
|
104
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
// The BullMQ adapter: the production default wherever an Open Mercato app already runs Redis.
|
|
2
|
+
//
|
|
3
|
+
// `bullmq` and `ioredis` are optional peers and are imported lazily, so an app that only uses
|
|
4
|
+
// the pg-boss adapter never has to install them.
|
|
5
|
+
//
|
|
6
|
+
// Supports BullMQ 5 and 6. Both carry everything the mechanism needs — a caller-supplied job
|
|
7
|
+
// id, the three-argument processor (so a real AbortSignal), `moveToDelayed` for a hand-back
|
|
8
|
+
// that spends no attempt, and job schedulers for the tick. The range matches
|
|
9
|
+
// `@open-mercato/queue`'s own peer range deliberately: a host on 5 must be able to install
|
|
10
|
+
// this package, because two BullMQ majors against one Redis is not a thing to arrange by
|
|
11
|
+
// accident.
|
|
12
|
+
|
|
13
|
+
import { deliveryId } from '../core/ids'
|
|
14
|
+
import type { Delivery, RetrySettings } from '../core/types'
|
|
15
|
+
import type {
|
|
16
|
+
BindOptions,
|
|
17
|
+
BoundWorker,
|
|
18
|
+
DeliveryHandler,
|
|
19
|
+
DeliveryState,
|
|
20
|
+
EnqueueOptions,
|
|
21
|
+
TransportAdapter,
|
|
22
|
+
} from './types'
|
|
23
|
+
|
|
24
|
+
type BullMQModule = typeof import('bullmq')
|
|
25
|
+
type BullQueue = InstanceType<BullMQModule['Queue']>
|
|
26
|
+
type BullJob = InstanceType<BullMQModule['Job']>
|
|
27
|
+
// Structural rather than `InstanceType<Worker>`: the concrete worker type is parameterised by
|
|
28
|
+
// the processor's return type and by a backend generic that differs between BullMQ 5 and 6,
|
|
29
|
+
// and nothing here needs more of it than shutdown.
|
|
30
|
+
type ClosableWorker = { close(force?: boolean): Promise<void>; on(event: 'error', listener: () => void): unknown }
|
|
31
|
+
|
|
32
|
+
export type BullMQTransportOptions = {
|
|
33
|
+
/** An ioredis connection or the options to build one. Passed through untouched. */
|
|
34
|
+
connection: unknown
|
|
35
|
+
prefix?: string
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
let cached: BullMQModule | null = null
|
|
39
|
+
async function bullmq(): Promise<BullMQModule> {
|
|
40
|
+
if (cached) return cached
|
|
41
|
+
try {
|
|
42
|
+
cached = await import('bullmq')
|
|
43
|
+
return cached
|
|
44
|
+
} catch (error) {
|
|
45
|
+
throw new Error(
|
|
46
|
+
'The bullmq transport requires the optional peer dependencies `bullmq` and `ioredis`. Install them, or use DURABLE_WORK_TRANSPORT=pgboss.',
|
|
47
|
+
{ cause: error },
|
|
48
|
+
)
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function backoffFor(retry: RetrySettings) {
|
|
53
|
+
return retry.backoff.type === 'fixed'
|
|
54
|
+
? { type: 'fixed' as const, delay: retry.backoff.delayMs }
|
|
55
|
+
: { type: 'exponential' as const, delay: retry.backoff.delayMs }
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export class BullMQTransport implements TransportAdapter {
|
|
59
|
+
readonly name = 'bullmq' as const
|
|
60
|
+
/** BullMQ writes to Redis, so it cannot join a Postgres transaction. Callers that need a
|
|
61
|
+
* job row and its delivery to commit together must enqueue after commit — and accept the
|
|
62
|
+
* gap the reconciler exists to close. */
|
|
63
|
+
readonly supportsTransactionalEnqueue = false
|
|
64
|
+
|
|
65
|
+
private readonly queues = new Map<string, BullQueue>()
|
|
66
|
+
private readonly workers: ClosableWorker[] = []
|
|
67
|
+
private readonly shutdown = new AbortController()
|
|
68
|
+
|
|
69
|
+
constructor(private readonly options: BullMQTransportOptions) {}
|
|
70
|
+
|
|
71
|
+
private async queue(name: string): Promise<BullQueue> {
|
|
72
|
+
const existing = this.queues.get(name)
|
|
73
|
+
if (existing) return existing
|
|
74
|
+
const { Queue } = await bullmq()
|
|
75
|
+
const queue = new Queue(name, { connection: this.options.connection as never, prefix: this.options.prefix })
|
|
76
|
+
this.queues.set(name, queue)
|
|
77
|
+
return queue
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async enqueue(queueName: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }> {
|
|
81
|
+
const queue = await this.queue(queueName)
|
|
82
|
+
const jobId = deliveryId(delivery)
|
|
83
|
+
// A caller-supplied job id makes the broker deduplicate a re-enqueue of the same delivery.
|
|
84
|
+
// It is a convenience, not the guarantee: the lease refuses a duplicate regardless, which
|
|
85
|
+
// is the version of this that survives a Redis flush.
|
|
86
|
+
await queue.add('delivery', delivery, {
|
|
87
|
+
jobId,
|
|
88
|
+
delay: opts.delayMs && opts.delayMs > 0 ? opts.delayMs : undefined,
|
|
89
|
+
attempts: opts.retry.attempts,
|
|
90
|
+
backoff: backoffFor(opts.retry),
|
|
91
|
+
removeOnComplete: { age: 3_600, count: 1_000 },
|
|
92
|
+
removeOnFail: { age: 86_400 },
|
|
93
|
+
})
|
|
94
|
+
return { transportJobId: jobId }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async remove(queueName: string, transportJobId: string): Promise<void> {
|
|
98
|
+
const queue = await this.queue(queueName)
|
|
99
|
+
// A job that is currently active cannot be removed; that is fine — the lease is what stops
|
|
100
|
+
// it, and this is only an optimisation to keep a cancelled job from being delivered.
|
|
101
|
+
await queue.remove(transportJobId).catch(() => undefined)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async getState(queueName: string, transportJobId: string): Promise<DeliveryState> {
|
|
105
|
+
const queue = await this.queue(queueName)
|
|
106
|
+
const job = await queue.getJob(transportJobId)
|
|
107
|
+
if (!job) return 'unknown'
|
|
108
|
+
const state = await job.getState()
|
|
109
|
+
switch (state) {
|
|
110
|
+
case 'waiting':
|
|
111
|
+
case 'waiting-children':
|
|
112
|
+
case 'prioritized':
|
|
113
|
+
return 'waiting'
|
|
114
|
+
case 'delayed':
|
|
115
|
+
return 'delayed'
|
|
116
|
+
case 'active':
|
|
117
|
+
return 'active'
|
|
118
|
+
case 'completed':
|
|
119
|
+
return 'completed'
|
|
120
|
+
case 'failed':
|
|
121
|
+
return 'failed'
|
|
122
|
+
default:
|
|
123
|
+
return 'unknown'
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void> {
|
|
128
|
+
const queue = await this.queue(opts.queue)
|
|
129
|
+
// A job scheduler rather than a self-re-enqueueing job: the schedule lives in Redis, so it
|
|
130
|
+
// survives every worker restarting at once, and one missed tick does not end the loop.
|
|
131
|
+
await queue.upsertJobScheduler(
|
|
132
|
+
opts.id,
|
|
133
|
+
{ every: opts.everyMs },
|
|
134
|
+
// Delivery-shaped like every other payload, so a bound handler never has to tell a tick
|
|
135
|
+
// from a delivery — and no adapter has to inspect a payload to decide.
|
|
136
|
+
{ name: 'tick', data: { jobId: opts.id, seq: 0, redrives: 0 } },
|
|
137
|
+
)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async bind(queueName: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker> {
|
|
141
|
+
const { Worker, DelayedError, UnrecoverableError } = await bullmq()
|
|
142
|
+
|
|
143
|
+
const worker = new Worker(
|
|
144
|
+
queueName,
|
|
145
|
+
async (job: BullJob, token?: string, signal?: AbortSignal) => {
|
|
146
|
+
const delivery = job.data as Delivery
|
|
147
|
+
let handedBack = false
|
|
148
|
+
|
|
149
|
+
// BullMQ's signal aborts when the job's lock is lost; ours aborts on shutdown. A slice
|
|
150
|
+
// needs to stop for either reason, so it watches both.
|
|
151
|
+
const combined = new AbortController()
|
|
152
|
+
const relay = () => combined.abort()
|
|
153
|
+
signal?.addEventListener('abort', relay, { once: true })
|
|
154
|
+
this.shutdown.signal.addEventListener('abort', relay, { once: true })
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
await handler(delivery, {
|
|
158
|
+
transportJobId: job.id ?? deliveryId(delivery),
|
|
159
|
+
attempt: job.attemptsMade + 1,
|
|
160
|
+
maxAttempts: job.opts.attempts ?? 1,
|
|
161
|
+
signal: combined.signal,
|
|
162
|
+
handBack: async (next, handBackOpts) => {
|
|
163
|
+
handedBack = true
|
|
164
|
+
// Native hand-back: the job keeps its id and its attempt count, and only its
|
|
165
|
+
// payload moves on. That is what makes yielding free — a re-enqueue would
|
|
166
|
+
// start a new job, and a retry would spend an attempt.
|
|
167
|
+
await job.updateData(next as unknown as never)
|
|
168
|
+
await job.moveToDelayed(Date.now() + (handBackOpts?.delayMs ?? 0), token)
|
|
169
|
+
},
|
|
170
|
+
})
|
|
171
|
+
} catch (error) {
|
|
172
|
+
// BullMQ retries on any throw, so "there is nothing left to attempt" has to be said
|
|
173
|
+
// in its own vocabulary. Without this translation a job that has already reached a
|
|
174
|
+
// terminal state is redelivered for every remaining attempt — each one refused by
|
|
175
|
+
// the claim, each one a wasted slice and a misleading log line.
|
|
176
|
+
if ((error as { name?: string })?.name === 'NoFurtherAttempts') {
|
|
177
|
+
throw new UnrecoverableError((error as Error).message)
|
|
178
|
+
}
|
|
179
|
+
throw error
|
|
180
|
+
} finally {
|
|
181
|
+
signal?.removeEventListener('abort', relay)
|
|
182
|
+
this.shutdown.signal.removeEventListener('abort', relay)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// BullMQ requires this to propagate out of the processor for the move to take effect.
|
|
186
|
+
if (handedBack) throw new DelayedError()
|
|
187
|
+
},
|
|
188
|
+
{
|
|
189
|
+
connection: this.options.connection as never,
|
|
190
|
+
prefix: this.options.prefix,
|
|
191
|
+
concurrency: opts.concurrency,
|
|
192
|
+
// Must exceed a whole slice, or BullMQ redelivers work that is still running — which
|
|
193
|
+
// the lease then refuses, wasting the slice and inflating the stall counter.
|
|
194
|
+
lockDuration: opts.activeTimeoutMs,
|
|
195
|
+
},
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
// Errors here are the broker's, not a job's; swallowing them would make a broken Redis
|
|
199
|
+
// look like an idle queue.
|
|
200
|
+
worker.on('error', () => undefined)
|
|
201
|
+
this.workers.push(worker as unknown as ClosableWorker)
|
|
202
|
+
return { queue: queueName, close: async (o) => void (await worker.close(o?.timeoutMs === 0)) }
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async close(opts: { timeoutMs?: number } = {}): Promise<void> {
|
|
206
|
+
// Abort first: in-flight slices see the signal and hand back at their next boundary,
|
|
207
|
+
// rather than being cut off wherever they happen to be.
|
|
208
|
+
this.shutdown.abort()
|
|
209
|
+
const deadline = new Promise<void>((resolve) => setTimeout(resolve, opts.timeoutMs ?? 30_000).unref?.())
|
|
210
|
+
await Promise.race([Promise.allSettled(this.workers.map((w) => w.close())).then(() => undefined), deadline])
|
|
211
|
+
await Promise.allSettled([...this.queues.values()].map((q) => q.close()))
|
|
212
|
+
}
|
|
213
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
// One suite, three adapters.
|
|
2
|
+
//
|
|
3
|
+
// Shipped from the package rather than written in the harness so that anyone adding a fourth
|
|
4
|
+
// adapter runs exactly the checks the existing three pass. A scenario that passes on one
|
|
5
|
+
// transport and not another is a bug, not a caveat — and the only way to keep that true is for
|
|
6
|
+
// there to be a single definition of what passing means.
|
|
7
|
+
//
|
|
8
|
+
// Framework-agnostic: the caller supplies `it` and `expect`, so the same suite runs under
|
|
9
|
+
// vitest here and under whatever a downstream host uses.
|
|
10
|
+
|
|
11
|
+
import type { Delivery, RetrySettings } from '../core/types'
|
|
12
|
+
import type { TransportAdapter } from './types'
|
|
13
|
+
|
|
14
|
+
export type ConformanceHooks = {
|
|
15
|
+
it: (name: string, fn: () => Promise<void>, timeoutMs?: number) => void
|
|
16
|
+
expect: (actual: unknown) => {
|
|
17
|
+
toBe(expected: unknown): void
|
|
18
|
+
toEqual(expected: unknown): void
|
|
19
|
+
toBeGreaterThan(expected: number): void
|
|
20
|
+
toBeGreaterThanOrEqual(expected: number): void
|
|
21
|
+
toBeLessThan(expected: number): void
|
|
22
|
+
}
|
|
23
|
+
/** A fresh adapter and a queue name nothing else uses. */
|
|
24
|
+
make: () => Promise<{ transport: TransportAdapter; queue: string; close: () => Promise<void> }>
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const RETRY: RetrySettings = { attempts: 3, backoff: { type: 'fixed', delayMs: 200, maxDelayMs: 200 } }
|
|
28
|
+
const ONCE: RetrySettings = { attempts: 1, backoff: { type: 'fixed', delayMs: 0, maxDelayMs: 0 } }
|
|
29
|
+
|
|
30
|
+
const delivery = (jobId: string, seq = 0, redrives = 0): Delivery => ({ jobId, seq, redrives })
|
|
31
|
+
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
32
|
+
|
|
33
|
+
/** Waits for a condition rather than for a duration, so a slow CI runner makes the suite
|
|
34
|
+
* slower rather than flaky. */
|
|
35
|
+
async function until(predicate: () => boolean, timeoutMs = 15_000): Promise<void> {
|
|
36
|
+
const deadline = Date.now() + timeoutMs
|
|
37
|
+
while (Date.now() < deadline) {
|
|
38
|
+
if (predicate()) return
|
|
39
|
+
await sleep(25)
|
|
40
|
+
}
|
|
41
|
+
throw new Error('condition was never met')
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function transportConformance(hooks: ConformanceHooks): void {
|
|
45
|
+
const { it, expect, make } = hooks
|
|
46
|
+
|
|
47
|
+
it('delivers an enqueued job to a bound handler', async () => {
|
|
48
|
+
const { transport, queue, close } = await make()
|
|
49
|
+
try {
|
|
50
|
+
const seen: Delivery[] = []
|
|
51
|
+
await transport.bind(queue, async (d) => void seen.push(d), { concurrency: 1, activeTimeoutMs: 30_000 })
|
|
52
|
+
await transport.enqueue(queue, delivery('job-1'), { retry: ONCE })
|
|
53
|
+
await until(() => seen.length === 1)
|
|
54
|
+
expect(seen[0]).toEqual(delivery('job-1'))
|
|
55
|
+
} finally {
|
|
56
|
+
await close()
|
|
57
|
+
}
|
|
58
|
+
}, 60_000)
|
|
59
|
+
|
|
60
|
+
it('honours a delay before making the delivery available', async () => {
|
|
61
|
+
const { transport, queue, close } = await make()
|
|
62
|
+
try {
|
|
63
|
+
let firstSeenAt = 0
|
|
64
|
+
const start = Date.now()
|
|
65
|
+
await transport.bind(queue, async () => void (firstSeenAt = Date.now()), { concurrency: 1, activeTimeoutMs: 30_000 })
|
|
66
|
+
await transport.enqueue(queue, delivery('job-delay'), { retry: ONCE, delayMs: 1_500 })
|
|
67
|
+
await until(() => firstSeenAt > 0)
|
|
68
|
+
expect(firstSeenAt - start).toBeGreaterThanOrEqual(1_000)
|
|
69
|
+
} finally {
|
|
70
|
+
await close()
|
|
71
|
+
}
|
|
72
|
+
}, 60_000)
|
|
73
|
+
|
|
74
|
+
it('retries a handler that throws, up to the attempt limit', async () => {
|
|
75
|
+
const { transport, queue, close } = await make()
|
|
76
|
+
try {
|
|
77
|
+
let attempts = 0
|
|
78
|
+
await transport.bind(
|
|
79
|
+
queue,
|
|
80
|
+
async () => {
|
|
81
|
+
attempts += 1
|
|
82
|
+
throw new Error('nope')
|
|
83
|
+
},
|
|
84
|
+
{ concurrency: 1, activeTimeoutMs: 30_000 },
|
|
85
|
+
)
|
|
86
|
+
await transport.enqueue(queue, delivery('job-retry'), { retry: RETRY })
|
|
87
|
+
await until(() => attempts >= 2, 20_000)
|
|
88
|
+
expect(attempts).toBeGreaterThanOrEqual(2)
|
|
89
|
+
} finally {
|
|
90
|
+
await close()
|
|
91
|
+
}
|
|
92
|
+
}, 60_000)
|
|
93
|
+
|
|
94
|
+
it('stops retrying when the handler signals there is nothing left to attempt', async () => {
|
|
95
|
+
const { transport, queue, close } = await make()
|
|
96
|
+
try {
|
|
97
|
+
let attempts = 0
|
|
98
|
+
await transport.bind(
|
|
99
|
+
queue,
|
|
100
|
+
async () => {
|
|
101
|
+
attempts += 1
|
|
102
|
+
const error = new Error('settled')
|
|
103
|
+
error.name = 'NoFurtherAttempts'
|
|
104
|
+
throw error
|
|
105
|
+
},
|
|
106
|
+
{ concurrency: 1, activeTimeoutMs: 30_000 },
|
|
107
|
+
)
|
|
108
|
+
await transport.enqueue(queue, delivery('job-final'), { retry: RETRY })
|
|
109
|
+
await until(() => attempts >= 1)
|
|
110
|
+
await sleep(1_500) // long enough for a retry to have landed if one were coming
|
|
111
|
+
expect(attempts).toBe(1)
|
|
112
|
+
} finally {
|
|
113
|
+
await close()
|
|
114
|
+
}
|
|
115
|
+
}, 60_000)
|
|
116
|
+
|
|
117
|
+
it('hands work back as a new delivery without spending an attempt', async () => {
|
|
118
|
+
const { transport, queue, close } = await make()
|
|
119
|
+
try {
|
|
120
|
+
const seen: Delivery[] = []
|
|
121
|
+
const attemptsSeen: number[] = []
|
|
122
|
+
await transport.bind(
|
|
123
|
+
queue,
|
|
124
|
+
async (d, ctx) => {
|
|
125
|
+
seen.push(d)
|
|
126
|
+
attemptsSeen.push(ctx.attempt)
|
|
127
|
+
if (d.seq < 2) await ctx.handBack({ ...d, seq: d.seq + 1 })
|
|
128
|
+
},
|
|
129
|
+
{ concurrency: 1, activeTimeoutMs: 30_000 },
|
|
130
|
+
)
|
|
131
|
+
await transport.enqueue(queue, delivery('job-yield'), { retry: RETRY })
|
|
132
|
+
|
|
133
|
+
await until(() => seen.length === 3, 20_000)
|
|
134
|
+
expect(seen.map((d) => d.seq)).toEqual([0, 1, 2])
|
|
135
|
+
// The point of a hand-back: three slices, none of them a retry.
|
|
136
|
+
expect(Math.max(...attemptsSeen)).toBe(1)
|
|
137
|
+
} finally {
|
|
138
|
+
await close()
|
|
139
|
+
}
|
|
140
|
+
}, 60_000)
|
|
141
|
+
|
|
142
|
+
it('collapses a re-enqueue of the same delivery instead of delivering it twice', async () => {
|
|
143
|
+
const { transport, queue, close } = await make()
|
|
144
|
+
try {
|
|
145
|
+
const seen: Delivery[] = []
|
|
146
|
+
await transport.bind(queue, async (d) => void seen.push(d), { concurrency: 1, activeTimeoutMs: 30_000 })
|
|
147
|
+
const d = delivery('job-dedupe')
|
|
148
|
+
// Enqueued before binding drains it, so both land while the job is still waiting.
|
|
149
|
+
await Promise.all([
|
|
150
|
+
transport.enqueue(queue, d, { retry: ONCE, delayMs: 800 }),
|
|
151
|
+
transport.enqueue(queue, d, { retry: ONCE, delayMs: 800 }),
|
|
152
|
+
])
|
|
153
|
+
await until(() => seen.length >= 1)
|
|
154
|
+
await sleep(1_000)
|
|
155
|
+
expect(seen.length).toBe(1)
|
|
156
|
+
} finally {
|
|
157
|
+
await close()
|
|
158
|
+
}
|
|
159
|
+
}, 60_000)
|
|
160
|
+
|
|
161
|
+
it('reports a delivery it does not hold as unknown, and removes one it does', async () => {
|
|
162
|
+
const { transport, queue, close } = await make()
|
|
163
|
+
try {
|
|
164
|
+
expect(await transport.getState(queue, 'nothing-here')).toBe('unknown')
|
|
165
|
+
const { transportJobId } = await transport.enqueue(queue, delivery('job-remove'), { retry: ONCE, delayMs: 60_000 })
|
|
166
|
+
await transport.remove(queue, transportJobId)
|
|
167
|
+
expect(await transport.getState(queue, transportJobId)).toBe('unknown')
|
|
168
|
+
} finally {
|
|
169
|
+
await close()
|
|
170
|
+
}
|
|
171
|
+
}, 60_000)
|
|
172
|
+
|
|
173
|
+
it('fires a tick repeatedly', async () => {
|
|
174
|
+
const { transport, queue, close } = await make()
|
|
175
|
+
try {
|
|
176
|
+
let ticks = 0
|
|
177
|
+
await transport.bind(queue, async () => void (ticks += 1), { concurrency: 1, activeTimeoutMs: 30_000 })
|
|
178
|
+
await transport.upsertTick({ id: 'conformance-tick', queue, everyMs: 1_000 })
|
|
179
|
+
await until(() => ticks >= 2, 20_000)
|
|
180
|
+
expect(ticks).toBeGreaterThanOrEqual(2)
|
|
181
|
+
} finally {
|
|
182
|
+
await close()
|
|
183
|
+
}
|
|
184
|
+
}, 60_000)
|
|
185
|
+
|
|
186
|
+
it('aborts in-flight work on close, and returns within its timeout', async () => {
|
|
187
|
+
const { transport, queue, close } = await make()
|
|
188
|
+
try {
|
|
189
|
+
let aborted = false
|
|
190
|
+
let started = false
|
|
191
|
+
await transport.bind(
|
|
192
|
+
queue,
|
|
193
|
+
async (_d, ctx) => {
|
|
194
|
+
started = true
|
|
195
|
+
await new Promise<void>((resolve) => {
|
|
196
|
+
if (ctx.signal.aborted) return resolve()
|
|
197
|
+
ctx.signal.addEventListener('abort', () => {
|
|
198
|
+
aborted = true
|
|
199
|
+
resolve()
|
|
200
|
+
})
|
|
201
|
+
// Long enough that only the abort can end it inside the close timeout.
|
|
202
|
+
setTimeout(resolve, 60_000).unref?.()
|
|
203
|
+
})
|
|
204
|
+
},
|
|
205
|
+
{ concurrency: 1, activeTimeoutMs: 120_000 },
|
|
206
|
+
)
|
|
207
|
+
await transport.enqueue(queue, delivery('job-drain'), { retry: ONCE })
|
|
208
|
+
await until(() => started)
|
|
209
|
+
|
|
210
|
+
const startedAt = Date.now()
|
|
211
|
+
await transport.close({ timeoutMs: 10_000 })
|
|
212
|
+
expect(Date.now() - startedAt).toBeLessThan(15_000)
|
|
213
|
+
expect(aborted).toBe(true)
|
|
214
|
+
} finally {
|
|
215
|
+
await close()
|
|
216
|
+
}
|
|
217
|
+
}, 90_000)
|
|
218
|
+
}
|