@fullstackhouse/open-mercato-durable-work 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -0
- package/dist/core/errors.js +81 -0
- package/dist/core/errors.js.map +7 -0
- package/dist/core/ids.js +30 -0
- package/dist/core/ids.js.map +7 -0
- package/dist/core/reconciler.js +149 -0
- package/dist/core/reconciler.js.map +7 -0
- package/dist/core/registry.js +72 -0
- package/dist/core/registry.js.map +7 -0
- package/dist/core/run-slice.js +210 -0
- package/dist/core/run-slice.js.map +7 -0
- package/dist/core/schema.js +100 -0
- package/dist/core/schema.js.map +7 -0
- package/dist/core/service.js +161 -0
- package/dist/core/service.js.map +7 -0
- package/dist/core/store.js +516 -0
- package/dist/core/store.js.map +7 -0
- package/dist/core/terminal.js +53 -0
- package/dist/core/terminal.js.map +7 -0
- package/dist/core/types.js +1 -0
- package/dist/core/types.js.map +7 -0
- package/dist/core/worker.js +111 -0
- package/dist/core/worker.js.map +7 -0
- package/dist/index.js +99 -0
- package/dist/index.js.map +7 -0
- package/dist/modules/durable_work/acl.js +10 -0
- package/dist/modules/durable_work/acl.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/[id]/redrive.js +20 -0
- package/dist/modules/durable_work/api/jobs/[id]/redrive.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/[id]/route.js +26 -0
- package/dist/modules/durable_work/api/jobs/[id]/route.js.map +7 -0
- package/dist/modules/durable_work/api/jobs/route.js +24 -0
- package/dist/modules/durable_work/api/jobs/route.js.map +7 -0
- package/dist/modules/durable_work/cli.js +72 -0
- package/dist/modules/durable_work/cli.js.map +7 -0
- package/dist/modules/durable_work/data/entities.js +163 -0
- package/dist/modules/durable_work/data/entities.js.map +7 -0
- package/dist/modules/durable_work/di.js +34 -0
- package/dist/modules/durable_work/di.js.map +7 -0
- package/dist/modules/durable_work/events.js +26 -0
- package/dist/modules/durable_work/events.js.map +7 -0
- package/dist/modules/durable_work/index.js +17 -0
- package/dist/modules/durable_work/index.js.map +7 -0
- package/dist/modules/durable_work/lib/route-helpers.js +66 -0
- package/dist/modules/durable_work/lib/route-helpers.js.map +7 -0
- package/dist/modules/durable_work/migrations/Migration20260908120000.js +17 -0
- package/dist/modules/durable_work/migrations/Migration20260908120000.js.map +7 -0
- package/dist/modules/durable_work/setup.js +12 -0
- package/dist/modules/durable_work/setup.js.map +7 -0
- package/dist/om/config.js +49 -0
- package/dist/om/config.js.map +7 -0
- package/dist/om/progress-mirror.js +49 -0
- package/dist/om/progress-mirror.js.map +7 -0
- package/dist/om/sql-executor-mikro.js +48 -0
- package/dist/om/sql-executor-mikro.js.map +7 -0
- package/dist/transport/bullmq.js +144 -0
- package/dist/transport/bullmq.js.map +7 -0
- package/dist/transport/conformance.js +177 -0
- package/dist/transport/conformance.js.map +7 -0
- package/dist/transport/memory.js +139 -0
- package/dist/transport/memory.js.map +7 -0
- package/dist/transport/pgboss.js +176 -0
- package/dist/transport/pgboss.js.map +7 -0
- package/dist/transport/types.js +1 -0
- package/dist/transport/types.js.map +7 -0
- package/generated/entities/durable_work_job/index.ts +42 -0
- package/generated/entities.ids.generated.ts +9 -0
- package/package.json +145 -0
- package/src/core/__tests__/registry.test.ts +43 -0
- package/src/core/errors.ts +104 -0
- package/src/core/ids.ts +58 -0
- package/src/core/reconciler.ts +242 -0
- package/src/core/registry.ts +199 -0
- package/src/core/run-slice.ts +343 -0
- package/src/core/schema.ts +114 -0
- package/src/core/service.ts +222 -0
- package/src/core/store.ts +786 -0
- package/src/core/terminal.ts +107 -0
- package/src/core/types.ts +120 -0
- package/src/core/worker.ts +169 -0
- package/src/index.ts +100 -0
- package/src/modules/durable_work/__integration__/TC-DW-001.spec.ts +51 -0
- package/src/modules/durable_work/__tests__/metadata.test.ts +13 -0
- package/src/modules/durable_work/__tests__/schema-agreement.test.ts +52 -0
- package/src/modules/durable_work/acl.ts +6 -0
- package/src/modules/durable_work/api/jobs/[id]/redrive.ts +27 -0
- package/src/modules/durable_work/api/jobs/[id]/route.ts +27 -0
- package/src/modules/durable_work/api/jobs/route.ts +26 -0
- package/src/modules/durable_work/cli.ts +91 -0
- package/src/modules/durable_work/data/entities.ts +158 -0
- package/src/modules/durable_work/di.ts +41 -0
- package/src/modules/durable_work/events.ts +30 -0
- package/src/modules/durable_work/index.ts +16 -0
- package/src/modules/durable_work/lib/route-helpers.ts +83 -0
- package/src/modules/durable_work/migrations/Migration20260908120000.ts +24 -0
- package/src/modules/durable_work/setup.ts +10 -0
- package/src/om/__tests__/sql-executor-mikro.test.ts +83 -0
- package/src/om/config.ts +65 -0
- package/src/om/progress-mirror.ts +80 -0
- package/src/om/sql-executor-mikro.ts +104 -0
- package/src/transport/bullmq.ts +213 -0
- package/src/transport/conformance.ts +218 -0
- package/src/transport/memory.ts +191 -0
- package/src/transport/pgboss.ts +250 -0
- package/src/transport/types.ts +81 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { NextResponse } from 'next/server'
|
|
2
|
+
|
|
3
|
+
import { routeContext, toDto } from '../../../lib/route-helpers'
|
|
4
|
+
|
|
5
|
+
export const metadata = {
|
|
6
|
+
GET: { requireAuth: true, requireFeatures: ['durable_work.view'] },
|
|
7
|
+
DELETE: { requireAuth: true, requireFeatures: ['durable_work.operate'] },
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export async function GET(req: Request, { params }: { params: { id: string } }) {
|
|
11
|
+
const ctx = await routeContext(req)
|
|
12
|
+
if (ctx instanceof NextResponse) return ctx
|
|
13
|
+
const job = await ctx.service.get(params.id, ctx.scope)
|
|
14
|
+
if (!job) return NextResponse.json({ error: 'Not found' }, { status: 404 })
|
|
15
|
+
return NextResponse.json(toDto(job))
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Asks the job to stop. Answers with what actually happened — a running job is `cancelling`
|
|
19
|
+
* until its driver observes the request, and saying `cancelled` before that would be a lie
|
|
20
|
+
* an operator might act on. */
|
|
21
|
+
export async function DELETE(req: Request, { params }: { params: { id: string } }) {
|
|
22
|
+
const ctx = await routeContext(req)
|
|
23
|
+
if (ctx instanceof NextResponse) return ctx
|
|
24
|
+
const job = await ctx.service.cancel(params.id, ctx.scope, ctx.userId)
|
|
25
|
+
if (!job) return NextResponse.json({ error: 'Not found or already finished' }, { status: 404 })
|
|
26
|
+
return NextResponse.json({ ...toDto(job), state: job.status === 'cancelled' ? 'cancelled' : 'cancelling' })
|
|
27
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { NextResponse } from 'next/server'
|
|
2
|
+
|
|
3
|
+
import { routeContext, toDto } from '../../lib/route-helpers'
|
|
4
|
+
import type { DurableJobStatus } from '../../../../core/types'
|
|
5
|
+
|
|
6
|
+
export const metadata = {
|
|
7
|
+
GET: { requireAuth: true, requireFeatures: ['durable_work.view'] },
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
const STATUSES = new Set<DurableJobStatus>(['pending', 'running', 'completed', 'failed', 'cancelled'])
|
|
11
|
+
|
|
12
|
+
export async function GET(req: Request) {
|
|
13
|
+
const ctx = await routeContext(req)
|
|
14
|
+
if (ctx instanceof NextResponse) return ctx
|
|
15
|
+
|
|
16
|
+
const url = new URL(req.url)
|
|
17
|
+
const status = url.searchParams.get('status')
|
|
18
|
+
const { items, total } = await ctx.service.list(ctx.scope, {
|
|
19
|
+
kind: url.searchParams.get('kind') ?? undefined,
|
|
20
|
+
status: status && STATUSES.has(status as DurableJobStatus) ? (status as DurableJobStatus) : undefined,
|
|
21
|
+
page: Number(url.searchParams.get('page') ?? '1'),
|
|
22
|
+
pageSize: Number(url.searchParams.get('pageSize') ?? '20'),
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
return NextResponse.json({ items: items.map(toDto), total })
|
|
26
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// The worker process, and the operator's command line.
|
|
2
|
+
//
|
|
3
|
+
// `mercato durable_work worker` is how the mechanism runs in production. It is deliberately its
|
|
4
|
+
// own process: a slice can run for minutes, and hosting that inside the web process means a
|
|
5
|
+
// deploy either kills work mid-batch or holds the deploy open for the length of a slice.
|
|
6
|
+
|
|
7
|
+
import type { ModuleCli } from '@open-mercato/shared/modules/registry'
|
|
8
|
+
import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
|
|
9
|
+
|
|
10
|
+
import type { DurableWorkService } from '../../core/service'
|
|
11
|
+
import { registry } from '../../core/registry'
|
|
12
|
+
import { startWorker } from '../../core/worker'
|
|
13
|
+
import { readConfig } from '../../om/config'
|
|
14
|
+
import type { SqlTransactor } from '../../core/types'
|
|
15
|
+
import type { TransportAdapter } from '../../transport/types'
|
|
16
|
+
|
|
17
|
+
const flag = (argv: string[], name: string): string | undefined => {
|
|
18
|
+
const index = argv.indexOf(`--${name}`)
|
|
19
|
+
return index >= 0 ? argv[index + 1] : undefined
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const emit = (event: string, fields: Record<string, unknown> = {}) => console.log(JSON.stringify({ event, ...fields }))
|
|
23
|
+
|
|
24
|
+
const workerCommand: ModuleCli = {
|
|
25
|
+
command: 'worker',
|
|
26
|
+
async run(argv: string[]) {
|
|
27
|
+
const config = readConfig()
|
|
28
|
+
const container = await createRequestContainer()
|
|
29
|
+
const sql = container.resolve('durableWorkSql') as SqlTransactor
|
|
30
|
+
const transport = container.resolve('durableWorkTransport') as TransportAdapter
|
|
31
|
+
|
|
32
|
+
const worker = await startWorker({
|
|
33
|
+
sql,
|
|
34
|
+
transport,
|
|
35
|
+
registry,
|
|
36
|
+
kinds: flag(argv, 'kinds')?.split(','),
|
|
37
|
+
concurrency: flag(argv, 'concurrency') ? Number(flag(argv, 'concurrency')) : undefined,
|
|
38
|
+
tickMs: config.tickMs,
|
|
39
|
+
reconcilerGraceMs: config.reconcilerGraceMs,
|
|
40
|
+
drainTimeoutMs: config.drainTimeoutMs,
|
|
41
|
+
log: (event, fields) => emit(event, fields),
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
emit('durable_work.worker_started', {
|
|
45
|
+
owner: worker.owner,
|
|
46
|
+
transport: transport.name,
|
|
47
|
+
kinds: registry.list().map((kind) => kind.kind),
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
// SIGTERM is what a deploy sends. Draining rather than exiting is the difference between a
|
|
51
|
+
// slice handing its remaining work back and a slice being killed between two writes.
|
|
52
|
+
let stopping = false
|
|
53
|
+
const stop = async (signal: string) => {
|
|
54
|
+
if (stopping) return
|
|
55
|
+
stopping = true
|
|
56
|
+
emit('durable_work.worker_draining', { signal, timeoutMs: config.drainTimeoutMs })
|
|
57
|
+
await worker.stop()
|
|
58
|
+
emit('durable_work.worker_stopped')
|
|
59
|
+
process.exit(0)
|
|
60
|
+
}
|
|
61
|
+
process.on('SIGTERM', () => void stop('SIGTERM'))
|
|
62
|
+
process.on('SIGINT', () => void stop('SIGINT'))
|
|
63
|
+
|
|
64
|
+
await new Promise(() => undefined) // run until signalled
|
|
65
|
+
},
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const reconcileCommand: ModuleCli = {
|
|
69
|
+
command: 'reconcile',
|
|
70
|
+
async run() {
|
|
71
|
+
const container = await createRequestContainer()
|
|
72
|
+
const service = container.resolve('durableWorkService') as DurableWorkService
|
|
73
|
+
console.log(JSON.stringify(await service.reconcile(), null, 2))
|
|
74
|
+
},
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const helpCommand: ModuleCli = {
|
|
78
|
+
command: 'help',
|
|
79
|
+
async run() {
|
|
80
|
+
console.log(
|
|
81
|
+
[
|
|
82
|
+
'mercato durable_work worker [--kinds a,b] [--concurrency n]',
|
|
83
|
+
' Bind every registered kind, own the reconciler tick, drain on SIGTERM.',
|
|
84
|
+
'mercato durable_work reconcile',
|
|
85
|
+
' Run one reconciler pass and print what it repaired.',
|
|
86
|
+
].join('\n'),
|
|
87
|
+
)
|
|
88
|
+
},
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export default [workerCommand, reconcileCommand, helpCommand]
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// The MikroORM view of the job table.
|
|
2
|
+
//
|
|
3
|
+
// The mechanism itself never goes through this entity — every statement in `core/store.ts` is
|
|
4
|
+
// hand-written SQL, because each is a compare-and-set whose predicate is the guarantee. The
|
|
5
|
+
// entity exists so the table is discoverable to the host: migrations, the entity registry,
|
|
6
|
+
// query tooling and anything an app wants to join against.
|
|
7
|
+
//
|
|
8
|
+
// It must therefore stay in step with `core/schema.ts`, which is the definition. A test
|
|
9
|
+
// asserts they agree column for column.
|
|
10
|
+
|
|
11
|
+
import { OptionalProps } from '@mikro-orm/core'
|
|
12
|
+
import { Entity, Index, PrimaryKey, Property } from '@mikro-orm/decorators/legacy'
|
|
13
|
+
|
|
14
|
+
export type DurableWorkJobStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
|
|
15
|
+
|
|
16
|
+
@Entity({ tableName: 'durable_work_jobs' })
|
|
17
|
+
@Index({ name: 'durable_work_jobs_running_idx', properties: ['tenantId'] })
|
|
18
|
+
@Index({ name: 'durable_work_jobs_subject_idx', properties: ['subjectType', 'subjectId'] })
|
|
19
|
+
export class DurableWorkJob {
|
|
20
|
+
[OptionalProps]?:
|
|
21
|
+
| 'status'
|
|
22
|
+
| 'leaseEpoch'
|
|
23
|
+
| 'continuationSeq'
|
|
24
|
+
| 'redrives'
|
|
25
|
+
| 'redrivesSinceCommit'
|
|
26
|
+
| 'consecutiveFailures'
|
|
27
|
+
| 'interruptions'
|
|
28
|
+
| 'mirrorAttempts'
|
|
29
|
+
| 'processedCount'
|
|
30
|
+
| 'createdAt'
|
|
31
|
+
| 'updatedAt'
|
|
32
|
+
|
|
33
|
+
@PrimaryKey({ type: 'uuid', defaultRaw: 'gen_random_uuid()' })
|
|
34
|
+
id!: string
|
|
35
|
+
|
|
36
|
+
@Property({ name: 'tenant_id', type: 'uuid' })
|
|
37
|
+
tenantId!: string
|
|
38
|
+
|
|
39
|
+
@Property({ name: 'organization_id', type: 'uuid', nullable: true })
|
|
40
|
+
organizationId?: string | null
|
|
41
|
+
|
|
42
|
+
@Property({ name: 'kind', type: 'text' })
|
|
43
|
+
kind!: string
|
|
44
|
+
|
|
45
|
+
@Property({ name: 'status', type: 'text' })
|
|
46
|
+
status: DurableWorkJobStatus = 'pending'
|
|
47
|
+
|
|
48
|
+
@Property({ name: 'created_by', type: 'uuid', nullable: true })
|
|
49
|
+
createdBy?: string | null
|
|
50
|
+
|
|
51
|
+
@Property({ name: 'created_at', type: 'timestamptz' })
|
|
52
|
+
createdAt: Date = new Date()
|
|
53
|
+
|
|
54
|
+
@Property({ name: 'updated_at', type: 'timestamptz', onUpdate: () => new Date() })
|
|
55
|
+
updatedAt: Date = new Date()
|
|
56
|
+
|
|
57
|
+
@Property({ name: 'input', type: 'jsonb', nullable: true })
|
|
58
|
+
input?: unknown
|
|
59
|
+
|
|
60
|
+
@Property({ name: 'checkpoint', type: 'jsonb', nullable: true })
|
|
61
|
+
checkpoint?: unknown
|
|
62
|
+
|
|
63
|
+
@Property({ name: 'meta', type: 'jsonb', nullable: true })
|
|
64
|
+
meta?: Record<string, unknown> | null
|
|
65
|
+
|
|
66
|
+
@Property({ name: 'idempotency_key', type: 'text', nullable: true })
|
|
67
|
+
idempotencyKey?: string | null
|
|
68
|
+
|
|
69
|
+
@Property({ name: 'lock_key', type: 'text', nullable: true })
|
|
70
|
+
lockKey?: string | null
|
|
71
|
+
|
|
72
|
+
@Property({ name: 'subject_type', type: 'text', nullable: true })
|
|
73
|
+
subjectType?: string | null
|
|
74
|
+
|
|
75
|
+
@Property({ name: 'subject_id', type: 'text', nullable: true })
|
|
76
|
+
subjectId?: string | null
|
|
77
|
+
|
|
78
|
+
@Property({ name: 'progress_job_id', type: 'uuid', nullable: true })
|
|
79
|
+
progressJobId?: string | null
|
|
80
|
+
|
|
81
|
+
@Property({ name: 'lease_owner', type: 'text', nullable: true })
|
|
82
|
+
leaseOwner?: string | null
|
|
83
|
+
|
|
84
|
+
@Property({ name: 'lease_epoch', type: 'bigint' })
|
|
85
|
+
leaseEpoch: number = 0
|
|
86
|
+
|
|
87
|
+
@Property({ name: 'lease_expires_at', type: 'timestamptz', nullable: true })
|
|
88
|
+
leaseExpiresAt?: Date | null
|
|
89
|
+
|
|
90
|
+
@Property({ name: 'heartbeat_at', type: 'timestamptz', nullable: true })
|
|
91
|
+
heartbeatAt?: Date | null
|
|
92
|
+
|
|
93
|
+
@Property({ name: 'queue_name', type: 'text', nullable: true })
|
|
94
|
+
queueName?: string | null
|
|
95
|
+
|
|
96
|
+
@Property({ name: 'queue_job_id', type: 'text', nullable: true })
|
|
97
|
+
queueJobId?: string | null
|
|
98
|
+
|
|
99
|
+
@Property({ name: 'continuation_seq', type: 'int' })
|
|
100
|
+
continuationSeq: number = 0
|
|
101
|
+
|
|
102
|
+
@Property({ name: 'redrives', type: 'int' })
|
|
103
|
+
redrives: number = 0
|
|
104
|
+
|
|
105
|
+
@Property({ name: 'next_run_at', type: 'timestamptz', nullable: true })
|
|
106
|
+
nextRunAt?: Date | null
|
|
107
|
+
|
|
108
|
+
@Property({ name: 'pending_since', type: 'timestamptz', nullable: true })
|
|
109
|
+
pendingSince?: Date | null
|
|
110
|
+
|
|
111
|
+
@Property({ name: 'redrives_since_commit', type: 'int' })
|
|
112
|
+
redrivesSinceCommit: number = 0
|
|
113
|
+
|
|
114
|
+
@Property({ name: 'consecutive_failures', type: 'int' })
|
|
115
|
+
consecutiveFailures: number = 0
|
|
116
|
+
|
|
117
|
+
@Property({ name: 'interruptions', type: 'int' })
|
|
118
|
+
interruptions: number = 0
|
|
119
|
+
|
|
120
|
+
@Property({ name: 'mirror_attempts', type: 'int' })
|
|
121
|
+
mirrorAttempts: number = 0
|
|
122
|
+
|
|
123
|
+
@Property({ name: 'last_committed_at', type: 'timestamptz', nullable: true })
|
|
124
|
+
lastCommittedAt?: Date | null
|
|
125
|
+
|
|
126
|
+
@Property({ name: 'started_at', type: 'timestamptz', nullable: true })
|
|
127
|
+
startedAt?: Date | null
|
|
128
|
+
|
|
129
|
+
@Property({ name: 'finished_at', type: 'timestamptz', nullable: true })
|
|
130
|
+
finishedAt?: Date | null
|
|
131
|
+
|
|
132
|
+
@Property({ name: 'parked_at', type: 'timestamptz', nullable: true })
|
|
133
|
+
parkedAt?: Date | null
|
|
134
|
+
|
|
135
|
+
@Property({ name: 'cancel_requested_at', type: 'timestamptz', nullable: true })
|
|
136
|
+
cancelRequestedAt?: Date | null
|
|
137
|
+
|
|
138
|
+
@Property({ name: 'cancelled_by', type: 'uuid', nullable: true })
|
|
139
|
+
cancelledBy?: string | null
|
|
140
|
+
|
|
141
|
+
@Property({ name: 'error_class', type: 'text', nullable: true })
|
|
142
|
+
errorClass?: string | null
|
|
143
|
+
|
|
144
|
+
@Property({ name: 'error_code', type: 'text', nullable: true })
|
|
145
|
+
errorCode?: string | null
|
|
146
|
+
|
|
147
|
+
@Property({ name: 'error_message', type: 'text', nullable: true })
|
|
148
|
+
errorMessage?: string | null
|
|
149
|
+
|
|
150
|
+
@Property({ name: 'domain_mirrored_at', type: 'timestamptz', nullable: true })
|
|
151
|
+
domainMirroredAt?: Date | null
|
|
152
|
+
|
|
153
|
+
@Property({ name: 'processed_count', type: 'int' })
|
|
154
|
+
processedCount: number = 0
|
|
155
|
+
|
|
156
|
+
@Property({ name: 'total_count', type: 'int', nullable: true })
|
|
157
|
+
totalCount?: number | null
|
|
158
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { AppContainer } from '@open-mercato/shared/lib/di/container'
|
|
2
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
3
|
+
|
|
4
|
+
import { DurableWorkService } from '../../core/service'
|
|
5
|
+
import { registry } from '../../core/registry'
|
|
6
|
+
import { createTransport, readConfig } from '../../om/config'
|
|
7
|
+
import { mikroExecutor } from '../../om/sql-executor-mikro'
|
|
8
|
+
import type { TransportAdapter } from '../../transport/types'
|
|
9
|
+
|
|
10
|
+
// One transport per process, not per request.
|
|
11
|
+
//
|
|
12
|
+
// A container is built per request, and a transport owns broker connections and bound workers.
|
|
13
|
+
// Building one per request would open a Redis connection per HTTP call — the kind of leak that
|
|
14
|
+
// looks like a memory problem for a week before anyone finds it.
|
|
15
|
+
let transport: TransportAdapter | null = null
|
|
16
|
+
function sharedTransport(): TransportAdapter {
|
|
17
|
+
if (!transport) transport = createTransport(readConfig())
|
|
18
|
+
return transport
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function register(container: AppContainer) {
|
|
22
|
+
container.register({
|
|
23
|
+
durableWorkService: {
|
|
24
|
+
resolve: (c) => {
|
|
25
|
+
const em = c.resolve<EntityManager>('em')
|
|
26
|
+
const config = readConfig()
|
|
27
|
+
return new DurableWorkService({
|
|
28
|
+
sql: mikroExecutor(em),
|
|
29
|
+
transport: sharedTransport(),
|
|
30
|
+
registry,
|
|
31
|
+
graceMs: config.reconcilerGraceMs,
|
|
32
|
+
})
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
// The raw executor, for the worker command: it drives the mechanism directly rather than
|
|
36
|
+
// through the service, and needs to open its own transactions.
|
|
37
|
+
durableWorkSql: { resolve: (c) => mikroExecutor(c.resolve<EntityManager>('em')) },
|
|
38
|
+
durableWorkRegistry: { resolve: () => registry },
|
|
39
|
+
durableWorkTransport: { resolve: () => sharedTransport() },
|
|
40
|
+
})
|
|
41
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { createModuleEvents } from '@open-mercato/shared/modules/events'
|
|
2
|
+
|
|
3
|
+
// The lifecycle a host can subscribe to.
|
|
4
|
+
//
|
|
5
|
+
// Every one is emitted after its transaction commits, and never inside it: a subscriber that
|
|
6
|
+
// throws must not be able to roll back the fact the event describes, and a slow subscriber
|
|
7
|
+
// must not hold a row lock for the length of its work.
|
|
8
|
+
export const events = [
|
|
9
|
+
{ id: 'durable_work.job.created', label: 'Job created', entity: 'job', category: 'crud', clientBroadcast: true },
|
|
10
|
+
{ id: 'durable_work.job.started', label: 'Job started', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
11
|
+
{ id: 'durable_work.job.yielded', label: 'Slice handed back', entity: 'job', category: 'lifecycle', clientBroadcast: false },
|
|
12
|
+
{ id: 'durable_work.job.completed', label: 'Job completed', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
13
|
+
{ id: 'durable_work.job.failed', label: 'Job failed', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
14
|
+
{ id: 'durable_work.job.parked', label: 'Job parked for an operator', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
15
|
+
{ id: 'durable_work.job.cancelled', label: 'Job cancelled', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
16
|
+
{ id: 'durable_work.job.redriven', label: 'Job re-driven', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
17
|
+
{ id: 'durable_work.job.lease_lost', label: 'Lease lost', entity: 'job', category: 'lifecycle', clientBroadcast: false },
|
|
18
|
+
{ id: 'durable_work.job.mirror_stuck', label: 'Domain mirror stuck', entity: 'job', category: 'lifecycle', clientBroadcast: true },
|
|
19
|
+
] as const
|
|
20
|
+
|
|
21
|
+
export const eventsConfig = createModuleEvents({
|
|
22
|
+
moduleId: 'durable_work',
|
|
23
|
+
events,
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
export const emitDurableWorkEvent = eventsConfig.emit
|
|
27
|
+
|
|
28
|
+
export type DurableWorkEventId = (typeof events)[number]['id']
|
|
29
|
+
|
|
30
|
+
export default eventsConfig
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ModuleInfo } from '@open-mercato/shared/modules/registry'
|
|
2
|
+
|
|
3
|
+
export const metadata: ModuleInfo = {
|
|
4
|
+
name: 'durable_work',
|
|
5
|
+
title: 'Durable Work',
|
|
6
|
+
version: '0.0.1',
|
|
7
|
+
description:
|
|
8
|
+
'Durable at-least-once background work: a leased job record with epoch fencing, bounded resumable slices, a server-side reconciler, fenced cancel and an operator API. Other modules register job kinds; this module runs them.',
|
|
9
|
+
author: 'Full Stack House',
|
|
10
|
+
license: 'MIT',
|
|
11
|
+
ejectable: true,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export { features } from './acl'
|
|
15
|
+
|
|
16
|
+
export default metadata
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Shared plumbing for the operator routes.
|
|
2
|
+
|
|
3
|
+
import { NextResponse } from 'next/server'
|
|
4
|
+
import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
|
|
5
|
+
import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
|
|
6
|
+
|
|
7
|
+
import type { DurableWorkService } from '../../../core/service'
|
|
8
|
+
import type { DurableJob, Scope } from '../../../core/types'
|
|
9
|
+
|
|
10
|
+
export type RouteContext = { service: DurableWorkService; scope: Scope; userId: string | null }
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Resolves the caller's scope and the service, or the response to return instead.
|
|
14
|
+
*
|
|
15
|
+
* The scope comes from the session, never from the request body or the path. An operator API
|
|
16
|
+
* that let a caller name a tenant would be a way to re-drive somebody else's work.
|
|
17
|
+
*/
|
|
18
|
+
export async function routeContext(req: Request): Promise<RouteContext | NextResponse> {
|
|
19
|
+
const auth = await getAuthFromRequest(req)
|
|
20
|
+
if (!auth || !auth.tenantId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
|
21
|
+
const container = await createRequestContainer()
|
|
22
|
+
return {
|
|
23
|
+
service: container.resolve('durableWorkService') as DurableWorkService,
|
|
24
|
+
scope: { tenantId: auth.tenantId, organizationId: auth.orgId ?? null },
|
|
25
|
+
userId: auth.sub ?? null,
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The wire shape. Timestamps as ISO strings, plus two things only the server can decide. */
|
|
30
|
+
export function toDto(job: DurableJob) {
|
|
31
|
+
return {
|
|
32
|
+
id: job.id,
|
|
33
|
+
kind: job.kind,
|
|
34
|
+
status: job.status,
|
|
35
|
+
input: job.input,
|
|
36
|
+
checkpoint: job.checkpoint,
|
|
37
|
+
subject: job.subjectType ? { type: job.subjectType, id: job.subjectId } : null,
|
|
38
|
+
lockKey: job.lockKey,
|
|
39
|
+
idempotencyKey: job.idempotencyKey,
|
|
40
|
+
processedCount: job.processedCount,
|
|
41
|
+
totalCount: job.totalCount,
|
|
42
|
+
redrives: job.redrives,
|
|
43
|
+
interruptions: job.interruptions,
|
|
44
|
+
consecutiveFailures: job.consecutiveFailures,
|
|
45
|
+
mirrorAttempts: job.mirrorAttempts,
|
|
46
|
+
leaseOwner: job.leaseOwner,
|
|
47
|
+
leaseEpoch: job.leaseEpoch,
|
|
48
|
+
leaseExpiresAt: iso(job.leaseExpiresAt),
|
|
49
|
+
heartbeatAt: iso(job.heartbeatAt),
|
|
50
|
+
nextRunAt: iso(job.nextRunAt),
|
|
51
|
+
startedAt: iso(job.startedAt),
|
|
52
|
+
finishedAt: iso(job.finishedAt),
|
|
53
|
+
parkedAt: iso(job.parkedAt),
|
|
54
|
+
cancelRequestedAt: iso(job.cancelRequestedAt),
|
|
55
|
+
errorClass: job.errorClass,
|
|
56
|
+
errorCode: job.errorCode,
|
|
57
|
+
errorMessage: job.errorMessage,
|
|
58
|
+
createdAt: iso(job.createdAt),
|
|
59
|
+
updatedAt: iso(job.updatedAt),
|
|
60
|
+
// Derived on the server because the client cannot see the kind's grace, its pending TTL
|
|
61
|
+
// or its mirror budget — and a UI that guessed would offer buttons that then 409.
|
|
62
|
+
redrivable: isRedrivable(job),
|
|
63
|
+
stuck: isStuck(job),
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const iso = (value: Date | null): string | null => (value ? value.toISOString() : null)
|
|
68
|
+
|
|
69
|
+
/** `completed` and `cancelled` are done on purpose: the first succeeded, the second was asked
|
|
70
|
+
* for. Everything else that has stopped can be re-driven. */
|
|
71
|
+
function isRedrivable(job: DurableJob): boolean {
|
|
72
|
+
if (job.status === 'completed' || job.status === 'cancelled') return false
|
|
73
|
+
if (job.status === 'failed') return true
|
|
74
|
+
return job.status === 'running' && job.leaseExpiresAt != null && job.leaseExpiresAt.getTime() < Date.now()
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** "Nobody is driving this and nothing is scheduled" — the state an operator needs to see. */
|
|
78
|
+
function isStuck(job: DurableJob): boolean {
|
|
79
|
+
if (job.status !== 'running') return false
|
|
80
|
+
const expired = job.leaseExpiresAt != null && job.leaseExpiresAt.getTime() < Date.now() - 20_000
|
|
81
|
+
const nothingScheduled = job.nextRunAt == null || job.nextRunAt.getTime() < Date.now() - 900_000
|
|
82
|
+
return expired && nothingScheduled
|
|
83
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Migration } from '@mikro-orm/migrations'
|
|
2
|
+
|
|
3
|
+
import { CREATE_INDEXES, CREATE_TABLE, DROP_INDEXES, DROP_TABLE, SET_FILLFACTOR } from '../../../core/schema'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Creates `durable_work_jobs`.
|
|
7
|
+
*
|
|
8
|
+
* The statements come from `core/schema.ts` rather than being written out again here, so what
|
|
9
|
+
* a host migrates and what the failure harness exercises are the same DDL. Two copies of a
|
|
10
|
+
* schema drift, and the way that drift surfaces is a predicate silently not being enforced in
|
|
11
|
+
* production while every test still passes.
|
|
12
|
+
*/
|
|
13
|
+
export class Migration20260908120000 extends Migration {
|
|
14
|
+
override async up(): Promise<void> {
|
|
15
|
+
this.addSql(CREATE_TABLE)
|
|
16
|
+
this.addSql(SET_FILLFACTOR)
|
|
17
|
+
for (const statement of CREATE_INDEXES) this.addSql(statement)
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
override async down(): Promise<void> {
|
|
21
|
+
for (const statement of DROP_INDEXES) this.addSql(statement)
|
|
22
|
+
this.addSql(DROP_TABLE)
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ModuleSetupConfig } from '@open-mercato/shared/modules/setup'
|
|
2
|
+
|
|
3
|
+
export const setup: ModuleSetupConfig = {
|
|
4
|
+
defaultRoleFeatures: {
|
|
5
|
+
superadmin: ['durable_work.view', 'durable_work.operate'],
|
|
6
|
+
admin: ['durable_work.view', 'durable_work.operate'],
|
|
7
|
+
},
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export default setup
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { toPositional } from '../sql-executor-mikro'
|
|
2
|
+
|
|
3
|
+
// The statements are written in Postgres's own `$n` form so the SQL the failure harness
|
|
4
|
+
// exercises is character-for-character the SQL a host runs. MikroORM binds `?` positionally,
|
|
5
|
+
// and this is the whole of the translation between them — so it is the whole of what can go
|
|
6
|
+
// silently wrong between a tested statement and a deployed one.
|
|
7
|
+
describe('toPositional', () => {
|
|
8
|
+
it('rewrites placeholders in order', () => {
|
|
9
|
+
expect(toPositional('select $1, $2', ['a', 'b'])).toEqual({ text: 'select ?, ?', params: ['a', 'b'] })
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
it('duplicates a parameter that the statement reads twice', () => {
|
|
13
|
+
// The scope predicate does exactly this: compares the organization and tests it for null.
|
|
14
|
+
const sql = 'where organization_id = $2 or ($2::uuid is null and organization_id is null)'
|
|
15
|
+
expect(toPositional(sql, ['tenant', null])).toEqual({
|
|
16
|
+
text: 'where organization_id = ? or (?::uuid is null and organization_id is null)',
|
|
17
|
+
params: [null, null],
|
|
18
|
+
})
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
it('handles placeholders that are not in ascending order', () => {
|
|
22
|
+
expect(toPositional('select $3, $1', ['a', 'b', 'c'])).toEqual({ text: 'select ?, ?', params: ['c', 'a'] })
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
it('refuses a statement that references a parameter nobody supplied', () => {
|
|
26
|
+
// Silently binding undefined would turn a fenced predicate into one that matches nothing,
|
|
27
|
+
// which reads exactly like "the lease was lost" and would be debugged as such.
|
|
28
|
+
expect(() => toPositional('select $2', ['only-one'])).toThrow(/references \$2 but 1 parameter/)
|
|
29
|
+
})
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
// The two bugs that made the OM adapter differ from the harness — where every statement runs
|
|
33
|
+
// through node-postgres, which reports affected rows and honours transactions for free.
|
|
34
|
+
describe('mikroExecutor', () => {
|
|
35
|
+
const stub = (result: unknown) => {
|
|
36
|
+
const calls: Array<{ sql: string; method?: string; ctx?: unknown }> = []
|
|
37
|
+
const connection = {
|
|
38
|
+
execute: async (sql: string, _params?: unknown[], method?: string, ctx?: unknown) => {
|
|
39
|
+
calls.push({ sql, method, ctx })
|
|
40
|
+
return result
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
const em = {
|
|
44
|
+
getConnection: () => connection,
|
|
45
|
+
getTransactionContext: () => 'the-transaction',
|
|
46
|
+
fork: () => em,
|
|
47
|
+
transactional: async (fn: (trx: unknown) => Promise<unknown>) => fn(em),
|
|
48
|
+
}
|
|
49
|
+
return { em, calls }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
it('counts rows a statement affected, not rows it returned', async () => {
|
|
53
|
+
// An UPDATE with no RETURNING returns nothing. Reporting that as zero affected rows makes
|
|
54
|
+
// every domain mirror look like it matched nothing — which is treated exactly like a throw,
|
|
55
|
+
// so the mirror is retried forever over a write that already landed.
|
|
56
|
+
const { em, calls } = stub({ affectedRows: 1 })
|
|
57
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
58
|
+
const { mikroExecutor } = await import('../sql-executor-mikro')
|
|
59
|
+
const sql = mikroExecutor(em as any)
|
|
60
|
+
expect(await sql.query('update sync_runs set status = $1 where id = $2', ['completed', 'x'])).toEqual({ rows: [], rowCount: 1 })
|
|
61
|
+
expect(calls[0]!.method).toBe('run')
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('reads rows from a statement that returns them', async () => {
|
|
65
|
+
const { em, calls } = stub([{ id: 'a' }])
|
|
66
|
+
const { mikroExecutor } = await import('../sql-executor-mikro')
|
|
67
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
68
|
+
const sql = mikroExecutor(em as any)
|
|
69
|
+
expect(await sql.query('update t set a = $1 returning id', [1])).toEqual({ rows: [{ id: 'a' }], rowCount: 1 })
|
|
70
|
+
expect(calls[0]!.method).toBe('all')
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it('passes the transaction context to every statement', async () => {
|
|
74
|
+
// Without it the statements run on a pooled connection outside the transaction: nothing
|
|
75
|
+
// fails, it simply stops being atomic — so `fencedWrite` no longer rolls back a stale
|
|
76
|
+
// worker's writes and a terminal transition no longer moves both rows together.
|
|
77
|
+
const { em, calls } = stub([{ id: 'a' }])
|
|
78
|
+
const { mikroExecutor } = await import('../sql-executor-mikro')
|
|
79
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
80
|
+
await mikroExecutor(em as any).transaction(async (tx) => tx.query('select 1'))
|
|
81
|
+
expect(calls[0]!.ctx).toBe('the-transaction')
|
|
82
|
+
})
|
|
83
|
+
})
|
package/src/om/config.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// How a host configures the mechanism: environment variables in, a transport out.
|
|
2
|
+
|
|
3
|
+
import { BullMQTransport } from '../transport/bullmq'
|
|
4
|
+
import { MemoryTransport } from '../transport/memory'
|
|
5
|
+
import { PgBossTransport } from '../transport/pgboss'
|
|
6
|
+
import type { TransportAdapter, TransportName } from '../transport/types'
|
|
7
|
+
|
|
8
|
+
export type DurableWorkConfig = {
|
|
9
|
+
transport: TransportName
|
|
10
|
+
redisUrl: string | null
|
|
11
|
+
databaseUrl: string | null
|
|
12
|
+
pgBossSchema: string
|
|
13
|
+
tickMs: number
|
|
14
|
+
drainTimeoutMs: number
|
|
15
|
+
reconcilerGraceMs: number
|
|
16
|
+
/** Hosts the worker inside the app process. Dev and ephemeral tests only. */
|
|
17
|
+
inProcessWorker: boolean
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const num = (value: string | undefined, fallback: number): number => {
|
|
21
|
+
const parsed = Number(value)
|
|
22
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const bool = (value: string | undefined): boolean => value === '1' || value?.toLowerCase() === 'true'
|
|
26
|
+
|
|
27
|
+
export function readConfig(env: NodeJS.ProcessEnv = process.env): DurableWorkConfig {
|
|
28
|
+
const raw = (env.DURABLE_WORK_TRANSPORT ?? 'pgboss').trim().toLowerCase()
|
|
29
|
+
if (raw !== 'memory' && raw !== 'bullmq' && raw !== 'pgboss') {
|
|
30
|
+
throw new Error(`DURABLE_WORK_TRANSPORT must be memory | bullmq | pgboss, got ${JSON.stringify(raw)}`)
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
transport: raw,
|
|
34
|
+
// Falls back to the queue module's Redis, because an app that already runs one should not
|
|
35
|
+
// have to configure a second.
|
|
36
|
+
redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,
|
|
37
|
+
databaseUrl: env.DATABASE_URL ?? null,
|
|
38
|
+
pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? 'durable_work_boss',
|
|
39
|
+
tickMs: num(env.DURABLE_WORK_TICK_MS, 15_000),
|
|
40
|
+
drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 30_000),
|
|
41
|
+
reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 20_000),
|
|
42
|
+
inProcessWorker: bool(env.DURABLE_WORK_INPROCESS_WORKER),
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function createTransport(config: DurableWorkConfig, deps: { redisConnection?: unknown } = {}): TransportAdapter {
|
|
47
|
+
switch (config.transport) {
|
|
48
|
+
case 'memory':
|
|
49
|
+
// Nothing survives the process, so this is a development convenience and is refused in
|
|
50
|
+
// production rather than quietly losing every job on the next deploy.
|
|
51
|
+
if (process.env.NODE_ENV === 'production') {
|
|
52
|
+
throw new Error('DURABLE_WORK_TRANSPORT=memory keeps jobs in process memory and cannot be used in production.')
|
|
53
|
+
}
|
|
54
|
+
return new MemoryTransport()
|
|
55
|
+
case 'bullmq': {
|
|
56
|
+
const connection = deps.redisConnection ?? config.redisUrl
|
|
57
|
+
if (!connection) throw new Error('DURABLE_WORK_TRANSPORT=bullmq requires DURABLE_WORK_REDIS_URL (or QUEUE_REDIS_URL).')
|
|
58
|
+
return new BullMQTransport({ connection })
|
|
59
|
+
}
|
|
60
|
+
case 'pgboss': {
|
|
61
|
+
if (!config.databaseUrl) throw new Error('DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.')
|
|
62
|
+
return new PgBossTransport({ connectionString: config.databaseUrl, schema: config.pgBossSchema })
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|