@fullstackhouse/open-mercato-durable-work 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/README.md +136 -0
  2. package/dist/core/errors.js +81 -0
  3. package/dist/core/errors.js.map +7 -0
  4. package/dist/core/ids.js +30 -0
  5. package/dist/core/ids.js.map +7 -0
  6. package/dist/core/reconciler.js +149 -0
  7. package/dist/core/reconciler.js.map +7 -0
  8. package/dist/core/registry.js +72 -0
  9. package/dist/core/registry.js.map +7 -0
  10. package/dist/core/run-slice.js +210 -0
  11. package/dist/core/run-slice.js.map +7 -0
  12. package/dist/core/schema.js +100 -0
  13. package/dist/core/schema.js.map +7 -0
  14. package/dist/core/service.js +161 -0
  15. package/dist/core/service.js.map +7 -0
  16. package/dist/core/store.js +516 -0
  17. package/dist/core/store.js.map +7 -0
  18. package/dist/core/terminal.js +53 -0
  19. package/dist/core/terminal.js.map +7 -0
  20. package/dist/core/types.js +1 -0
  21. package/dist/core/types.js.map +7 -0
  22. package/dist/core/worker.js +111 -0
  23. package/dist/core/worker.js.map +7 -0
  24. package/dist/index.js +99 -0
  25. package/dist/index.js.map +7 -0
  26. package/dist/modules/durable_work/acl.js +10 -0
  27. package/dist/modules/durable_work/acl.js.map +7 -0
  28. package/dist/modules/durable_work/api/jobs/[id]/redrive.js +20 -0
  29. package/dist/modules/durable_work/api/jobs/[id]/redrive.js.map +7 -0
  30. package/dist/modules/durable_work/api/jobs/[id]/route.js +26 -0
  31. package/dist/modules/durable_work/api/jobs/[id]/route.js.map +7 -0
  32. package/dist/modules/durable_work/api/jobs/route.js +24 -0
  33. package/dist/modules/durable_work/api/jobs/route.js.map +7 -0
  34. package/dist/modules/durable_work/cli.js +72 -0
  35. package/dist/modules/durable_work/cli.js.map +7 -0
  36. package/dist/modules/durable_work/data/entities.js +163 -0
  37. package/dist/modules/durable_work/data/entities.js.map +7 -0
  38. package/dist/modules/durable_work/di.js +34 -0
  39. package/dist/modules/durable_work/di.js.map +7 -0
  40. package/dist/modules/durable_work/events.js +26 -0
  41. package/dist/modules/durable_work/events.js.map +7 -0
  42. package/dist/modules/durable_work/index.js +17 -0
  43. package/dist/modules/durable_work/index.js.map +7 -0
  44. package/dist/modules/durable_work/lib/route-helpers.js +66 -0
  45. package/dist/modules/durable_work/lib/route-helpers.js.map +7 -0
  46. package/dist/modules/durable_work/migrations/Migration20260908120000.js +17 -0
  47. package/dist/modules/durable_work/migrations/Migration20260908120000.js.map +7 -0
  48. package/dist/modules/durable_work/setup.js +12 -0
  49. package/dist/modules/durable_work/setup.js.map +7 -0
  50. package/dist/om/config.js +49 -0
  51. package/dist/om/config.js.map +7 -0
  52. package/dist/om/progress-mirror.js +49 -0
  53. package/dist/om/progress-mirror.js.map +7 -0
  54. package/dist/om/sql-executor-mikro.js +48 -0
  55. package/dist/om/sql-executor-mikro.js.map +7 -0
  56. package/dist/transport/bullmq.js +144 -0
  57. package/dist/transport/bullmq.js.map +7 -0
  58. package/dist/transport/conformance.js +177 -0
  59. package/dist/transport/conformance.js.map +7 -0
  60. package/dist/transport/memory.js +139 -0
  61. package/dist/transport/memory.js.map +7 -0
  62. package/dist/transport/pgboss.js +176 -0
  63. package/dist/transport/pgboss.js.map +7 -0
  64. package/dist/transport/types.js +1 -0
  65. package/dist/transport/types.js.map +7 -0
  66. package/generated/entities/durable_work_job/index.ts +42 -0
  67. package/generated/entities.ids.generated.ts +9 -0
  68. package/package.json +145 -0
  69. package/src/core/__tests__/registry.test.ts +43 -0
  70. package/src/core/errors.ts +104 -0
  71. package/src/core/ids.ts +58 -0
  72. package/src/core/reconciler.ts +242 -0
  73. package/src/core/registry.ts +199 -0
  74. package/src/core/run-slice.ts +343 -0
  75. package/src/core/schema.ts +114 -0
  76. package/src/core/service.ts +222 -0
  77. package/src/core/store.ts +786 -0
  78. package/src/core/terminal.ts +107 -0
  79. package/src/core/types.ts +120 -0
  80. package/src/core/worker.ts +169 -0
  81. package/src/index.ts +100 -0
  82. package/src/modules/durable_work/__integration__/TC-DW-001.spec.ts +51 -0
  83. package/src/modules/durable_work/__tests__/metadata.test.ts +13 -0
  84. package/src/modules/durable_work/__tests__/schema-agreement.test.ts +52 -0
  85. package/src/modules/durable_work/acl.ts +6 -0
  86. package/src/modules/durable_work/api/jobs/[id]/redrive.ts +27 -0
  87. package/src/modules/durable_work/api/jobs/[id]/route.ts +27 -0
  88. package/src/modules/durable_work/api/jobs/route.ts +26 -0
  89. package/src/modules/durable_work/cli.ts +91 -0
  90. package/src/modules/durable_work/data/entities.ts +158 -0
  91. package/src/modules/durable_work/di.ts +41 -0
  92. package/src/modules/durable_work/events.ts +30 -0
  93. package/src/modules/durable_work/index.ts +16 -0
  94. package/src/modules/durable_work/lib/route-helpers.ts +83 -0
  95. package/src/modules/durable_work/migrations/Migration20260908120000.ts +24 -0
  96. package/src/modules/durable_work/setup.ts +10 -0
  97. package/src/om/__tests__/sql-executor-mikro.test.ts +83 -0
  98. package/src/om/config.ts +65 -0
  99. package/src/om/progress-mirror.ts +80 -0
  100. package/src/om/sql-executor-mikro.ts +104 -0
  101. package/src/transport/bullmq.ts +213 -0
  102. package/src/transport/conformance.ts +218 -0
  103. package/src/transport/memory.ts +191 -0
  104. package/src/transport/pgboss.ts +250 -0
  105. package/src/transport/types.ts +81 -0
@@ -0,0 +1,27 @@
1
+ import { NextResponse } from 'next/server'
2
+
3
+ import { routeContext, toDto } from '../../../lib/route-helpers'
4
+
5
+ export const metadata = {
6
+ GET: { requireAuth: true, requireFeatures: ['durable_work.view'] },
7
+ DELETE: { requireAuth: true, requireFeatures: ['durable_work.operate'] },
8
+ }
9
+
10
+ export async function GET(req: Request, { params }: { params: { id: string } }) {
11
+ const ctx = await routeContext(req)
12
+ if (ctx instanceof NextResponse) return ctx
13
+ const job = await ctx.service.get(params.id, ctx.scope)
14
+ if (!job) return NextResponse.json({ error: 'Not found' }, { status: 404 })
15
+ return NextResponse.json(toDto(job))
16
+ }
17
+
18
+ /** Asks the job to stop. Answers with what actually happened — a running job is `cancelling`
19
+ * until its driver observes the request, and saying `cancelled` before that would be a lie
20
+ * an operator might act on. */
21
+ export async function DELETE(req: Request, { params }: { params: { id: string } }) {
22
+ const ctx = await routeContext(req)
23
+ if (ctx instanceof NextResponse) return ctx
24
+ const job = await ctx.service.cancel(params.id, ctx.scope, ctx.userId)
25
+ if (!job) return NextResponse.json({ error: 'Not found or already finished' }, { status: 404 })
26
+ return NextResponse.json({ ...toDto(job), state: job.status === 'cancelled' ? 'cancelled' : 'cancelling' })
27
+ }
@@ -0,0 +1,26 @@
1
+ import { NextResponse } from 'next/server'
2
+
3
+ import { routeContext, toDto } from '../../lib/route-helpers'
4
+ import type { DurableJobStatus } from '../../../../core/types'
5
+
6
+ export const metadata = {
7
+ GET: { requireAuth: true, requireFeatures: ['durable_work.view'] },
8
+ }
9
+
10
+ const STATUSES = new Set<DurableJobStatus>(['pending', 'running', 'completed', 'failed', 'cancelled'])
11
+
12
+ export async function GET(req: Request) {
13
+ const ctx = await routeContext(req)
14
+ if (ctx instanceof NextResponse) return ctx
15
+
16
+ const url = new URL(req.url)
17
+ const status = url.searchParams.get('status')
18
+ const { items, total } = await ctx.service.list(ctx.scope, {
19
+ kind: url.searchParams.get('kind') ?? undefined,
20
+ status: status && STATUSES.has(status as DurableJobStatus) ? (status as DurableJobStatus) : undefined,
21
+ page: Number(url.searchParams.get('page') ?? '1'),
22
+ pageSize: Number(url.searchParams.get('pageSize') ?? '20'),
23
+ })
24
+
25
+ return NextResponse.json({ items: items.map(toDto), total })
26
+ }
@@ -0,0 +1,91 @@
1
+ // The worker process, and the operator's command line.
2
+ //
3
+ // `mercato durable_work worker` is how the mechanism runs in production. It is deliberately its
4
+ // own process: a slice can run for minutes, and hosting that inside the web process means a
5
+ // deploy either kills work mid-batch or holds the deploy open for the length of a slice.
6
+
7
+ import type { ModuleCli } from '@open-mercato/shared/modules/registry'
8
+ import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
9
+
10
+ import type { DurableWorkService } from '../../core/service'
11
+ import { registry } from '../../core/registry'
12
+ import { startWorker } from '../../core/worker'
13
+ import { readConfig } from '../../om/config'
14
+ import type { SqlTransactor } from '../../core/types'
15
+ import type { TransportAdapter } from '../../transport/types'
16
+
17
+ const flag = (argv: string[], name: string): string | undefined => {
18
+ const index = argv.indexOf(`--${name}`)
19
+ return index >= 0 ? argv[index + 1] : undefined
20
+ }
21
+
22
+ const emit = (event: string, fields: Record<string, unknown> = {}) => console.log(JSON.stringify({ event, ...fields }))
23
+
24
+ const workerCommand: ModuleCli = {
25
+ command: 'worker',
26
+ async run(argv: string[]) {
27
+ const config = readConfig()
28
+ const container = await createRequestContainer()
29
+ const sql = container.resolve('durableWorkSql') as SqlTransactor
30
+ const transport = container.resolve('durableWorkTransport') as TransportAdapter
31
+
32
+ const worker = await startWorker({
33
+ sql,
34
+ transport,
35
+ registry,
36
+ kinds: flag(argv, 'kinds')?.split(','),
37
+ concurrency: flag(argv, 'concurrency') ? Number(flag(argv, 'concurrency')) : undefined,
38
+ tickMs: config.tickMs,
39
+ reconcilerGraceMs: config.reconcilerGraceMs,
40
+ drainTimeoutMs: config.drainTimeoutMs,
41
+ log: (event, fields) => emit(event, fields),
42
+ })
43
+
44
+ emit('durable_work.worker_started', {
45
+ owner: worker.owner,
46
+ transport: transport.name,
47
+ kinds: registry.list().map((kind) => kind.kind),
48
+ })
49
+
50
+ // SIGTERM is what a deploy sends. Draining rather than exiting is the difference between a
51
+ // slice handing its remaining work back and a slice being killed between two writes.
52
+ let stopping = false
53
+ const stop = async (signal: string) => {
54
+ if (stopping) return
55
+ stopping = true
56
+ emit('durable_work.worker_draining', { signal, timeoutMs: config.drainTimeoutMs })
57
+ await worker.stop()
58
+ emit('durable_work.worker_stopped')
59
+ process.exit(0)
60
+ }
61
+ process.on('SIGTERM', () => void stop('SIGTERM'))
62
+ process.on('SIGINT', () => void stop('SIGINT'))
63
+
64
+ await new Promise(() => undefined) // run until signalled
65
+ },
66
+ }
67
+
68
+ const reconcileCommand: ModuleCli = {
69
+ command: 'reconcile',
70
+ async run() {
71
+ const container = await createRequestContainer()
72
+ const service = container.resolve('durableWorkService') as DurableWorkService
73
+ console.log(JSON.stringify(await service.reconcile(), null, 2))
74
+ },
75
+ }
76
+
77
+ const helpCommand: ModuleCli = {
78
+ command: 'help',
79
+ async run() {
80
+ console.log(
81
+ [
82
+ 'mercato durable_work worker [--kinds a,b] [--concurrency n]',
83
+ ' Bind every registered kind, own the reconciler tick, drain on SIGTERM.',
84
+ 'mercato durable_work reconcile',
85
+ ' Run one reconciler pass and print what it repaired.',
86
+ ].join('\n'),
87
+ )
88
+ },
89
+ }
90
+
91
+ export default [workerCommand, reconcileCommand, helpCommand]
@@ -0,0 +1,158 @@
1
+ // The MikroORM view of the job table.
2
+ //
3
+ // The mechanism itself never goes through this entity — every statement in `core/store.ts` is
4
+ // hand-written SQL, because each is a compare-and-set whose predicate is the guarantee. The
5
+ // entity exists so the table is discoverable to the host: migrations, the entity registry,
6
+ // query tooling and anything an app wants to join against.
7
+ //
8
+ // It must therefore stay in step with `core/schema.ts`, which is the definition. A test
9
+ // asserts they agree column for column.
10
+
11
+ import { OptionalProps } from '@mikro-orm/core'
12
+ import { Entity, Index, PrimaryKey, Property } from '@mikro-orm/decorators/legacy'
13
+
14
+ export type DurableWorkJobStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
15
+
16
+ @Entity({ tableName: 'durable_work_jobs' })
17
+ @Index({ name: 'durable_work_jobs_running_idx', properties: ['tenantId'] })
18
+ @Index({ name: 'durable_work_jobs_subject_idx', properties: ['subjectType', 'subjectId'] })
19
+ export class DurableWorkJob {
20
+ [OptionalProps]?:
21
+ | 'status'
22
+ | 'leaseEpoch'
23
+ | 'continuationSeq'
24
+ | 'redrives'
25
+ | 'redrivesSinceCommit'
26
+ | 'consecutiveFailures'
27
+ | 'interruptions'
28
+ | 'mirrorAttempts'
29
+ | 'processedCount'
30
+ | 'createdAt'
31
+ | 'updatedAt'
32
+
33
+ @PrimaryKey({ type: 'uuid', defaultRaw: 'gen_random_uuid()' })
34
+ id!: string
35
+
36
+ @Property({ name: 'tenant_id', type: 'uuid' })
37
+ tenantId!: string
38
+
39
+ @Property({ name: 'organization_id', type: 'uuid', nullable: true })
40
+ organizationId?: string | null
41
+
42
+ @Property({ name: 'kind', type: 'text' })
43
+ kind!: string
44
+
45
+ @Property({ name: 'status', type: 'text' })
46
+ status: DurableWorkJobStatus = 'pending'
47
+
48
+ @Property({ name: 'created_by', type: 'uuid', nullable: true })
49
+ createdBy?: string | null
50
+
51
+ @Property({ name: 'created_at', type: 'timestamptz' })
52
+ createdAt: Date = new Date()
53
+
54
+ @Property({ name: 'updated_at', type: 'timestamptz', onUpdate: () => new Date() })
55
+ updatedAt: Date = new Date()
56
+
57
+ @Property({ name: 'input', type: 'jsonb', nullable: true })
58
+ input?: unknown
59
+
60
+ @Property({ name: 'checkpoint', type: 'jsonb', nullable: true })
61
+ checkpoint?: unknown
62
+
63
+ @Property({ name: 'meta', type: 'jsonb', nullable: true })
64
+ meta?: Record<string, unknown> | null
65
+
66
+ @Property({ name: 'idempotency_key', type: 'text', nullable: true })
67
+ idempotencyKey?: string | null
68
+
69
+ @Property({ name: 'lock_key', type: 'text', nullable: true })
70
+ lockKey?: string | null
71
+
72
+ @Property({ name: 'subject_type', type: 'text', nullable: true })
73
+ subjectType?: string | null
74
+
75
+ @Property({ name: 'subject_id', type: 'text', nullable: true })
76
+ subjectId?: string | null
77
+
78
+ @Property({ name: 'progress_job_id', type: 'uuid', nullable: true })
79
+ progressJobId?: string | null
80
+
81
+ @Property({ name: 'lease_owner', type: 'text', nullable: true })
82
+ leaseOwner?: string | null
83
+
84
+ @Property({ name: 'lease_epoch', type: 'bigint' })
85
+ leaseEpoch: number = 0
86
+
87
+ @Property({ name: 'lease_expires_at', type: 'timestamptz', nullable: true })
88
+ leaseExpiresAt?: Date | null
89
+
90
+ @Property({ name: 'heartbeat_at', type: 'timestamptz', nullable: true })
91
+ heartbeatAt?: Date | null
92
+
93
+ @Property({ name: 'queue_name', type: 'text', nullable: true })
94
+ queueName?: string | null
95
+
96
+ @Property({ name: 'queue_job_id', type: 'text', nullable: true })
97
+ queueJobId?: string | null
98
+
99
+ @Property({ name: 'continuation_seq', type: 'int' })
100
+ continuationSeq: number = 0
101
+
102
+ @Property({ name: 'redrives', type: 'int' })
103
+ redrives: number = 0
104
+
105
+ @Property({ name: 'next_run_at', type: 'timestamptz', nullable: true })
106
+ nextRunAt?: Date | null
107
+
108
+ @Property({ name: 'pending_since', type: 'timestamptz', nullable: true })
109
+ pendingSince?: Date | null
110
+
111
+ @Property({ name: 'redrives_since_commit', type: 'int' })
112
+ redrivesSinceCommit: number = 0
113
+
114
+ @Property({ name: 'consecutive_failures', type: 'int' })
115
+ consecutiveFailures: number = 0
116
+
117
+ @Property({ name: 'interruptions', type: 'int' })
118
+ interruptions: number = 0
119
+
120
+ @Property({ name: 'mirror_attempts', type: 'int' })
121
+ mirrorAttempts: number = 0
122
+
123
+ @Property({ name: 'last_committed_at', type: 'timestamptz', nullable: true })
124
+ lastCommittedAt?: Date | null
125
+
126
+ @Property({ name: 'started_at', type: 'timestamptz', nullable: true })
127
+ startedAt?: Date | null
128
+
129
+ @Property({ name: 'finished_at', type: 'timestamptz', nullable: true })
130
+ finishedAt?: Date | null
131
+
132
+ @Property({ name: 'parked_at', type: 'timestamptz', nullable: true })
133
+ parkedAt?: Date | null
134
+
135
+ @Property({ name: 'cancel_requested_at', type: 'timestamptz', nullable: true })
136
+ cancelRequestedAt?: Date | null
137
+
138
+ @Property({ name: 'cancelled_by', type: 'uuid', nullable: true })
139
+ cancelledBy?: string | null
140
+
141
+ @Property({ name: 'error_class', type: 'text', nullable: true })
142
+ errorClass?: string | null
143
+
144
+ @Property({ name: 'error_code', type: 'text', nullable: true })
145
+ errorCode?: string | null
146
+
147
+ @Property({ name: 'error_message', type: 'text', nullable: true })
148
+ errorMessage?: string | null
149
+
150
+ @Property({ name: 'domain_mirrored_at', type: 'timestamptz', nullable: true })
151
+ domainMirroredAt?: Date | null
152
+
153
+ @Property({ name: 'processed_count', type: 'int' })
154
+ processedCount: number = 0
155
+
156
+ @Property({ name: 'total_count', type: 'int', nullable: true })
157
+ totalCount?: number | null
158
+ }
@@ -0,0 +1,41 @@
1
+ import type { AppContainer } from '@open-mercato/shared/lib/di/container'
2
+ import type { EntityManager } from '@mikro-orm/postgresql'
3
+
4
+ import { DurableWorkService } from '../../core/service'
5
+ import { registry } from '../../core/registry'
6
+ import { createTransport, readConfig } from '../../om/config'
7
+ import { mikroExecutor } from '../../om/sql-executor-mikro'
8
+ import type { TransportAdapter } from '../../transport/types'
9
+
10
+ // One transport per process, not per request.
11
+ //
12
+ // A container is built per request, and a transport owns broker connections and bound workers.
13
+ // Building one per request would open a Redis connection per HTTP call — the kind of leak that
14
+ // looks like a memory problem for a week before anyone finds it.
15
+ let transport: TransportAdapter | null = null
16
+ function sharedTransport(): TransportAdapter {
17
+ if (!transport) transport = createTransport(readConfig())
18
+ return transport
19
+ }
20
+
21
+ export function register(container: AppContainer) {
22
+ container.register({
23
+ durableWorkService: {
24
+ resolve: (c) => {
25
+ const em = c.resolve<EntityManager>('em')
26
+ const config = readConfig()
27
+ return new DurableWorkService({
28
+ sql: mikroExecutor(em),
29
+ transport: sharedTransport(),
30
+ registry,
31
+ graceMs: config.reconcilerGraceMs,
32
+ })
33
+ },
34
+ },
35
+ // The raw executor, for the worker command: it drives the mechanism directly rather than
36
+ // through the service, and needs to open its own transactions.
37
+ durableWorkSql: { resolve: (c) => mikroExecutor(c.resolve<EntityManager>('em')) },
38
+ durableWorkRegistry: { resolve: () => registry },
39
+ durableWorkTransport: { resolve: () => sharedTransport() },
40
+ })
41
+ }
@@ -0,0 +1,30 @@
1
+ import { createModuleEvents } from '@open-mercato/shared/modules/events'
2
+
3
+ // The lifecycle a host can subscribe to.
4
+ //
5
+ // Every one is emitted after its transaction commits, and never inside it: a subscriber that
6
+ // throws must not be able to roll back the fact the event describes, and a slow subscriber
7
+ // must not hold a row lock for the length of its work.
8
+ export const events = [
9
+ { id: 'durable_work.job.created', label: 'Job created', entity: 'job', category: 'crud', clientBroadcast: true },
10
+ { id: 'durable_work.job.started', label: 'Job started', entity: 'job', category: 'lifecycle', clientBroadcast: true },
11
+ { id: 'durable_work.job.yielded', label: 'Slice handed back', entity: 'job', category: 'lifecycle', clientBroadcast: false },
12
+ { id: 'durable_work.job.completed', label: 'Job completed', entity: 'job', category: 'lifecycle', clientBroadcast: true },
13
+ { id: 'durable_work.job.failed', label: 'Job failed', entity: 'job', category: 'lifecycle', clientBroadcast: true },
14
+ { id: 'durable_work.job.parked', label: 'Job parked for an operator', entity: 'job', category: 'lifecycle', clientBroadcast: true },
15
+ { id: 'durable_work.job.cancelled', label: 'Job cancelled', entity: 'job', category: 'lifecycle', clientBroadcast: true },
16
+ { id: 'durable_work.job.redriven', label: 'Job re-driven', entity: 'job', category: 'lifecycle', clientBroadcast: true },
17
+ { id: 'durable_work.job.lease_lost', label: 'Lease lost', entity: 'job', category: 'lifecycle', clientBroadcast: false },
18
+ { id: 'durable_work.job.mirror_stuck', label: 'Domain mirror stuck', entity: 'job', category: 'lifecycle', clientBroadcast: true },
19
+ ] as const
20
+
21
+ export const eventsConfig = createModuleEvents({
22
+ moduleId: 'durable_work',
23
+ events,
24
+ })
25
+
26
+ export const emitDurableWorkEvent = eventsConfig.emit
27
+
28
+ export type DurableWorkEventId = (typeof events)[number]['id']
29
+
30
+ export default eventsConfig
@@ -0,0 +1,16 @@
1
+ import type { ModuleInfo } from '@open-mercato/shared/modules/registry'
2
+
3
+ export const metadata: ModuleInfo = {
4
+ name: 'durable_work',
5
+ title: 'Durable Work',
6
+ version: '0.0.1',
7
+ description:
8
+ 'Durable at-least-once background work: a leased job record with epoch fencing, bounded resumable slices, a server-side reconciler, fenced cancel and an operator API. Other modules register job kinds; this module runs them.',
9
+ author: 'Full Stack House',
10
+ license: 'MIT',
11
+ ejectable: true,
12
+ }
13
+
14
+ export { features } from './acl'
15
+
16
+ export default metadata
@@ -0,0 +1,83 @@
1
+ // Shared plumbing for the operator routes.
2
+
3
+ import { NextResponse } from 'next/server'
4
+ import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
5
+ import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
6
+
7
+ import type { DurableWorkService } from '../../../core/service'
8
+ import type { DurableJob, Scope } from '../../../core/types'
9
+
10
+ export type RouteContext = { service: DurableWorkService; scope: Scope; userId: string | null }
11
+
12
+ /**
13
+ * Resolves the caller's scope and the service, or the response to return instead.
14
+ *
15
+ * The scope comes from the session, never from the request body or the path. An operator API
16
+ * that let a caller name a tenant would be a way to re-drive somebody else's work.
17
+ */
18
+ export async function routeContext(req: Request): Promise<RouteContext | NextResponse> {
19
+ const auth = await getAuthFromRequest(req)
20
+ if (!auth || !auth.tenantId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
21
+ const container = await createRequestContainer()
22
+ return {
23
+ service: container.resolve('durableWorkService') as DurableWorkService,
24
+ scope: { tenantId: auth.tenantId, organizationId: auth.orgId ?? null },
25
+ userId: auth.sub ?? null,
26
+ }
27
+ }
28
+
29
+ /** The wire shape. Timestamps as ISO strings, plus two things only the server can decide. */
30
+ export function toDto(job: DurableJob) {
31
+ return {
32
+ id: job.id,
33
+ kind: job.kind,
34
+ status: job.status,
35
+ input: job.input,
36
+ checkpoint: job.checkpoint,
37
+ subject: job.subjectType ? { type: job.subjectType, id: job.subjectId } : null,
38
+ lockKey: job.lockKey,
39
+ idempotencyKey: job.idempotencyKey,
40
+ processedCount: job.processedCount,
41
+ totalCount: job.totalCount,
42
+ redrives: job.redrives,
43
+ interruptions: job.interruptions,
44
+ consecutiveFailures: job.consecutiveFailures,
45
+ mirrorAttempts: job.mirrorAttempts,
46
+ leaseOwner: job.leaseOwner,
47
+ leaseEpoch: job.leaseEpoch,
48
+ leaseExpiresAt: iso(job.leaseExpiresAt),
49
+ heartbeatAt: iso(job.heartbeatAt),
50
+ nextRunAt: iso(job.nextRunAt),
51
+ startedAt: iso(job.startedAt),
52
+ finishedAt: iso(job.finishedAt),
53
+ parkedAt: iso(job.parkedAt),
54
+ cancelRequestedAt: iso(job.cancelRequestedAt),
55
+ errorClass: job.errorClass,
56
+ errorCode: job.errorCode,
57
+ errorMessage: job.errorMessage,
58
+ createdAt: iso(job.createdAt),
59
+ updatedAt: iso(job.updatedAt),
60
+ // Derived on the server because the client cannot see the kind's grace, its pending TTL
61
+ // or its mirror budget — and a UI that guessed would offer buttons that then 409.
62
+ redrivable: isRedrivable(job),
63
+ stuck: isStuck(job),
64
+ }
65
+ }
66
+
67
+ const iso = (value: Date | null): string | null => (value ? value.toISOString() : null)
68
+
69
+ /** `completed` and `cancelled` are done on purpose: the first succeeded, the second was asked
70
+ * for. Everything else that has stopped can be re-driven. */
71
+ function isRedrivable(job: DurableJob): boolean {
72
+ if (job.status === 'completed' || job.status === 'cancelled') return false
73
+ if (job.status === 'failed') return true
74
+ return job.status === 'running' && job.leaseExpiresAt != null && job.leaseExpiresAt.getTime() < Date.now()
75
+ }
76
+
77
+ /** "Nobody is driving this and nothing is scheduled" — the state an operator needs to see. */
78
+ function isStuck(job: DurableJob): boolean {
79
+ if (job.status !== 'running') return false
80
+ const expired = job.leaseExpiresAt != null && job.leaseExpiresAt.getTime() < Date.now() - 20_000
81
+ const nothingScheduled = job.nextRunAt == null || job.nextRunAt.getTime() < Date.now() - 900_000
82
+ return expired && nothingScheduled
83
+ }
@@ -0,0 +1,24 @@
1
+ import { Migration } from '@mikro-orm/migrations'
2
+
3
+ import { CREATE_INDEXES, CREATE_TABLE, DROP_INDEXES, DROP_TABLE, SET_FILLFACTOR } from '../../../core/schema'
4
+
5
+ /**
6
+ * Creates `durable_work_jobs`.
7
+ *
8
+ * The statements come from `core/schema.ts` rather than being written out again here, so what
9
+ * a host migrates and what the failure harness exercises are the same DDL. Two copies of a
10
+ * schema drift, and the way that drift surfaces is a predicate silently not being enforced in
11
+ * production while every test still passes.
12
+ */
13
+ export class Migration20260908120000 extends Migration {
14
+ override async up(): Promise<void> {
15
+ this.addSql(CREATE_TABLE)
16
+ this.addSql(SET_FILLFACTOR)
17
+ for (const statement of CREATE_INDEXES) this.addSql(statement)
18
+ }
19
+
20
+ override async down(): Promise<void> {
21
+ for (const statement of DROP_INDEXES) this.addSql(statement)
22
+ this.addSql(DROP_TABLE)
23
+ }
24
+ }
@@ -0,0 +1,10 @@
1
+ import type { ModuleSetupConfig } from '@open-mercato/shared/modules/setup'
2
+
3
+ export const setup: ModuleSetupConfig = {
4
+ defaultRoleFeatures: {
5
+ superadmin: ['durable_work.view', 'durable_work.operate'],
6
+ admin: ['durable_work.view', 'durable_work.operate'],
7
+ },
8
+ }
9
+
10
+ export default setup
@@ -0,0 +1,83 @@
1
+ import { toPositional } from '../sql-executor-mikro'
2
+
3
+ // The statements are written in Postgres's own `$n` form so the SQL the failure harness
4
+ // exercises is character-for-character the SQL a host runs. MikroORM binds `?` positionally,
5
+ // and this is the whole of the translation between them — so it is the whole of what can go
6
+ // silently wrong between a tested statement and a deployed one.
7
+ describe('toPositional', () => {
8
+ it('rewrites placeholders in order', () => {
9
+ expect(toPositional('select $1, $2', ['a', 'b'])).toEqual({ text: 'select ?, ?', params: ['a', 'b'] })
10
+ })
11
+
12
+ it('duplicates a parameter that the statement reads twice', () => {
13
+ // The scope predicate does exactly this: compares the organization and tests it for null.
14
+ const sql = 'where organization_id = $2 or ($2::uuid is null and organization_id is null)'
15
+ expect(toPositional(sql, ['tenant', null])).toEqual({
16
+ text: 'where organization_id = ? or (?::uuid is null and organization_id is null)',
17
+ params: [null, null],
18
+ })
19
+ })
20
+
21
+ it('handles placeholders that are not in ascending order', () => {
22
+ expect(toPositional('select $3, $1', ['a', 'b', 'c'])).toEqual({ text: 'select ?, ?', params: ['c', 'a'] })
23
+ })
24
+
25
+ it('refuses a statement that references a parameter nobody supplied', () => {
26
+ // Silently binding undefined would turn a fenced predicate into one that matches nothing,
27
+ // which reads exactly like "the lease was lost" and would be debugged as such.
28
+ expect(() => toPositional('select $2', ['only-one'])).toThrow(/references \$2 but 1 parameter/)
29
+ })
30
+ })
31
+
32
+ // The two bugs that made the OM adapter differ from the harness — where every statement runs
33
+ // through node-postgres, which reports affected rows and honours transactions for free.
34
+ describe('mikroExecutor', () => {
35
+ const stub = (result: unknown) => {
36
+ const calls: Array<{ sql: string; method?: string; ctx?: unknown }> = []
37
+ const connection = {
38
+ execute: async (sql: string, _params?: unknown[], method?: string, ctx?: unknown) => {
39
+ calls.push({ sql, method, ctx })
40
+ return result
41
+ },
42
+ }
43
+ const em = {
44
+ getConnection: () => connection,
45
+ getTransactionContext: () => 'the-transaction',
46
+ fork: () => em,
47
+ transactional: async (fn: (trx: unknown) => Promise<unknown>) => fn(em),
48
+ }
49
+ return { em, calls }
50
+ }
51
+
52
+ it('counts rows a statement affected, not rows it returned', async () => {
53
+ // An UPDATE with no RETURNING returns nothing. Reporting that as zero affected rows makes
54
+ // every domain mirror look like it matched nothing — which is treated exactly like a throw,
55
+ // so the mirror is retried forever over a write that already landed.
56
+ const { em, calls } = stub({ affectedRows: 1 })
57
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
58
+ const { mikroExecutor } = await import('../sql-executor-mikro')
59
+ const sql = mikroExecutor(em as any)
60
+ expect(await sql.query('update sync_runs set status = $1 where id = $2', ['completed', 'x'])).toEqual({ rows: [], rowCount: 1 })
61
+ expect(calls[0]!.method).toBe('run')
62
+ })
63
+
64
+ it('reads rows from a statement that returns them', async () => {
65
+ const { em, calls } = stub([{ id: 'a' }])
66
+ const { mikroExecutor } = await import('../sql-executor-mikro')
67
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
68
+ const sql = mikroExecutor(em as any)
69
+ expect(await sql.query('update t set a = $1 returning id', [1])).toEqual({ rows: [{ id: 'a' }], rowCount: 1 })
70
+ expect(calls[0]!.method).toBe('all')
71
+ })
72
+
73
+ it('passes the transaction context to every statement', async () => {
74
+ // Without it the statements run on a pooled connection outside the transaction: nothing
75
+ // fails, it simply stops being atomic — so `fencedWrite` no longer rolls back a stale
76
+ // worker's writes and a terminal transition no longer moves both rows together.
77
+ const { em, calls } = stub([{ id: 'a' }])
78
+ const { mikroExecutor } = await import('../sql-executor-mikro')
79
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
80
+ await mikroExecutor(em as any).transaction(async (tx) => tx.query('select 1'))
81
+ expect(calls[0]!.ctx).toBe('the-transaction')
82
+ })
83
+ })
@@ -0,0 +1,65 @@
1
+ // How a host configures the mechanism: environment variables in, a transport out.
2
+
3
+ import { BullMQTransport } from '../transport/bullmq'
4
+ import { MemoryTransport } from '../transport/memory'
5
+ import { PgBossTransport } from '../transport/pgboss'
6
+ import type { TransportAdapter, TransportName } from '../transport/types'
7
+
8
+ export type DurableWorkConfig = {
9
+ transport: TransportName
10
+ redisUrl: string | null
11
+ databaseUrl: string | null
12
+ pgBossSchema: string
13
+ tickMs: number
14
+ drainTimeoutMs: number
15
+ reconcilerGraceMs: number
16
+ /** Hosts the worker inside the app process. Dev and ephemeral tests only. */
17
+ inProcessWorker: boolean
18
+ }
19
+
20
+ const num = (value: string | undefined, fallback: number): number => {
21
+ const parsed = Number(value)
22
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
23
+ }
24
+
25
+ const bool = (value: string | undefined): boolean => value === '1' || value?.toLowerCase() === 'true'
26
+
27
+ export function readConfig(env: NodeJS.ProcessEnv = process.env): DurableWorkConfig {
28
+ const raw = (env.DURABLE_WORK_TRANSPORT ?? 'pgboss').trim().toLowerCase()
29
+ if (raw !== 'memory' && raw !== 'bullmq' && raw !== 'pgboss') {
30
+ throw new Error(`DURABLE_WORK_TRANSPORT must be memory | bullmq | pgboss, got ${JSON.stringify(raw)}`)
31
+ }
32
+ return {
33
+ transport: raw,
34
+ // Falls back to the queue module's Redis, because an app that already runs one should not
35
+ // have to configure a second.
36
+ redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,
37
+ databaseUrl: env.DATABASE_URL ?? null,
38
+ pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? 'durable_work_boss',
39
+ tickMs: num(env.DURABLE_WORK_TICK_MS, 15_000),
40
+ drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 30_000),
41
+ reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 20_000),
42
+ inProcessWorker: bool(env.DURABLE_WORK_INPROCESS_WORKER),
43
+ }
44
+ }
45
+
46
+ export function createTransport(config: DurableWorkConfig, deps: { redisConnection?: unknown } = {}): TransportAdapter {
47
+ switch (config.transport) {
48
+ case 'memory':
49
+ // Nothing survives the process, so this is a development convenience and is refused in
50
+ // production rather than quietly losing every job on the next deploy.
51
+ if (process.env.NODE_ENV === 'production') {
52
+ throw new Error('DURABLE_WORK_TRANSPORT=memory keeps jobs in process memory and cannot be used in production.')
53
+ }
54
+ return new MemoryTransport()
55
+ case 'bullmq': {
56
+ const connection = deps.redisConnection ?? config.redisUrl
57
+ if (!connection) throw new Error('DURABLE_WORK_TRANSPORT=bullmq requires DURABLE_WORK_REDIS_URL (or QUEUE_REDIS_URL).')
58
+ return new BullMQTransport({ connection })
59
+ }
60
+ case 'pgboss': {
61
+ if (!config.databaseUrl) throw new Error('DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.')
62
+ return new PgBossTransport({ connectionString: config.databaseUrl, schema: config.pgBossSchema })
63
+ }
64
+ }
65
+ }