bunderstack 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/config.ts +5 -2
- package/src/index.ts +128 -6
- package/src/internal-tables-pg.ts +23 -0
- package/src/internal-tables.ts +34 -0
- package/src/jobs/cron.ts +87 -0
- package/src/jobs/define.ts +171 -0
- package/src/jobs/index.ts +20 -0
- package/src/jobs/queue.ts +57 -0
- package/src/jobs/worker.ts +290 -0
- package/src/manifest.ts +7 -0
- package/src/schema-export-pg.ts +1 -0
- package/src/schema-export.ts +1 -0
- package/src/trpc.ts +2 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bunderstack",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Batteries-included backend framework for Bun: CRUD APIs, auth, file storage, realtime, tRPC, email, and validated env from a single Drizzle schema and config object.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"backend",
|
package/src/config.ts
CHANGED
|
@@ -40,6 +40,8 @@ export const BunderstackOptionsSchema = z.object({
|
|
|
40
40
|
email: z.unknown().optional(),
|
|
41
41
|
// Loose: a tRPC router or builder callback. Resolved in createBunderstack.
|
|
42
42
|
trpc: z.unknown().optional(),
|
|
43
|
+
// Loose: holds handler functions and zod schemas. Resolved in createBunderstack.
|
|
44
|
+
jobs: z.unknown().optional(),
|
|
43
45
|
rateLimit: z
|
|
44
46
|
.union([
|
|
45
47
|
z.boolean(),
|
|
@@ -80,7 +82,7 @@ export type BunderstackConfig<
|
|
|
80
82
|
TEnv extends EnvConfigInput | undefined = EnvConfigInput | undefined,
|
|
81
83
|
> = Omit<
|
|
82
84
|
z.input<typeof BunderstackOptionsSchema>,
|
|
83
|
-
'schema' | 'access' | 'auth' | 'storage' | 'env' | 'email' | 'trpc'
|
|
85
|
+
'schema' | 'access' | 'auth' | 'storage' | 'env' | 'email' | 'trpc' | 'jobs'
|
|
84
86
|
> & {
|
|
85
87
|
schema: TSchema
|
|
86
88
|
access?: TAccess
|
|
@@ -90,7 +92,8 @@ export type BunderstackConfig<
|
|
|
90
92
|
email?: EmailConfigInput
|
|
91
93
|
// `trpc` is intentionally NOT declared here: createBunderstack intersects
|
|
92
94
|
// its own inference-friendly `trpc` declaration (router | builder callback)
|
|
93
|
-
// so the callback's `t` parameter gets contextual typing.
|
|
95
|
+
// so the callback's `t` parameter gets contextual typing. `jobs` follows the
|
|
96
|
+
// same pattern (defs map | builder callback receiving `j`).
|
|
94
97
|
rateLimit?: boolean | RateLimitConfig
|
|
95
98
|
idempotency?: boolean | IdempotencyConfig
|
|
96
99
|
realtime?:
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,18 @@ import { buildCrudRouter } from './crud'
|
|
|
22
22
|
import { createDb } from './db'
|
|
23
23
|
import { buildHandler } from './handler'
|
|
24
24
|
import { withInternalTables } from './internal-tables'
|
|
25
|
+
import {
|
|
26
|
+
createJobsBuilder,
|
|
27
|
+
createJobRunner,
|
|
28
|
+
enqueueJob,
|
|
29
|
+
validateJobsDefs,
|
|
30
|
+
} from './jobs/index'
|
|
31
|
+
import type {
|
|
32
|
+
BunderstackJobsBuilder,
|
|
33
|
+
EnqueueOptions,
|
|
34
|
+
JobsDefs,
|
|
35
|
+
JobsFacade,
|
|
36
|
+
} from './jobs/index'
|
|
25
37
|
import {
|
|
26
38
|
PROVISION_INTERNALS,
|
|
27
39
|
type WithProvisionInternals,
|
|
@@ -40,6 +52,8 @@ type AuthInstance = ReturnType<typeof createAuth>
|
|
|
40
52
|
const DEFAULT_PENDING_TTL_MS = 30 * 60_000
|
|
41
53
|
/** How often the auto-started orphan sweep runs. */
|
|
42
54
|
const SWEEP_INTERVAL_MS = 10 * 60_000
|
|
55
|
+
/** How often the in-process job worker polls for claimable jobs. */
|
|
56
|
+
const JOBS_POLL_INTERVAL_MS = 1000
|
|
43
57
|
|
|
44
58
|
/**
|
|
45
59
|
* Public storage facade exposed as `app.storage`. Object-level operations live
|
|
@@ -73,6 +87,7 @@ export type BunderstackApp<
|
|
|
73
87
|
TBuckets extends string = string,
|
|
74
88
|
TEnv extends EnvConfigInput | undefined = undefined,
|
|
75
89
|
TRouter = undefined,
|
|
90
|
+
TJobsDefs extends JobsDefs | undefined = undefined,
|
|
76
91
|
> = {
|
|
77
92
|
handler: (req: Request) => Promise<Response>
|
|
78
93
|
db: DbFor<TSchema>
|
|
@@ -85,6 +100,8 @@ export type BunderstackApp<
|
|
|
85
100
|
env: ValidatedEnv<TEnv>
|
|
86
101
|
/** Email facade; always present — send() throws when email isn't configured. */
|
|
87
102
|
email: EmailFacade
|
|
103
|
+
/** Job queue facade; always present — enqueue throws when jobs aren't configured. */
|
|
104
|
+
jobs: JobsFacade<TJobsDefs extends JobsDefs ? TJobsDefs : Record<never, never>>
|
|
88
105
|
/** Deploy-time introspection: what this app needs provisioned. */
|
|
89
106
|
manifest: BunderstackManifest
|
|
90
107
|
/**
|
|
@@ -102,7 +119,11 @@ export type BunderstackApp<
|
|
|
102
119
|
// Overloads: the builder-callback form and the prebuilt-router/none form are
|
|
103
120
|
// separate signatures so the callback's `t` parameter gets contextual typing
|
|
104
121
|
// and the router type lands on `$inferClient` without conditional-type
|
|
105
|
-
// inference (which breaks under contextual return types).
|
|
122
|
+
// inference (which breaks under contextual return types). `jobs` needs the
|
|
123
|
+
// same split against BOTH trpc forms — a union parameter type (`TJobsDefs |
|
|
124
|
+
// (callback => TJobsDefs)`) defeats inference (TS widens TJobsDefs to its
|
|
125
|
+
// constraint when a function literal could match either union arm) — hence
|
|
126
|
+
// four overloads covering the trpc × jobs option cross product.
|
|
106
127
|
export function createBunderstack<
|
|
107
128
|
TSchema extends Record<string, unknown>,
|
|
108
129
|
const TAccess extends Record<string, TableAccessInput> | undefined =
|
|
@@ -110,12 +131,47 @@ export function createBunderstack<
|
|
|
110
131
|
const TStorage extends StorageConfigInput | undefined = undefined,
|
|
111
132
|
const TEnv extends EnvConfigInput | undefined = undefined,
|
|
112
133
|
TRouter extends AnyRouter = AnyRouter,
|
|
134
|
+
const TJobsDefs extends JobsDefs | undefined = undefined,
|
|
113
135
|
>(
|
|
114
136
|
options: BunderstackConfig<TSchema, TAccess, TStorage, TEnv> & {
|
|
115
137
|
/** Builder callback receiving the pre-wired `t` instance. */
|
|
116
138
|
trpc: (t: BunderstackTRPC<TSchema, ValidatedEnv<TEnv>>) => TRouter
|
|
139
|
+
/** Builder callback receiving the pre-wired `j` instance. */
|
|
140
|
+
jobs: (j: BunderstackJobsBuilder<TSchema, ValidatedEnv<TEnv>>) => TJobsDefs
|
|
141
|
+
},
|
|
142
|
+
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter, TJobsDefs>>
|
|
143
|
+
export function createBunderstack<
|
|
144
|
+
TSchema extends Record<string, unknown>,
|
|
145
|
+
const TAccess extends Record<string, TableAccessInput> | undefined =
|
|
146
|
+
undefined,
|
|
147
|
+
const TStorage extends StorageConfigInput | undefined = undefined,
|
|
148
|
+
const TEnv extends EnvConfigInput | undefined = undefined,
|
|
149
|
+
TRouter extends AnyRouter = AnyRouter,
|
|
150
|
+
const TJobsDefs extends JobsDefs | undefined = undefined,
|
|
151
|
+
>(
|
|
152
|
+
options: BunderstackConfig<TSchema, TAccess, TStorage, TEnv> & {
|
|
153
|
+
/** Builder callback receiving the pre-wired `t` instance. */
|
|
154
|
+
trpc: (t: BunderstackTRPC<TSchema, ValidatedEnv<TEnv>>) => TRouter
|
|
155
|
+
/** Prebuilt job definitions (escape hatch for multi-file setups). */
|
|
156
|
+
jobs?: TJobsDefs
|
|
157
|
+
},
|
|
158
|
+
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter, TJobsDefs>>
|
|
159
|
+
export function createBunderstack<
|
|
160
|
+
TSchema extends Record<string, unknown>,
|
|
161
|
+
const TAccess extends Record<string, TableAccessInput> | undefined =
|
|
162
|
+
undefined,
|
|
163
|
+
const TStorage extends StorageConfigInput | undefined = undefined,
|
|
164
|
+
const TEnv extends EnvConfigInput | undefined = undefined,
|
|
165
|
+
TRouter extends AnyRouter | undefined = undefined,
|
|
166
|
+
const TJobsDefs extends JobsDefs | undefined = undefined,
|
|
167
|
+
>(
|
|
168
|
+
options: BunderstackConfig<TSchema, TAccess, TStorage, TEnv> & {
|
|
169
|
+
/** Prebuilt tRPC router (escape hatch for multi-file setups). */
|
|
170
|
+
trpc?: TRouter
|
|
171
|
+
/** Builder callback receiving the pre-wired `j` instance. */
|
|
172
|
+
jobs: (j: BunderstackJobsBuilder<TSchema, ValidatedEnv<TEnv>>) => TJobsDefs
|
|
117
173
|
},
|
|
118
|
-
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter>>
|
|
174
|
+
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter, TJobsDefs>>
|
|
119
175
|
export function createBunderstack<
|
|
120
176
|
TSchema extends Record<string, unknown>,
|
|
121
177
|
const TAccess extends Record<string, TableAccessInput> | undefined =
|
|
@@ -123,12 +179,15 @@ export function createBunderstack<
|
|
|
123
179
|
const TStorage extends StorageConfigInput | undefined = undefined,
|
|
124
180
|
const TEnv extends EnvConfigInput | undefined = undefined,
|
|
125
181
|
TRouter extends AnyRouter | undefined = undefined,
|
|
182
|
+
const TJobsDefs extends JobsDefs | undefined = undefined,
|
|
126
183
|
>(
|
|
127
184
|
options: BunderstackConfig<TSchema, TAccess, TStorage, TEnv> & {
|
|
128
185
|
/** Prebuilt tRPC router (escape hatch for multi-file setups). */
|
|
129
186
|
trpc?: TRouter
|
|
187
|
+
/** Prebuilt job definitions (escape hatch for multi-file setups). */
|
|
188
|
+
jobs?: TJobsDefs
|
|
130
189
|
},
|
|
131
|
-
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter>>
|
|
190
|
+
): Promise<BunderstackApp<TSchema, TAccess, BucketNamesOf<TStorage>, TEnv, TRouter, TJobsDefs>>
|
|
132
191
|
export async function createBunderstack<
|
|
133
192
|
TSchema extends Record<string, unknown>,
|
|
134
193
|
const TAccess extends Record<string, TableAccessInput> | undefined =
|
|
@@ -140,9 +199,19 @@ export async function createBunderstack<
|
|
|
140
199
|
trpc?:
|
|
141
200
|
| AnyRouter
|
|
142
201
|
| ((t: BunderstackTRPC<TSchema, ValidatedEnv<TEnv>>) => AnyRouter)
|
|
202
|
+
jobs?:
|
|
203
|
+
| JobsDefs
|
|
204
|
+
| ((j: BunderstackJobsBuilder<TSchema, ValidatedEnv<TEnv>>) => JobsDefs)
|
|
143
205
|
},
|
|
144
206
|
): Promise<
|
|
145
|
-
BunderstackApp<
|
|
207
|
+
BunderstackApp<
|
|
208
|
+
TSchema,
|
|
209
|
+
TAccess,
|
|
210
|
+
BucketNamesOf<TStorage>,
|
|
211
|
+
TEnv,
|
|
212
|
+
AnyRouter | undefined,
|
|
213
|
+
JobsDefs | undefined
|
|
214
|
+
>
|
|
146
215
|
> {
|
|
147
216
|
const dialect = detectDialect(options.schema)
|
|
148
217
|
// Env is validated FIRST: the app refuses to boot on missing/invalid vars,
|
|
@@ -271,6 +340,42 @@ export async function createBunderstack<
|
|
|
271
340
|
void sweepOrphans(registry, db, DEFAULT_PENDING_TTL_MS).catch(() => {})
|
|
272
341
|
}, SWEEP_INTERVAL_MS)
|
|
273
342
|
sweepTimer.unref?.()
|
|
343
|
+
const jobsDefs: JobsDefs | undefined = options.jobs
|
|
344
|
+
? typeof options.jobs === 'function'
|
|
345
|
+
? options.jobs(createJobsBuilder<TSchema, ValidatedEnv<TEnv>>())
|
|
346
|
+
: (options.jobs as JobsDefs)
|
|
347
|
+
: undefined
|
|
348
|
+
if (jobsDefs) validateJobsDefs(jobsDefs)
|
|
349
|
+
const jobRunner = jobsDefs
|
|
350
|
+
? createJobRunner({
|
|
351
|
+
db,
|
|
352
|
+
defs: jobsDefs,
|
|
353
|
+
ctx: { db: userDb, env, email, storage },
|
|
354
|
+
})
|
|
355
|
+
: undefined
|
|
356
|
+
const jobs = {
|
|
357
|
+
async enqueue(name: string, input?: unknown, opts?: EnqueueOptions) {
|
|
358
|
+
if (!jobsDefs) {
|
|
359
|
+
throw new Error(
|
|
360
|
+
'[bunderstack] no jobs configured — add a `jobs` key to createBunderstack',
|
|
361
|
+
)
|
|
362
|
+
}
|
|
363
|
+
const result = await enqueueJob(db, jobsDefs, name, input, opts)
|
|
364
|
+
// Wake the local worker so same-process jobs start without poll latency.
|
|
365
|
+
if (jobRunner && !introspect) void jobRunner.tick().catch(() => {})
|
|
366
|
+
return result
|
|
367
|
+
},
|
|
368
|
+
tick(now?: number) {
|
|
369
|
+
return jobRunner ? jobRunner.tick(now) : Promise.resolve()
|
|
370
|
+
},
|
|
371
|
+
}
|
|
372
|
+
if (jobRunner) jobRunner.setJobsFacade(jobs)
|
|
373
|
+
if (jobRunner && !introspect) {
|
|
374
|
+
const jobsTimer = setInterval(() => {
|
|
375
|
+
void jobRunner.tick().catch(() => {})
|
|
376
|
+
}, JOBS_POLL_INTERVAL_MS)
|
|
377
|
+
jobsTimer.unref?.()
|
|
378
|
+
}
|
|
274
379
|
const trpcRouter: AnyRouter | undefined =
|
|
275
380
|
typeof options.trpc === 'function'
|
|
276
381
|
? options.trpc(createTRPC<TSchema, ValidatedEnv<TEnv>>())
|
|
@@ -286,6 +391,7 @@ export async function createBunderstack<
|
|
|
286
391
|
user: await resolveAccessUser(authResolver, req.headers),
|
|
287
392
|
env,
|
|
288
393
|
email,
|
|
394
|
+
jobs,
|
|
289
395
|
req,
|
|
290
396
|
}),
|
|
291
397
|
})
|
|
@@ -304,7 +410,8 @@ export async function createBunderstack<
|
|
|
304
410
|
TAccess,
|
|
305
411
|
BucketNamesOf<TStorage>,
|
|
306
412
|
TEnv,
|
|
307
|
-
AnyRouter | undefined
|
|
413
|
+
AnyRouter | undefined,
|
|
414
|
+
JobsDefs | undefined
|
|
308
415
|
> = {
|
|
309
416
|
handler,
|
|
310
417
|
// Internal tables live on the runtime db but stay out of the public type.
|
|
@@ -314,6 +421,10 @@ export async function createBunderstack<
|
|
|
314
421
|
router,
|
|
315
422
|
env,
|
|
316
423
|
email,
|
|
424
|
+
// Runtime facade is untyped (JobsRuntimeFacade); the generic-typed field
|
|
425
|
+
// narrows `enqueue` per-app from the declared job defs — same relationship
|
|
426
|
+
// as `userDb` above.
|
|
427
|
+
jobs: jobs as never,
|
|
317
428
|
trpcRouter,
|
|
318
429
|
manifest: buildManifest({
|
|
319
430
|
schema: options.schema,
|
|
@@ -321,6 +432,7 @@ export async function createBunderstack<
|
|
|
321
432
|
storage: config.storage,
|
|
322
433
|
envConfig: options.env as EnvConfigInput | undefined,
|
|
323
434
|
realtime: Boolean(config.realtime),
|
|
435
|
+
jobs: jobsDefs,
|
|
324
436
|
}),
|
|
325
437
|
}
|
|
326
438
|
|
|
@@ -349,7 +461,7 @@ export type {
|
|
|
349
461
|
export { validateEnv, createClientEnv, BunderstackEnvError } from './env'
|
|
350
462
|
export type { EnvConfigInput, BaseEnv, ValidatedEnv } from './env'
|
|
351
463
|
export { buildManifest } from './manifest'
|
|
352
|
-
export type { BunderstackManifest, ManifestEnvVar } from './manifest'
|
|
464
|
+
export type { BunderstackManifest, ManifestEnvVar, ManifestJob } from './manifest'
|
|
353
465
|
export { createEmail } from './email'
|
|
354
466
|
export type {
|
|
355
467
|
EmailMessage,
|
|
@@ -359,6 +471,16 @@ export type {
|
|
|
359
471
|
} from './email'
|
|
360
472
|
export { createTRPC } from './trpc'
|
|
361
473
|
export type { BunderstackTRPC, TRPCContext } from './trpc'
|
|
474
|
+
export { createJobsBuilder } from './jobs/index'
|
|
475
|
+
export type {
|
|
476
|
+
BunderstackJobsBuilder,
|
|
477
|
+
EnqueueOptions,
|
|
478
|
+
JobContext,
|
|
479
|
+
JobDefinition,
|
|
480
|
+
JobsDefs,
|
|
481
|
+
JobsFacade,
|
|
482
|
+
JobsRuntimeFacade,
|
|
483
|
+
} from './jobs/index'
|
|
362
484
|
export {
|
|
363
485
|
defineAccess,
|
|
364
486
|
validateAndResolveAccess,
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
pgTable,
|
|
9
9
|
primaryKey,
|
|
10
10
|
text,
|
|
11
|
+
uniqueIndex,
|
|
11
12
|
} from 'drizzle-orm/pg-core'
|
|
12
13
|
|
|
13
14
|
export const bunderstackFilesPg = pgTable(
|
|
@@ -43,3 +44,25 @@ export const bunderstackIdempotencyPg = pgTable(
|
|
|
43
44
|
},
|
|
44
45
|
(t) => [primaryKey({ columns: [t.key, t.tableName] })],
|
|
45
46
|
)
|
|
47
|
+
|
|
48
|
+
export const bunderstackJobsPg = pgTable(
|
|
49
|
+
'_bunderstack_jobs',
|
|
50
|
+
{
|
|
51
|
+
id: text('id').primaryKey(),
|
|
52
|
+
type: text('type').notNull(),
|
|
53
|
+
payloadJson: text('payload_json').notNull(),
|
|
54
|
+
status: text('status').notNull(),
|
|
55
|
+
attempts: integer('attempts').notNull().default(0),
|
|
56
|
+
runAt: bigint('run_at', { mode: 'number' }).notNull(),
|
|
57
|
+
lockedUntil: bigint('locked_until', { mode: 'number' }),
|
|
58
|
+
dedupeKey: text('dedupe_key'),
|
|
59
|
+
lastError: text('last_error'),
|
|
60
|
+
createdAt: bigint('created_at', { mode: 'number' }).notNull(),
|
|
61
|
+
finishedAt: bigint('finished_at', { mode: 'number' }),
|
|
62
|
+
},
|
|
63
|
+
(t) => [
|
|
64
|
+
index('bjq_claim').on(t.status, t.runAt),
|
|
65
|
+
index('bjq_type_status').on(t.type, t.status),
|
|
66
|
+
uniqueIndex('bjq_dedupe').on(t.type, t.dedupeKey),
|
|
67
|
+
],
|
|
68
|
+
)
|
package/src/internal-tables.ts
CHANGED
|
@@ -6,12 +6,14 @@ import {
|
|
|
6
6
|
primaryKey,
|
|
7
7
|
sqliteTable,
|
|
8
8
|
text,
|
|
9
|
+
uniqueIndex,
|
|
9
10
|
} from 'drizzle-orm/sqlite-core'
|
|
10
11
|
|
|
11
12
|
import { detectDialect } from './dialect'
|
|
12
13
|
import {
|
|
13
14
|
bunderstackFilesPg,
|
|
14
15
|
bunderstackIdempotencyPg,
|
|
16
|
+
bunderstackJobsPg,
|
|
15
17
|
} from './internal-tables-pg'
|
|
16
18
|
|
|
17
19
|
export const bunderstackFiles = sqliteTable(
|
|
@@ -48,19 +50,45 @@ export const bunderstackIdempotency = sqliteTable(
|
|
|
48
50
|
(t) => [primaryKey({ columns: [t.key, t.tableName] })],
|
|
49
51
|
)
|
|
50
52
|
|
|
53
|
+
export const bunderstackJobs = sqliteTable(
|
|
54
|
+
'_bunderstack_jobs',
|
|
55
|
+
{
|
|
56
|
+
id: text('id').primaryKey(),
|
|
57
|
+
type: text('type').notNull(),
|
|
58
|
+
payloadJson: text('payload_json').notNull(),
|
|
59
|
+
status: text('status').notNull(), // pending | running | succeeded | failed
|
|
60
|
+
attempts: integer('attempts').notNull().default(0),
|
|
61
|
+
runAt: integer('run_at').notNull(),
|
|
62
|
+
lockedUntil: integer('locked_until'),
|
|
63
|
+
dedupeKey: text('dedupe_key'),
|
|
64
|
+
lastError: text('last_error'),
|
|
65
|
+
createdAt: integer('created_at').notNull(),
|
|
66
|
+
finishedAt: integer('finished_at'),
|
|
67
|
+
},
|
|
68
|
+
(t) => [
|
|
69
|
+
index('bjq_claim').on(t.status, t.runAt),
|
|
70
|
+
index('bjq_type_status').on(t.type, t.status),
|
|
71
|
+
// NULL dedupe keys are distinct in both dialects, so keyless jobs never collide.
|
|
72
|
+
uniqueIndex('bjq_dedupe').on(t.type, t.dedupeKey),
|
|
73
|
+
],
|
|
74
|
+
)
|
|
75
|
+
|
|
51
76
|
export const INTERNAL_TABLES = {
|
|
52
77
|
bunderstackFiles,
|
|
53
78
|
bunderstackIdempotency,
|
|
79
|
+
bunderstackJobs,
|
|
54
80
|
} as const
|
|
55
81
|
|
|
56
82
|
export const INTERNAL_TABLE_NAMES: ReadonlySet<string> = new Set([
|
|
57
83
|
'bunderstack_file_meta',
|
|
58
84
|
'_bunderstack_idempotency',
|
|
85
|
+
'_bunderstack_jobs',
|
|
59
86
|
])
|
|
60
87
|
|
|
61
88
|
export const INTERNAL_TABLES_PG = {
|
|
62
89
|
bunderstackFiles: bunderstackFilesPg,
|
|
63
90
|
bunderstackIdempotency: bunderstackIdempotencyPg,
|
|
91
|
+
bunderstackJobs: bunderstackJobsPg,
|
|
64
92
|
} as const
|
|
65
93
|
|
|
66
94
|
// Both dialect twins count as "ours" for the re-export identity check.
|
|
@@ -70,6 +98,7 @@ const INTERNAL_TABLE_CANDIDATES = new Map<string, readonly unknown[]>([
|
|
|
70
98
|
getTableName(bunderstackIdempotency),
|
|
71
99
|
[bunderstackIdempotency, bunderstackIdempotencyPg],
|
|
72
100
|
],
|
|
101
|
+
[getTableName(bunderstackJobs), [bunderstackJobs, bunderstackJobsPg]],
|
|
73
102
|
])
|
|
74
103
|
|
|
75
104
|
/** Internal file-meta table matching the db's dialect. */
|
|
@@ -82,6 +111,11 @@ export function idempotencyTableFor(db: unknown) {
|
|
|
82
111
|
return is(db, PgDatabase) ? bunderstackIdempotencyPg : bunderstackIdempotency
|
|
83
112
|
}
|
|
84
113
|
|
|
114
|
+
/** Internal jobs table matching the db's dialect. */
|
|
115
|
+
export function jobsTableFor(db: unknown) {
|
|
116
|
+
return is(db, PgDatabase) ? bunderstackJobsPg : bunderstackJobs
|
|
117
|
+
}
|
|
118
|
+
|
|
85
119
|
export function withInternalTables<TSchema extends Record<string, unknown>>(
|
|
86
120
|
schema: TSchema,
|
|
87
121
|
): TSchema & typeof INTERNAL_TABLES {
|
package/src/jobs/cron.ts
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// src/jobs/cron.ts — minimal 5-field cron parser, UTC, minute granularity.
|
|
2
|
+
// Supports: * , lists (a,b) , ranges (a-b) , steps (*/n, a-b/n, a/n). No
|
|
3
|
+
// month/day names, no seconds field, no @-shortcuts — YAGNI for v1.
|
|
4
|
+
|
|
5
|
+
export type CronField = { any: boolean; values: ReadonlySet<number> }
|
|
6
|
+
|
|
7
|
+
export type ParsedCron = {
|
|
8
|
+
minute: CronField
|
|
9
|
+
hour: CronField
|
|
10
|
+
dayOfMonth: CronField
|
|
11
|
+
month: CronField
|
|
12
|
+
dayOfWeek: CronField
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const PART_RE = /^(\*|\d+(?:-\d+)?)(?:\/(\d+))?$/
|
|
16
|
+
|
|
17
|
+
function parseField(
|
|
18
|
+
spec: string,
|
|
19
|
+
min: number,
|
|
20
|
+
max: number,
|
|
21
|
+
expr: string,
|
|
22
|
+
): CronField {
|
|
23
|
+
if (spec === '*') return { any: true, values: new Set() }
|
|
24
|
+
const values = new Set<number>()
|
|
25
|
+
for (const part of spec.split(',')) {
|
|
26
|
+
const m = PART_RE.exec(part)
|
|
27
|
+
if (!m) throw new Error(`[bunderstack] invalid cron "${expr}": "${part}"`)
|
|
28
|
+
const step = m[2] !== undefined ? Number(m[2]) : 1
|
|
29
|
+
let lo: number
|
|
30
|
+
let hi: number
|
|
31
|
+
if (m[1] === '*') {
|
|
32
|
+
lo = min
|
|
33
|
+
hi = max
|
|
34
|
+
} else if (m[1]!.includes('-')) {
|
|
35
|
+
const [a, b] = m[1]!.split('-')
|
|
36
|
+
lo = Number(a)
|
|
37
|
+
hi = Number(b)
|
|
38
|
+
} else {
|
|
39
|
+
lo = Number(m[1])
|
|
40
|
+
// "5/15" means "starting at 5, every 15" per cron convention.
|
|
41
|
+
hi = step > 1 ? max : lo
|
|
42
|
+
}
|
|
43
|
+
if (step < 1 || lo < min || hi > max || lo > hi) {
|
|
44
|
+
throw new Error(`[bunderstack] invalid cron "${expr}": "${part}"`)
|
|
45
|
+
}
|
|
46
|
+
for (let v = lo; v <= hi; v += step) values.add(v)
|
|
47
|
+
}
|
|
48
|
+
return { any: false, values }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function parseCron(expr: string): ParsedCron {
|
|
52
|
+
const parts = expr.trim().split(/\s+/)
|
|
53
|
+
if (parts.length !== 5) {
|
|
54
|
+
throw new Error(
|
|
55
|
+
`[bunderstack] invalid cron "${expr}": expected 5 fields (minute hour day-of-month month day-of-week)`,
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
const dow = parseField(parts[4]!, 0, 7, expr)
|
|
59
|
+
return {
|
|
60
|
+
minute: parseField(parts[0]!, 0, 59, expr),
|
|
61
|
+
hour: parseField(parts[1]!, 0, 23, expr),
|
|
62
|
+
dayOfMonth: parseField(parts[2]!, 1, 31, expr),
|
|
63
|
+
month: parseField(parts[3]!, 1, 12, expr),
|
|
64
|
+
// 7 is an alias for Sunday (0).
|
|
65
|
+
dayOfWeek: dow.any
|
|
66
|
+
? dow
|
|
67
|
+
: { any: false, values: new Set([...dow.values].map((v) => v % 7)) },
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function inField(field: CronField, value: number): boolean {
|
|
72
|
+
return field.any || field.values.has(value)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Whether the minute containing `epochMs` matches, evaluated in UTC. */
|
|
76
|
+
export function cronMatches(cron: ParsedCron, epochMs: number): boolean {
|
|
77
|
+
const d = new Date(epochMs)
|
|
78
|
+
if (!inField(cron.minute, d.getUTCMinutes())) return false
|
|
79
|
+
if (!inField(cron.hour, d.getUTCHours())) return false
|
|
80
|
+
if (!inField(cron.month, d.getUTCMonth() + 1)) return false
|
|
81
|
+
const domOk = inField(cron.dayOfMonth, d.getUTCDate())
|
|
82
|
+
const dowOk = inField(cron.dayOfWeek, d.getUTCDay())
|
|
83
|
+
// Standard cron rule: when BOTH day fields are restricted, either may match.
|
|
84
|
+
return !cron.dayOfMonth.any && !cron.dayOfWeek.any
|
|
85
|
+
? domOk || dowOk
|
|
86
|
+
: domOk && dowOk
|
|
87
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// src/jobs/define.ts — job definition types and the typed builder.
|
|
2
|
+
// `createJobsBuilder` mirrors `createTRPC`: it exists purely to carry
|
|
3
|
+
// TSchema/TEnvResult typing into inline callbacks and extracted files.
|
|
4
|
+
import type { ZodType } from 'zod'
|
|
5
|
+
|
|
6
|
+
import type { DbFor } from '../db'
|
|
7
|
+
import type { EmailFacade } from '../email'
|
|
8
|
+
import type { StorageFacade } from '../index'
|
|
9
|
+
|
|
10
|
+
import { parseCron } from './cron'
|
|
11
|
+
|
|
12
|
+
export const DEFAULT_RETRIES = 3
|
|
13
|
+
export const DEFAULT_TIMEOUT_MS = 60_000
|
|
14
|
+
|
|
15
|
+
export type EnqueueOptions = {
|
|
16
|
+
/** Collapse duplicate enqueues; see spec for cron vs non-cron lifetime. */
|
|
17
|
+
dedupeKey?: string
|
|
18
|
+
/** Milliseconds from now until the job becomes claimable. */
|
|
19
|
+
delay?: number
|
|
20
|
+
/** Absolute time the job becomes claimable; wins over `delay`. */
|
|
21
|
+
runAt?: Date | number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The untyped runtime facade. Handler ctx and tRPC ctx expose this shape;
|
|
26
|
+
* `app.jobs` narrows `enqueue` to the declared job names/payloads.
|
|
27
|
+
*/
|
|
28
|
+
export type JobsRuntimeFacade = {
|
|
29
|
+
enqueue(
|
|
30
|
+
name: string,
|
|
31
|
+
input?: unknown,
|
|
32
|
+
opts?: EnqueueOptions,
|
|
33
|
+
): Promise<{ id: string }>
|
|
34
|
+
/** Run one poll cycle deterministically (tests). `now` defaults to Date.now(). */
|
|
35
|
+
tick(now?: number): Promise<void>
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type JobContext<
|
|
39
|
+
TSchema extends Record<string, unknown> = Record<string, unknown>,
|
|
40
|
+
TEnvResult = Record<string, unknown>,
|
|
41
|
+
> = {
|
|
42
|
+
db: DbFor<TSchema>
|
|
43
|
+
env: TEnvResult
|
|
44
|
+
email: EmailFacade
|
|
45
|
+
storage: StorageFacade
|
|
46
|
+
jobs: JobsRuntimeFacade
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export type JobDefinition<
|
|
50
|
+
TInput,
|
|
51
|
+
TSchema extends Record<string, unknown> = Record<string, unknown>,
|
|
52
|
+
TEnvResult = Record<string, unknown>,
|
|
53
|
+
> = {
|
|
54
|
+
/** zod schema for the payload; parsed at enqueue AND before the handler runs. */
|
|
55
|
+
input?: ZodType<TInput>
|
|
56
|
+
/** Attempts after the first failure. Default 3 (so 4 total attempts). */
|
|
57
|
+
retries?: number
|
|
58
|
+
/** Delay before retry N (1-based). Default exponential: 1s, 2s, 4s, … */
|
|
59
|
+
backoff?: ((attempt: number) => number) | { baseMs?: number; factor?: number }
|
|
60
|
+
/** Max simultaneous `running` rows of this type, enforced cross-replica. */
|
|
61
|
+
concurrency?: number
|
|
62
|
+
/** Lease duration in ms; an expired lease sends the job back to pending. */
|
|
63
|
+
timeout?: number
|
|
64
|
+
/** 5-field UTC cron expression. Cron jobs cannot declare `input`. */
|
|
65
|
+
cron?: string
|
|
66
|
+
handler: (
|
|
67
|
+
input: TInput,
|
|
68
|
+
ctx: JobContext<TSchema, TEnvResult>,
|
|
69
|
+
) => Promise<void> | void
|
|
70
|
+
/** Fires once, after the final attempt fails. Errors here are logged, never retried. */
|
|
71
|
+
onFailed?: (
|
|
72
|
+
input: TInput,
|
|
73
|
+
error: Error,
|
|
74
|
+
ctx: JobContext<TSchema, TEnvResult>,
|
|
75
|
+
) => Promise<void> | void
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
79
|
+
export type AnyJobDefinition = JobDefinition<any, any, any>
|
|
80
|
+
export type JobsDefs = Record<string, AnyJobDefinition>
|
|
81
|
+
|
|
82
|
+
/** Throws when a definition is unusable. Safe to call more than once. */
|
|
83
|
+
export function validateJobsDefs(defs: JobsDefs): void {
|
|
84
|
+
for (const [name, def] of Object.entries(defs)) {
|
|
85
|
+
if (typeof def.handler !== 'function') {
|
|
86
|
+
throw new Error(`[bunderstack] job "${name}" has no handler`)
|
|
87
|
+
}
|
|
88
|
+
if (def.cron !== undefined) {
|
|
89
|
+
parseCron(def.cron) // throws with a clear message on invalid expressions
|
|
90
|
+
if (def.input !== undefined) {
|
|
91
|
+
throw new Error(
|
|
92
|
+
`[bunderstack] job "${name}": cron jobs cannot declare input (nothing enqueues a payload for a schedule)`,
|
|
93
|
+
)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (def.retries !== undefined && (def.retries < 0 || !Number.isInteger(def.retries))) {
|
|
97
|
+
throw new Error(`[bunderstack] job "${name}": retries must be a non-negative integer`)
|
|
98
|
+
}
|
|
99
|
+
if (def.concurrency !== undefined && (def.concurrency < 1 || !Number.isInteger(def.concurrency))) {
|
|
100
|
+
throw new Error(`[bunderstack] job "${name}": concurrency must be a positive integer`)
|
|
101
|
+
}
|
|
102
|
+
if (def.timeout !== undefined && def.timeout <= 0) {
|
|
103
|
+
throw new Error(`[bunderstack] job "${name}": timeout must be positive`)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Delay in ms before retry `attempt` (1-based = the attempt that just failed). */
|
|
109
|
+
export function backoffMs(def: AnyJobDefinition, attempt: number): number {
|
|
110
|
+
const b = def.backoff
|
|
111
|
+
if (typeof b === 'function') return b(attempt)
|
|
112
|
+
const baseMs = b?.baseMs ?? 1000
|
|
113
|
+
const factor = b?.factor ?? 2
|
|
114
|
+
return baseMs * factor ** (attempt - 1)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Build the `j` instance bunderstack hands to the config's `jobs` builder
|
|
119
|
+
* callback (and exports for multi-file job setups).
|
|
120
|
+
*/
|
|
121
|
+
export function createJobsBuilder<
|
|
122
|
+
TSchema extends Record<string, unknown>,
|
|
123
|
+
TEnvResult = Record<string, unknown>,
|
|
124
|
+
>() {
|
|
125
|
+
return {
|
|
126
|
+
/** Identity with inference: pins TInput from the zod schema. */
|
|
127
|
+
job<TInput = undefined>(
|
|
128
|
+
def: JobDefinition<TInput, TSchema, TEnvResult>,
|
|
129
|
+
): JobDefinition<TInput, TSchema, TEnvResult> {
|
|
130
|
+
return def
|
|
131
|
+
},
|
|
132
|
+
/** Identity with validation: returns the defs map, typed. */
|
|
133
|
+
define<TDefs extends JobsDefs>(defs: TDefs): TDefs {
|
|
134
|
+
validateJobsDefs(defs)
|
|
135
|
+
return defs
|
|
136
|
+
},
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Type of the `j` instance — for builder callbacks declared in separate files. */
|
|
141
|
+
export type BunderstackJobsBuilder<
|
|
142
|
+
TSchema extends Record<string, unknown>,
|
|
143
|
+
TEnvResult = Record<string, unknown>,
|
|
144
|
+
> = ReturnType<typeof createJobsBuilder<TSchema, TEnvResult>>
|
|
145
|
+
|
|
146
|
+
// Infers TInput from the JobDefinition's own type argument rather than
|
|
147
|
+
// pattern-matching the (optional, so union-with-undefined) `input` property —
|
|
148
|
+
// `TDef extends { input: ZodType<infer I> }` fails structurally because
|
|
149
|
+
// `input?: ZodType<TInput>` desugars to `ZodType<TInput> | undefined`, which
|
|
150
|
+
// can never satisfy a required-property pattern.
|
|
151
|
+
type JobInputOf<TDef> = TDef extends JobDefinition<infer TInput, any, any>
|
|
152
|
+
? TInput
|
|
153
|
+
: undefined
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* `app.jobs`: `enqueue` narrowed to declared names + payloads. `Omit`s the
|
|
157
|
+
* runtime facade's loose `enqueue` first — intersecting two same-named
|
|
158
|
+
* methods instead would make TS treat them as overloaded, so the loose
|
|
159
|
+
* `(name: string, ...)` signature would still accept any name.
|
|
160
|
+
*/
|
|
161
|
+
export type JobsFacade<TDefs extends JobsDefs> = Omit<
|
|
162
|
+
JobsRuntimeFacade,
|
|
163
|
+
'enqueue'
|
|
164
|
+
> & {
|
|
165
|
+
enqueue<K extends keyof TDefs & string>(
|
|
166
|
+
name: K,
|
|
167
|
+
...rest: JobInputOf<TDefs[K]> extends undefined
|
|
168
|
+
? [input?: undefined, opts?: EnqueueOptions]
|
|
169
|
+
: [input: JobInputOf<TDefs[K]>, opts?: EnqueueOptions]
|
|
170
|
+
): Promise<{ id: string }>
|
|
171
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// src/jobs/index.ts — module surface consumed by createBunderstack.
|
|
2
|
+
export {
|
|
3
|
+
createJobsBuilder,
|
|
4
|
+
validateJobsDefs,
|
|
5
|
+
DEFAULT_RETRIES,
|
|
6
|
+
DEFAULT_TIMEOUT_MS,
|
|
7
|
+
} from './define'
|
|
8
|
+
export type {
|
|
9
|
+
AnyJobDefinition,
|
|
10
|
+
BunderstackJobsBuilder,
|
|
11
|
+
EnqueueOptions,
|
|
12
|
+
JobContext,
|
|
13
|
+
JobDefinition,
|
|
14
|
+
JobsDefs,
|
|
15
|
+
JobsFacade,
|
|
16
|
+
JobsRuntimeFacade,
|
|
17
|
+
} from './define'
|
|
18
|
+
export { enqueueJob } from './queue'
|
|
19
|
+
export { createJobRunner } from './worker'
|
|
20
|
+
export { parseCron, cronMatches } from './cron'
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// src/jobs/queue.ts — durable enqueue with constraint-backed dedupe.
|
|
2
|
+
import { and, eq } from 'drizzle-orm'
|
|
3
|
+
|
|
4
|
+
import type { AnyDb } from '../dialect'
|
|
5
|
+
import type { EnqueueOptions, JobsDefs } from './define'
|
|
6
|
+
|
|
7
|
+
import { jobsTableFor } from '../internal-tables'
|
|
8
|
+
import { generate } from '../typeid'
|
|
9
|
+
|
|
10
|
+
export async function enqueueJob(
|
|
11
|
+
db: AnyDb,
|
|
12
|
+
defs: JobsDefs,
|
|
13
|
+
name: string,
|
|
14
|
+
input: unknown,
|
|
15
|
+
opts: EnqueueOptions = {},
|
|
16
|
+
): Promise<{ id: string }> {
|
|
17
|
+
const def = defs[name]
|
|
18
|
+
if (!def) throw new Error(`[bunderstack] unknown job type "${name}"`)
|
|
19
|
+
// Fail fast: a bad payload should throw at the call site, not in the worker.
|
|
20
|
+
const parsed = def.input ? def.input.parse(input) : null
|
|
21
|
+
const t = jobsTableFor(db)
|
|
22
|
+
const now = Date.now()
|
|
23
|
+
const runAt =
|
|
24
|
+
opts.runAt !== undefined
|
|
25
|
+
? new Date(opts.runAt).getTime()
|
|
26
|
+
: now + (opts.delay ?? 0)
|
|
27
|
+
|
|
28
|
+
// Two rounds cover the race where the deduping row reaches a terminal state
|
|
29
|
+
// (clearing its key) between our failed insert and our read.
|
|
30
|
+
for (let round = 0; round < 2; round++) {
|
|
31
|
+
const id = generate('job')
|
|
32
|
+
const insertedRows = await db
|
|
33
|
+
.insert(t)
|
|
34
|
+
.values({
|
|
35
|
+
id,
|
|
36
|
+
type: name,
|
|
37
|
+
payloadJson: JSON.stringify(parsed),
|
|
38
|
+
status: 'pending',
|
|
39
|
+
attempts: 0,
|
|
40
|
+
runAt,
|
|
41
|
+
dedupeKey: opts.dedupeKey ?? null,
|
|
42
|
+
createdAt: now,
|
|
43
|
+
})
|
|
44
|
+
.onConflictDoNothing({ target: [t.type, t.dedupeKey] })
|
|
45
|
+
.returning({ id: t.id })
|
|
46
|
+
if (insertedRows[0]) return { id: String(insertedRows[0].id) }
|
|
47
|
+
const existing = await db
|
|
48
|
+
.select({ id: t.id })
|
|
49
|
+
.from(t)
|
|
50
|
+
.where(and(eq(t.type, name), eq(t.dedupeKey, opts.dedupeKey ?? '')))
|
|
51
|
+
.limit(1)
|
|
52
|
+
if (existing[0]) return { id: String(existing[0].id) }
|
|
53
|
+
}
|
|
54
|
+
throw new Error(
|
|
55
|
+
`[bunderstack] enqueue of "${name}" lost a dedupe race twice — please retry`,
|
|
56
|
+
)
|
|
57
|
+
}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
// src/jobs/worker.ts — the in-process worker. One `tick()` is a full cycle:
|
|
2
|
+
// recover expired leases → schedule cron slots → reap old succeeded rows →
|
|
3
|
+
// claim and run claimable jobs (awaiting handlers, so tests drive `tick()`
|
|
4
|
+
// deterministically with an injected `now`). Multiple replicas run the same
|
|
5
|
+
// loop safely: claims are atomic and cron slots dedupe on a unique index.
|
|
6
|
+
import { and, eq, inArray, is, isNotNull, lt, lte, sql } from 'drizzle-orm'
|
|
7
|
+
import { PgDatabase } from 'drizzle-orm/pg-core'
|
|
8
|
+
|
|
9
|
+
import type { AnyDb } from '../dialect'
|
|
10
|
+
import type {
|
|
11
|
+
AnyJobDefinition,
|
|
12
|
+
JobsDefs,
|
|
13
|
+
JobsRuntimeFacade,
|
|
14
|
+
} from './define'
|
|
15
|
+
import type { ParsedCron } from './cron'
|
|
16
|
+
|
|
17
|
+
import { jobsTableFor } from '../internal-tables'
|
|
18
|
+
import { cronMatches, parseCron } from './cron'
|
|
19
|
+
import { backoffMs, DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from './define'
|
|
20
|
+
import { enqueueJob } from './queue'
|
|
21
|
+
|
|
22
|
+
const CLAIM_BATCH = 10
|
|
23
|
+
const SUCCEEDED_RETENTION_MS = 24 * 60 * 60 * 1000
|
|
24
|
+
|
|
25
|
+
type JobRow = {
|
|
26
|
+
id: string
|
|
27
|
+
type: string
|
|
28
|
+
payloadJson: string
|
|
29
|
+
attempts: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function toError(err: unknown): Error {
|
|
33
|
+
return err instanceof Error ? err : new Error(String(err))
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function maxAttempts(def: AnyJobDefinition): number {
|
|
37
|
+
return 1 + (def.retries ?? DEFAULT_RETRIES)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Terminal-status column patch: non-cron jobs release their dedupe key. */
|
|
41
|
+
function terminalPatch(def: AnyJobDefinition | undefined) {
|
|
42
|
+
return def?.cron ? {} : { dedupeKey: null }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function createJobRunner(deps: {
|
|
46
|
+
db: AnyDb
|
|
47
|
+
defs: JobsDefs
|
|
48
|
+
/** Handler ctx WITHOUT `jobs`; the facade is injected via setJobsFacade. */
|
|
49
|
+
ctx: Record<string, unknown>
|
|
50
|
+
}) {
|
|
51
|
+
const { db, defs } = deps
|
|
52
|
+
const t = jobsTableFor(db)
|
|
53
|
+
const ctx = { ...deps.ctx } as Record<string, unknown>
|
|
54
|
+
const crons = new Map<string, ParsedCron>()
|
|
55
|
+
for (const [name, def] of Object.entries(defs)) {
|
|
56
|
+
if (def.cron) crons.set(name, parseCron(def.cron))
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function fireOnFailed(
|
|
60
|
+
def: AnyJobDefinition,
|
|
61
|
+
payloadJson: string,
|
|
62
|
+
error: Error,
|
|
63
|
+
) {
|
|
64
|
+
if (!def.onFailed) return
|
|
65
|
+
let input: unknown
|
|
66
|
+
try {
|
|
67
|
+
const raw = JSON.parse(payloadJson)
|
|
68
|
+
input = def.input ? def.input.parse(raw) : undefined
|
|
69
|
+
} catch {
|
|
70
|
+
input = undefined // payload unusable; the hook still gets the error
|
|
71
|
+
}
|
|
72
|
+
try {
|
|
73
|
+
await def.onFailed(input, error, ctx as never)
|
|
74
|
+
} catch (hookErr) {
|
|
75
|
+
console.error('[bunderstack] onFailed hook threw:', hookErr)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** running rows whose lease expired → pending (or failed when exhausted). */
|
|
80
|
+
async function recoverExpiredLeases(now: number) {
|
|
81
|
+
const expired: (JobRow & { lastError: string | null })[] = await db
|
|
82
|
+
.select({
|
|
83
|
+
id: t.id,
|
|
84
|
+
type: t.type,
|
|
85
|
+
payloadJson: t.payloadJson,
|
|
86
|
+
attempts: t.attempts,
|
|
87
|
+
lastError: t.lastError,
|
|
88
|
+
})
|
|
89
|
+
.from(t)
|
|
90
|
+
.where(
|
|
91
|
+
and(eq(t.status, 'running'), isNotNull(t.lockedUntil), lt(t.lockedUntil, now)),
|
|
92
|
+
)
|
|
93
|
+
for (const row of expired) {
|
|
94
|
+
const def = defs[row.type]
|
|
95
|
+
const error = new Error('lease expired (worker crashed or timed out)')
|
|
96
|
+
if (!def) {
|
|
97
|
+
await db
|
|
98
|
+
.update(t)
|
|
99
|
+
.set({
|
|
100
|
+
status: 'failed',
|
|
101
|
+
finishedAt: now,
|
|
102
|
+
lockedUntil: null,
|
|
103
|
+
lastError: `unknown job type "${row.type}"`,
|
|
104
|
+
dedupeKey: null,
|
|
105
|
+
})
|
|
106
|
+
.where(eq(t.id, row.id))
|
|
107
|
+
continue
|
|
108
|
+
}
|
|
109
|
+
if (Number(row.attempts) >= maxAttempts(def)) {
|
|
110
|
+
await db
|
|
111
|
+
.update(t)
|
|
112
|
+
.set({
|
|
113
|
+
status: 'failed',
|
|
114
|
+
finishedAt: now,
|
|
115
|
+
lockedUntil: null,
|
|
116
|
+
lastError: error.message,
|
|
117
|
+
...terminalPatch(def),
|
|
118
|
+
})
|
|
119
|
+
.where(eq(t.id, row.id))
|
|
120
|
+
await fireOnFailed(def, row.payloadJson, error)
|
|
121
|
+
} else {
|
|
122
|
+
await db
|
|
123
|
+
.update(t)
|
|
124
|
+
.set({
|
|
125
|
+
status: 'pending',
|
|
126
|
+
lockedUntil: null,
|
|
127
|
+
runAt: now + backoffMs(def, Number(row.attempts)),
|
|
128
|
+
lastError: error.message,
|
|
129
|
+
})
|
|
130
|
+
.where(eq(t.id, row.id))
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Enqueue the current minute's slot for every cron definition. */
|
|
136
|
+
async function scheduleCronSlots(now: number) {
|
|
137
|
+
const minute = Math.floor(now / 60_000) * 60_000
|
|
138
|
+
for (const [name, cron] of crons) {
|
|
139
|
+
if (!cronMatches(cron, minute)) continue
|
|
140
|
+
// The unique (type, dedupe_key) index collapses concurrent replicas'
|
|
141
|
+
// enqueues of the same slot into one row.
|
|
142
|
+
await enqueueJob(db, defs, name, undefined, {
|
|
143
|
+
dedupeKey: `cron:${name}:${minute}`,
|
|
144
|
+
runAt: minute,
|
|
145
|
+
})
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function reapSucceeded(now: number) {
|
|
150
|
+
await db
|
|
151
|
+
.delete(t)
|
|
152
|
+
.where(
|
|
153
|
+
and(
|
|
154
|
+
eq(t.status, 'succeeded'),
|
|
155
|
+
lt(t.finishedAt, now - SUCCEEDED_RETENTION_MS),
|
|
156
|
+
),
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Atomically claim up to `limit` runnable jobs of one type. */
|
|
161
|
+
async function claim(
|
|
162
|
+
type: string,
|
|
163
|
+
limit: number,
|
|
164
|
+
now: number,
|
|
165
|
+
leaseUntil: number,
|
|
166
|
+
): Promise<JobRow[]> {
|
|
167
|
+
const pendingIds = db
|
|
168
|
+
.select({ id: t.id })
|
|
169
|
+
.from(t)
|
|
170
|
+
.where(and(eq(t.type, type), eq(t.status, 'pending'), lte(t.runAt, now)))
|
|
171
|
+
.orderBy(t.runAt)
|
|
172
|
+
.limit(limit)
|
|
173
|
+
// PG: lock the selected rows so concurrent replicas skip them. SQLite's
|
|
174
|
+
// single-writer model makes the one-statement UPDATE atomic on its own.
|
|
175
|
+
const sub = is(db, PgDatabase)
|
|
176
|
+
? (pendingIds as unknown as { for: (m: string, o: object) => typeof pendingIds })
|
|
177
|
+
.for('update', { skipLocked: true })
|
|
178
|
+
: pendingIds
|
|
179
|
+
const rows: JobRow[] = await db
|
|
180
|
+
.update(t)
|
|
181
|
+
.set({
|
|
182
|
+
status: 'running',
|
|
183
|
+
lockedUntil: leaseUntil,
|
|
184
|
+
attempts: sql`${t.attempts} + 1`,
|
|
185
|
+
})
|
|
186
|
+
.where(and(inArray(t.id, sub), eq(t.status, 'pending')))
|
|
187
|
+
.returning({
|
|
188
|
+
id: t.id,
|
|
189
|
+
type: t.type,
|
|
190
|
+
payloadJson: t.payloadJson,
|
|
191
|
+
attempts: t.attempts,
|
|
192
|
+
})
|
|
193
|
+
return rows
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// `now` is the tick's injected clock: retry runAt math uses it so tests can
|
|
197
|
+
// drive backoff deterministically. finishedAt uses the real clock (a handler
|
|
198
|
+
// may run long past the tick's start).
|
|
199
|
+
async function runJob(row: JobRow, def: AnyJobDefinition, now: number) {
|
|
200
|
+
let input: unknown
|
|
201
|
+
try {
|
|
202
|
+
const raw = JSON.parse(row.payloadJson)
|
|
203
|
+
input = def.input ? def.input.parse(raw) : undefined
|
|
204
|
+
} catch (err) {
|
|
205
|
+
// Stored payload no longer parses (schema drift): retrying can't help.
|
|
206
|
+
const e = toError(err)
|
|
207
|
+
await db
|
|
208
|
+
.update(t)
|
|
209
|
+
.set({
|
|
210
|
+
status: 'failed',
|
|
211
|
+
finishedAt: Date.now(),
|
|
212
|
+
lockedUntil: null,
|
|
213
|
+
lastError: e.message,
|
|
214
|
+
...terminalPatch(def),
|
|
215
|
+
})
|
|
216
|
+
.where(eq(t.id, row.id))
|
|
217
|
+
await fireOnFailed(def, row.payloadJson, e)
|
|
218
|
+
return
|
|
219
|
+
}
|
|
220
|
+
try {
|
|
221
|
+
await def.handler(input, ctx as never)
|
|
222
|
+
await db
|
|
223
|
+
.update(t)
|
|
224
|
+
.set({
|
|
225
|
+
status: 'succeeded',
|
|
226
|
+
finishedAt: Date.now(),
|
|
227
|
+
lockedUntil: null,
|
|
228
|
+
...terminalPatch(def),
|
|
229
|
+
})
|
|
230
|
+
.where(eq(t.id, row.id))
|
|
231
|
+
} catch (err) {
|
|
232
|
+
const e = toError(err)
|
|
233
|
+
if (Number(row.attempts) < maxAttempts(def)) {
|
|
234
|
+
await db
|
|
235
|
+
.update(t)
|
|
236
|
+
.set({
|
|
237
|
+
status: 'pending',
|
|
238
|
+
lockedUntil: null,
|
|
239
|
+
runAt: now + backoffMs(def, Number(row.attempts)),
|
|
240
|
+
lastError: e.message,
|
|
241
|
+
})
|
|
242
|
+
.where(eq(t.id, row.id))
|
|
243
|
+
} else {
|
|
244
|
+
await db
|
|
245
|
+
.update(t)
|
|
246
|
+
.set({
|
|
247
|
+
status: 'failed',
|
|
248
|
+
finishedAt: Date.now(),
|
|
249
|
+
lockedUntil: null,
|
|
250
|
+
lastError: e.message,
|
|
251
|
+
...terminalPatch(def),
|
|
252
|
+
})
|
|
253
|
+
.where(eq(t.id, row.id))
|
|
254
|
+
await fireOnFailed(def, row.payloadJson, e)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async function runClaimable(now: number) {
|
|
260
|
+
const work: Promise<void>[] = []
|
|
261
|
+
for (const [type, def] of Object.entries(defs)) {
|
|
262
|
+
let limit = CLAIM_BATCH
|
|
263
|
+
if (def.concurrency !== undefined) {
|
|
264
|
+
const runningRows = await db
|
|
265
|
+
.select({ id: t.id })
|
|
266
|
+
.from(t)
|
|
267
|
+
.where(and(eq(t.type, type), eq(t.status, 'running')))
|
|
268
|
+
const capacity = def.concurrency - runningRows.length
|
|
269
|
+
if (capacity <= 0) continue
|
|
270
|
+
limit = Math.min(limit, capacity)
|
|
271
|
+
}
|
|
272
|
+
const leaseUntil = now + (def.timeout ?? DEFAULT_TIMEOUT_MS)
|
|
273
|
+
const claimed = await claim(type, limit, now, leaseUntil)
|
|
274
|
+
for (const row of claimed) work.push(runJob(row, def, now))
|
|
275
|
+
}
|
|
276
|
+
await Promise.all(work)
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return {
|
|
280
|
+
async tick(now: number = Date.now()) {
|
|
281
|
+
await recoverExpiredLeases(now)
|
|
282
|
+
await scheduleCronSlots(now)
|
|
283
|
+
await reapSucceeded(now)
|
|
284
|
+
await runClaimable(now)
|
|
285
|
+
},
|
|
286
|
+
setJobsFacade(f: JobsRuntimeFacade) {
|
|
287
|
+
ctx.jobs = f
|
|
288
|
+
},
|
|
289
|
+
}
|
|
290
|
+
}
|
package/src/manifest.ts
CHANGED
|
@@ -6,9 +6,11 @@ import type { ZodType } from 'zod'
|
|
|
6
6
|
|
|
7
7
|
import type { Dialect } from './dialect'
|
|
8
8
|
import type { EnvConfigInput } from './env'
|
|
9
|
+
import type { JobsDefs } from './jobs/define'
|
|
9
10
|
import type { ResolvedBucket, ResolvedStorageBuckets } from './storage/buckets'
|
|
10
11
|
|
|
11
12
|
export type ManifestEnvVar = { key: string; required: boolean }
|
|
13
|
+
export type ManifestJob = { name: string; cron?: string }
|
|
12
14
|
|
|
13
15
|
export type BunderstackManifest = {
|
|
14
16
|
dialect: Dialect
|
|
@@ -17,6 +19,7 @@ export type BunderstackManifest = {
|
|
|
17
19
|
buckets: { name: string; visibility: ResolvedBucket['visibility'] }[]
|
|
18
20
|
realtime: boolean
|
|
19
21
|
env: { server: ManifestEnvVar[]; client: ManifestEnvVar[] }
|
|
22
|
+
jobs: ManifestJob[]
|
|
20
23
|
}
|
|
21
24
|
|
|
22
25
|
function describeSection(
|
|
@@ -34,6 +37,7 @@ export function buildManifest(args: {
|
|
|
34
37
|
storage: ResolvedStorageBuckets
|
|
35
38
|
envConfig: EnvConfigInput | undefined
|
|
36
39
|
realtime: boolean
|
|
40
|
+
jobs: JobsDefs | undefined
|
|
37
41
|
}): BunderstackManifest {
|
|
38
42
|
return {
|
|
39
43
|
dialect: args.dialect,
|
|
@@ -48,5 +52,8 @@ export function buildManifest(args: {
|
|
|
48
52
|
server: describeSection(args.envConfig?.server),
|
|
49
53
|
client: describeSection(args.envConfig?.client),
|
|
50
54
|
},
|
|
55
|
+
jobs: Object.entries(args.jobs ?? {}).map(([name, def]) =>
|
|
56
|
+
def.cron !== undefined ? { name, cron: def.cron } : { name },
|
|
57
|
+
),
|
|
51
58
|
}
|
|
52
59
|
}
|
package/src/schema-export-pg.ts
CHANGED
package/src/schema-export.ts
CHANGED
package/src/trpc.ts
CHANGED
|
@@ -5,6 +5,7 @@ import superjson from 'superjson'
|
|
|
5
5
|
import type { AccessUser } from './access'
|
|
6
6
|
import type { DbFor } from './db'
|
|
7
7
|
import type { EmailFacade } from './email'
|
|
8
|
+
import type { JobsRuntimeFacade } from './jobs/index'
|
|
8
9
|
|
|
9
10
|
export type TRPCContext<
|
|
10
11
|
TSchema extends Record<string, unknown>,
|
|
@@ -14,6 +15,7 @@ export type TRPCContext<
|
|
|
14
15
|
user: AccessUser | null
|
|
15
16
|
env: TEnvResult
|
|
16
17
|
email: EmailFacade
|
|
18
|
+
jobs: JobsRuntimeFacade
|
|
17
19
|
req: Request
|
|
18
20
|
}
|
|
19
21
|
|