bunderstack 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/package.json +2 -1
- package/src/access.ts +18 -0
- package/src/config.ts +22 -44
- package/src/cron.ts +2 -1
- package/src/crud.ts +1 -9
- package/src/env.ts +9 -8
- package/src/handler.ts +5 -5
- package/src/index.ts +86 -86
- package/src/internal-tables-pg.ts +1 -17
- package/src/internal-tables.ts +0 -31
- package/src/jobs/define.ts +68 -12
- package/src/jobs/index.ts +3 -9
- package/src/jobs/queue.ts +10 -6
- package/src/jobs/slots.ts +52 -0
- package/src/jobs/worker.ts +139 -42
- package/src/manifest.ts +0 -6
- package/src/realtime/index.ts +3 -11
- package/src/realtime/redis.ts +3 -12
- package/src/routes.ts +137 -0
- package/src/jobs/cron-auth.ts +0 -28
- package/src/jobs/cron-router.ts +0 -135
- package/src/jobs/cron-runner.ts +0 -224
- package/src/jobs/local-cron.ts +0 -78
package/src/internal-tables.ts
CHANGED
|
@@ -12,7 +12,6 @@ import {
|
|
|
12
12
|
import { detectDialect } from './dialect'
|
|
13
13
|
import {
|
|
14
14
|
bunderstackFilesPg,
|
|
15
|
-
bunderstackCronRunsPg,
|
|
16
15
|
bunderstackIdempotencyPg,
|
|
17
16
|
bunderstackJobsPg,
|
|
18
17
|
} from './internal-tables-pg'
|
|
@@ -74,43 +73,22 @@ export const bunderstackJobs = sqliteTable(
|
|
|
74
73
|
],
|
|
75
74
|
)
|
|
76
75
|
|
|
77
|
-
export const bunderstackCronRuns = sqliteTable(
|
|
78
|
-
'_bunderstack_cron_runs',
|
|
79
|
-
{
|
|
80
|
-
taskId: text('task_id').notNull(),
|
|
81
|
-
scheduledAt: integer('scheduled_at').notNull(),
|
|
82
|
-
status: text('status').notNull(),
|
|
83
|
-
attempts: integer('attempts').notNull().default(0),
|
|
84
|
-
lockedUntil: integer('locked_until'),
|
|
85
|
-
lastError: text('last_error'),
|
|
86
|
-
startedAt: integer('started_at'),
|
|
87
|
-
finishedAt: integer('finished_at'),
|
|
88
|
-
},
|
|
89
|
-
(t) => [
|
|
90
|
-
primaryKey({ columns: [t.taskId, t.scheduledAt] }),
|
|
91
|
-
index('bcr_claim').on(t.status, t.lockedUntil),
|
|
92
|
-
],
|
|
93
|
-
)
|
|
94
|
-
|
|
95
76
|
export const INTERNAL_TABLES = {
|
|
96
77
|
bunderstackFiles,
|
|
97
78
|
bunderstackIdempotency,
|
|
98
79
|
bunderstackJobs,
|
|
99
|
-
bunderstackCronRuns,
|
|
100
80
|
} as const
|
|
101
81
|
|
|
102
82
|
export const INTERNAL_TABLE_NAMES: ReadonlySet<string> = new Set([
|
|
103
83
|
'bunderstack_file_meta',
|
|
104
84
|
'_bunderstack_idempotency',
|
|
105
85
|
'_bunderstack_jobs',
|
|
106
|
-
'_bunderstack_cron_runs',
|
|
107
86
|
])
|
|
108
87
|
|
|
109
88
|
export const INTERNAL_TABLES_PG = {
|
|
110
89
|
bunderstackFiles: bunderstackFilesPg,
|
|
111
90
|
bunderstackIdempotency: bunderstackIdempotencyPg,
|
|
112
91
|
bunderstackJobs: bunderstackJobsPg,
|
|
113
|
-
bunderstackCronRuns: bunderstackCronRunsPg,
|
|
114
92
|
} as const
|
|
115
93
|
|
|
116
94
|
// Both dialect twins count as "ours" for the re-export identity check.
|
|
@@ -121,10 +99,6 @@ const INTERNAL_TABLE_CANDIDATES = new Map<string, readonly unknown[]>([
|
|
|
121
99
|
[bunderstackIdempotency, bunderstackIdempotencyPg],
|
|
122
100
|
],
|
|
123
101
|
[getTableName(bunderstackJobs), [bunderstackJobs, bunderstackJobsPg]],
|
|
124
|
-
[
|
|
125
|
-
getTableName(bunderstackCronRuns),
|
|
126
|
-
[bunderstackCronRuns, bunderstackCronRunsPg],
|
|
127
|
-
],
|
|
128
102
|
])
|
|
129
103
|
|
|
130
104
|
/** Internal file-meta table matching the db's dialect. */
|
|
@@ -142,11 +116,6 @@ export function jobsTableFor(db: unknown) {
|
|
|
142
116
|
return is(db, PgDatabase) ? bunderstackJobsPg : bunderstackJobs
|
|
143
117
|
}
|
|
144
118
|
|
|
145
|
-
/** Internal cron-run table matching the db's dialect. */
|
|
146
|
-
export function cronRunsTableFor(db: unknown) {
|
|
147
|
-
return is(db, PgDatabase) ? bunderstackCronRunsPg : bunderstackCronRuns
|
|
148
|
-
}
|
|
149
|
-
|
|
150
119
|
export function withInternalTables<TSchema extends Record<string, unknown>>(
|
|
151
120
|
schema: TSchema,
|
|
152
121
|
): TSchema & typeof INTERNAL_TABLES {
|
package/src/jobs/define.ts
CHANGED
|
@@ -9,6 +9,8 @@ import type { StorageFacade } from '../index'
|
|
|
9
9
|
|
|
10
10
|
import { parseCron } from './cron'
|
|
11
11
|
|
|
12
|
+
import { CRON_PREFIX, type CatchUp } from './slots'
|
|
13
|
+
|
|
12
14
|
export const DEFAULT_RETRIES = 3
|
|
13
15
|
export const DEFAULT_TIMEOUT_MS = 60_000
|
|
14
16
|
|
|
@@ -21,6 +23,15 @@ export type EnqueueOptions = {
|
|
|
21
23
|
runAt?: Date | number
|
|
22
24
|
}
|
|
23
25
|
|
|
26
|
+
export type TickResult = {
|
|
27
|
+
/** Rows moved from pending to running this tick. */
|
|
28
|
+
claimed: number
|
|
29
|
+
/** Handlers that completed successfully. */
|
|
30
|
+
ran: number
|
|
31
|
+
/** Handlers that threw, whether or not they will be retried. */
|
|
32
|
+
failed: number
|
|
33
|
+
}
|
|
34
|
+
|
|
24
35
|
/**
|
|
25
36
|
* The untyped runtime facade. Handler ctx and tRPC ctx expose this shape;
|
|
26
37
|
* `app.jobs` narrows `enqueue` to the declared job names/payloads.
|
|
@@ -32,7 +43,7 @@ export type JobsRuntimeFacade = {
|
|
|
32
43
|
opts?: EnqueueOptions,
|
|
33
44
|
): Promise<{ id: string }>
|
|
34
45
|
/** Run one poll cycle deterministically (tests). `now` defaults to Date.now(). */
|
|
35
|
-
tick(now?: number): Promise<
|
|
46
|
+
tick(now?: number): Promise<TickResult>
|
|
36
47
|
}
|
|
37
48
|
|
|
38
49
|
import type { RealtimeFacade } from '../realtime/facade'
|
|
@@ -66,7 +77,7 @@ export type QueueJobDefinition<
|
|
|
66
77
|
retries?: number
|
|
67
78
|
/** Delay before retry N (1-based). Default exponential: 1s, 2s, 4s, … */
|
|
68
79
|
backoff?: ((attempt: number) => number) | { baseMs?: number; factor?: number }
|
|
69
|
-
/** Max simultaneous `running` rows of this type, enforced
|
|
80
|
+
/** Max simultaneous `running` rows of this type, enforced per worker. */
|
|
70
81
|
concurrency?: number
|
|
71
82
|
/** Lease duration in ms; an expired lease sends the job back to pending. */
|
|
72
83
|
timeout?: number
|
|
@@ -91,10 +102,26 @@ export type CronDefinition<
|
|
|
91
102
|
> = {
|
|
92
103
|
kind: 'cron'
|
|
93
104
|
schedule: TSchedule
|
|
105
|
+
/** Attempts after the first failure. Default 3 (so 4 total attempts). */
|
|
106
|
+
retries?: number
|
|
107
|
+
/** Delay before retry N (1-based). Default exponential: 1s, 2s, 4s, … */
|
|
108
|
+
backoff?: ((attempt: number) => number) | { baseMs?: number; factor?: number }
|
|
109
|
+
/** Lease duration in ms; an expired lease sends the slot back to pending. */
|
|
110
|
+
timeout?: number
|
|
111
|
+
/** How missed slots are handled on wake. Default 'latest'. */
|
|
112
|
+
catchUp?: CatchUp
|
|
113
|
+
/** How far back catch-up looks, in ms. Default 1 hour. */
|
|
114
|
+
catchUpWindow?: number
|
|
94
115
|
handler: (
|
|
95
116
|
invocation: CronInvocation,
|
|
96
117
|
ctx: JobContext<TSchema, TEnvResult>,
|
|
97
118
|
) => Promise<void> | void
|
|
119
|
+
/** Fires once, after the final attempt fails. Errors here are logged, never retried. */
|
|
120
|
+
onFailed?: (
|
|
121
|
+
invocation: CronInvocation,
|
|
122
|
+
error: Error,
|
|
123
|
+
ctx: JobContext<TSchema, TEnvResult>,
|
|
124
|
+
) => Promise<void> | void
|
|
98
125
|
}
|
|
99
126
|
|
|
100
127
|
export type BackgroundDefinition =
|
|
@@ -111,6 +138,10 @@ export type JobDefinition<
|
|
|
111
138
|
|
|
112
139
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
113
140
|
export type AnyJobDefinition = QueueJobDefinition<any, any, any>
|
|
141
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
142
|
+
export type AnyBackgroundDefinition =
|
|
143
|
+
| QueueJobDefinition<any, any, any>
|
|
144
|
+
| CronDefinition<any, any, any>
|
|
114
145
|
export type JobsDefs = BackgroundDefs
|
|
115
146
|
|
|
116
147
|
export type QueueJobKeys<TDefs extends BackgroundDefs> = {
|
|
@@ -129,18 +160,38 @@ export function validateBackgroundDefs(defs: BackgroundDefs): void {
|
|
|
129
160
|
if (typeof def.handler !== 'function') {
|
|
130
161
|
throw new Error(`[bunderstack] background task "${name}" has no handler`)
|
|
131
162
|
}
|
|
132
|
-
if (def.kind === '
|
|
133
|
-
|
|
134
|
-
|
|
163
|
+
if (def.kind === 'job' && name.startsWith(CRON_PREFIX)) {
|
|
164
|
+
throw new Error(
|
|
165
|
+
`[bunderstack] job "${name}": the "${CRON_PREFIX}" prefix is reserved for cron tasks`,
|
|
166
|
+
)
|
|
135
167
|
}
|
|
136
168
|
if (
|
|
137
169
|
def.retries !== undefined &&
|
|
138
170
|
(def.retries < 0 || !Number.isInteger(def.retries))
|
|
139
171
|
) {
|
|
140
172
|
throw new Error(
|
|
141
|
-
`[bunderstack]
|
|
173
|
+
`[bunderstack] background task "${name}": retries must be a non-negative integer`,
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
if (def.timeout !== undefined && def.timeout <= 0) {
|
|
177
|
+
throw new Error(
|
|
178
|
+
`[bunderstack] background task "${name}": timeout must be positive`,
|
|
142
179
|
)
|
|
143
180
|
}
|
|
181
|
+
if (def.kind === 'cron') {
|
|
182
|
+
parseCron(def.schedule)
|
|
183
|
+
if ((def as { concurrency?: number }).concurrency !== undefined) {
|
|
184
|
+
throw new Error(
|
|
185
|
+
`[bunderstack] cron "${name}": concurrency is not supported for cron tasks — slots are already unique`,
|
|
186
|
+
)
|
|
187
|
+
}
|
|
188
|
+
if (def.catchUpWindow !== undefined && def.catchUpWindow <= 0) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`[bunderstack] cron "${name}": catchUpWindow must be positive`,
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
continue
|
|
194
|
+
}
|
|
144
195
|
if (
|
|
145
196
|
def.concurrency !== undefined &&
|
|
146
197
|
(def.concurrency < 1 || !Number.isInteger(def.concurrency))
|
|
@@ -149,22 +200,27 @@ export function validateBackgroundDefs(defs: BackgroundDefs): void {
|
|
|
149
200
|
`[bunderstack] job "${name}": concurrency must be a positive integer`,
|
|
150
201
|
)
|
|
151
202
|
}
|
|
152
|
-
if (def.timeout !== undefined && def.timeout <= 0) {
|
|
153
|
-
throw new Error(`[bunderstack] job "${name}": timeout must be positive`)
|
|
154
|
-
}
|
|
155
203
|
}
|
|
156
204
|
}
|
|
157
205
|
|
|
158
206
|
/** @deprecated Use validateBackgroundDefs. */
|
|
159
207
|
export const validateJobsDefs = validateBackgroundDefs
|
|
160
208
|
|
|
161
|
-
/**
|
|
162
|
-
|
|
209
|
+
/**
|
|
210
|
+
* Delay in ms before retry `attempt` (1-based = the attempt that just failed).
|
|
211
|
+
* Jittered by ±20% so a shared outage does not retry every job in lockstep.
|
|
212
|
+
* A caller-supplied backoff function is returned verbatim — the caller owns it.
|
|
213
|
+
*/
|
|
214
|
+
export function backoffMs(
|
|
215
|
+
def: AnyBackgroundDefinition,
|
|
216
|
+
attempt: number,
|
|
217
|
+
): number {
|
|
163
218
|
const b = def.backoff
|
|
164
219
|
if (typeof b === 'function') return b(attempt)
|
|
165
220
|
const baseMs = b?.baseMs ?? 1000
|
|
166
221
|
const factor = b?.factor ?? 2
|
|
167
|
-
|
|
222
|
+
const flat = baseMs * factor ** (attempt - 1)
|
|
223
|
+
return Math.round(flat * (0.8 + Math.random() * 0.4))
|
|
168
224
|
}
|
|
169
225
|
|
|
170
226
|
/**
|
package/src/jobs/index.ts
CHANGED
|
@@ -25,19 +25,13 @@ export type {
|
|
|
25
25
|
} from './define'
|
|
26
26
|
export { enqueueJob } from './queue'
|
|
27
27
|
export { createJobRunner } from './worker'
|
|
28
|
-
export { runCronSlot, runScheduledSlot } from './cron-runner'
|
|
29
|
-
export type { CronRunResult } from './cron-runner'
|
|
30
|
-
export { buildCronRouter } from './cron-router'
|
|
31
|
-
export { signScheduleRequest, verifyScheduleRequest } from './cron-auth'
|
|
32
28
|
export { startJobWorker } from './runtime'
|
|
33
29
|
export type {
|
|
34
30
|
StartWorkerOptions,
|
|
35
31
|
RunWorkerOptions,
|
|
36
32
|
WorkerHandle,
|
|
37
33
|
} from './runtime'
|
|
38
|
-
export { startLocalCronScheduler } from './local-cron'
|
|
39
|
-
export type {
|
|
40
|
-
LocalCronScheduler,
|
|
41
|
-
LocalCronSchedulerOptions,
|
|
42
|
-
} from './local-cron'
|
|
43
34
|
export { parseCron, cronMatches } from './cron'
|
|
35
|
+
export { slotsDue, floorSlot, CRON_PREFIX, SLOT_MS } from './slots'
|
|
36
|
+
export type { CatchUp } from './slots'
|
|
37
|
+
export type { TickResult } from './define'
|
package/src/jobs/queue.ts
CHANGED
|
@@ -7,6 +7,8 @@ import type { EnqueueOptions, JobsDefs } from './define'
|
|
|
7
7
|
import { jobsTableFor } from '../internal-tables'
|
|
8
8
|
import { generate } from '../typeid'
|
|
9
9
|
|
|
10
|
+
import { CRON_PREFIX } from './slots'
|
|
11
|
+
|
|
10
12
|
export async function enqueueJob(
|
|
11
13
|
db: AnyDb,
|
|
12
14
|
defs: JobsDefs,
|
|
@@ -15,11 +17,13 @@ export async function enqueueJob(
|
|
|
15
17
|
opts: EnqueueOptions = {},
|
|
16
18
|
): Promise<{ id: string }> {
|
|
17
19
|
const def = defs[name]
|
|
18
|
-
if (!def
|
|
19
|
-
throw new Error(`[bunderstack] unknown
|
|
20
|
+
if (!def) {
|
|
21
|
+
throw new Error(`[bunderstack] unknown background task "${name}"`)
|
|
20
22
|
}
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
+
const isCron = def.kind === 'cron'
|
|
24
|
+
const type = isCron ? `${CRON_PREFIX}${name}` : name
|
|
25
|
+
// Cron slots carry no payload; queue jobs validate theirs at the call site.
|
|
26
|
+
const parsed = isCron ? null : def.input ? def.input.parse(input) : null
|
|
23
27
|
const t = jobsTableFor(db)
|
|
24
28
|
const now = Date.now()
|
|
25
29
|
const runAt =
|
|
@@ -35,7 +39,7 @@ export async function enqueueJob(
|
|
|
35
39
|
.insert(t)
|
|
36
40
|
.values({
|
|
37
41
|
id,
|
|
38
|
-
type
|
|
42
|
+
type,
|
|
39
43
|
payloadJson: JSON.stringify(parsed),
|
|
40
44
|
status: 'pending',
|
|
41
45
|
attempts: 0,
|
|
@@ -49,7 +53,7 @@ export async function enqueueJob(
|
|
|
49
53
|
const existing = await db
|
|
50
54
|
.select({ id: t.id })
|
|
51
55
|
.from(t)
|
|
52
|
-
.where(and(eq(t.type,
|
|
56
|
+
.where(and(eq(t.type, type), eq(t.dedupeKey, opts.dedupeKey ?? '')))
|
|
53
57
|
.limit(1)
|
|
54
58
|
if (existing[0]) return { id: String(existing[0].id) }
|
|
55
59
|
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// src/jobs/slots.ts — cron slot enumeration. Pure; no db, no clock reads.
|
|
2
|
+
import { cronMatches, type ParsedCron } from './cron'
|
|
3
|
+
|
|
4
|
+
/** Slot granularity. Every slot timestamp satisfies `slot % SLOT_MS === 0`. */
|
|
5
|
+
export const SLOT_MS = 60_000
|
|
6
|
+
|
|
7
|
+
/** Reserved job-type prefix for cron occurrences. */
|
|
8
|
+
export const CRON_PREFIX = 'cron:'
|
|
9
|
+
|
|
10
|
+
/** How far back either catch-up mode will look. */
|
|
11
|
+
export const DEFAULT_CATCH_UP_WINDOW_MS = 60 * SLOT_MS
|
|
12
|
+
|
|
13
|
+
export type CatchUp = 'latest' | 'all'
|
|
14
|
+
|
|
15
|
+
/** Aligns `ms` down to its containing slot. */
|
|
16
|
+
export function floorSlot(ms: number): number {
|
|
17
|
+
return Math.floor(ms / SLOT_MS) * SLOT_MS
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Slots matching `cron` in the half-open range `(from, to]`, oldest first.
|
|
22
|
+
*
|
|
23
|
+
* `from` is exclusive so a stored watermark is never re-emitted. Both modes are
|
|
24
|
+
* clamped to `catchUpWindowMs` — without it a watermark far in the past would
|
|
25
|
+
* make this iterate unbounded minutes.
|
|
26
|
+
*/
|
|
27
|
+
export function slotsDue(args: {
|
|
28
|
+
cron: ParsedCron
|
|
29
|
+
from: number
|
|
30
|
+
to: number
|
|
31
|
+
catchUp?: CatchUp
|
|
32
|
+
catchUpWindowMs?: number
|
|
33
|
+
}): number[] {
|
|
34
|
+
const catchUp = args.catchUp ?? 'latest'
|
|
35
|
+
const windowMs = args.catchUpWindowMs ?? DEFAULT_CATCH_UP_WINDOW_MS
|
|
36
|
+
const to = floorSlot(args.to)
|
|
37
|
+
const from = Math.max(floorSlot(args.from), to - windowMs)
|
|
38
|
+
if (to <= from) return []
|
|
39
|
+
|
|
40
|
+
if (catchUp === 'latest') {
|
|
41
|
+
for (let s = to; s > from; s -= SLOT_MS) {
|
|
42
|
+
if (cronMatches(args.cron, s)) return [s]
|
|
43
|
+
}
|
|
44
|
+
return []
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const slots: number[] = []
|
|
48
|
+
for (let s = from + SLOT_MS; s <= to; s += SLOT_MS) {
|
|
49
|
+
if (cronMatches(args.cron, s)) slots.push(s)
|
|
50
|
+
}
|
|
51
|
+
return slots
|
|
52
|
+
}
|
package/src/jobs/worker.ts
CHANGED
|
@@ -1,32 +1,56 @@
|
|
|
1
1
|
// src/jobs/worker.ts — the queue worker. One `tick()` is a full cycle:
|
|
2
2
|
// recover expired leases → reap old succeeded rows → claim and run queue jobs.
|
|
3
|
-
import { and, eq, inArray, is, isNotNull, lt, lte, sql } from 'drizzle-orm'
|
|
3
|
+
import { and, eq, inArray, is, isNotNull, lt, lte, max, sql } from 'drizzle-orm'
|
|
4
4
|
import { PgDatabase } from 'drizzle-orm/pg-core'
|
|
5
5
|
|
|
6
6
|
import type { AnyDb } from '../dialect'
|
|
7
|
-
import type {
|
|
7
|
+
import type {
|
|
8
|
+
AnyBackgroundDefinition,
|
|
9
|
+
JobsDefs,
|
|
10
|
+
JobsRuntimeFacade,
|
|
11
|
+
TickResult,
|
|
12
|
+
} from './define'
|
|
8
13
|
|
|
9
14
|
import { jobsTableFor } from '../internal-tables'
|
|
15
|
+
import { parseCron } from './cron'
|
|
10
16
|
import { backoffMs, DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from './define'
|
|
17
|
+
import { enqueueJob } from './queue'
|
|
18
|
+
import { CRON_PREFIX, floorSlot, slotsDue, SLOT_MS } from './slots'
|
|
11
19
|
|
|
12
20
|
const CLAIM_BATCH = 10
|
|
13
21
|
const SUCCEEDED_RETENTION_MS = 24 * 60 * 60 * 1000
|
|
22
|
+
const REAP_INTERVAL_MS = 60 * 60_000
|
|
14
23
|
|
|
15
24
|
type JobRow = {
|
|
16
25
|
id: string
|
|
17
26
|
type: string
|
|
18
27
|
payloadJson: string
|
|
19
28
|
attempts: number
|
|
29
|
+
runAt: number
|
|
20
30
|
}
|
|
21
31
|
|
|
22
32
|
function toError(err: unknown): Error {
|
|
23
33
|
return err instanceof Error ? err : new Error(String(err))
|
|
24
34
|
}
|
|
25
35
|
|
|
26
|
-
function maxAttempts(def:
|
|
36
|
+
function maxAttempts(def: AnyBackgroundDefinition): number {
|
|
27
37
|
return 1 + (def.retries ?? DEFAULT_RETRIES)
|
|
28
38
|
}
|
|
29
39
|
|
|
40
|
+
/** Resolve a stored row type back to its definition. Cron rows carry the
|
|
41
|
+
* reserved prefix; queue rows use the definition key verbatim. */
|
|
42
|
+
function definitionFor(
|
|
43
|
+
defs: JobsDefs,
|
|
44
|
+
type: string,
|
|
45
|
+
): AnyBackgroundDefinition | undefined {
|
|
46
|
+
if (type.startsWith(CRON_PREFIX)) {
|
|
47
|
+
const def = defs[type.slice(CRON_PREFIX.length)]
|
|
48
|
+
return def?.kind === 'cron' ? def : undefined
|
|
49
|
+
}
|
|
50
|
+
const def = defs[type]
|
|
51
|
+
return def?.kind === 'job' ? def : undefined
|
|
52
|
+
}
|
|
53
|
+
|
|
30
54
|
/** Terminal queue rows release their dedupe key. */
|
|
31
55
|
function terminalPatch() {
|
|
32
56
|
return { dedupeKey: null }
|
|
@@ -41,21 +65,66 @@ export function createJobRunner(deps: {
|
|
|
41
65
|
const { db, defs } = deps
|
|
42
66
|
const t = jobsTableFor(db)
|
|
43
67
|
const ctx = { ...deps.ctx } as Record<string, unknown>
|
|
68
|
+
let lastReapAt = 0
|
|
69
|
+
|
|
70
|
+
/** Cron rows carry no payload — their handler input is the slot itself. */
|
|
71
|
+
function resolveInput(def: AnyBackgroundDefinition, row: JobRow): unknown {
|
|
72
|
+
if (def.kind === 'cron') {
|
|
73
|
+
return { scheduledFor: new Date(Number(row.runAt)) }
|
|
74
|
+
}
|
|
75
|
+
const raw = JSON.parse(row.payloadJson)
|
|
76
|
+
return def.input ? def.input.parse(raw) : undefined
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The watermark is the newest slot we already stored for this cron. When no
|
|
81
|
+
* rows exist — a newly declared cron, or one whose rows were reaped — anchor
|
|
82
|
+
* one slot before now so the current minute is eligible and nothing older is.
|
|
83
|
+
*/
|
|
84
|
+
async function cronWatermark(type: string, now: number): Promise<number> {
|
|
85
|
+
const rows = await db
|
|
86
|
+
.select({ latest: max(t.runAt) })
|
|
87
|
+
.from(t)
|
|
88
|
+
.where(eq(t.type, type))
|
|
89
|
+
const latest = rows[0]?.latest
|
|
90
|
+
return latest == null ? floorSlot(now) - SLOT_MS : Number(latest)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Enqueue a row per due slot. The unique(type, dedupeKey) constraint makes
|
|
94
|
+
* this safe to run concurrently in any number of processes. */
|
|
95
|
+
async function materializeCronSlots(now: number) {
|
|
96
|
+
for (const [name, def] of Object.entries(defs)) {
|
|
97
|
+
if (def.kind !== 'cron') continue
|
|
98
|
+
const type = `${CRON_PREFIX}${name}`
|
|
99
|
+
const from = await cronWatermark(type, now)
|
|
100
|
+
const slots = slotsDue({
|
|
101
|
+
cron: parseCron(def.schedule),
|
|
102
|
+
from,
|
|
103
|
+
to: now,
|
|
104
|
+
catchUp: def.catchUp,
|
|
105
|
+
catchUpWindowMs: def.catchUpWindow,
|
|
106
|
+
})
|
|
107
|
+
for (const slot of slots) {
|
|
108
|
+
await enqueueJob(db, defs, name, null, {
|
|
109
|
+
runAt: slot,
|
|
110
|
+
dedupeKey: String(slot),
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
44
116
|
async function fireOnFailed(
|
|
45
|
-
def:
|
|
46
|
-
|
|
117
|
+
def: AnyBackgroundDefinition,
|
|
118
|
+
input: unknown,
|
|
47
119
|
error: Error,
|
|
48
120
|
) {
|
|
49
121
|
if (!def.onFailed) return
|
|
50
|
-
let input: unknown
|
|
51
122
|
try {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
try {
|
|
58
|
-
await def.onFailed(input, error, ctx as never)
|
|
123
|
+
await (def.onFailed as (i: unknown, e: Error, c: unknown) => unknown)(
|
|
124
|
+
input,
|
|
125
|
+
error,
|
|
126
|
+
ctx,
|
|
127
|
+
)
|
|
59
128
|
} catch (hookErr) {
|
|
60
129
|
console.error('[bunderstack] onFailed hook threw:', hookErr)
|
|
61
130
|
}
|
|
@@ -70,6 +139,7 @@ export function createJobRunner(deps: {
|
|
|
70
139
|
payloadJson: t.payloadJson,
|
|
71
140
|
attempts: t.attempts,
|
|
72
141
|
lastError: t.lastError,
|
|
142
|
+
runAt: t.runAt,
|
|
73
143
|
})
|
|
74
144
|
.from(t)
|
|
75
145
|
.where(
|
|
@@ -80,9 +150,9 @@ export function createJobRunner(deps: {
|
|
|
80
150
|
),
|
|
81
151
|
)
|
|
82
152
|
for (const row of expired) {
|
|
83
|
-
const def = defs
|
|
153
|
+
const def = definitionFor(defs, row.type)
|
|
84
154
|
const error = new Error('lease expired (worker crashed or timed out)')
|
|
85
|
-
if (!def
|
|
155
|
+
if (!def) {
|
|
86
156
|
await db
|
|
87
157
|
.update(t)
|
|
88
158
|
.set({
|
|
@@ -106,7 +176,7 @@ export function createJobRunner(deps: {
|
|
|
106
176
|
...terminalPatch(),
|
|
107
177
|
})
|
|
108
178
|
.where(eq(t.id, row.id))
|
|
109
|
-
await fireOnFailed(def, row
|
|
179
|
+
await fireOnFailed(def, resolveInput(def, row), error)
|
|
110
180
|
} else {
|
|
111
181
|
await db
|
|
112
182
|
.update(t)
|
|
@@ -167,6 +237,7 @@ export function createJobRunner(deps: {
|
|
|
167
237
|
type: t.type,
|
|
168
238
|
payloadJson: t.payloadJson,
|
|
169
239
|
attempts: t.attempts,
|
|
240
|
+
runAt: t.runAt,
|
|
170
241
|
})
|
|
171
242
|
return rows
|
|
172
243
|
}
|
|
@@ -174,15 +245,19 @@ export function createJobRunner(deps: {
|
|
|
174
245
|
// `now` is the tick's injected clock: retry runAt math uses it so tests can
|
|
175
246
|
// drive backoff deterministically. finishedAt uses the real clock (a handler
|
|
176
247
|
// may run long past the tick's start).
|
|
177
|
-
async function runJob(
|
|
248
|
+
async function runJob(
|
|
249
|
+
row: JobRow,
|
|
250
|
+
def: AnyBackgroundDefinition,
|
|
251
|
+
now: number,
|
|
252
|
+
leaseUntil: number,
|
|
253
|
+
): Promise<'ran' | 'failed' | 'lost'> {
|
|
178
254
|
let input: unknown
|
|
179
255
|
try {
|
|
180
|
-
|
|
181
|
-
input = def.input ? def.input.parse(raw) : undefined
|
|
256
|
+
input = resolveInput(def, row)
|
|
182
257
|
} catch (err) {
|
|
183
258
|
// Stored payload no longer parses (schema drift): retrying can't help.
|
|
184
259
|
const e = toError(err)
|
|
185
|
-
await db
|
|
260
|
+
const updated = await db
|
|
186
261
|
.update(t)
|
|
187
262
|
.set({
|
|
188
263
|
status: 'failed',
|
|
@@ -191,13 +266,15 @@ export function createJobRunner(deps: {
|
|
|
191
266
|
lastError: e.message,
|
|
192
267
|
...terminalPatch(),
|
|
193
268
|
})
|
|
194
|
-
.where(eq(t.id, row.id))
|
|
195
|
-
|
|
196
|
-
return
|
|
269
|
+
.where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
|
|
270
|
+
.returning({ id: t.id })
|
|
271
|
+
if (!updated[0]) return 'lost'
|
|
272
|
+
await fireOnFailed(def, undefined, e)
|
|
273
|
+
return 'failed'
|
|
197
274
|
}
|
|
198
275
|
try {
|
|
199
|
-
await def.handler(
|
|
200
|
-
await db
|
|
276
|
+
await (def.handler as (i: unknown, c: unknown) => unknown)(input, ctx)
|
|
277
|
+
const updated = await db
|
|
201
278
|
.update(t)
|
|
202
279
|
.set({
|
|
203
280
|
status: 'succeeded',
|
|
@@ -205,11 +282,14 @@ export function createJobRunner(deps: {
|
|
|
205
282
|
lockedUntil: null,
|
|
206
283
|
...terminalPatch(),
|
|
207
284
|
})
|
|
208
|
-
.where(eq(t.id, row.id))
|
|
285
|
+
.where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
|
|
286
|
+
.returning({ id: t.id })
|
|
287
|
+
if (!updated[0]) return 'lost'
|
|
288
|
+
return 'ran'
|
|
209
289
|
} catch (err) {
|
|
210
290
|
const e = toError(err)
|
|
211
291
|
if (Number(row.attempts) < maxAttempts(def)) {
|
|
212
|
-
await db
|
|
292
|
+
const updated = await db
|
|
213
293
|
.update(t)
|
|
214
294
|
.set({
|
|
215
295
|
status: 'pending',
|
|
@@ -217,9 +297,11 @@ export function createJobRunner(deps: {
|
|
|
217
297
|
runAt: now + backoffMs(def, Number(row.attempts)),
|
|
218
298
|
lastError: e.message,
|
|
219
299
|
})
|
|
220
|
-
.where(eq(t.id, row.id))
|
|
300
|
+
.where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
|
|
301
|
+
.returning({ id: t.id })
|
|
302
|
+
if (!updated[0]) return 'lost'
|
|
221
303
|
} else {
|
|
222
|
-
await db
|
|
304
|
+
const updated = await db
|
|
223
305
|
.update(t)
|
|
224
306
|
.set({
|
|
225
307
|
status: 'failed',
|
|
@@ -228,19 +310,22 @@ export function createJobRunner(deps: {
|
|
|
228
310
|
lastError: e.message,
|
|
229
311
|
...terminalPatch(),
|
|
230
312
|
})
|
|
231
|
-
.where(eq(t.id, row.id))
|
|
232
|
-
|
|
313
|
+
.where(and(eq(t.id, row.id), eq(t.lockedUntil, leaseUntil)))
|
|
314
|
+
.returning({ id: t.id })
|
|
315
|
+
if (!updated[0]) return 'lost'
|
|
316
|
+
await fireOnFailed(def, input, e)
|
|
233
317
|
}
|
|
318
|
+
return 'failed'
|
|
234
319
|
}
|
|
235
320
|
}
|
|
236
321
|
|
|
237
|
-
async function runClaimable(now: number) {
|
|
238
|
-
const work: Promise<
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
const
|
|
322
|
+
async function runClaimable(now: number): Promise<TickResult> {
|
|
323
|
+
const work: Promise<'ran' | 'failed' | 'lost'>[] = []
|
|
324
|
+
let totalClaimed = 0
|
|
325
|
+
for (const [name, def] of Object.entries(defs)) {
|
|
326
|
+
const type = def.kind === 'cron' ? `${CRON_PREFIX}${name}` : name
|
|
242
327
|
let limit = CLAIM_BATCH
|
|
243
|
-
if (def.concurrency !== undefined) {
|
|
328
|
+
if (def.kind === 'job' && def.concurrency !== undefined) {
|
|
244
329
|
const runningRows = await db
|
|
245
330
|
.select({ id: t.id })
|
|
246
331
|
.from(t)
|
|
@@ -251,16 +336,28 @@ export function createJobRunner(deps: {
|
|
|
251
336
|
}
|
|
252
337
|
const leaseUntil = now + (def.timeout ?? DEFAULT_TIMEOUT_MS)
|
|
253
338
|
const claimed = await claim(type, limit, now, leaseUntil)
|
|
254
|
-
|
|
339
|
+
totalClaimed += claimed.length
|
|
340
|
+
for (const row of claimed) work.push(runJob(row, def, now, leaseUntil))
|
|
341
|
+
}
|
|
342
|
+
const outcomes = await Promise.all(work)
|
|
343
|
+
let ran = 0
|
|
344
|
+
let failed = 0
|
|
345
|
+
for (const outcome of outcomes) {
|
|
346
|
+
if (outcome === 'ran') ran++
|
|
347
|
+
else if (outcome === 'failed') failed++
|
|
255
348
|
}
|
|
256
|
-
|
|
349
|
+
return { claimed: totalClaimed, ran, failed }
|
|
257
350
|
}
|
|
258
351
|
|
|
259
352
|
return {
|
|
260
|
-
async tick(now: number = Date.now()) {
|
|
353
|
+
async tick(now: number = Date.now()): Promise<TickResult> {
|
|
354
|
+
await materializeCronSlots(now)
|
|
261
355
|
await recoverExpiredLeases(now)
|
|
262
|
-
|
|
263
|
-
|
|
356
|
+
if (now - lastReapAt >= REAP_INTERVAL_MS) {
|
|
357
|
+
lastReapAt = now
|
|
358
|
+
await reapSucceeded(now)
|
|
359
|
+
}
|
|
360
|
+
return runClaimable(now)
|
|
264
361
|
},
|
|
265
362
|
setJobsFacade(f: JobsRuntimeFacade) {
|
|
266
363
|
ctx.jobs = f
|
package/src/manifest.ts
CHANGED
|
@@ -7,7 +7,6 @@ import type { JobsDefs } from './jobs/define'
|
|
|
7
7
|
import type { ResolvedBucket, ResolvedStorageBuckets } from './storage/buckets'
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
-
bunderstackCronRuns,
|
|
11
10
|
bunderstackFiles,
|
|
12
11
|
bunderstackIdempotency,
|
|
13
12
|
bunderstackJobs,
|
|
@@ -189,11 +188,6 @@ function systemTables() {
|
|
|
189
188
|
physicalName: getTableName(bunderstackJobs),
|
|
190
189
|
system: true,
|
|
191
190
|
},
|
|
192
|
-
{
|
|
193
|
-
exportName: '_system.scheduledRuns',
|
|
194
|
-
physicalName: getTableName(bunderstackCronRuns),
|
|
195
|
-
system: true,
|
|
196
|
-
},
|
|
197
191
|
]
|
|
198
192
|
}
|
|
199
193
|
|