@fabrika/app 0.9.0-cloudflare.2 → 0.9.0-cloudflare.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +20 -18
- package/skill/references/app-setup.md +29 -16
- package/skill/references/endpoint.md +24 -22
- package/skill/references/errors.md +3 -2
- package/skill/references/jobs.md +13 -3
- package/skill/references/tests.md +7 -0
- package/src/bun.ts +159 -6
- package/src/cloudflare/application.ts +543 -73
- package/src/cloudflare/export.ts +621 -0
- package/src/cloudflare/import.ts +1097 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/ops.ts +147 -18
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +3 -2
- package/src/contract/artifact-schema.ts +7 -1
- package/src/contract/build.ts +6 -2
- package/src/contract/context.ts +26 -9
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +1 -1
- package/src/contract/group.ts +1 -1
- package/src/files/kinds.ts +54 -24
- package/src/implement/anonymous.ts +1 -10
- package/src/implement/boot.ts +21 -9
- package/src/implement/dispatch.ts +171 -33
- package/src/implement/egress.ts +34 -8
- package/src/implement/invocation.ts +50 -10
- package/src/implement/workload.ts +18 -3
- package/src/jobs/runner.ts +639 -96
- package/src/testing/kit.ts +60 -0
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +36 -5
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +58 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/ops.d.ts +39 -4
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +2 -1
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +24 -9
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +1 -1
- package/types/src/contract/group.d.ts +1 -1
- package/types/src/files/kinds.d.ts +17 -6
- package/types/src/implement/anonymous.d.ts +0 -4
- package/types/src/implement/boot.d.ts +11 -0
- package/types/src/implement/dispatch.d.ts +31 -0
- package/types/src/implement/invocation.d.ts +17 -7
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +60 -2
- package/types/src/testing/kit.d.ts +16 -0
package/src/jobs/runner.ts
CHANGED
|
@@ -3,12 +3,23 @@
|
|
|
3
3
|
// that timed out, failed or lost the lease never commits (`@fabrika/db`'s fence). A site's `ctx.kick` sends a run in
|
|
4
4
|
// the background after its commit, without waiting for any lock; a kick that finds the lease held leaves the shared pending flag, and the run
|
|
5
5
|
// holding it runs once more after it succeeded. Failures, crashed runs and a job that stopped succeeding are reported
|
|
6
|
-
// to Operations, each under a fixed failure code.
|
|
6
|
+
// to Operations, each under a fixed failure code. On Cloudflare the object's alarm drives the jobs through `scheduler`:
|
|
7
|
+
// a run it starts owns the job's due wakes, and when it fails they move to a durable retry instead of an inline one.
|
|
7
8
|
|
|
8
9
|
import type { IamMembership } from '@fabrika/auth'
|
|
9
10
|
import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type JobMailer, type PermissionEntry, permits } from '@fabrika/auth-core'
|
|
10
11
|
import type { CapabilityAuthority } from '@fabrika/db/capability'
|
|
11
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
type DataAccess,
|
|
14
|
+
type FinishWakes,
|
|
15
|
+
type FrameworkTables,
|
|
16
|
+
type JobLeases,
|
|
17
|
+
type JobWakes,
|
|
18
|
+
type LeaseFence,
|
|
19
|
+
LeaseLostError,
|
|
20
|
+
type Session,
|
|
21
|
+
type Wake,
|
|
22
|
+
} from '@fabrika/db/framework'
|
|
12
23
|
import { EmailDeliveryError, type EmailSenderCapabilities, mayRetryDelivery } from '@fabrika/email'
|
|
13
24
|
import { uuidv7 } from '@fabrika/platform'
|
|
14
25
|
import { NotFoundError } from '../errors'
|
|
@@ -22,6 +33,7 @@ import {
|
|
|
22
33
|
runInRequest,
|
|
23
34
|
type Within,
|
|
24
35
|
} from '../implement/capability'
|
|
36
|
+
import { admitInvocation, type InvocationAdmission } from '../implement/dispatch'
|
|
25
37
|
import { type PublicEgressRules, runAsSite, siteEgress } from '../implement/egress'
|
|
26
38
|
import { failureCode, type FailureReason, failureReason } from '../implement/failures'
|
|
27
39
|
import { ServerFault } from '../implement/faults'
|
|
@@ -34,6 +46,7 @@ import {
|
|
|
34
46
|
jobHandles,
|
|
35
47
|
jobSql,
|
|
36
48
|
jobSqlOutsideTransaction,
|
|
49
|
+
type KickRecorder,
|
|
37
50
|
kickRecorder,
|
|
38
51
|
siteData,
|
|
39
52
|
type SiteDeclaration,
|
|
@@ -42,10 +55,10 @@ import {
|
|
|
42
55
|
import { jobLinkIssuing, type JobLinkMount } from '../implement/membership'
|
|
43
56
|
import type { SignedTokens } from '../implement/tokens'
|
|
44
57
|
import { type AppUrl, appUrl } from '../implement/urls'
|
|
45
|
-
import { type WorkloadCredential, workloadSite } from '../implement/workload'
|
|
58
|
+
import { type WorkloadAuthority, type WorkloadCredential, workloadSite } from '../implement/workload'
|
|
46
59
|
import { jobMail, type MailTransports } from './mail'
|
|
47
60
|
import type { OperationsReporter } from './report'
|
|
48
|
-
import { type CronSchedule, parseCron, periodAt } from './schedule'
|
|
61
|
+
import { type CronSchedule, isDueAt, parseCron, periodAt } from './schedule'
|
|
49
62
|
import type { AppMail, JobContext, JobPrincipal } from './types'
|
|
50
63
|
|
|
51
64
|
/**
|
|
@@ -55,6 +68,11 @@ import type { AppMail, JobContext, JobPrincipal } from './types'
|
|
|
55
68
|
export const LEASE_GRACE_MS = 60_000
|
|
56
69
|
/** How often the serving process checks every job for a crashed run and for a schedule that stopped. */
|
|
57
70
|
export const JOB_CHECK_INTERVAL_MS = 60_000
|
|
71
|
+
/**
|
|
72
|
+
* A run the Cloudflare alarm started has one segment (`CLOUDFLARE_MAX_JOB_SEGMENT_MS`): its attempt ends this much
|
|
73
|
+
* before the segment, at most a tenth of it, so the finish runs inside the segment too.
|
|
74
|
+
*/
|
|
75
|
+
export const SEGMENT_FINISH_RESERVE_MS = 30_000
|
|
58
76
|
const BACKOFF_BASE_MS = 1_000
|
|
59
77
|
const BACKOFF_MAX_MS = 5 * 60_000
|
|
60
78
|
|
|
@@ -76,7 +94,7 @@ export type JobFailureCode =
|
|
|
76
94
|
| 'unauthorized'
|
|
77
95
|
/** IAM could not be asked for the job's token. */
|
|
78
96
|
| 'iam_unavailable'
|
|
79
|
-
/** The process stopped a kicked run while shutting down; the schedule runs
|
|
97
|
+
/** The process stopped a kicked run while shutting down, or the object's maintenance stopped a run; the schedule runs it again. */
|
|
80
98
|
| 'stopped'
|
|
81
99
|
/** The transport refused a message the handler did not catch; `reason` is the `EmailDeliveryError` code. */
|
|
82
100
|
| 'mail_failed'
|
|
@@ -94,6 +112,8 @@ export interface JobClock {
|
|
|
94
112
|
wait(ms: number, signal: AbortSignal): Promise<void>
|
|
95
113
|
/** In [0, 1): the jitter of a retry's backoff. */
|
|
96
114
|
random(): number
|
|
115
|
+
/** Epoch milliseconds: what an attempt's deadline is measured on; `Date.now()` when absent. */
|
|
116
|
+
now?(): number
|
|
97
117
|
}
|
|
98
118
|
|
|
99
119
|
export interface JobRunOptions {
|
|
@@ -126,6 +146,59 @@ export interface ContractJobs {
|
|
|
126
146
|
* twice its period. The serving process calls it every `JOB_CHECK_INTERVAL_MS`; it never throws.
|
|
127
147
|
*/
|
|
128
148
|
check(options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
|
|
149
|
+
/**
|
|
150
|
+
* The serving process's wake timer on Bun, where crontab runs the schedule: reaps a crashed holder of every job and
|
|
151
|
+
* kicks each contract job with a due wake through the kick path, handing each kick to `track` until it ended. Answers
|
|
152
|
+
* how long until a wake is due next on the database's clock (0 while one still is), `null` when none is pending. The
|
|
153
|
+
* first call deletes the wakes of jobs the contract no longer declares.
|
|
154
|
+
*/
|
|
155
|
+
wake(track: (kick: Promise<void>) => void): Promise<number | null>
|
|
156
|
+
/** What the Cloudflare object's alarm drives the jobs through; `null` without `@fabrika/db`. */
|
|
157
|
+
readonly scheduler: JobScheduler | null
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** What the scheduler's `take` did for one job in the alarm's transaction. */
|
|
161
|
+
export interface TakenLease {
|
|
162
|
+
/** The lease a run of the job now holds, for `run`; `null` when nothing was due or another run holds it. */
|
|
163
|
+
readonly fence: LeaseFence | null
|
|
164
|
+
/** Whether a crashed holder's lease was freed: `crashed` reports it once the transaction committed. */
|
|
165
|
+
readonly reaped: boolean
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* The jobs as the Cloudflare object's alarm drives them (ADR-0065 §5): over the wakes and leases of the object's
|
|
170
|
+
* storage, every time on the database's clock. The object holds the transactions and arms the alarm.
|
|
171
|
+
*/
|
|
172
|
+
export interface JobScheduler {
|
|
173
|
+
/** Every job bound to run, the framework's included. */
|
|
174
|
+
readonly jobs: readonly string[]
|
|
175
|
+
/**
|
|
176
|
+
* Startup repair: reaps and reports every crashed run, of a job the contract no longer declares too; registers every
|
|
177
|
+
* job and surveils it (as `check`), deletes the wakes of jobs the contract no longer declares and the cron wakes of a
|
|
178
|
+
* changed expression, and keeps one cron wake per job at its next fire; an overdue one stays, so missed runs coalesce
|
|
179
|
+
* into one. `false` when the object's barrier admits no job.
|
|
180
|
+
*/
|
|
181
|
+
repair(options?: Pick<JobRunOptions, 'reporter'>): Promise<boolean>
|
|
182
|
+
/** The jobs the contract no longer declares whose lease a crashed run still holds: `take` reaps them. */
|
|
183
|
+
orphans(): Promise<readonly string[]>
|
|
184
|
+
/** How long until every wake due within `withinMs` of the database's clock is due; 0 when none is early. */
|
|
185
|
+
earlyBy(withinMs: number): Promise<number>
|
|
186
|
+
/**
|
|
187
|
+
* Reaps a crashed holder of `job`'s lease, then, when one of its wakes is due, takes the lease, or sets the pending
|
|
188
|
+
* flag for the run holding it; of a job no longer declared (an orphan), only reaps. Statements only, for the caller's
|
|
189
|
+
* one transaction.
|
|
190
|
+
*/
|
|
191
|
+
take(job: string): Promise<TakenLease>
|
|
192
|
+
/** Runs `job` under the lease `take` took. A failed run retries the wakes it owns durably. */
|
|
193
|
+
run(job: string, fence: LeaseFence, options?: JobRunOptions): Promise<JobRunResult>
|
|
194
|
+
/** Reports the crashed run `take` reaped. */
|
|
195
|
+
crashed(job: string, options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
|
|
196
|
+
/** When the scheduler wakes next: `JobWakes.next`, read in `using` when given. */
|
|
197
|
+
next(using?: Session): Promise<Date | null>
|
|
198
|
+
/** How far the next wake is from the database's now, in milliseconds: `JobWakes.untilNext`, read in `using` when given. */
|
|
199
|
+
untilNext(using?: Session): Promise<number | null>
|
|
200
|
+
/** Reports a failure of the scheduler itself: the alarm's or the repair's. Never rejects. */
|
|
201
|
+
failed(step: 'alarm' | 'repair', error: unknown): Promise<void>
|
|
129
202
|
}
|
|
130
203
|
|
|
131
204
|
/** A run that cannot start: an unknown job. Its message is framework-authored. */
|
|
@@ -134,6 +207,7 @@ export class JobError extends Error {
|
|
|
134
207
|
}
|
|
135
208
|
|
|
136
209
|
export const systemClock: JobClock = Object.freeze({
|
|
210
|
+
now: () => Date.now(),
|
|
137
211
|
wait: (ms: number, signal: AbortSignal) =>
|
|
138
212
|
new Promise<void>((resolve) => {
|
|
139
213
|
if (signal.aborted) {
|
|
@@ -174,6 +248,8 @@ interface RunnerContext {
|
|
|
174
248
|
readonly database: ContractDatabase | null
|
|
175
249
|
readonly ownedDimensions: ReadonlySet<string>
|
|
176
250
|
readonly leases: JobLeases
|
|
251
|
+
/** The jobs' pending wakes, which a run's finish consumes, retries or schedules. */
|
|
252
|
+
readonly wakes: JobWakes
|
|
177
253
|
readonly reporter: OperationsReporter
|
|
178
254
|
readonly mint: CapabilityMint
|
|
179
255
|
readonly data: DataAccess | null
|
|
@@ -314,11 +390,22 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
|
|
|
314
390
|
}
|
|
315
391
|
const names = Object.freeze(entries.map(([name]) => name))
|
|
316
392
|
const framework = input.framework
|
|
317
|
-
const
|
|
393
|
+
const declaredJobs: readonly (Scheduled & { readonly timeoutMs: number })[] = [
|
|
394
|
+
...[...bound.values()].map((job) => ({ name: job.name, expression: job.artifact.schedule, schedule: job.schedule, timeoutMs: job.timeoutMs })),
|
|
395
|
+
...[...frameworkJobs.values()].map((own) => ({
|
|
396
|
+
name: own.name,
|
|
397
|
+
expression: own.job.schedule,
|
|
398
|
+
schedule: own.schedule,
|
|
399
|
+
timeoutMs: own.job.timeoutMs,
|
|
400
|
+
})),
|
|
401
|
+
]
|
|
402
|
+
let reconciled = false
|
|
403
|
+
const contextFor = (tables: FrameworkTables, chosen: OperationsReporter | undefined): RunnerContext => ({
|
|
318
404
|
artifact: input.artifact,
|
|
319
405
|
database: input.database,
|
|
320
406
|
ownedDimensions: input.ownedDimensions,
|
|
321
|
-
leases,
|
|
407
|
+
leases: tables.jobs,
|
|
408
|
+
wakes: tables.wakes,
|
|
322
409
|
reporter: chosen ?? input.reporter,
|
|
323
410
|
mint: input.mint,
|
|
324
411
|
data: input.data,
|
|
@@ -340,57 +427,147 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
|
|
|
340
427
|
run(name: string, options: JobRunOptions = {}): Promise<JobRunResult> {
|
|
341
428
|
const own = frameworkJobs.get(name)
|
|
342
429
|
if (own !== undefined && framework !== null) {
|
|
343
|
-
return
|
|
430
|
+
return admitted(
|
|
431
|
+
'framework-job',
|
|
432
|
+
name,
|
|
433
|
+
(control) => runFrameworkJob(own, contextFor(framework, options.reporter), options.clock ?? systemClock, control, null),
|
|
434
|
+
)
|
|
344
435
|
}
|
|
345
436
|
const job = bound.get(name)
|
|
346
437
|
if (job === undefined || framework === null) {
|
|
347
438
|
throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
|
|
348
439
|
}
|
|
349
|
-
return
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
440
|
+
return admitted(
|
|
441
|
+
'job',
|
|
442
|
+
name,
|
|
443
|
+
(control) => runJob(job, contextFor(framework, options.reporter), options.clock ?? systemClock, 'wait', control, 'inline'),
|
|
444
|
+
)
|
|
445
|
+
.then((result) => {
|
|
446
|
+
if (result.status === 'skipped') {
|
|
447
|
+
console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
|
|
448
|
+
}
|
|
449
|
+
return result
|
|
450
|
+
})
|
|
355
451
|
},
|
|
356
452
|
async check(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
|
|
357
|
-
|
|
453
|
+
const admission = framework === null ? null : admitInvocation('framework-job')
|
|
454
|
+
if (framework === null || admission === null) {
|
|
358
455
|
return
|
|
359
456
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
457
|
+
try {
|
|
458
|
+
const context = contextFor(framework, options.reporter)
|
|
459
|
+
for (const job of [...bound.values(), ...frameworkJobs.values()]) {
|
|
460
|
+
try {
|
|
461
|
+
await context.leases.register(job.name)
|
|
462
|
+
await surveil(job, context)
|
|
463
|
+
} catch {
|
|
464
|
+
// A driver error may quote the database URL.
|
|
465
|
+
console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
} finally {
|
|
469
|
+
admission.release()
|
|
470
|
+
}
|
|
471
|
+
},
|
|
472
|
+
async wake(track: (kick: Promise<void>) => void): Promise<number | null> {
|
|
473
|
+
if (framework === null) {
|
|
474
|
+
return null
|
|
475
|
+
}
|
|
476
|
+
if (!reconciled) {
|
|
477
|
+
await framework.wakes.reconcile(declaredJobs.map((job) => ({ job: job.name, expression: job.expression })))
|
|
478
|
+
reconciled = true
|
|
479
|
+
}
|
|
480
|
+
// A crashed holder's expiry is a wake too, and a kick never takes a lease over.
|
|
481
|
+
for (const job of declaredJobs) {
|
|
482
|
+
if (await framework.jobs.reapCrashed(job.name)) {
|
|
483
|
+
await reportCrashed(job.name, input.reporter)
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
const now = (await framework.jobs.now()).getTime()
|
|
487
|
+
for (const job of bound.values()) {
|
|
488
|
+
const due = (await framework.wakes.pending(job.name)).filter((wake) => wake.dueAt.getTime() <= now)
|
|
489
|
+
if (due.length > 0 && !(await ownedByHolder(job.name, now, framework.jobs, framework.wakes))) {
|
|
490
|
+
track(kick(job.name))
|
|
368
491
|
}
|
|
369
492
|
}
|
|
493
|
+
const next = await framework.wakes.next()
|
|
494
|
+
return next === null ? null : Math.max(0, next.getTime() - now)
|
|
370
495
|
},
|
|
496
|
+
scheduler: framework === null ? null : jobScheduler({
|
|
497
|
+
framework,
|
|
498
|
+
declared: declaredJobs,
|
|
499
|
+
contextFor: (reporter) => contextFor(framework, reporter),
|
|
500
|
+
run(name: string, fence: LeaseFence, options: JobRunOptions): Promise<JobRunResult> {
|
|
501
|
+
const clock = options.clock ?? systemClock
|
|
502
|
+
const context = contextFor(framework, options.reporter)
|
|
503
|
+
const own = frameworkJobs.get(name)
|
|
504
|
+
if (own !== undefined) {
|
|
505
|
+
return admitted('framework-job', name, (control) => runFrameworkJob(own, context, clock, control, fence))
|
|
506
|
+
}
|
|
507
|
+
const job = bound.get(name)
|
|
508
|
+
if (job === undefined) {
|
|
509
|
+
throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
|
|
510
|
+
}
|
|
511
|
+
return admitted('job', name, (control) => runJob(job, context, clock, { fence }, control, 'durable'))
|
|
512
|
+
},
|
|
513
|
+
}),
|
|
371
514
|
})
|
|
372
515
|
const stopping = new AbortController()
|
|
373
516
|
const kick = async (name: string): Promise<void> => {
|
|
374
517
|
const job = bound.get(name)
|
|
375
|
-
if (job
|
|
376
|
-
|
|
518
|
+
if (job === undefined || framework === null) {
|
|
519
|
+
return
|
|
520
|
+
}
|
|
521
|
+
const admission = admitInvocation('kick')
|
|
522
|
+
if (admission === null) {
|
|
523
|
+
// A kick hurries a run; the schedule still runs the job once the object opens.
|
|
524
|
+
console.error(`fabrika: job '${name}' was not kicked: the object is in maintenance`)
|
|
525
|
+
return
|
|
526
|
+
}
|
|
527
|
+
try {
|
|
528
|
+
await kickOnce(job, contextFor(framework, undefined), {
|
|
529
|
+
signal: AbortSignal.any([admission.signal, stopping.signal]),
|
|
530
|
+
track: admission.track,
|
|
531
|
+
})
|
|
532
|
+
} finally {
|
|
533
|
+
admission.release()
|
|
377
534
|
}
|
|
378
535
|
}
|
|
379
536
|
return { jobs, declared, kick, stopKicks: () => stopping.abort() }
|
|
380
537
|
}
|
|
381
538
|
|
|
539
|
+
/**
|
|
540
|
+
* What stops one run, and where the handler work that outlives its attempt is counted: the run's admission at the
|
|
541
|
+
* object's barrier, which a drain aborts, and for a kick also the process's stop.
|
|
542
|
+
*/
|
|
543
|
+
type RunControl = Pick<InvocationAdmission, 'signal' | 'track'>
|
|
544
|
+
|
|
545
|
+
/** A run admitted at the current event's gate, released once it ended; a refused one is `stopped` and starts nothing. */
|
|
546
|
+
async function admitted(kind: 'job' | 'framework-job', name: string, run: (control: RunControl) => Promise<JobRunResult>): Promise<JobRunResult> {
|
|
547
|
+
const admission = admitInvocation(kind)
|
|
548
|
+
if (admission === null) {
|
|
549
|
+
console.error(`fabrika: job '${name}' did not start: the object is in maintenance; its schedule runs it again`)
|
|
550
|
+
return { status: 'failed', attempts: 0, code: 'stopped' }
|
|
551
|
+
}
|
|
552
|
+
try {
|
|
553
|
+
return await run(admission)
|
|
554
|
+
} finally {
|
|
555
|
+
admission.release()
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
382
559
|
/**
|
|
383
560
|
* One kick: a run while the lease is free, else the shared pending flag. Neither waits for a lock, so a kick never
|
|
384
561
|
* holds a connection requests share while a run writes. After a failed run the flag stays for the schedule.
|
|
385
562
|
*/
|
|
386
|
-
async function kickOnce(job: BoundJob, context: RunnerContext,
|
|
387
|
-
if (
|
|
563
|
+
async function kickOnce(job: BoundJob, context: RunnerContext, control: RunControl): Promise<void> {
|
|
564
|
+
if (control.signal.aborted) {
|
|
388
565
|
// The flag waits for no lock, and another process that is not stopping runs the job.
|
|
389
566
|
await markPending(job, context)
|
|
390
567
|
return
|
|
391
568
|
}
|
|
392
|
-
if ((await runJob(job, context, systemClock, 'try',
|
|
393
|
-
await handOver(job, context, systemClock,
|
|
569
|
+
if ((await runJob(job, context, systemClock, 'try', control, 'inline')).status === 'skipped') {
|
|
570
|
+
await handOver(job, context, systemClock, control, 'inline')
|
|
394
571
|
}
|
|
395
572
|
}
|
|
396
573
|
|
|
@@ -398,11 +575,11 @@ async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal
|
|
|
398
575
|
* Leaves the flag to the run holding the lease, which takes it after it succeeded. That run may have finished before
|
|
399
576
|
* the flag was set, so the lease is tried once more; `null` when another run still holds it.
|
|
400
577
|
*/
|
|
401
|
-
async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock,
|
|
578
|
+
async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, control: RunControl, retries: Retries): Promise<JobRunResult | null> {
|
|
402
579
|
if (!(await markPending(job, context))) {
|
|
403
580
|
return null
|
|
404
581
|
}
|
|
405
|
-
const result = await runJob(job, context, clock, 'try',
|
|
582
|
+
const result = await runJob(job, context, clock, 'try', control, retries)
|
|
406
583
|
return result.status === 'skipped' ? null : result
|
|
407
584
|
}
|
|
408
585
|
|
|
@@ -449,13 +626,123 @@ function backoffMs(attempt: number, random: number): number {
|
|
|
449
626
|
return Math.round(ceiling / 2 + (ceiling / 2) * random)
|
|
450
627
|
}
|
|
451
628
|
|
|
629
|
+
const nowOf = (clock: JobClock): number => clock.now?.() ?? Date.now()
|
|
630
|
+
|
|
631
|
+
/** How much of a segment that ends at `deadline` is left, never below zero. */
|
|
632
|
+
const remainingMs = (clock: JobClock, deadline: number): number => Math.max(0, deadline - nowOf(clock))
|
|
633
|
+
|
|
634
|
+
const finishReserveMs = (timeoutMs: number): number => Math.min(SEGMENT_FINISH_RESERVE_MS, Math.floor(timeoutMs / 10))
|
|
635
|
+
|
|
636
|
+
const OUT_OF_SEGMENT: unique symbol = Symbol('fabrika.job.outOfSegment')
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* `step` while the segment that ends at `end` lasts (always, without one). Past its end the run stops waiting for it;
|
|
640
|
+
* only a step that is safe to abandon goes here: a read, or the refusal of the attempt's writes, never a finish.
|
|
641
|
+
*/
|
|
642
|
+
async function withinSegment<T>(clock: JobClock, end: number | null, step: Promise<T>): Promise<T | typeof OUT_OF_SEGMENT> {
|
|
643
|
+
if (end === null) {
|
|
644
|
+
return await step
|
|
645
|
+
}
|
|
646
|
+
const timer = new AbortController()
|
|
647
|
+
// Abandoned past the segment, its rejection is nobody's: the race below still answers it while it is awaited.
|
|
648
|
+
step.catch(() => undefined)
|
|
649
|
+
try {
|
|
650
|
+
return await Promise.race([step, clock.wait(remainingMs(clock, end), timer.signal).then((): typeof OUT_OF_SEGMENT => OUT_OF_SEGMENT)])
|
|
651
|
+
} finally {
|
|
652
|
+
timer.abort()
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** A run whose segment ended before its finish started: its lease expires, the reap reports the crash, and the alarm runs it again. */
|
|
657
|
+
function outOfSegment(job: { readonly name: string }, attempts: number): JobRunResult {
|
|
658
|
+
console.error(`fabrika: job '${job.name}' reached the end of its segment before its finish; its lease expires and the alarm runs it again`)
|
|
659
|
+
return { status: 'failed', attempts, code: 'timeout' }
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
const MINUTE_MS = 60_000
|
|
663
|
+
/** How far ahead a next fire is looked for, as far as `previousDue` looks back. */
|
|
664
|
+
const NEXT_FIRE_DAYS = 366 * 5
|
|
665
|
+
|
|
666
|
+
/** The first due minute of `schedule` after `after`, or `null` when none falls in the window. */
|
|
667
|
+
function nextFire(schedule: CronSchedule, after: Date): Date | null {
|
|
668
|
+
const hours = [...schedule.hours].sort((left, right) => left - right)
|
|
669
|
+
const minutes = [...schedule.minutes].sort((left, right) => left - right)
|
|
670
|
+
const [firstHour, firstMinute] = [hours[0], minutes[0]]
|
|
671
|
+
if (firstHour === undefined || firstMinute === undefined) {
|
|
672
|
+
return null
|
|
673
|
+
}
|
|
674
|
+
const today = Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate())
|
|
675
|
+
for (let ahead = 0; ahead <= NEXT_FIRE_DAYS; ahead++) {
|
|
676
|
+
const day = today + ahead * 24 * 60 * MINUTE_MS
|
|
677
|
+
// The day's first scheduled minute is due exactly when the day matches.
|
|
678
|
+
if (!isDueAt(schedule, new Date(day + (firstHour * 60 + firstMinute) * MINUTE_MS))) {
|
|
679
|
+
continue
|
|
680
|
+
}
|
|
681
|
+
for (const hour of hours) {
|
|
682
|
+
for (const minute of minutes) {
|
|
683
|
+
const time = day + (hour * 60 + minute) * MINUTE_MS
|
|
684
|
+
if (time > after.getTime()) {
|
|
685
|
+
return new Date(time)
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
return null
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/** A cron wake's dedupe key: one pending wake per job and expression. */
|
|
694
|
+
const cronKey = (job: string, expression: string): string => `cron:${job}:${expression}`
|
|
695
|
+
|
|
696
|
+
/** A job's schedule as the scheduler and a finish read it. */
|
|
697
|
+
interface Scheduled {
|
|
698
|
+
readonly name: string
|
|
699
|
+
/** The cron expression as declared: its wake's key. */
|
|
700
|
+
readonly expression: string
|
|
701
|
+
readonly schedule: CronSchedule
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Whether a run holds `job`'s lease and owns every wake due by `now`, as it took them with the lease: its finish consumes
|
|
706
|
+
* them, so a kick or a pending run for them would only repeat that run. A post-commit kick's run usually owns its wake.
|
|
707
|
+
*/
|
|
708
|
+
async function ownedByHolder(job: string, now: number, leases: JobLeases, wakes: JobWakes): Promise<boolean> {
|
|
709
|
+
const state = await leases.state(job)
|
|
710
|
+
if (state === null || state.holder === null) {
|
|
711
|
+
return false
|
|
712
|
+
}
|
|
713
|
+
return (await wakes.pending(job)).filter((wake) => wake.dueAt.getTime() <= now).every((wake) => wake.owned)
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/** The wakes the run holding `job`'s lease owns: those it took with the lease. */
|
|
717
|
+
async function ownedWakes(job: string, context: RunnerContext): Promise<readonly Wake[]> {
|
|
718
|
+
return (await context.wakes.pending(job)).filter((wake) => wake.owned)
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* What a finish does with the owned wakes when the run is done with them, succeeded or failed past its retries: they
|
|
723
|
+
* are consumed, and when the job keeps a cron wake (always, for a run the alarm started) its next fire is scheduled
|
|
724
|
+
* under its key. On Bun, where crontab runs the schedule, a job keeps none.
|
|
725
|
+
*/
|
|
726
|
+
async function completion(job: Scheduled, context: RunnerContext, durable: boolean): Promise<FinishWakes> {
|
|
727
|
+
if (!durable && !(await context.wakes.pending(job.name)).some((wake) => wake.reason === 'cron')) {
|
|
728
|
+
return { kind: 'completed' }
|
|
729
|
+
}
|
|
730
|
+
const dueAt = nextFire(job.schedule, await context.leases.now())
|
|
731
|
+
return dueAt === null ? { kind: 'completed' } : { kind: 'completed', nextCron: { expression: job.expression, dueAt } }
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
/** A crashed run (its lease expired before it finished): one report. */
|
|
735
|
+
async function reportCrashed(job: string, reporter: OperationsReporter): Promise<void> {
|
|
736
|
+
const message = `job '${job}' failed: crashed (its run's lease expired before the run finished)`
|
|
737
|
+
console.error(`fabrika: ${message}`)
|
|
738
|
+
await reporter.report({ fingerprint: ['fabrika-job-failed', job], type: 'JobFailed', message, tags: { job, code: 'crashed' } })
|
|
739
|
+
}
|
|
740
|
+
|
|
452
741
|
/** A crashed run (its lease expired before it finished) and a job overdue by twice its period: one report each. */
|
|
453
742
|
async function surveil(job: { readonly name: string; readonly schedule: CronSchedule }, context: RunnerContext): Promise<void> {
|
|
454
743
|
const tags = { job: job.name }
|
|
455
744
|
if (await context.leases.reapCrashed(job.name)) {
|
|
456
|
-
|
|
457
|
-
console.error(`fabrika: ${message}`)
|
|
458
|
-
await context.reporter.report({ fingerprint: ['fabrika-job-failed', job.name], type: 'JobFailed', message, tags: { ...tags, code: 'crashed' } })
|
|
745
|
+
await reportCrashed(job.name, context.reporter)
|
|
459
746
|
}
|
|
460
747
|
const period = periodAt(job.schedule, await context.leases.now())
|
|
461
748
|
const claim = period === null ? null : await context.leases.claimOverdue(job.name, period)
|
|
@@ -473,30 +760,156 @@ async function surveil(job: { readonly name: string; readonly schedule: CronSche
|
|
|
473
760
|
})
|
|
474
761
|
}
|
|
475
762
|
|
|
763
|
+
interface SchedulerInput {
|
|
764
|
+
readonly framework: FrameworkTables
|
|
765
|
+
/** Every job bound to run, with the attempt time its lease is taken for. */
|
|
766
|
+
readonly declared: readonly (Scheduled & { readonly timeoutMs: number })[]
|
|
767
|
+
readonly contextFor: (reporter: OperationsReporter | undefined) => RunnerContext
|
|
768
|
+
readonly run: (job: string, fence: LeaseFence, options: JobRunOptions) => Promise<JobRunResult>
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
function jobScheduler(input: SchedulerInput): JobScheduler {
|
|
772
|
+
const { framework, declared } = input
|
|
773
|
+
const byName = new Map(declared.map((job) => [job.name, job]))
|
|
774
|
+
return Object.freeze({
|
|
775
|
+
jobs: Object.freeze(declared.map((job) => job.name)),
|
|
776
|
+
async repair(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<boolean> {
|
|
777
|
+
const admission = admitInvocation('framework-job')
|
|
778
|
+
if (admission === null) {
|
|
779
|
+
return false
|
|
780
|
+
}
|
|
781
|
+
try {
|
|
782
|
+
const context = input.contextFor(options.reporter)
|
|
783
|
+
for (const job of await framework.wakes.expiredHolders()) {
|
|
784
|
+
if (await context.leases.reapCrashed(job)) {
|
|
785
|
+
await reportCrashed(job, context.reporter)
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
for (const job of declared) {
|
|
789
|
+
await context.leases.register(job.name)
|
|
790
|
+
await surveil(job, context)
|
|
791
|
+
}
|
|
792
|
+
await framework.wakes.reconcile(declared.map((job) => ({ job: job.name, expression: job.expression })))
|
|
793
|
+
const now = await context.leases.now()
|
|
794
|
+
for (const job of declared) {
|
|
795
|
+
const dueAt = nextFire(job.schedule, now)
|
|
796
|
+
if (dueAt !== null) {
|
|
797
|
+
await framework.wakes.schedule({ job: job.name, reason: 'cron', dueAt, dedupe: cronKey(job.name, job.expression) })
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return true
|
|
801
|
+
} finally {
|
|
802
|
+
admission.release()
|
|
803
|
+
}
|
|
804
|
+
},
|
|
805
|
+
async orphans(): Promise<readonly string[]> {
|
|
806
|
+
return (await framework.wakes.expiredHolders()).filter((job) => !byName.has(job))
|
|
807
|
+
},
|
|
808
|
+
async earlyBy(withinMs: number): Promise<number> {
|
|
809
|
+
const now = (await framework.jobs.now()).getTime()
|
|
810
|
+
let early = 0
|
|
811
|
+
for (const job of declared) {
|
|
812
|
+
for (const wake of await framework.wakes.pending(job.name)) {
|
|
813
|
+
const ahead = wake.dueAt.getTime() - now
|
|
814
|
+
if (ahead <= withinMs) {
|
|
815
|
+
early = Math.max(early, ahead)
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
return early
|
|
820
|
+
},
|
|
821
|
+
async take(name: string): Promise<TakenLease> {
|
|
822
|
+
const job = byName.get(name)
|
|
823
|
+
const leases = framework.jobs
|
|
824
|
+
if (job === undefined) {
|
|
825
|
+
// A removed job runs no more; only a crashed run's lease is freed, so it stops arming the alarm.
|
|
826
|
+
return { fence: null, reaped: await leases.reapCrashed(name) }
|
|
827
|
+
}
|
|
828
|
+
await leases.register(name)
|
|
829
|
+
const reaped = await leases.reapCrashed(name)
|
|
830
|
+
const now = (await leases.now()).getTime()
|
|
831
|
+
const due = (await framework.wakes.pending(name)).filter((wake) => wake.dueAt.getTime() <= now)
|
|
832
|
+
if (due.length === 0) {
|
|
833
|
+
return { fence: null, reaped }
|
|
834
|
+
}
|
|
835
|
+
const fence = await leases.tryAcquire(name, attemptId(), job.timeoutMs, job.timeoutMs + LEASE_GRACE_MS)
|
|
836
|
+
if (fence === null && !(await ownedByHolder(name, now, leases, framework.wakes))) {
|
|
837
|
+
// The run holding the lease runs once more after it succeeded; a wake it did not own stays pending.
|
|
838
|
+
await leases.markPending(name)
|
|
839
|
+
}
|
|
840
|
+
return { fence, reaped }
|
|
841
|
+
},
|
|
842
|
+
run(job: string, fence: LeaseFence, options: JobRunOptions = {}): Promise<JobRunResult> {
|
|
843
|
+
return input.run(job, fence, options)
|
|
844
|
+
},
|
|
845
|
+
async crashed(job: string, options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
|
|
846
|
+
await reportCrashed(job, input.contextFor(options.reporter).reporter)
|
|
847
|
+
},
|
|
848
|
+
next(using?: Session): Promise<Date | null> {
|
|
849
|
+
return framework.wakes.next(using)
|
|
850
|
+
},
|
|
851
|
+
untilNext(using?: Session): Promise<number | null> {
|
|
852
|
+
return framework.wakes.untilNext(using)
|
|
853
|
+
},
|
|
854
|
+
async failed(step: 'alarm' | 'repair', error: unknown): Promise<void> {
|
|
855
|
+
try {
|
|
856
|
+
await input.contextFor(undefined).reporter.report({
|
|
857
|
+
fingerprint: ['fabrika-alarm-failed', step],
|
|
858
|
+
type: 'AlarmFailed',
|
|
859
|
+
message: `the job scheduler's ${step} failed`,
|
|
860
|
+
tags: { step },
|
|
861
|
+
error,
|
|
862
|
+
})
|
|
863
|
+
} catch {
|
|
864
|
+
console.error(`fabrika: the failure of the job scheduler's ${step} could not be reported`)
|
|
865
|
+
}
|
|
866
|
+
},
|
|
867
|
+
})
|
|
868
|
+
}
|
|
869
|
+
|
|
476
870
|
interface Progress {
|
|
477
871
|
attempts: number
|
|
478
872
|
}
|
|
479
873
|
|
|
874
|
+
/** What the attempt that succeeded kicked: `null` until a handler returned. */
|
|
875
|
+
interface Kicked {
|
|
876
|
+
recorder: KickRecorder | null
|
|
877
|
+
}
|
|
878
|
+
|
|
480
879
|
/**
|
|
481
880
|
* How a run takes the lease: `wait` is `fabrika job run`'s, in a process of its own; `try` is a kick's and a follow-up
|
|
482
|
-
* run's, which never wait for a lock.
|
|
881
|
+
* run's, which never wait for a lock; a fence is one the alarm's scheduler already took.
|
|
882
|
+
*/
|
|
883
|
+
type Acquisition = 'wait' | 'try' | { readonly fence: LeaseFence }
|
|
884
|
+
|
|
885
|
+
/**
|
|
886
|
+
* How a failed attempt is retried: `inline` in the same run, after a backoff, under a renewed lease; `durable` (a run the
|
|
887
|
+
* Cloudflare alarm started) by moving the wakes it owns to a later due time, so the alarm runs it again. A durable run
|
|
888
|
+
* that owns no wake retries inline: there is nothing to move.
|
|
483
889
|
*/
|
|
484
|
-
type
|
|
890
|
+
type Retries = 'inline' | 'durable'
|
|
485
891
|
|
|
486
892
|
/**
|
|
487
|
-
* A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once
|
|
488
|
-
* aborted, no follow-up run starts: the pending flag stays for the schedule.
|
|
893
|
+
* A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once the run's
|
|
894
|
+
* control aborted, no follow-up run starts: the pending flag stays for the schedule.
|
|
489
895
|
*/
|
|
490
|
-
async function runJob(
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
896
|
+
async function runJob(
|
|
897
|
+
job: BoundJob,
|
|
898
|
+
context: RunnerContext,
|
|
899
|
+
clock: JobClock,
|
|
900
|
+
acquisition: Acquisition,
|
|
901
|
+
control: RunControl,
|
|
902
|
+
retries: Retries,
|
|
903
|
+
): Promise<JobRunResult> {
|
|
904
|
+
let result = await runOnce(job, context, clock, acquisition, control, retries)
|
|
905
|
+
while (result.status === 'succeeded' && !control.signal.aborted && (await pendingTaken(job, context))) {
|
|
906
|
+
const next = await runOnce(job, context, clock, 'try', control, retries)
|
|
494
907
|
if (next.status !== 'skipped') {
|
|
495
908
|
result = next
|
|
496
909
|
continue
|
|
497
910
|
}
|
|
498
911
|
// A newer run holds the lease and may have read before the kick's row: the flag goes back to it.
|
|
499
|
-
const handed = await handOver(job, context, clock,
|
|
912
|
+
const handed = await handOver(job, context, clock, control, retries)
|
|
500
913
|
return handed ?? result
|
|
501
914
|
}
|
|
502
915
|
return result
|
|
@@ -512,11 +925,23 @@ async function pendingTaken(job: BoundJob, context: RunnerContext): Promise<bool
|
|
|
512
925
|
}
|
|
513
926
|
}
|
|
514
927
|
|
|
515
|
-
async function runOnce(
|
|
928
|
+
async function runOnce(
|
|
929
|
+
job: BoundJob,
|
|
930
|
+
context: RunnerContext,
|
|
931
|
+
clock: JobClock,
|
|
932
|
+
acquisition: Acquisition,
|
|
933
|
+
control: RunControl,
|
|
934
|
+
retries: Retries,
|
|
935
|
+
): Promise<JobRunResult> {
|
|
516
936
|
const progress: Progress = { attempts: 0 }
|
|
517
937
|
try {
|
|
518
|
-
return await runLeased(job, context, clock, progress, acquisition,
|
|
938
|
+
return await runLeased(job, context, clock, progress, acquisition, control, retries)
|
|
519
939
|
} catch {
|
|
940
|
+
if (control.signal.aborted) {
|
|
941
|
+
// A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
|
|
942
|
+
console.error(`fabrika: job '${job.name}' stopped before it released its lease; its schedule runs it again`)
|
|
943
|
+
return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
|
|
944
|
+
}
|
|
520
945
|
// Only the lease store throws here: the handler's errors stay inside its attempt. Its message may quote the URL.
|
|
521
946
|
return failed(job, progress.attempts, failure('driver_failed'), context.reporter)
|
|
522
947
|
}
|
|
@@ -528,32 +953,47 @@ async function runLeased(
|
|
|
528
953
|
clock: JobClock,
|
|
529
954
|
progress: Progress,
|
|
530
955
|
acquisition: Acquisition,
|
|
531
|
-
|
|
956
|
+
control: RunControl,
|
|
957
|
+
retries: Retries,
|
|
532
958
|
): Promise<JobRunResult> {
|
|
533
959
|
const { leases } = context
|
|
534
|
-
await leases.register(job.name)
|
|
535
960
|
const leaseMs = job.timeoutMs + LEASE_GRACE_MS
|
|
536
|
-
if (stop?.aborted === true) {
|
|
537
|
-
return { status: 'skipped' }
|
|
538
|
-
}
|
|
539
961
|
let fence: LeaseFence | null
|
|
540
|
-
if (acquisition === '
|
|
541
|
-
|
|
542
|
-
fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
962
|
+
if (typeof acquisition === 'object') {
|
|
963
|
+
fence = acquisition.fence
|
|
543
964
|
} else {
|
|
544
|
-
|
|
965
|
+
await leases.register(job.name)
|
|
966
|
+
if (control.signal.aborted) {
|
|
967
|
+
return { status: 'skipped' }
|
|
968
|
+
}
|
|
969
|
+
if (acquisition === 'wait') {
|
|
970
|
+
await surveil(job, context)
|
|
971
|
+
fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
972
|
+
} else {
|
|
973
|
+
fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
974
|
+
}
|
|
545
975
|
}
|
|
546
976
|
if (fence === null) {
|
|
547
977
|
return { status: 'skipped' }
|
|
548
978
|
}
|
|
979
|
+
// The segment is measured from the lease, as the database's attempt deadline is. A run the alarm started has one
|
|
980
|
+
// segment: its attempts end before it by the finish's reserve, and no finish starts past it.
|
|
981
|
+
const segmentEnd = retries === 'durable' ? nowOf(clock) + job.timeoutMs : null
|
|
982
|
+
const attemptEnd = (
|
|
983
|
+
start: number,
|
|
984
|
+
): number => (segmentEnd === null ? start + job.timeoutMs : Math.min(start + job.timeoutMs, segmentEnd - finishReserveMs(job.timeoutMs)))
|
|
985
|
+
let deadline = attemptEnd(nowOf(clock))
|
|
549
986
|
// This run covers every kick before it: their rows committed before it reads.
|
|
550
987
|
await pendingTaken(job, context)
|
|
988
|
+
const owned = retries === 'durable' ? await ownedWakes(job.name, context) : []
|
|
989
|
+
const durable = owned.length > 0
|
|
990
|
+
const scheduled: Scheduled = { name: job.name, expression: job.artifact.schedule, schedule: job.schedule }
|
|
551
991
|
const attempts = job.artifact.retries + 1
|
|
552
|
-
progress.attempts = 1
|
|
553
|
-
// The
|
|
554
|
-
const kicked:
|
|
555
|
-
let outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked,
|
|
556
|
-
while (outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
|
|
992
|
+
progress.attempts = durable ? Math.max(...owned.map((wake) => wake.attempt)) + 1 : 1
|
|
993
|
+
// The kicks of the succeeding attempt; a failed attempt's kicks are dropped with its writes.
|
|
994
|
+
const kicked: Kicked = { recorder: null }
|
|
995
|
+
let outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
|
|
996
|
+
while (!durable && outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
|
|
557
997
|
console.error(`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}`)
|
|
558
998
|
const delay = backoffMs(progress.attempts, clock.random())
|
|
559
999
|
// A new generation at once: nothing the failed attempt still has in flight commits from here on.
|
|
@@ -562,24 +1002,63 @@ async function runLeased(
|
|
|
562
1002
|
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
563
1003
|
}
|
|
564
1004
|
fence = next
|
|
565
|
-
|
|
1005
|
+
deadline = attemptEnd(nowOf(clock) + delay)
|
|
1006
|
+
await clock.wait(delay, control.signal)
|
|
566
1007
|
progress.attempts++
|
|
567
|
-
outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked,
|
|
1008
|
+
outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
|
|
1009
|
+
}
|
|
1010
|
+
const held = fence
|
|
1011
|
+
// The finish, unless the segment ended first; a finish once started is awaited, as it may be committing.
|
|
1012
|
+
const finish = async (status: 'succeeded' | 'failed', wakes: Promise<FinishWakes | undefined>): Promise<boolean | typeof OUT_OF_SEGMENT> => {
|
|
1013
|
+
const applied = await withinSegment(clock, segmentEnd, wakes)
|
|
1014
|
+
if (applied === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
|
|
1015
|
+
return OUT_OF_SEGMENT
|
|
1016
|
+
}
|
|
1017
|
+
return await leases.finish(held, status, applied)
|
|
568
1018
|
}
|
|
569
1019
|
if (outcome === null) {
|
|
570
|
-
|
|
1020
|
+
// Before the finish, so a crash between it and a kick's start leaves the kick's wake to the scheduler.
|
|
1021
|
+
const kicks = kicked.recorder
|
|
1022
|
+
for (const wake of kicks?.wakes(await leases.now()) ?? []) {
|
|
1023
|
+
await context.wakes.schedule(wake)
|
|
1024
|
+
}
|
|
1025
|
+
const finished = await finish('succeeded', completion(scheduled, context, durable))
|
|
1026
|
+
if (finished === OUT_OF_SEGMENT) {
|
|
1027
|
+
return outOfSegment(job, progress.attempts)
|
|
1028
|
+
}
|
|
1029
|
+
if (finished) {
|
|
571
1030
|
// After the lease is released, so a job that kicks another never waits on its own lease; a kick never rejects.
|
|
572
|
-
await Promise.all(kicked.map((name) => context.kick(name)))
|
|
1031
|
+
await Promise.all((kicks?.kicked() ?? []).map((name) => context.kick(name)))
|
|
573
1032
|
return { status: 'succeeded', attempts: progress.attempts }
|
|
574
1033
|
}
|
|
575
1034
|
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
576
1035
|
}
|
|
577
|
-
await leases.finish(fence, 'failed')
|
|
578
1036
|
if (outcome.code === 'stopped') {
|
|
579
|
-
//
|
|
1037
|
+
// Its wakes stay, so the alarm or the schedule runs it again. Not a failure of the job: Operations hears nothing.
|
|
1038
|
+
await leases.finish(fence, 'failed')
|
|
580
1039
|
console.error(`fabrika: job '${job.name}' stopped: the process is shutting down; its schedule runs it again`)
|
|
581
1040
|
return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
|
|
582
1041
|
}
|
|
1042
|
+
if (durable && !outcome.final && progress.attempts < attempts) {
|
|
1043
|
+
const at = new Date(nowOf(clock) + backoffMs(progress.attempts, clock.random()))
|
|
1044
|
+
console.error(
|
|
1045
|
+
`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}; it runs again at ${at.toISOString()}`,
|
|
1046
|
+
)
|
|
1047
|
+
const finished = await finish('failed', Promise.resolve({ kind: 'retry', at }))
|
|
1048
|
+
if (finished === OUT_OF_SEGMENT) {
|
|
1049
|
+
return outOfSegment(job, progress.attempts)
|
|
1050
|
+
}
|
|
1051
|
+
if (!finished) {
|
|
1052
|
+
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
1053
|
+
}
|
|
1054
|
+
return outcome.why === null
|
|
1055
|
+
? { status: 'failed', attempts: progress.attempts, code: outcome.code }
|
|
1056
|
+
: { status: 'failed', attempts: progress.attempts, code: outcome.code, reason: outcome.why.reason }
|
|
1057
|
+
}
|
|
1058
|
+
// Past its retries a run is done with its wakes: the next cron fire is scheduled, a kick's wake removed.
|
|
1059
|
+
if ((await finish('failed', completion(scheduled, context, durable))) === OUT_OF_SEGMENT) {
|
|
1060
|
+
return outOfSegment(job, progress.attempts)
|
|
1061
|
+
}
|
|
583
1062
|
return failed(job, progress.attempts, outcome, context.reporter)
|
|
584
1063
|
}
|
|
585
1064
|
|
|
@@ -618,14 +1097,27 @@ async function failed(job: { readonly name: string }, attempts: number, outcome:
|
|
|
618
1097
|
|
|
619
1098
|
const TIMED_OUT: unique symbol = Symbol('fabrika.job.timedOut')
|
|
620
1099
|
|
|
621
|
-
/**
|
|
622
|
-
|
|
1100
|
+
/**
|
|
1101
|
+
* One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's.
|
|
1102
|
+
* A run its control stopped is not a failure: the schedule runs it again. `acquired` is the lease the alarm's scheduler
|
|
1103
|
+
* took: such a run is done with the wakes it owns whether it succeeded or failed, as a framework job has no retries.
|
|
1104
|
+
*/
|
|
1105
|
+
async function runFrameworkJob(
|
|
1106
|
+
bound: BoundFrameworkJob,
|
|
1107
|
+
context: RunnerContext,
|
|
1108
|
+
clock: JobClock,
|
|
1109
|
+
control: RunControl,
|
|
1110
|
+
acquired: LeaseFence | null,
|
|
1111
|
+
): Promise<JobRunResult> {
|
|
623
1112
|
const { leases } = context
|
|
624
|
-
|
|
1113
|
+
const scheduled: Scheduled = { name: bound.name, expression: bound.job.schedule, schedule: bound.schedule }
|
|
1114
|
+
let fence: LeaseFence | null = acquired
|
|
625
1115
|
try {
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
1116
|
+
if (fence === null) {
|
|
1117
|
+
await leases.register(bound.name)
|
|
1118
|
+
await surveil(bound, context)
|
|
1119
|
+
fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
|
|
1120
|
+
}
|
|
629
1121
|
} catch {
|
|
630
1122
|
return failed(bound, 0, failure('driver_failed'), context.reporter)
|
|
631
1123
|
}
|
|
@@ -633,14 +1125,18 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
|
|
|
633
1125
|
console.error(`fabrika: a previous run of job '${bound.name}' still holds its lease; this run does not start`)
|
|
634
1126
|
return { status: 'skipped' }
|
|
635
1127
|
}
|
|
1128
|
+
// A run the alarm took the lease for has one segment, whose end is kept for the finish, as a contract job's.
|
|
1129
|
+
const segmentEnd = acquired === null ? null : nowOf(clock) + bound.job.timeoutMs
|
|
636
1130
|
const frame = new RequestFrame(fence.holder, { fence })
|
|
637
1131
|
const cancel = new AbortController()
|
|
638
1132
|
const timer = new AbortController()
|
|
639
1133
|
let outcome: JobFailure | null
|
|
640
1134
|
try {
|
|
641
1135
|
const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, (error: unknown) => failureOf(error, null))
|
|
642
|
-
|
|
643
|
-
const
|
|
1136
|
+
control.track(ran)
|
|
1137
|
+
const budget = segmentEnd === null ? bound.job.timeoutMs : bound.job.timeoutMs - finishReserveMs(bound.job.timeoutMs)
|
|
1138
|
+
const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
|
|
1139
|
+
const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
|
|
644
1140
|
outcome = result === TIMED_OUT ? failure('timeout') : result
|
|
645
1141
|
} finally {
|
|
646
1142
|
frame.end()
|
|
@@ -649,52 +1145,103 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
|
|
|
649
1145
|
try {
|
|
650
1146
|
// Writes are refused before the run hears it was stopped: a scanner that answers on abort commits nothing.
|
|
651
1147
|
try {
|
|
652
|
-
await leases.cancelInFlight(fence)
|
|
1148
|
+
await withinSegment(clock, segmentEnd, leases.cancelInFlight(fence))
|
|
653
1149
|
} finally {
|
|
654
1150
|
cancel.abort()
|
|
655
1151
|
}
|
|
656
|
-
|
|
1152
|
+
// A stopped run keeps its wakes, so the alarm runs it again once the object opens.
|
|
1153
|
+
const done = outcome === null || (acquired !== null && outcome.code !== 'stopped')
|
|
1154
|
+
const wakes = await withinSegment(clock, segmentEnd, done ? completion(scheduled, context, acquired !== null) : Promise.resolve(undefined))
|
|
1155
|
+
if (wakes === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
|
|
1156
|
+
return outOfSegment(bound, 1)
|
|
1157
|
+
}
|
|
1158
|
+
// Awaited once started: a finish may be committing.
|
|
1159
|
+
const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed', wakes)
|
|
657
1160
|
if (outcome === null) {
|
|
658
1161
|
return finished ? { status: 'succeeded', attempts: 1 } : failed(bound, 1, failure('lease_lost'), context.reporter)
|
|
659
1162
|
}
|
|
660
1163
|
} catch {
|
|
661
|
-
|
|
1164
|
+
// A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
|
|
1165
|
+
if (!control.signal.aborted) {
|
|
1166
|
+
return failed(bound, 1, failure('driver_failed'), context.reporter)
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
// Here a run without a failure of its own is one whose stop refused its lease write.
|
|
1170
|
+
if (outcome === null || outcome.code === 'stopped' || control.signal.aborted) {
|
|
1171
|
+
console.error(`fabrika: job '${bound.name}' stopped: the object entered maintenance; its schedule runs it again`)
|
|
1172
|
+
return { status: 'failed', attempts: 1, code: 'stopped' }
|
|
662
1173
|
}
|
|
663
1174
|
return failed(bound, 1, outcome, context.reporter)
|
|
664
1175
|
}
|
|
665
1176
|
|
|
666
1177
|
/** Read afresh at each call: the signal may abort while the caller awaits. */
|
|
667
|
-
const isStopped = (
|
|
1178
|
+
const isStopped = (control: RunControl): boolean => control.signal.aborted
|
|
1179
|
+
|
|
1180
|
+
/** Resolves `stopped` once the run's control aborts; the listener goes when `done` aborts. */
|
|
1181
|
+
function stoppedBy(control: RunControl, done: AbortSignal): Promise<JobFailure> {
|
|
1182
|
+
return new Promise<JobFailure>((resolve) => {
|
|
1183
|
+
if (control.signal.aborted) {
|
|
1184
|
+
resolve(failure('stopped'))
|
|
1185
|
+
}
|
|
1186
|
+
control.signal.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: done })
|
|
1187
|
+
})
|
|
1188
|
+
}
|
|
668
1189
|
|
|
669
1190
|
/**
|
|
670
|
-
* `null` when the handler finished, else the attempt's failure code.
|
|
671
|
-
*
|
|
1191
|
+
* `null` when the handler finished, else the attempt's failure code. One deadline bounds the attempt: IAM's answer for
|
|
1192
|
+
* the job's key, then the handler with only what is left of it. Either way the attempt's frame ends and the statements
|
|
1193
|
+
* it still has running are cancelled before the runner touches the lease again, waited for only while the segment lasts.
|
|
672
1194
|
*/
|
|
673
1195
|
async function attemptOnce(
|
|
674
1196
|
job: BoundJob,
|
|
675
1197
|
fence: LeaseFence,
|
|
676
1198
|
attempt: number,
|
|
1199
|
+
deadline: number,
|
|
1200
|
+
segmentEnd: number | null,
|
|
677
1201
|
clock: JobClock,
|
|
678
1202
|
context: RunnerContext,
|
|
679
|
-
kicked:
|
|
680
|
-
|
|
1203
|
+
kicked: Kicked,
|
|
1204
|
+
control: RunControl,
|
|
681
1205
|
): Promise<JobFailure | null> {
|
|
682
|
-
kicked.
|
|
683
|
-
if (isStopped(
|
|
1206
|
+
kicked.recorder = null
|
|
1207
|
+
if (isStopped(control)) {
|
|
684
1208
|
return failure('stopped')
|
|
685
1209
|
}
|
|
686
1210
|
// The job's authority is what IAM signs for its key at the start of the attempt; a revoked key stops the next one.
|
|
687
|
-
const
|
|
688
|
-
|
|
1211
|
+
const setup = new AbortController()
|
|
1212
|
+
const exchange = job.credential.authority()
|
|
1213
|
+
// Its settlement, not only the attempt's: a drain waits for an exchange the attempt stopped waiting for.
|
|
1214
|
+
control.track(exchange)
|
|
1215
|
+
let authority: WorkloadAuthority
|
|
1216
|
+
try {
|
|
1217
|
+
const answered = await Promise.race([
|
|
1218
|
+
exchange,
|
|
1219
|
+
clock.wait(remainingMs(clock, deadline), setup.signal).then((): typeof TIMED_OUT => TIMED_OUT),
|
|
1220
|
+
stoppedBy(control, setup.signal),
|
|
1221
|
+
])
|
|
1222
|
+
if (answered === TIMED_OUT || 'code' in answered) {
|
|
1223
|
+
// The next attempt asks IAM afresh rather than wait on an exchange that outlived this one.
|
|
1224
|
+
job.credential.abandon(exchange)
|
|
1225
|
+
return answered === TIMED_OUT ? failure('timeout') : answered
|
|
1226
|
+
}
|
|
1227
|
+
authority = answered
|
|
1228
|
+
} finally {
|
|
1229
|
+
setup.abort()
|
|
1230
|
+
}
|
|
1231
|
+
if (isStopped(control)) {
|
|
689
1232
|
return failure('stopped')
|
|
690
1233
|
}
|
|
691
1234
|
if (!authority.ok) {
|
|
692
1235
|
return failure(authority.reason === 'unavailable' ? 'iam_unavailable' : 'unauthorized')
|
|
693
1236
|
}
|
|
1237
|
+
const budget = remainingMs(clock, deadline)
|
|
1238
|
+
if (budget === 0) {
|
|
1239
|
+
return failure('timeout')
|
|
1240
|
+
}
|
|
694
1241
|
const frame = new RequestFrame(fence.holder, { fence })
|
|
695
1242
|
const cancel = new AbortController()
|
|
696
1243
|
const timer = new AbortController()
|
|
697
|
-
const signal =
|
|
1244
|
+
const signal = AbortSignal.any([cancel.signal, control.signal])
|
|
698
1245
|
const kicks = kickRecorder(`job '${job.name}'`, job.site?.kicks ?? [])
|
|
699
1246
|
const ctx = { ...jobContext(job, frame, attempt, signal, context, authority.perms), kick: kicks.kick }
|
|
700
1247
|
try {
|
|
@@ -705,26 +1252,22 @@ async function attemptOnce(
|
|
|
705
1252
|
} else {
|
|
706
1253
|
await runAsSite(`job '${job.name}'`, siteEgress(site.egress, context.publicEgress), async () => await job.handler({ ctx }))
|
|
707
1254
|
}
|
|
708
|
-
kicked.
|
|
1255
|
+
kicked.recorder = kicks
|
|
709
1256
|
}).then(() => null, (error: unknown) => {
|
|
710
1257
|
const failed = failureOf(error, job.mailer === null ? null : (context.mail[job.mailer]?.sender.capabilities ?? null))
|
|
711
1258
|
if (failed.code === 'handler_failed') context.revealHandlerErrors?.(`job '${job.name}' attempt ${attempt}`, error)
|
|
712
1259
|
return failed
|
|
713
1260
|
})
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
}
|
|
719
|
-
stop?.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: timer.signal })
|
|
720
|
-
})
|
|
721
|
-
const result = await Promise.race([ran, timeout, stopped])
|
|
1261
|
+
// Its settlement, not only the end of its frame: a drain waits for a handler that outlived the attempt.
|
|
1262
|
+
control.track(ran)
|
|
1263
|
+
const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
|
|
1264
|
+
const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
|
|
722
1265
|
return result === TIMED_OUT ? failure('timeout') : result
|
|
723
1266
|
} finally {
|
|
724
1267
|
frame.end()
|
|
725
1268
|
cancel.abort()
|
|
726
1269
|
timer.abort()
|
|
727
|
-
await context.leases.cancelInFlight(fence)
|
|
1270
|
+
await withinSegment(clock, segmentEnd, context.leases.cancelInFlight(fence))
|
|
728
1271
|
}
|
|
729
1272
|
}
|
|
730
1273
|
|